diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl.sln b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl.sln deleted file mode 100644 index 207c24511b..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl.sln +++ /dev/null @@ -1,37 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29920.165 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ADPControl", "ADPControl\ADPControl.csproj", "{6309D4C5-F118-4C89-A67E-E557CA41ABA2}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BatchWrapper", "BatchWrapper\BatchWrapper.csproj", "{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlPackageWrapper", "SqlPackageWrapper\SqlPackageWrapper.csproj", "{A19335D3-9D80-43BE-9351-C3C3704689B0}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Release|Any CPU.ActiveCfg = Debug|Any CPU - {6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Release|Any CPU.Build.0 = Debug|Any CPU - {E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Release|Any CPU.ActiveCfg = Debug|Any CPU - {E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Release|Any CPU.Build.0 = Debug|Any CPU - {A19335D3-9D80-43BE-9351-C3C3704689B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A19335D3-9D80-43BE-9351-C3C3704689B0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A19335D3-9D80-43BE-9351-C3C3704689B0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A19335D3-9D80-43BE-9351-C3C3704689B0}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {3DB5F0CB-1D26-4E33-801D-7BBAE823C39D} - EndGlobalSection -EndGlobal diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/ADPControl.csproj b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/ADPControl.csproj deleted file mode 100644 index 730cc4b42b..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/ADPControl.csproj +++ /dev/null @@ -1,26 +0,0 @@ - - - netcoreapp3.1 - v3 - - - - - - - - - - - - - - - PreserveNewest - - - PreserveNewest - Never - - - \ No newline at end of file diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/AzureResourceManagerActivity.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/AzureResourceManagerActivity.cs deleted file mode 100644 index 21830d76d1..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/AzureResourceManagerActivity.cs +++ /dev/null @@ -1,244 +0,0 @@ -using Microsoft.Azure.Management.ResourceManager; -using Microsoft.Azure.Management.ResourceManager.Models; -using Microsoft.Azure.Management.Storage; -using Microsoft.Azure.Management.Storage.Models; -using Microsoft.Azure.Services.AppAuthentication; -using Microsoft.Azure.WebJobs; -using Microsoft.Azure.WebJobs.Extensions.DurableTask; -using Microsoft.Extensions.Logging; -using Microsoft.Rest; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Auth; -using Microsoft.WindowsAzure.Storage.Blob; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using static ADPControl.HttpSurface; - -namespace ADPControl -{ - public static class AzureResourceManagerActivity - { - private const string ArmTemplateFileName = "template.json"; - - private static string[] AllowedImportSubServerResourceTypes = new string[] { - "Microsoft.Sql/servers/firewallRules", - "Microsoft.Sql/servers/databases", - "Microsoft.Sql/servers/elasticPools", - //"Microsoft.Sql/servers/keys", - //"Microsoft.Sql/servers/databases/transparentDataEncryption", - "Microsoft.Sql/servers/databases/backupShortTermRetentionPolicies", - "Microsoft.Sql/servers/administrators" - }; - - // Deploy the ARM template - [FunctionName(nameof(BeginDeployArmTemplateForImport))] - public static async Task BeginDeployArmTemplateForImport([ActivityTrigger] ImportRequest request, ILogger log) - { - var azureServiceTokenProvider = new AzureServiceTokenProvider(); - TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/")); - - ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() }; - StorageManagementClient storageMgmtClient = new StorageManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() }; - - // Get the storage account keys for a given account and resource group - IList acctKeys = storageMgmtClient.StorageAccounts.ListKeys(request.ResourceGroupName, request.StorageAccountName).Keys; - - // Get a Storage account using account creds: - StorageCredentials storageCred = new StorageCredentials(request.StorageAccountName, acctKeys.FirstOrDefault().Value); - CloudStorageAccount linkedStorageAccount = new CloudStorageAccount(storageCred, true); - CloudBlobContainer container = linkedStorageAccount - .CreateCloudBlobClient() - .GetContainerReference(request.ContainerName); - - CloudBlockBlob blob = container.GetBlockBlobReference(ArmTemplateFileName); - string json = await blob.DownloadTextAsync(); - - JObject originalTemplate = JObject.Parse(json); - JObject importTemplate = UpdateArmTemplateForImport(originalTemplate, request); - - var deployParams = new Deployment - { - Properties = new DeploymentProperties - { - Mode = DeploymentMode.Incremental, - Template = importTemplate - } - }; - - string deploymentName = request.TargetSqlServerName + "_" + DateTime.UtcNow.ToFileTimeUtc(); - - try - { - await resourcesClient.Deployments.BeginCreateOrUpdateAsync(request.TargetSqlServerResourceGroupName, deploymentName, deployParams); - } - catch (Exception ex) - { - log.LogError(ex.ToString()); - throw ex; - } - - return deploymentName; - } - - // Get the ARM deployment status - [FunctionName(nameof(GetArmDeploymentForImport))] - public static async Task GetArmDeploymentForImport([ActivityTrigger] (Guid, string, string) input) - { - Guid subscriptionId = input.Item1; - string resourceGroupName = input.Item2; - string deploymentName = input.Item3; - - var azureServiceTokenProvider = new AzureServiceTokenProvider(); - TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/")); - ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = subscriptionId.ToString() }; - - DeploymentExtended result = await resourcesClient.Deployments.GetAsync(resourceGroupName, deploymentName); - return result.Properties.ProvisioningState; - } - - // Get the ARM template without the parameter of the resource name - [FunctionName(nameof(GetArmTemplateForExportSkipParameterization))] - public static async Task GetArmTemplateForExportSkipParameterization([ActivityTrigger] ExportRequest request, ILogger log) - { - log.LogInformation("GetArmTemplateForExportSkipParameterization: entering"); - - var azureServiceTokenProvider = new AzureServiceTokenProvider(); - TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/")); - if (tokenArmCredential != null) - { - log.LogInformation("GetArmTemplateForExportSkipParameterization: acquired access token"); - ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() }; - - string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.SourceSqlServerResourceGroupName, request.SourceSqlServerName); - ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.SourceSqlServerResourceGroupName, new ExportTemplateRequest(new List { sourceSqlServerResourceId }, "SkipResourceNameParameterization")); - - log.LogInformation("GetArmTemplateForExportSkipParameterization: server template exported. Size: {0} bytes", exportedTemplate.Template.ToString().Length); - dynamic template = (dynamic)exportedTemplate.Template; - - // Filtering the list of databases - dynamic databases = template.resources.SelectTokens("$.[?(@.type == 'Microsoft.Sql/servers/databases')]"); - int numberOfDatabases = 0; - foreach (var db in databases) - { - numberOfDatabases++; - } - log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with database list. Databases count: {0}", numberOfDatabases); - - return databases; - } - - log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with empty database list"); - return null; - } - - // Get the ARM template without the parameter of the resource name - [FunctionName(nameof(GetArmTemplateForImportSkipParameterization))] - public static async Task GetArmTemplateForImportSkipParameterization([ActivityTrigger] ImportRequest request, ILogger log) - { - var azureServiceTokenProvider = new AzureServiceTokenProvider(); - TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/")); - if (tokenArmCredential != null) - { - log.LogInformation("GetArmTemplateForImportSkipParameterization: acquired access token"); - ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() }; - - string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.TargetSqlServerResourceGroupName, request.TargetSqlServerName); - ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.TargetSqlServerResourceGroupName, new ExportTemplateRequest(new List { sourceSqlServerResourceId }, "SkipResourceNameParameterization")); - - log.LogInformation("GetArmTemplateForImportSkipParameterization: server template exported. Size: {0} bytes", exportedTemplate.Template.ToString().Length); - dynamic template = (dynamic)exportedTemplate.Template; - - // Filtering the list of databases - dynamic databases = template.resources.SelectTokens("$.[?(@.type == 'Microsoft.Sql/servers/databases')]"); - - int numberOfDatabases = 0; - foreach (var db in databases) - { - numberOfDatabases++; - } - log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with database list. Databases count: {0}", numberOfDatabases); - return databases; - } - - return null; - } - - [FunctionName(nameof(GetArmTemplateForExport))] - public static async Task GetArmTemplateForExport([ActivityTrigger] ExportRequest request) - { - var azureServiceTokenProvider = new AzureServiceTokenProvider(); - TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/")); - - ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() }; - - string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.SourceSqlServerResourceGroupName, request.SourceSqlServerName); - ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.SourceSqlServerResourceGroupName, new ExportTemplateRequest(new List { sourceSqlServerResourceId }, "IncludeParameterDefaultValue")); - return exportedTemplate.Template; - } - - private static JObject UpdateArmTemplateForImport(JObject originalTemplate, ImportRequest request) - { - string serverNameParameterName = null; - - // Go through every parameter to find the property name is like 'server_%_name' - using (JsonTextReader reader = new JsonTextReader(new StringReader(originalTemplate["parameters"].ToString()))) - { - while (reader.Read()) - { - if (reader.TokenType.ToString().Equals("PropertyName") - && reader.ValueType.ToString().Equals("System.String") - && reader.Value.ToString().StartsWith("servers_") - && reader.Value.ToString().EndsWith("_name")) - { - serverNameParameterName = reader.Value.ToString(); - break; - } - } - } - - // 1. Replacing the default value to the target server name, appending to the new template - originalTemplate["parameters"][serverNameParameterName]["defaultValue"] = request.TargetSqlServerName; - JObject serverNameParameterValue = (JObject)originalTemplate["parameters"][serverNameParameterName]; - - // 2. Cleanup all the parameters except the updated server name - ((JObject)originalTemplate["parameters"]).RemoveAll(); - ((JObject)originalTemplate["parameters"]).Add(serverNameParameterName, serverNameParameterValue); - - // 3. Adjust the servers resource by adding password after the login - JObject server = (JObject)originalTemplate["resources"] - .SelectToken("$.[?(@.type == 'Microsoft.Sql/servers')]"); - - server.Remove("identity"); - - JObject serverProperties = (JObject)server["properties"]; - serverProperties.Property("administratorLogin") - .AddAfterSelf(new JProperty("administratorLoginPassword", request.SqlAdminPassword)); - - JArray newResources = new JArray(); - - // 4. Getting the whitelisted resources and adding them to the new template later. - foreach (string resourceType in AllowedImportSubServerResourceTypes) - { - List resources = originalTemplate["resources"] - .SelectTokens(string.Format("$.[?(@.type == '{0}')]", resourceType)).ToList(); - newResources.Add(resources); - } - - // 5. Clean up all the resources excepted the new server and whitelisted resource type. - ((JArray)originalTemplate["resources"]).Clear(); - ((JArray)originalTemplate["resources"]).Add(server); - - foreach (var resource in newResources) - { - ((JArray)originalTemplate["resources"]).Add(resource); - } - - return originalTemplate; - } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/BatchActivity.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/BatchActivity.cs deleted file mode 100644 index 2b84b66c99..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/BatchActivity.cs +++ /dev/null @@ -1,214 +0,0 @@ -using Microsoft.Azure.Batch; -using Microsoft.Azure.Batch.Auth; -using Microsoft.Azure.Batch.Common; -using Microsoft.Azure.Services.AppAuthentication; -using Microsoft.Azure.WebJobs; -using Microsoft.Azure.WebJobs.Extensions.DurableTask; -using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using static ADPControl.HttpSurface; - -namespace ADPControl -{ - public static class BatchActivity - { - // Batch resource settings - private const string PoolVMSize = "Standard_D8s_v3"; - private const string PoolId = PoolVMSize; - private const int PoolNodeCount = 2; - private const string AppPackageName = "SqlPackageWrapper"; - public const string AppPackageVersion = "1"; - - [FunctionName(nameof(CreateBatchPoolAndExportJob))] - public static async Task CreateBatchPoolAndExportJob([ActivityTrigger] ExportRequest request, ILogger log) - { - var azureServiceTokenProvider = new AzureServiceTokenProvider(); - - // Get a Batch client using function identity - BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/")); - - string jobId = request.SourceSqlServerName + "-Export-" + DateTime.UtcNow.ToString("MMddHHmmss"); - using (BatchClient batchClient = BatchClient.Open(batchCred)) - { - ImageReference imageReference = CreateImageReference(); - VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference); - - await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId); - await CreateBatchJob(batchClient, jobId, log); - } - - return jobId; - } - - [FunctionName(nameof(CreateBatchPoolAndImportJob))] - public static async Task CreateBatchPoolAndImportJob([ActivityTrigger] ImportRequest request, ILogger log) - { - var azureServiceTokenProvider = new AzureServiceTokenProvider(); - - // Get a Batch client using function identity - BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/")); - - string jobId = request.TargetSqlServerName + "-Import-" + DateTime.UtcNow.ToString("MMddHHmmss"); - using (BatchClient batchClient = BatchClient.Open(batchCred)) - { - ImageReference imageReference = CreateImageReference(); - VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference); - - await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId); - await CreateBatchJob(batchClient, jobId, log); - } - - return jobId; - } - - public static async Task CreateBatchJob(BatchClient batchClient, string jobId, ILogger log) - { - // Create a Batch job - log.LogInformation("Creating job [{0}]...", jobId); - CloudJob job = null; - - try - { - job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = PoolId }); - job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob; - - // Commit the job to the Batch service - await job.CommitAsync(); - - log.LogInformation($"Created job {jobId}"); - - // Obtain the bound job from the Batch service - await job.RefreshAsync(); - } - catch (BatchException be) - { - // Accept the specific error code JobExists as that is expected if the job already exists - if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists) - { - log.LogWarning("The job {0} already existed when we tried to create it", jobId); - } - else - { - log.LogError("Exception creating job: {0}", be.Message); - throw be; // Any other exception is unexpected - } - } - - return job; - } - - // Create the Compute Pool of the Batch Account - public static async Task CreateBatchPoolIfNotExist(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration, string vnetSubnetId) - { - Console.WriteLine("Creating pool [{0}]...", PoolId); - - try - { - CloudPool pool = batchClient.PoolOperations.CreatePool( - poolId: PoolId, - targetDedicatedComputeNodes: PoolNodeCount, - virtualMachineSize: PoolVMSize, - virtualMachineConfiguration: vmConfiguration); - - // Specify the application and version to install on the compute nodes - pool.ApplicationPackageReferences = new List - { - new ApplicationPackageReference { - ApplicationId = AppPackageName, - Version = AppPackageVersion } - }; - - // Initial the first data disk for each VM in the pool - StartTask startTask = new StartTask("cmd /c Powershell -command \"Get-Disk | Where partitionstyle -eq 'raw' | sort number | Select-Object -first 1 |" + - " Initialize-Disk -PartitionStyle MBR -PassThru | New-Partition -UseMaximumSize -DriveLetter F |" + - " Format-Volume -FileSystem NTFS -NewFileSystemLabel data1 -Confirm:$false -Force\""); - - startTask.MaxTaskRetryCount = 1; - startTask.UserIdentity = new UserIdentity(new AutoUserSpecification(AutoUserScope.Pool, ElevationLevel.Admin)); - startTask.WaitForSuccess = true; - - pool.StartTask = startTask; - - // Create the Pool within the vnet subnet if it's specified. - if (vnetSubnetId != null) - { - pool.NetworkConfiguration = new NetworkConfiguration(); - pool.NetworkConfiguration.SubnetId = vnetSubnetId; - } - - await pool.CommitAsync(); - await pool.RefreshAsync(); - } - catch (BatchException be) - { - // Accept the specific error code PoolExists as that is expected if the pool already exists - if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) - { - Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId); - } - else - { - throw; // Any other exception is unexpected - } - } - } - - public static VirtualMachineConfiguration CreateVirtualMachineConfiguration(ImageReference imageReference) - { - VirtualMachineConfiguration config = new VirtualMachineConfiguration( - imageReference: imageReference, - nodeAgentSkuId: "batch.node.windows amd64"); - - config.DataDisks = new List(); - config.DataDisks.Add(new DataDisk(0, 2048, CachingType.ReadOnly, StorageAccountType.PremiumLrs)); - - return config; - } - - public static ImageReference CreateImageReference() - { - return new ImageReference( - publisher: "MicrosoftWindowsServer", - offer: "WindowsServer", - sku: "2019-datacenter-smalldisk", - version: "latest"); - } - - public static void CreateBatchTasks(string action, string jobId, string containerUrl, string batchAccountUrl, string sqlServerName, string accessToken, dynamic databases, ILogger log) - { - // Get a Batch client using function identity - log.LogInformation("CreateBatchTasks: entering"); - var azureServiceTokenProvider = new AzureServiceTokenProvider(); - BatchTokenCredentials batchCred = new BatchTokenCredentials(batchAccountUrl, azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/").Result); - using (BatchClient batchClient = BatchClient.Open(batchCred)) - { - // For each database, submit the Exporting job to Azure Batch Compute Pool. - log.LogInformation("CreateBatchTasks: enumerating databases"); - List tasks = new List(); - foreach (var db in databases) - { - string serverDatabaseName = db.name.ToString(); - string logicalDatabase = serverDatabaseName.Remove(0, sqlServerName.Length + 1); - - log.LogInformation("CreateBatchTasks: creating task for database {0}", logicalDatabase); - string taskId = sqlServerName + "_" + logicalDatabase; - string command = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_{0}#{1}%\\BatchWrapper {2}", AppPackageName.ToUpper(), AppPackageVersion, action); - command += string.Format(" {0} {1} {2} {3} {4}", sqlServerName, logicalDatabase, accessToken, AppPackageName.ToUpper(), AppPackageVersion); - string taskCommandLine = string.Format(command); - - CloudTask singleTask = new CloudTask(taskId, taskCommandLine); - singleTask.EnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) }; - - Console.WriteLine(string.Format("Adding task {0} to job ...", taskId)); - tasks.Add(singleTask); - } - - // Add all tasks to the job. - batchClient.JobOperations.AddTask(jobId, tasks); - } - log.LogInformation("CreateBatchTasks: exiting"); - } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/HttpSurface.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/HttpSurface.cs deleted file mode 100644 index 2dcacb9142..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/HttpSurface.cs +++ /dev/null @@ -1,103 +0,0 @@ -using Microsoft.Azure.WebJobs; -using Microsoft.Azure.WebJobs.Extensions.DurableTask; -using Microsoft.Azure.WebJobs.Extensions.Http; -using Microsoft.Extensions.Logging; -using System; -using System.Net.Http; -using System.Threading.Tasks; - -namespace ADPControl -{ - public static class HttpSurface - { - public class ExportRequest - { - public Guid SubscriptionId { get; set; } - - public string ResourceGroupName { get; set; } - - public string SourceSqlServerResourceGroupName { get; set; } - - public string SourceSqlServerName { get; set; } - - public string BatchAccountUrl { get; set; } - - public string StorageAccountName { get; set; } - - public string AccessToken { get; set; } - - public string VNetSubnetId { get; set; } - } - - public class ImportRequest - { - public Guid SubscriptionId { get; set; } - - public string ResourceGroupName { get; set; } - - public string TargetSqlServerResourceGroupName { get; set; } - - public string TargetSqlServerName { get; set; } - - public string TargetAccessToken { get; set; } - - public string BatchAccountUrl { get; set; } - - public string StorageAccountName { get; set; } - - public string ContainerName { get; set; } - - public string SqlAdminPassword { get; set; } - - public string VNetSubnetId { get; set; } - } - - [FunctionName("Export")] - public static async Task PostExport( - [HttpTrigger(AuthorizationLevel.Function, "post", Route = "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/Export")] - HttpRequestMessage req, - [DurableClient] IDurableOrchestrationClient starter, - ILogger log, - Guid subscriptionId, - string resourceGroupName) - { - log.LogInformation("C# HTTP trigger function processed an Export request."); - ExportRequest request = await req.Content.ReadAsAsync(); - - request.SubscriptionId = subscriptionId; - request.ResourceGroupName = resourceGroupName; - - if (request.SourceSqlServerResourceGroupName == null) - request.SourceSqlServerResourceGroupName = resourceGroupName; - - string instanceId = await starter.StartNewAsync(nameof(Orchestrator.RunExportOrchestrator), request); - - log.LogInformation($"Started orchestration with ID = '{instanceId}'."); - return starter.CreateCheckStatusResponse(req, instanceId); - } - - [FunctionName("Import")] - public static async Task PostImport( - [HttpTrigger(AuthorizationLevel.Function, "post", Route = "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/Import")] - HttpRequestMessage req, - [DurableClient] IDurableOrchestrationClient starter, - ILogger log, - Guid subscriptionId, - string resourceGroupName) - { - log.LogInformation("C# HTTP trigger function processed an Import request."); - ImportRequest request = await req.Content.ReadAsAsync(); - - request.SubscriptionId = subscriptionId; - request.ResourceGroupName = resourceGroupName; - - if (request.TargetSqlServerResourceGroupName == null) - request.TargetSqlServerResourceGroupName = resourceGroupName; - - string instanceId = await starter.StartNewAsync(nameof(Orchestrator.RunImportOrchestrator), request); - - log.LogInformation($"Started orchestration with ID = '{instanceId}'."); - return starter.CreateCheckStatusResponse(req, instanceId); - } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/Orchestrator.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/Orchestrator.cs deleted file mode 100644 index 9794aefba2..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/Orchestrator.cs +++ /dev/null @@ -1,104 +0,0 @@ -using Microsoft.Azure.WebJobs; -using Microsoft.Azure.WebJobs.Extensions.DurableTask; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using System; -using System.Threading; -using System.Threading.Tasks; -using static ADPControl.HttpSurface; - -namespace ADPControl -{ - public static class Orchestrator - { - // The Import Orchestrator - [FunctionName(nameof(RunImportOrchestrator))] - public static async Task RunImportOrchestrator( - [OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log) - { - log.LogInformation("RunImportOrchestrator: entering"); - - try { - - ImportRequest importRequest = context.GetInput(); - // Deploy the ARM template to Create empty SQL resource - string deploymentName = await context.CallActivityAsync(nameof(AzureResourceManagerActivity.BeginDeployArmTemplateForImport), importRequest); - while (true) - { - log.LogInformation("RunImportOrchestrator: starting ARM deployment"); - string status = await context.CallActivityAsync(nameof(AzureResourceManagerActivity.GetArmDeploymentForImport), (importRequest.SubscriptionId, importRequest.TargetSqlServerResourceGroupName, deploymentName)); - if (status == "Succeeded") - { - log.LogInformation("RunImportOrchestrator: ARM deployment succeeded"); - break; - } - else if (status == "Failed") - { - log.LogInformation("RunImportOrchestrator: ARM deployment failed"); - throw new Exception("Failed ARM Deployment"); - } - - // Orchestration sleeps until this time. - var nextCheck = context.CurrentUtcDateTime.AddSeconds(10); - - if (!context.IsReplaying) { log.LogInformation($"RunImportOrchestrator: Replaying ARM deployment, next check at {nextCheck}."); } - await context.CreateTimer(nextCheck, CancellationToken.None); - } - - log.LogInformation("RunImportOrchestrator: Enumerating databases"); - var databases = await context.CallActivityAsync(nameof(AzureResourceManagerActivity.GetArmTemplateForImportSkipParameterization), importRequest); - - // Create BatchPool And Job - log.LogInformation("RunImportOrchestrator: Creating batch pool and import job"); - string jobId = await context.CallActivityAsync(nameof(BatchActivity.CreateBatchPoolAndImportJob), importRequest); - - string containerUrl = await context.CallActivityAsync(nameof(StorageActivity.GettingJobContainerUrl), (importRequest.SubscriptionId, importRequest.ResourceGroupName, importRequest.StorageAccountName, importRequest.ContainerName)); - - log.LogInformation("RunImportOrchestrator: Creating import database tasks"); - BatchActivity.CreateBatchTasks("Import", jobId, containerUrl, importRequest.BatchAccountUrl, importRequest.TargetSqlServerName, importRequest.TargetAccessToken, databases, log); - - // create output values - Tuple[] outputValues = { - Tuple.Create("Orchestration progress:", "Complete"), - Tuple.Create("deploymentName", deploymentName), - Tuple.Create("jobId", jobId), - Tuple.Create("containerUrl", containerUrl) - }; - context.SetOutput(outputValues); - } - finally { - log.LogInformation("RunImportOrchestrator: exiting"); - } - } - - // The Export Orchestrator - [FunctionName(nameof(RunExportOrchestrator))] - public static async Task RunExportOrchestrator( - [OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log) - { - ExportRequest exportRequest = context.GetInput(); - - // Getting the ARM template Skip ResourceName Parameterization. - var databases = await context.CallActivityAsync(nameof(AzureResourceManagerActivity.GetArmTemplateForExportSkipParameterization), exportRequest); - - // Getting the ARM template. - dynamic Template = await context.CallActivityAsync(nameof(AzureResourceManagerActivity.GetArmTemplateForExport), exportRequest); - string json = JsonConvert.SerializeObject(Template); - - // Create BatchPool And Job - string jobId = await context.CallActivityAsync(nameof(BatchActivity.CreateBatchPoolAndExportJob), exportRequest); - - string containerUrl = await context.CallActivityAsync(nameof(StorageActivity.GettingJobContainerUrl), (exportRequest.SubscriptionId, exportRequest.ResourceGroupName, exportRequest.StorageAccountName, jobId)); - await context.CallActivityAsync(nameof(StorageActivity.UploadingArmTemplate), (containerUrl, json)); - - BatchActivity.CreateBatchTasks("Export", jobId, containerUrl, exportRequest.BatchAccountUrl, exportRequest.SourceSqlServerName, exportRequest.AccessToken, databases, log); - - // create output values - Tuple[] outputValues = { - Tuple.Create("jobId", jobId), - Tuple.Create("containerUrl", containerUrl) - }; - context.SetOutput(outputValues); - } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/StorageActivity.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/StorageActivity.cs deleted file mode 100644 index d56791621c..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/StorageActivity.cs +++ /dev/null @@ -1,79 +0,0 @@ -using Microsoft.Azure.Management.Storage; -using Microsoft.Azure.Management.Storage.Models; -using Microsoft.Azure.Services.AppAuthentication; -using Microsoft.Azure.WebJobs; -using Microsoft.Azure.WebJobs.Extensions.DurableTask; -using Microsoft.Extensions.Logging; -using Microsoft.Rest; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Auth; -using Microsoft.WindowsAzure.Storage.Blob; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; - -namespace ADPControl -{ - public static class StorageActivity - { - private const string ArmTemplateFileName = "template.json"; - - [FunctionName(nameof(GettingJobContainerUrl))] - public static string GettingJobContainerUrl([ActivityTrigger] (Guid, string, string, string) input, ILogger log) - { - Guid SubscriptionId = input.Item1; - String ResourceGroupName = input.Item2; - String StorageAccountName = input.Item3; - String ContainerName = input.Item4; - - var azureServiceTokenProvider = new AzureServiceTokenProvider(); - TokenCredentials tokenArmCredential = new TokenCredentials(azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/").Result); - StorageManagementClient storageMgmtClient = new StorageManagementClient(tokenArmCredential) { SubscriptionId = SubscriptionId.ToString() }; - - // Get the storage account keys for a given account and resource group - IList acctKeys = storageMgmtClient.StorageAccounts.ListKeys(ResourceGroupName, StorageAccountName).Keys; - - // Get a Storage account using account creds: - StorageCredentials storageCred = new StorageCredentials(StorageAccountName, acctKeys.FirstOrDefault().Value); - CloudStorageAccount linkedStorageAccount = new CloudStorageAccount(storageCred, true); - - bool createContainer = false; - // Normalize the container name for the Export action. - if (ContainerName.Contains("-Export-")) - { - ContainerName = ContainerName.Replace("Export-", ""); - createContainer = true; - } - - CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(ContainerName); - - if(createContainer) - container.CreateIfNotExistsAsync().Wait(); - - string containerUrl = container.Uri.ToString() + - container.GetSharedAccessSignature(new SharedAccessBlobPolicy() - { - Permissions = SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List, - SharedAccessExpiryTime = DateTime.UtcNow.AddDays(7) - }); - return containerUrl; - } - - [FunctionName(nameof(UploadingArmTemplate))] - public static void UploadingArmTemplate([ActivityTrigger] (string, string) input, ILogger log) - { - string containerUrl = input.Item1; - string json = input.Item2; - - CloudBlobContainer container = new CloudBlobContainer(new Uri(containerUrl)); - CloudBlockBlob blob = container.GetBlockBlobReference(ArmTemplateFileName); - blob.Properties.ContentType = "application/json"; - using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(json))) - { - blob.UploadFromStreamAsync(stream).Wait(); - } - } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/host.json b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/host.json deleted file mode 100644 index ee986a759d..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/host.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "version": "2.0", - "extensions": { - "durableTask": { - "hubName": "adp" - } - }, - "logging": { - "applicationInsights": { - "samplingExcludedTypes": "Request", - "samplingSettings": { - "isEnabled": true - } - } - } -} \ No newline at end of file diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/ActionType.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/ActionType.cs deleted file mode 100644 index 28602c922e..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/ActionType.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace BatchWrapper -{ - /// - /// The type of sqlpackage action to perform. - /// - public enum ActionType - { - DefaultInvalid = -1, - Export, - Import - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/App.config b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/App.config deleted file mode 100644 index 3617b3e032..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/App.config +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/BatchWrapper.csproj b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/BatchWrapper.csproj deleted file mode 100644 index bbbcf923de..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/BatchWrapper.csproj +++ /dev/null @@ -1,98 +0,0 @@ - - - - - Debug - AnyCPU - {E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1} - Exe - BatchWrapper - BatchWrapper - v4.7.2 - 8.0 - 512 - true - true - false - publish\ - true - Disk - false - Foreground - 7 - Days - false - false - true - 0 - 1.0.0.%2a - false - true - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\Microsoft.Azure.Batch.Conventions.Files.3.5.1\lib\net461\Microsoft.Azure.Batch.Conventions.Files.dll - - - ..\packages\WindowsAzure.Storage.9.3.3\lib\net45\Microsoft.WindowsAzure.Storage.dll - - - - - - - - - - - - - - - - - - - - - - - - - - - - False - Microsoft .NET Framework 4.7.2 %28x86 and x64%29 - true - - - False - .NET Framework 3.5 SP1 - false - - - - - - - \ No newline at end of file diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Constants.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Constants.cs deleted file mode 100644 index ded7e29970..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Constants.cs +++ /dev/null @@ -1,34 +0,0 @@ -namespace BatchWrapper -{ - /// - /// Constants for the batch wrapper. - /// - public static class Constants - { - /// - /// Environment variable names present or needed during the batch task execution. - /// - public static class EnvironmentVariableNames - { - /// - /// Path to the directory containing the sqlpackage exe. - /// - internal const string AppPackagePrefix = "AZ_BATCH_APP_PACKAGE"; - - /// - /// Path to the working directory assigned to the batch task. - /// - internal const string TaskWorkingDir = "AZ_BATCH_TASK_WORKING_DIR"; - - /// - /// Path to the working directory assigned to the batch task. - /// - internal const string AzBatchTaskId = "AZ_BATCH_TASK_ID"; - - /// - /// Path to the working directory assigned to the batch task. - /// - internal const string JobContainerUrl = "JOB_CONTAINER_URL"; - } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Payload.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Payload.cs deleted file mode 100644 index 2a037d8511..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Payload.cs +++ /dev/null @@ -1,38 +0,0 @@ -namespace BatchWrapper -{ - /// - /// The top-level object stored in the key vault for an import/export operation. - /// - public sealed class Payload - { - /// - /// The Name of sqlpackage to use for performing the import/export operation. - /// - public string ApplicatonPackageName{ get; set; } - - /// - /// The Version of sqlpackage to use for performing the import/export operation. - /// - public string ApplicatonPackageVersion { get; set; } - - /// - /// The type of sqlpackage action to perform. - /// - public ActionType Action { get; set; } - - /// - /// The logical server name to export from or import to. - /// - public string LogicalServerName { get; set; } - - /// - /// The database name to export from or import to. - /// - public string DatabaseName { get; set; } - - /// - /// The server admin username. - /// - public string AccessToken { get; set; } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Program.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Program.cs deleted file mode 100644 index 72237b1a8d..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Program.cs +++ /dev/null @@ -1,178 +0,0 @@ -using Microsoft.Azure.Batch.Conventions.Files; -using Microsoft.WindowsAzure.Storage.Blob; -using System; -using System.Diagnostics; -using System.IO; -using System.Text; -using System.Threading.Tasks; - -namespace BatchWrapper -{ - public static class Program - { - private static string dataDirectory = "F:\\data"; - private static string tempDirectory = "F:\\temp"; - private static string[] directories = { dataDirectory, tempDirectory }; - - private static readonly TimeSpan stdoutFlushDelay = TimeSpan.FromSeconds(3); - - private static void WriteLine(string message) => WriteLineInternal(Console.Out, message); - private static void WriteErrorLine(string message) => WriteLineInternal(Console.Error, message); - private static void WriteLineInternal(TextWriter writer, string message) - { - var lines = message?.Split('\n') ?? new string[0]; - foreach (var line in lines) - { - writer.WriteLine($"[{DateTime.UtcNow:u}] {line?.TrimEnd()}"); - } - } - - public static async Task Main(string[] args) - { - var assembly = typeof(Program).Assembly; - WriteLine($"{assembly.ManifestModule.Name} v{assembly.GetName().Version.ToString(3)}"); - - // Get the command payload - var payload = new Payload(); - - if (args.Length > 0) - { - payload.Action = (ActionType)Enum.Parse(typeof(ActionType), args[0]); - payload.LogicalServerName = args[1] + ".database.windows.net"; - payload.DatabaseName = args[2]; - payload.AccessToken = args[3]; - payload.ApplicatonPackageName = args[4]; - payload.ApplicatonPackageVersion = args[5]; - } - - // Cleanup folders - foreach (string dir in directories) - { - if (Directory.Exists(dir)) - { - Directory.Delete(dir, true); - } - - Directory.CreateDirectory(dir); - } - - string sqlPackageBacpacFile = Path.Combine(dataDirectory, payload.DatabaseName + ".bacpac"); - string sqlPackageLogPath = payload.DatabaseName + ".log"; - - var targetDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AppPackagePrefix + "_" + payload.ApplicatonPackageName + "#" + payload.ApplicatonPackageVersion); - var workingDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.TaskWorkingDir); - - string taskId = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AzBatchTaskId); - string jobContainerUrl = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.JobContainerUrl); - - // Build the import/export command - var cmdBuilder = new StringBuilder(); - cmdBuilder.Append($"/Action:{payload.Action}"); - cmdBuilder.Append(" /MaxParallelism:16"); - cmdBuilder.Append(String.Format(" /DiagnosticsFile:{0}", sqlPackageLogPath)); - cmdBuilder.Append(" /p:CommandTimeout=604800"); - - switch (payload.Action) - { - case ActionType.Export: - cmdBuilder.Append($" /SourceServerName:{payload.LogicalServerName}"); - cmdBuilder.Append($" /SourceDatabaseName:{payload.DatabaseName}"); - cmdBuilder.Append($" /AccessToken:{payload.AccessToken}"); - cmdBuilder.Append($" /TargetFile:{sqlPackageBacpacFile}"); - cmdBuilder.Append($" /SourceTimeout:30"); - cmdBuilder.Append(String.Format(" /p:TempDirectoryForTableData=\"{0}\"", tempDirectory)); - cmdBuilder.Append(" /p:VerifyFullTextDocumentTypesSupported=false"); - break; - - case ActionType.Import: - cmdBuilder.Append($" /TargetServerName:{payload.LogicalServerName}"); - cmdBuilder.Append($" /TargetDatabaseName:{payload.DatabaseName}"); - cmdBuilder.Append($" /AccessToken:{payload.AccessToken}"); - cmdBuilder.Append($" /TargetTimeout:30"); - cmdBuilder.Append($" /SourceFile:{sqlPackageBacpacFile}"); - break; - - default: - throw new ArgumentException($"Invalid action type: {payload.Action}"); - } - - if (payload.Action == ActionType.Import) - { - WriteLine(string.Format("Downloading {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile)); - CloudBlobContainer container = new CloudBlobContainer(new Uri(jobContainerUrl)); - CloudBlockBlob blob = container.GetBlockBlobReference(String.Format("$JobOutput/{0}.bacpac", payload.DatabaseName)); - blob.DownloadToFile(sqlPackageBacpacFile, FileMode.CreateNew); - - if (File.Exists(sqlPackageBacpacFile)) - { - WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile)); - } - else - { - throw new Exception(string.Format("{0} didn't download", sqlPackageBacpacFile)); - } - } - - // Perform the import/export process - var startTime = DateTimeOffset.UtcNow; - var process = new Process - { - StartInfo = new ProcessStartInfo - { - WorkingDirectory = workingDir, - FileName = Path.Combine(targetDir, "sqlpackage.exe"), - Arguments = cmdBuilder.ToString(), - CreateNoWindow = true, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true - } - }; - - process.OutputDataReceived += (s, e) => WriteLine(e.Data); - process.ErrorDataReceived += (s, e) => WriteErrorLine(e.Data); - process.Start(); - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - process.WaitForExit(); - - WriteLine(String.Format("SqlPackage.exe exited with code: {0}", process.ExitCode)); - - if (payload.Action == ActionType.Export) - { - if (File.Exists(sqlPackageBacpacFile)) - { - WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile)); - } - else - { - throw new Exception(string.Format("{0} didn't downloaded", sqlPackageBacpacFile)); - } - - // Persist the Job Output - JobOutputStorage jobOutputStorage = new JobOutputStorage(new Uri(jobContainerUrl)); - - await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageLogPath); - WriteLine(String.Format("Uploaded {0} to job account", sqlPackageLogPath)); - - await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageBacpacFile, payload.DatabaseName + ".bacpac"); - WriteLine(String.Format("Uploaded {0} to job account", sqlPackageBacpacFile)); - } - - // We are tracking the disk file to save our standard output, but the node agent may take - // up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up. - await Task.Delay(stdoutFlushDelay); - - // Cleanup folders - foreach (string dir in directories) - { - if (Directory.Exists(dir)) - { - Directory.Delete(dir, true); - } - } - - return process.ExitCode; - } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/packages.config b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/packages.config deleted file mode 100644 index f8121f7edd..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/packages.config +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/.vscode/launch.json b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/.vscode/launch.json deleted file mode 100644 index 9035a71e76..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/.vscode/launch.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - // Use IntelliSense to find out which attributes exist for C# debugging - // Use hover for the description of the existing attributes - // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md - "version": "0.2.0", - "configurations": [ - { - "name": ".NET Core Launch (console)", - "type": "coreclr", - "request": "launch", - "preLaunchTask": "build", - // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/bin/Debug/netcoreapp3.1/SqlPackageWrapper.dll", - "args": [], - "cwd": "${workspaceFolder}", - // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console - "console": "internalConsole", - "stopAtEntry": false - }, - { - "name": ".NET Core Attach", - "type": "coreclr", - "request": "attach", - "processId": "${command:pickProcess}" - } - ] -} \ No newline at end of file diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/.vscode/tasks.json b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/.vscode/tasks.json deleted file mode 100644 index a6efb6b11c..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/.vscode/tasks.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "version": "2.0.0", - "tasks": [ - { - "label": "build", - "command": "dotnet", - "type": "process", - "args": [ - "build", - "${workspaceFolder}/SqlPackageWrapper.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary" - ], - "problemMatcher": "$msCompile" - }, - { - "label": "publish", - "command": "dotnet", - "type": "process", - "args": [ - "publish", - "${workspaceFolder}/SqlPackageWrapper.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary" - ], - "problemMatcher": "$msCompile" - }, - { - "label": "watch", - "command": "dotnet", - "type": "process", - "args": [ - "watch", - "run", - "${workspaceFolder}/SqlPackageWrapper.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary" - ], - "problemMatcher": "$msCompile" - } - ] -} \ No newline at end of file diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/ActionType.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/ActionType.cs deleted file mode 100644 index 279ce0f71c..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/ActionType.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace SqlPackageWrapper -{ - /// - /// The type of sqlpackage action to perform. - /// - public enum ActionType - { - DefaultInvalid = -1, - Export, - Import - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Constants.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Constants.cs deleted file mode 100644 index e94bc9b1c9..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Constants.cs +++ /dev/null @@ -1,39 +0,0 @@ -namespace SqlPackageWrapper -{ - /// - /// Constants for the batch wrapper. - /// - public static class Constants - { - /// - /// Environment variable names present or needed during the batch task execution. - /// - public static class EnvironmentVariableNames - { - /// - /// Path to the directory containing the batch wrapper exe. - /// - public const string WrapperLocation = "AZ_BATCH_APP_PACKAGE_BATCHWRAPPER"; - - /// - /// Path to the directory containing the sqlpackage exe. - /// - internal const string SqlPackageLocation = "AZ_BATCH_APP_PACKAGE_SQLPACKAGE"; - - /// - /// Path to the working directory assigned to the batch task. - /// - internal const string TaskWorkingDir = "AZ_BATCH_TASK_WORKING_DIR"; - - /// - /// Path to the working directory assigned to the batch task. - /// - internal const string AzBatchTaskId = "AZ_BATCH_TASK_ID"; - - /// - /// Path to the working directory assigned to the batch task. - /// - internal const string JobContainerUrl = "JOB_CONTAINER_URL"; - } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Payload.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Payload.cs deleted file mode 100644 index 170ec9d515..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Payload.cs +++ /dev/null @@ -1,38 +0,0 @@ -namespace SqlPackageWrapper -{ - /// - /// The top-level object stored in the key vault for an import/export operation. - /// - public sealed class Payload - { - /// - /// The version of sqlpackage to use for performing the import/export operation. - /// - public string SqlPackageVersion { get; set; } - - /// - /// The type of sqlpackage action to perform. - /// - public ActionType Action { get; set; } - - /// - /// The logical server name to export from or import to. - /// - public string LogicalServerName { get; set; } - - /// - /// The database name to export from or import to. - /// - public string DatabaseName { get; set; } - - /// - /// The server admin username. - /// - public string Username { get; set; } - - /// - /// The server admin password. - /// - public string Password { get; set; } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Program.cs b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Program.cs deleted file mode 100644 index 2ee94c255f..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Program.cs +++ /dev/null @@ -1,162 +0,0 @@ -using Microsoft.Azure.Batch.Conventions.Files; -using Microsoft.WindowsAzure.Storage.Blob; -using System; -using System.Diagnostics; -using System.IO; -using System.Text; -using System.Threading.Tasks; - -namespace SqlPackageWrapper -{ - public static class Program - { - private static string dataDirectory = "F:\\data"; - private static string tempDirectory = "F:\\temp"; - private static string[] directories = { dataDirectory, tempDirectory }; - - private static readonly TimeSpan stdoutFlushDelay = TimeSpan.FromSeconds(3); - - private static void WriteLine(string message) => WriteLineInternal(Console.Out, message); - private static void WriteErrorLine(string message) => WriteLineInternal(Console.Error, message); - private static void WriteLineInternal(TextWriter writer, string message) - { - var lines = message?.Split('\n') ?? new string[0]; - foreach (var line in lines) - { - writer.WriteLine($"[{DateTime.UtcNow:u}] {line?.TrimEnd()}"); - } - } - - public static async Task Main(string[] args) - { - var assembly = typeof(Program).Assembly; - WriteLine($"{assembly.ManifestModule.Name} v{assembly.GetName().Version.ToString(3)}"); - - // Get the command payload - var payload = new Payload(); - - if (args.Length > 0) - { - payload.Action = (ActionType)Enum.Parse(typeof(ActionType), args[0]); - payload.LogicalServerName = args[1] + ".database.windows.net"; - payload.DatabaseName = args[2]; - payload.Username = args[3]; - payload.Password = args[4]; - payload.SqlPackageVersion = args[5]; - } - - // Cleanup folders - foreach (string dir in directories) - { - if (Directory.Exists(dir)) - { - Directory.Delete(dir, true); - } - - Directory.CreateDirectory(dir); - } - - string sqlPackageDataPath = Path.Combine(dataDirectory, payload.DatabaseName + ".bacpac"); - string sqlPackageLogPath = Path.Combine(dataDirectory, payload.DatabaseName + ".log"); - - var targetDir = Environment.GetEnvironmentVariable($"{Constants.EnvironmentVariableNames.SqlPackageLocation}#{payload.SqlPackageVersion}"); - var workingDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.TaskWorkingDir); - - string taskId = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AzBatchTaskId); - string jobContainerUrl = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.JobContainerUrl); - - // Build the import/export command - var cmdBuilder = new StringBuilder(); - cmdBuilder.Append($"/Action:{payload.Action}"); - cmdBuilder.Append(" /MaxParallelism:16"); - cmdBuilder.Append(String.Format(" /DiagnosticsFile:{0}", sqlPackageLogPath)); - cmdBuilder.Append(" /p:CommandTimeout=86400"); - - switch (payload.Action) - { - case ActionType.Export: - cmdBuilder.Append($" /SourceServerName:{payload.LogicalServerName}"); - cmdBuilder.Append($" /SourceDatabaseName:{payload.DatabaseName}"); - cmdBuilder.Append($" /SourceUser:{payload.Username}"); - cmdBuilder.Append($" /SourcePassword:{payload.Password}"); - cmdBuilder.Append($" /TargetFile:{sqlPackageDataPath}"); - cmdBuilder.Append(String.Format(" /p:TempDirectoryForTableData=\"{0}\"", tempDirectory)); - cmdBuilder.Append(" /p:VerifyFullTextDocumentTypesSupported=false"); - break; - - case ActionType.Import: - cmdBuilder.Append($" /TargetServerName:{payload.LogicalServerName}"); - cmdBuilder.Append($" /TargetDatabaseName:{payload.DatabaseName}"); - cmdBuilder.Append($" /TargetUser:{payload.Username}"); - cmdBuilder.Append($" /TargetPassword:{payload.Password}"); - cmdBuilder.Append($" /SourceFile:{sqlPackageDataPath}"); - break; - - default: - throw new ArgumentException($"Invalid action type: {payload.Action}"); - } - - if (payload.Action == ActionType.Import) - { - WriteLine(string.Format("Downloading {0} bacpac file to {1}", payload.DatabaseName, sqlPackageDataPath)); - CloudBlobContainer container = new CloudBlobContainer(new Uri(jobContainerUrl)); - CloudBlockBlob blob = container.GetBlockBlobReference(String.Format("$JobOutput/{0}.bacpac", payload.DatabaseName)); - await blob.DownloadToFileAsync(sqlPackageDataPath, FileMode.CreateNew); - WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageDataPath)); - - await Task.Delay(stdoutFlushDelay); - } - - // Perform the import/export process - var startTime = DateTimeOffset.UtcNow; - var process = new Process - { - StartInfo = new ProcessStartInfo - { - WorkingDirectory = workingDir, - FileName = Path.Combine(targetDir, "sqlpackage.exe"), - Arguments = cmdBuilder.ToString(), - CreateNoWindow = true, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true - } - }; - process.OutputDataReceived += (s, e) => WriteLine(e.Data); - process.ErrorDataReceived += (s, e) => WriteErrorLine(e.Data); - process.Start(); - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - process.WaitForExit(); - - WriteLine(String.Format("SqlPackage.exe exited with code: {0}", process.ExitCode)); - - if (payload.Action == ActionType.Export) - { - // Persist the Job Output - JobOutputStorage jobOutputStorage = new JobOutputStorage(new Uri(jobContainerUrl)); - - await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageLogPath, payload.DatabaseName + ".log"); - WriteLine(String.Format("Uploaded {0} to job account", sqlPackageLogPath)); - - await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageDataPath, payload.DatabaseName + ".bacpac"); - WriteLine(String.Format("Uploaded {0} to job account", sqlPackageDataPath)); - - // We are tracking the disk file to save our standard output, but the node agent may take - // up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up. - await Task.Delay(stdoutFlushDelay); - } - - // Cleanup folders - foreach (string dir in directories) - { - if (Directory.Exists(dir)) - { - Directory.Delete(dir, true); - } - } - - return process.ExitCode; - } - } -} diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/SqlPackageWrapper.csproj b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/SqlPackageWrapper.csproj deleted file mode 100644 index c7d2d2657e..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/SqlPackageWrapper.csproj +++ /dev/null @@ -1,14 +0,0 @@ - - - - Exe - netcoreapp2.1;net452 - - - - - - - - - diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/readme.md b/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/readme.md deleted file mode 100644 index 87fe8db9a0..0000000000 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/readme.md +++ /dev/null @@ -1 +0,0 @@ -Folder for helper components used by notebooks in the Hybrid Cloud Toolkit diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/_data/toc.yml b/extensions/azurehybridtoolkit/notebooks/hybridbook/_data/toc.yml index 590a79d446..66c2be94f9 100644 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/_data/toc.yml +++ b/extensions/azurehybridtoolkit/notebooks/hybridbook/_data/toc.yml @@ -8,6 +8,13 @@ - title: Search search: true +- title: ADP + url: /ADP/readme + not_numbered: true + expand_sections: false + sections: + - title: ADP Control + url: ADP/ADPControl - title: Assessment url: /Assessments/readme not_numbered: true diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl.ipynb b/extensions/azurehybridtoolkit/notebooks/hybridbook/content/ADP/ADPControl.ipynb similarity index 100% rename from extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl.ipynb rename to extensions/azurehybridtoolkit/notebooks/hybridbook/content/ADP/ADPControl.ipynb diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/content/ADP/readme.md b/extensions/azurehybridtoolkit/notebooks/hybridbook/content/ADP/readme.md new file mode 100644 index 0000000000..8b5a29534d --- /dev/null +++ b/extensions/azurehybridtoolkit/notebooks/hybridbook/content/ADP/readme.md @@ -0,0 +1,8 @@ +# ADP Control + +[Home](../readme.md) + +Notebook to manage ADP Orchestrator import/export requests onto an SQL Server. + +## Notebooks in this Chapter +- [ADP Control](ADPControl.ipynb) diff --git a/extensions/azurehybridtoolkit/notebooks/hybridbook/content/readme.md b/extensions/azurehybridtoolkit/notebooks/hybridbook/content/readme.md index 7c10809669..e655d71b68 100644 --- a/extensions/azurehybridtoolkit/notebooks/hybridbook/content/readme.md +++ b/extensions/azurehybridtoolkit/notebooks/hybridbook/content/readme.md @@ -2,13 +2,15 @@ ## Chapters * [Prerequisites and Initial Setup](prereqs.ipynb) - Notebook installation of required modules. -* [Assessments](Assessments/readme.md) - Notebooks that contain examples to determine whether a given database or SQL Server instance is ready to migrate by utilizing SQL Assessments. SQL instances are scanned based on a "best practices" set of rules. +* [ADP](ADP/readme.md) - manage ADP Orchestrator import/export requests onto an SQL Server. + +* [Assessments](Assessments/readme.md) - Notebooks that contain examples to determine whether a given database or SQL Server instance is ready to migrate by utilizing SQL Assessments. SQL instances are scanned based on a "best practices" set of rules. * [Networking](networking/readme.md) - Setup secure Point-to-Site (P2S) or Site-to-Site (S2S) network connectivity to Microsoft Azure using a Virtual Private Network (VPN). This notebook serves as a building block for other notebooks as communicating securely between on-premise and Azure is essential for many tasks. * [Provisioning](provisioning/readme.md) - Creating and communicating with SQL Resources in Microsoft Azure. Includes common tasks such as creating SQL Virtual Machines or SQL Managed Instances in the cloud. -* [Data Portability](data-portability/readme.md) - Install a custom Azure function to facilitate importing and exporting cloud resources. The solution uses parallel tasks in Azure Batch to perform data storage work. Azure Batch is a process that runs large-scale parallel and high-performance computing jobs efficiently in Azure. +* [Data Portability](data-portability/readme.md) - Install a custom Azure function to facilitate importing and exporting cloud resources. The solution uses parallel tasks in Azure Batch to perform data storage work. Azure Batch is a process that runs large-scale parallel and high-performance computing jobs efficiently in Azure. * [High Availability and Disaster Recovery](hadr/readme.md) - Notebooks to leverage Azure SQL for business continuity in a hybrid cloud environment. @@ -20,16 +22,16 @@ ## About -The **Azure SQL Hybrid Cloud Toolkit** is a [Jupyter Book](https://jupyterbook.org/intro.html) extension of [Azure Data Studio](https://docs.microsoft.com/en-us/sql/azure-data-studio/download-azure-data-studio) (ADS) designed to help [Azure SQL Database](https://azure.microsoft.com/en-us/services/sql-database/) and ADS users deploy, migrate and configure for a hybrid cloud environment. The toolkit was designed with and intended to be executed within ADS. This is to ensure the best possible user experience for those without vast knowledge of Azure services while adhering closely to the software _best practices_ standards required by experienced cloud users. +The **Azure SQL Hybrid Cloud Toolkit** is a [Jupyter Book](https://jupyterbook.org/intro.html) extension of [Azure Data Studio](https://docs.microsoft.com/en-us/sql/azure-data-studio/download-azure-data-studio) (ADS) designed to help [Azure SQL Database](https://azure.microsoft.com/en-us/services/sql-database/) and ADS users deploy, migrate and configure for a hybrid cloud environment. The toolkit was designed with and intended to be executed within ADS. This is to ensure the best possible user experience for those without vast knowledge of Azure services while adhering closely to the software _best practices_ standards required by experienced cloud users. ## Goals and Methodology The toolkit better positions a customer with regards to planning, migrating, and thriving in a hybrid cloud environment by: * Providing SQL'zure users with reliable free software and content that is well-written and executable * Greatly simplifying the integration of Azure Data services into an existing environment -* Positioning Azure to be the natural cloud services choice with a low-friction experience +* Positioning Azure to be the natural cloud services choice with a low-friction experience * Notebooks are executable by a normal user (unless otherwise specificed) on minimal hardware * Most notebooks require some configuration. If so, the proper configurations should be clearly located towards the top of the notebook or cell, whichever is most appropriate * By design, Notebooks are written to be executed from top-to-bottom. Therefore, each notebook has a specific task to perform and should focus only on that task. It may contain several cells to execute but it will adhere to the one-task per notebook paradigm -**NOTE:** Executing notebooks could potentially create new Azure Resources which may incur charges to the Azure Subscription. Make sure the repercussions of executing any cells are understood. \ No newline at end of file +**NOTE:** Executing notebooks could potentially create new Azure Resources which may incur charges to the Azure Subscription. Make sure the repercussions of executing any cells are understood.