ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-14 18:46:34 -05:00
Code Issues Packages Projects Releases Wiki Activity

Removal of Components folder, moving ADP notebook to its own notebook. (#13848)

Browse Source 
* adp folder removed, notebook moved.

* moved ADP notebook to own folder.
This commit is contained in:
Alex Ma
2020-12-18 10:58:16 -08:00
committed by GitHub
parent 59ad601664
commit 72a3aa1207
27 changed files with 22 additions and 1555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl.sln
View File

@@ -1,37 +0,0 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29920.165
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ADPControl", "ADPControl\ADPControl.csproj", "{6309D4C5-F118-4C89-A67E-E557CA41ABA2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BatchWrapper", "BatchWrapper\BatchWrapper.csproj", "{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlPackageWrapper", "SqlPackageWrapper\SqlPackageWrapper.csproj", "{A19335D3-9D80-43BE-9351-C3C3704689B0}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Release|Any CPU.ActiveCfg = Debug|Any CPU
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Release|Any CPU.Build.0 = Debug|Any CPU
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Release|Any CPU.ActiveCfg = Debug|Any CPU
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Release|Any CPU.Build.0 = Debug|Any CPU
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {3DB5F0CB-1D26-4E33-801D-7BBAE823C39D}
EndGlobalSection
EndGlobal

26
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/ADPControl.csproj
View File

@@ -1,26 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<AzureFunctionsVersion>v3</AzureFunctionsVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.Batch" Version="13.0.0" />
<PackageReference Include="Microsoft.Azure.Management.ResourceManager" Version="3.7.0-preview" />
<PackageReference Include="Microsoft.Azure.Management.Storage" Version="17.0.0" />
<PackageReference Include="Microsoft.Azure.Services.AppAuthentication" Version="1.5.0" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="2.2.2" />
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.7" />
<PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.21" />
<PackageReference Include="Microsoft.Rest.ClientRuntime.Azure" Version="3.3.19" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
</ItemGroup>
<ItemGroup>
<None Update="host.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="local.settings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<CopyToPublishDirectory>Never</CopyToPublishDirectory>
</None>
</ItemGroup>
</Project>

244
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/AzureResourceManagerActivity.cs
View File

@@ -1,244 +0,0 @@
using Microsoft.Azure.Management.ResourceManager;
using Microsoft.Azure.Management.ResourceManager.Models;
using Microsoft.Azure.Management.Storage;
using Microsoft.Azure.Management.Storage.Models;
using Microsoft.Azure.Services.AppAuthentication;
using Microsoft.Azure.WebJobs;
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
using Microsoft.Extensions.Logging;
using Microsoft.Rest;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Auth;
using Microsoft.WindowsAzure.Storage.Blob;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using static ADPControl.HttpSurface;
namespace ADPControl
{
public static class AzureResourceManagerActivity
{
private const string ArmTemplateFileName = "template.json";
private static string[] AllowedImportSubServerResourceTypes = new string[] {
"Microsoft.Sql/servers/firewallRules",
"Microsoft.Sql/servers/databases",
"Microsoft.Sql/servers/elasticPools",
//"Microsoft.Sql/servers/keys",
//"Microsoft.Sql/servers/databases/transparentDataEncryption",
"Microsoft.Sql/servers/databases/backupShortTermRetentionPolicies",
"Microsoft.Sql/servers/administrators"
};
// Deploy the ARM template
[FunctionName(nameof(BeginDeployArmTemplateForImport))]
public static async Task<string> BeginDeployArmTemplateForImport([ActivityTrigger] ImportRequest request, ILogger log)
{
var azureServiceTokenProvider = new AzureServiceTokenProvider();
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
StorageManagementClient storageMgmtClient = new StorageManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
// Get the storage account keys for a given account and resource group
IList<StorageAccountKey> acctKeys = storageMgmtClient.StorageAccounts.ListKeys(request.ResourceGroupName, request.StorageAccountName).Keys;
// Get a Storage account using account creds:
StorageCredentials storageCred = new StorageCredentials(request.StorageAccountName, acctKeys.FirstOrDefault().Value);
CloudStorageAccount linkedStorageAccount = new CloudStorageAccount(storageCred, true);
CloudBlobContainer container = linkedStorageAccount
.CreateCloudBlobClient()
.GetContainerReference(request.ContainerName);
CloudBlockBlob blob = container.GetBlockBlobReference(ArmTemplateFileName);
string json = await blob.DownloadTextAsync();
JObject originalTemplate = JObject.Parse(json);
JObject importTemplate = UpdateArmTemplateForImport(originalTemplate, request);
var deployParams = new Deployment
{
Properties = new DeploymentProperties
{
Mode = DeploymentMode.Incremental,
Template = importTemplate
}
};
string deploymentName = request.TargetSqlServerName + "_" + DateTime.UtcNow.ToFileTimeUtc();
try
{
await resourcesClient.Deployments.BeginCreateOrUpdateAsync(request.TargetSqlServerResourceGroupName, deploymentName, deployParams);
}
catch (Exception ex)
{
log.LogError(ex.ToString());
throw ex;
}
return deploymentName;
}
// Get the ARM deployment status
[FunctionName(nameof(GetArmDeploymentForImport))]
public static async Task<string> GetArmDeploymentForImport([ActivityTrigger] (Guid, string, string) input)
{
Guid subscriptionId = input.Item1;
string resourceGroupName = input.Item2;
string deploymentName = input.Item3;
var azureServiceTokenProvider = new AzureServiceTokenProvider();
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = subscriptionId.ToString() };
DeploymentExtended result = await resourcesClient.Deployments.GetAsync(resourceGroupName, deploymentName);
return result.Properties.ProvisioningState;
}
// Get the ARM template without the parameter of the resource name
[FunctionName(nameof(GetArmTemplateForExportSkipParameterization))]
public static async Task<dynamic> GetArmTemplateForExportSkipParameterization([ActivityTrigger] ExportRequest request, ILogger log)
{
log.LogInformation("GetArmTemplateForExportSkipParameterization: entering");
var azureServiceTokenProvider = new AzureServiceTokenProvider();
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
if (tokenArmCredential != null)
{
log.LogInformation("GetArmTemplateForExportSkipParameterization: acquired access token");
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.SourceSqlServerResourceGroupName, request.SourceSqlServerName);
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.SourceSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "SkipResourceNameParameterization"));
log.LogInformation("GetArmTemplateForExportSkipParameterization: server template exported. Size: {0} bytes", exportedTemplate.Template.ToString().Length);
dynamic template = (dynamic)exportedTemplate.Template;
// Filtering the list of databases
dynamic databases = template.resources.SelectTokens("$.[?(@.type == 'Microsoft.Sql/servers/databases')]");
int numberOfDatabases = 0;
foreach (var db in databases)
{
numberOfDatabases++;
}
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with database list. Databases count: {0}", numberOfDatabases);
return databases;
}
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with empty database list");
return null;
}
// Get the ARM template without the parameter of the resource name
[FunctionName(nameof(GetArmTemplateForImportSkipParameterization))]
public static async Task<dynamic> GetArmTemplateForImportSkipParameterization([ActivityTrigger] ImportRequest request, ILogger log)
{
var azureServiceTokenProvider = new AzureServiceTokenProvider();
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
if (tokenArmCredential != null)
{
log.LogInformation("GetArmTemplateForImportSkipParameterization: acquired access token");
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.TargetSqlServerResourceGroupName, request.TargetSqlServerName);
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.TargetSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "SkipResourceNameParameterization"));
log.LogInformation("GetArmTemplateForImportSkipParameterization: server template exported. Size: {0} bytes", exportedTemplate.Template.ToString().Length);
dynamic template = (dynamic)exportedTemplate.Template;
// Filtering the list of databases
dynamic databases = template.resources.SelectTokens("$.[?(@.type == 'Microsoft.Sql/servers/databases')]");
int numberOfDatabases = 0;
foreach (var db in databases)
{
numberOfDatabases++;
}
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with database list. Databases count: {0}", numberOfDatabases);
return databases;
}
return null;
}
[FunctionName(nameof(GetArmTemplateForExport))]
public static async Task<dynamic> GetArmTemplateForExport([ActivityTrigger] ExportRequest request)
{
var azureServiceTokenProvider = new AzureServiceTokenProvider();
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.SourceSqlServerResourceGroupName, request.SourceSqlServerName);
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.SourceSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "IncludeParameterDefaultValue"));
return exportedTemplate.Template;
}
private static JObject UpdateArmTemplateForImport(JObject originalTemplate, ImportRequest request)
{
string serverNameParameterName = null;
// Go through every parameter to find the property name is like 'server_%_name'
using (JsonTextReader reader = new JsonTextReader(new StringReader(originalTemplate["parameters"].ToString())))
{
while (reader.Read())
{
if (reader.TokenType.ToString().Equals("PropertyName")
&& reader.ValueType.ToString().Equals("System.String")
&& reader.Value.ToString().StartsWith("servers_")
&& reader.Value.ToString().EndsWith("_name"))
{
serverNameParameterName = reader.Value.ToString();
break;
}
}
}
// 1. Replacing the default value to the target server name, appending to the new template
originalTemplate["parameters"][serverNameParameterName]["defaultValue"] = request.TargetSqlServerName;
JObject serverNameParameterValue = (JObject)originalTemplate["parameters"][serverNameParameterName];
// 2. Cleanup all the parameters except the updated server name
((JObject)originalTemplate["parameters"]).RemoveAll();
((JObject)originalTemplate["parameters"]).Add(serverNameParameterName, serverNameParameterValue);
// 3. Adjust the servers resource by adding password after the login
JObject server = (JObject)originalTemplate["resources"]
.SelectToken("$.[?(@.type == 'Microsoft.Sql/servers')]");
server.Remove("identity");
JObject serverProperties = (JObject)server["properties"];
serverProperties.Property("administratorLogin")
.AddAfterSelf(new JProperty("administratorLoginPassword", request.SqlAdminPassword));
JArray newResources = new JArray();
// 4. Getting the whitelisted resources and adding them to the new template later.
foreach (string resourceType in AllowedImportSubServerResourceTypes)
{
List<JToken> resources = originalTemplate["resources"]
.SelectTokens(string.Format("$.[?(@.type == '{0}')]", resourceType)).ToList();
newResources.Add(resources);
}
// 5. Clean up all the resources excepted the new server and whitelisted resource type.
((JArray)originalTemplate["resources"]).Clear();
((JArray)originalTemplate["resources"]).Add(server);
foreach (var resource in newResources)
{
((JArray)originalTemplate["resources"]).Add(resource);
}
return originalTemplate;
}
}
}

214
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/BatchActivity.cs
View File

@@ -1,214 +0,0 @@
using Microsoft.Azure.Batch;
using Microsoft.Azure.Batch.Auth;
using Microsoft.Azure.Batch.Common;
using Microsoft.Azure.Services.AppAuthentication;
using Microsoft.Azure.WebJobs;
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using static ADPControl.HttpSurface;
namespace ADPControl
{
public static class BatchActivity
{
// Batch resource settings
private const string PoolVMSize = "Standard_D8s_v3";
private const string PoolId = PoolVMSize;
private const int PoolNodeCount = 2;
private const string AppPackageName = "SqlPackageWrapper";
public const string AppPackageVersion = "1";
[FunctionName(nameof(CreateBatchPoolAndExportJob))]
public static async Task<string> CreateBatchPoolAndExportJob([ActivityTrigger] ExportRequest request, ILogger log)
{
var azureServiceTokenProvider = new AzureServiceTokenProvider();
// Get a Batch client using function identity
BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));
string jobId = request.SourceSqlServerName + "-Export-" + DateTime.UtcNow.ToString("MMddHHmmss");
using (BatchClient batchClient = BatchClient.Open(batchCred))
{
ImageReference imageReference = CreateImageReference();
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
await CreateBatchJob(batchClient, jobId, log);
}
return jobId;
}
[FunctionName(nameof(CreateBatchPoolAndImportJob))]
public static async Task<string> CreateBatchPoolAndImportJob([ActivityTrigger] ImportRequest request, ILogger log)
{
var azureServiceTokenProvider = new AzureServiceTokenProvider();
// Get a Batch client using function identity
BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));
string jobId = request.TargetSqlServerName + "-Import-" + DateTime.UtcNow.ToString("MMddHHmmss");
using (BatchClient batchClient = BatchClient.Open(batchCred))
{
ImageReference imageReference = CreateImageReference();
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
await CreateBatchJob(batchClient, jobId, log);
}
return jobId;
}
public static async Task<CloudJob> CreateBatchJob(BatchClient batchClient, string jobId, ILogger log)
{
// Create a Batch job
log.LogInformation("Creating job [{0}]...", jobId);
CloudJob job = null;
try
{
job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = PoolId });
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
// Commit the job to the Batch service
await job.CommitAsync();
log.LogInformation($"Created job {jobId}");
// Obtain the bound job from the Batch service
await job.RefreshAsync();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
log.LogWarning("The job {0} already existed when we tried to create it", jobId);
}
else
{
log.LogError("Exception creating job: {0}", be.Message);
throw be; // Any other exception is unexpected
}
}
return job;
}
// Create the Compute Pool of the Batch Account
public static async Task CreateBatchPoolIfNotExist(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration, string vnetSubnetId)
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
try
{
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: PoolId,
targetDedicatedComputeNodes: PoolNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: vmConfiguration);
// Specify the application and version to install on the compute nodes
pool.ApplicationPackageReferences = new List<ApplicationPackageReference>
{
new ApplicationPackageReference {
ApplicationId = AppPackageName,
Version = AppPackageVersion }
};
// Initial the first data disk for each VM in the pool
StartTask startTask = new StartTask("cmd /c Powershell -command \"Get-Disk | Where partitionstyle -eq 'raw' | sort number | Select-Object -first 1 |" +
" Initialize-Disk -PartitionStyle MBR -PassThru | New-Partition -UseMaximumSize -DriveLetter F |" +
" Format-Volume -FileSystem NTFS -NewFileSystemLabel data1 -Confirm:$false -Force\"");
startTask.MaxTaskRetryCount = 1;
startTask.UserIdentity = new UserIdentity(new AutoUserSpecification(AutoUserScope.Pool, ElevationLevel.Admin));
startTask.WaitForSuccess = true;
pool.StartTask = startTask;
// Create the Pool within the vnet subnet if it's specified.
if (vnetSubnetId != null)
{
pool.NetworkConfiguration = new NetworkConfiguration();
pool.NetworkConfiguration.SubnetId = vnetSubnetId;
}
await pool.CommitAsync();
await pool.RefreshAsync();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
}
else
{
throw; // Any other exception is unexpected
}
}
}
public static VirtualMachineConfiguration CreateVirtualMachineConfiguration(ImageReference imageReference)
{
VirtualMachineConfiguration config = new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
config.DataDisks = new List<DataDisk>();
config.DataDisks.Add(new DataDisk(0, 2048, CachingType.ReadOnly, StorageAccountType.PremiumLrs));
return config;
}
public static ImageReference CreateImageReference()
{
return new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2019-datacenter-smalldisk",
version: "latest");
}
public static void CreateBatchTasks(string action, string jobId, string containerUrl, string batchAccountUrl, string sqlServerName, string accessToken, dynamic databases, ILogger log)
{
// Get a Batch client using function identity
log.LogInformation("CreateBatchTasks: entering");
var azureServiceTokenProvider = new AzureServiceTokenProvider();
BatchTokenCredentials batchCred = new BatchTokenCredentials(batchAccountUrl, azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/").Result);
using (BatchClient batchClient = BatchClient.Open(batchCred))
{
// For each database, submit the Exporting job to Azure Batch Compute Pool.
log.LogInformation("CreateBatchTasks: enumerating databases");
List<CloudTask> tasks = new List<CloudTask>();
foreach (var db in databases)
{
string serverDatabaseName = db.name.ToString();
string logicalDatabase = serverDatabaseName.Remove(0, sqlServerName.Length + 1);
log.LogInformation("CreateBatchTasks: creating task for database {0}", logicalDatabase);
string taskId = sqlServerName + "_" + logicalDatabase;
string command = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_{0}#{1}%\\BatchWrapper {2}", AppPackageName.ToUpper(), AppPackageVersion, action);
command += string.Format(" {0} {1} {2} {3} {4}", sqlServerName, logicalDatabase, accessToken, AppPackageName.ToUpper(), AppPackageVersion);
string taskCommandLine = string.Format(command);
CloudTask singleTask = new CloudTask(taskId, taskCommandLine);
singleTask.EnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
Console.WriteLine(string.Format("Adding task {0} to job ...", taskId));
tasks.Add(singleTask);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(jobId, tasks);
}
log.LogInformation("CreateBatchTasks: exiting");
}
}
}

103
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/HttpSurface.cs
View File

@@ -1,103 +0,0 @@
using Microsoft.Azure.WebJobs;
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
using Microsoft.Azure.WebJobs.Extensions.Http;
using Microsoft.Extensions.Logging;
using System;
using System.Net.Http;
using System.Threading.Tasks;
namespace ADPControl
{
public static class HttpSurface
{
public class ExportRequest
{
public Guid SubscriptionId { get; set; }
public string ResourceGroupName { get; set; }
public string SourceSqlServerResourceGroupName { get; set; }
public string SourceSqlServerName { get; set; }
public string BatchAccountUrl { get; set; }
public string StorageAccountName { get; set; }
public string AccessToken { get; set; }
public string VNetSubnetId { get; set; }
}
public class ImportRequest
{
public Guid SubscriptionId { get; set; }
public string ResourceGroupName { get; set; }
public string TargetSqlServerResourceGroupName { get; set; }
public string TargetSqlServerName { get; set; }
public string TargetAccessToken { get; set; }
public string BatchAccountUrl { get; set; }
public string StorageAccountName { get; set; }
public string ContainerName { get; set; }
public string SqlAdminPassword { get; set; }
public string VNetSubnetId { get; set; }
}
[FunctionName("Export")]
public static async Task<HttpResponseMessage> PostExport(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/Export")]
HttpRequestMessage req,
[DurableClient] IDurableOrchestrationClient starter,
ILogger log,
Guid subscriptionId,
string resourceGroupName)
{
log.LogInformation("C# HTTP trigger function processed an Export request.");
ExportRequest request = await req.Content.ReadAsAsync<ExportRequest>();
request.SubscriptionId = subscriptionId;
request.ResourceGroupName = resourceGroupName;
if (request.SourceSqlServerResourceGroupName == null)
request.SourceSqlServerResourceGroupName = resourceGroupName;
string instanceId = await starter.StartNewAsync(nameof(Orchestrator.RunExportOrchestrator), request);
log.LogInformation($"Started orchestration with ID = '{instanceId}'.");
return starter.CreateCheckStatusResponse(req, instanceId);
}
[FunctionName("Import")]
public static async Task<HttpResponseMessage> PostImport(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/Import")]
HttpRequestMessage req,
[DurableClient] IDurableOrchestrationClient starter,
ILogger log,
Guid subscriptionId,
string resourceGroupName)
{
log.LogInformation("C# HTTP trigger function processed an Import request.");
ImportRequest request = await req.Content.ReadAsAsync<ImportRequest>();
request.SubscriptionId = subscriptionId;
request.ResourceGroupName = resourceGroupName;
if (request.TargetSqlServerResourceGroupName == null)
request.TargetSqlServerResourceGroupName = resourceGroupName;
string instanceId = await starter.StartNewAsync(nameof(Orchestrator.RunImportOrchestrator), request);
log.LogInformation($"Started orchestration with ID = '{instanceId}'.");
return starter.CreateCheckStatusResponse(req, instanceId);
}
}
}

104
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/Orchestrator.cs
View File

@@ -1,104 +0,0 @@
using Microsoft.Azure.WebJobs;
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using System;
using System.Threading;
using System.Threading.Tasks;
using static ADPControl.HttpSurface;
namespace ADPControl
{
public static class Orchestrator
{
// The Import Orchestrator
[FunctionName(nameof(RunImportOrchestrator))]
public static async Task RunImportOrchestrator(
[OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log)
{
log.LogInformation("RunImportOrchestrator: entering");
try {
ImportRequest importRequest = context.GetInput<ImportRequest>();
// Deploy the ARM template to Create empty SQL resource
string deploymentName = await context.CallActivityAsync<string>(nameof(AzureResourceManagerActivity.BeginDeployArmTemplateForImport), importRequest);
while (true)
{
log.LogInformation("RunImportOrchestrator: starting ARM deployment");
string status = await context.CallActivityAsync<string>(nameof(AzureResourceManagerActivity.GetArmDeploymentForImport), (importRequest.SubscriptionId, importRequest.TargetSqlServerResourceGroupName, deploymentName));
if (status == "Succeeded")
{
log.LogInformation("RunImportOrchestrator: ARM deployment succeeded");
break;
}
else if (status == "Failed")
{
log.LogInformation("RunImportOrchestrator: ARM deployment failed");
throw new Exception("Failed ARM Deployment");
}
// Orchestration sleeps until this time.
var nextCheck = context.CurrentUtcDateTime.AddSeconds(10);
if (!context.IsReplaying) { log.LogInformation($"RunImportOrchestrator: Replaying ARM deployment, next check at {nextCheck}."); }
await context.CreateTimer(nextCheck, CancellationToken.None);
}
log.LogInformation("RunImportOrchestrator: Enumerating databases");
var databases = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForImportSkipParameterization), importRequest);
// Create BatchPool And Job
log.LogInformation("RunImportOrchestrator: Creating batch pool and import job");
string jobId = await context.CallActivityAsync<string>(nameof(BatchActivity.CreateBatchPoolAndImportJob), importRequest);
string containerUrl = await context.CallActivityAsync<string>(nameof(StorageActivity.GettingJobContainerUrl), (importRequest.SubscriptionId, importRequest.ResourceGroupName, importRequest.StorageAccountName, importRequest.ContainerName));
log.LogInformation("RunImportOrchestrator: Creating import database tasks");
BatchActivity.CreateBatchTasks("Import", jobId, containerUrl, importRequest.BatchAccountUrl, importRequest.TargetSqlServerName, importRequest.TargetAccessToken, databases, log);
// create output values
Tuple<string, string>[] outputValues = {
Tuple.Create("Orchestration progress:", "Complete"),
Tuple.Create("deploymentName", deploymentName),
Tuple.Create("jobId", jobId),
Tuple.Create("containerUrl", containerUrl)
};
context.SetOutput(outputValues);
}
finally {
log.LogInformation("RunImportOrchestrator: exiting");
}
}
// The Export Orchestrator
[FunctionName(nameof(RunExportOrchestrator))]
public static async Task RunExportOrchestrator(
[OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log)
{
ExportRequest exportRequest = context.GetInput<ExportRequest>();
// Getting the ARM template Skip ResourceName Parameterization.
var databases = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForExportSkipParameterization), exportRequest);
// Getting the ARM template.
dynamic Template = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForExport), exportRequest);
string json = JsonConvert.SerializeObject(Template);
// Create BatchPool And Job
string jobId = await context.CallActivityAsync<string>(nameof(BatchActivity.CreateBatchPoolAndExportJob), exportRequest);
string containerUrl = await context.CallActivityAsync<string>(nameof(StorageActivity.GettingJobContainerUrl), (exportRequest.SubscriptionId, exportRequest.ResourceGroupName, exportRequest.StorageAccountName, jobId));
await context.CallActivityAsync<string>(nameof(StorageActivity.UploadingArmTemplate), (containerUrl, json));
BatchActivity.CreateBatchTasks("Export", jobId, containerUrl, exportRequest.BatchAccountUrl, exportRequest.SourceSqlServerName, exportRequest.AccessToken, databases, log);
// create output values
Tuple<string, string>[] outputValues = {
Tuple.Create("jobId", jobId),
Tuple.Create("containerUrl", containerUrl)
};
context.SetOutput(outputValues);
}
}
}

79
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/StorageActivity.cs
View File

@@ -1,79 +0,0 @@
using Microsoft.Azure.Management.Storage;
using Microsoft.Azure.Management.Storage.Models;
using Microsoft.Azure.Services.AppAuthentication;
using Microsoft.Azure.WebJobs;
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
using Microsoft.Extensions.Logging;
using Microsoft.Rest;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Auth;
using Microsoft.WindowsAzure.Storage.Blob;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace ADPControl
{
public static class StorageActivity
{
private const string ArmTemplateFileName = "template.json";
[FunctionName(nameof(GettingJobContainerUrl))]
public static string GettingJobContainerUrl([ActivityTrigger] (Guid, string, string, string) input, ILogger log)
{
Guid SubscriptionId = input.Item1;
String ResourceGroupName = input.Item2;
String StorageAccountName = input.Item3;
String ContainerName = input.Item4;
var azureServiceTokenProvider = new AzureServiceTokenProvider();
TokenCredentials tokenArmCredential = new TokenCredentials(azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/").Result);
StorageManagementClient storageMgmtClient = new StorageManagementClient(tokenArmCredential) { SubscriptionId = SubscriptionId.ToString() };
// Get the storage account keys for a given account and resource group
IList<StorageAccountKey> acctKeys = storageMgmtClient.StorageAccounts.ListKeys(ResourceGroupName, StorageAccountName).Keys;
// Get a Storage account using account creds:
StorageCredentials storageCred = new StorageCredentials(StorageAccountName, acctKeys.FirstOrDefault().Value);
CloudStorageAccount linkedStorageAccount = new CloudStorageAccount(storageCred, true);
bool createContainer = false;
// Normalize the container name for the Export action.
if (ContainerName.Contains("-Export-"))
{
ContainerName = ContainerName.Replace("Export-", "");
createContainer = true;
}
CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(ContainerName);
if(createContainer)
container.CreateIfNotExistsAsync().Wait();
string containerUrl = container.Uri.ToString() +
container.GetSharedAccessSignature(new SharedAccessBlobPolicy()
{
Permissions = SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List,
SharedAccessExpiryTime = DateTime.UtcNow.AddDays(7)
});
return containerUrl;
}
[FunctionName(nameof(UploadingArmTemplate))]
public static void UploadingArmTemplate([ActivityTrigger] (string, string) input, ILogger log)
{
string containerUrl = input.Item1;
string json = input.Item2;
CloudBlobContainer container = new CloudBlobContainer(new Uri(containerUrl));
CloudBlockBlob blob = container.GetBlockBlobReference(ArmTemplateFileName);
blob.Properties.ContentType = "application/json";
using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(json)))
{
blob.UploadFromStreamAsync(stream).Wait();
}
}
}
}

16
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl/host.json
View File

@@ -1,16 +0,0 @@
{
"version": "2.0",
"extensions": {
"durableTask": {
"hubName": "adp"
}
},
"logging": {
"applicationInsights": {
"samplingExcludedTypes": "Request",
"samplingSettings": {
"isEnabled": true
}
}
}
}

12
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/ActionType.cs
View File

@@ -1,12 +0,0 @@
namespace BatchWrapper
{
/// <summary>
/// The type of sqlpackage action to perform.
/// </summary>
public enum ActionType
{
DefaultInvalid = -1,
Export,
Import
}
}

22
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/App.config
View File

@@ -1,22 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
</startup>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-12.0.0.0" newVersion="12.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Azure.Batch" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Azure.KeyVault.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.0.5.0" newVersion="3.0.5.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

98
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/BatchWrapper.csproj
View File

@@ -1,98 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>BatchWrapper</RootNamespace>
<AssemblyName>BatchWrapper</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<LangVersion>8.0</LangVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<Deterministic>true</Deterministic>
<IsWebBootstrapper>false</IsWebBootstrapper>
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Azure.Batch.Conventions.Files, Version=3.5.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.Azure.Batch.Conventions.Files.3.5.1\lib\net461\Microsoft.Azure.Batch.Conventions.Files.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=9.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\WindowsAzure.Storage.9.3.3\lib\net45\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Runtime" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="ActionType.cs" />
<Compile Include="Constants.cs" />
<Compile Include="Payload.cs" />
<Compile Include="Program.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<BootstrapperPackage Include=".NETFramework,Version=v4.7.2">
<Visible>False</Visible>
<ProductName>Microsoft .NET Framework 4.7.2 %28x86 and x64%29</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>false</Install>
</BootstrapperPackage>
</ItemGroup>
<ItemGroup>
<Folder Include="Properties\" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>

34
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Constants.cs
View File

@@ -1,34 +0,0 @@
namespace BatchWrapper
{
/// <summary>
/// Constants for the batch wrapper.
/// </summary>
public static class Constants
{
/// <summary>
/// Environment variable names present or needed during the batch task execution.
/// </summary>
public static class EnvironmentVariableNames
{
/// <summary>
/// Path to the directory containing the sqlpackage exe.
/// </summary>
internal const string AppPackagePrefix = "AZ_BATCH_APP_PACKAGE";
/// <summary>
/// Path to the working directory assigned to the batch task.
/// </summary>
internal const string TaskWorkingDir = "AZ_BATCH_TASK_WORKING_DIR";
/// <summary>
/// Path to the working directory assigned to the batch task.
/// </summary>
internal const string AzBatchTaskId = "AZ_BATCH_TASK_ID";
/// <summary>
/// Path to the working directory assigned to the batch task.
/// </summary>
internal const string JobContainerUrl = "JOB_CONTAINER_URL";
}
}
}

38
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Payload.cs
View File

@@ -1,38 +0,0 @@
namespace BatchWrapper
{
/// <summary>
/// The top-level object stored in the key vault for an import/export operation.
/// </summary>
public sealed class Payload
{
/// <summary>
/// The Name of sqlpackage to use for performing the import/export operation.
/// </summary>
public string ApplicatonPackageName{ get; set; }
/// <summary>
/// The Version of sqlpackage to use for performing the import/export operation.
/// </summary>
public string ApplicatonPackageVersion { get; set; }
/// <summary>
/// The type of sqlpackage action to perform.
/// </summary>
public ActionType Action { get; set; }
/// <summary>
/// The logical server name to export from or import to.
/// </summary>
public string LogicalServerName { get; set; }
/// <summary>
/// The database name to export from or import to.
/// </summary>
public string DatabaseName { get; set; }
/// <summary>
/// The server admin username.
/// </summary>
public string AccessToken { get; set; }
}
}

178
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Program.cs
View File

@@ -1,178 +0,0 @@
using Microsoft.Azure.Batch.Conventions.Files;
using Microsoft.WindowsAzure.Storage.Blob;
using System;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
namespace BatchWrapper
{
public static class Program
{
private static string dataDirectory = "F:\\data";
private static string tempDirectory = "F:\\temp";
private static string[] directories = { dataDirectory, tempDirectory };
private static readonly TimeSpan stdoutFlushDelay = TimeSpan.FromSeconds(3);
private static void WriteLine(string message) => WriteLineInternal(Console.Out, message);
private static void WriteErrorLine(string message) => WriteLineInternal(Console.Error, message);
private static void WriteLineInternal(TextWriter writer, string message)
{
var lines = message?.Split('\n') ?? new string[0];
foreach (var line in lines)
{
writer.WriteLine($"[{DateTime.UtcNow:u}] {line?.TrimEnd()}");
}
}
public static async Task<int> Main(string[] args)
{
var assembly = typeof(Program).Assembly;
WriteLine($"{assembly.ManifestModule.Name} v{assembly.GetName().Version.ToString(3)}");
// Get the command payload
var payload = new Payload();
if (args.Length > 0)
{
payload.Action = (ActionType)Enum.Parse(typeof(ActionType), args[0]);
payload.LogicalServerName = args[1] + ".database.windows.net";
payload.DatabaseName = args[2];
payload.AccessToken = args[3];
payload.ApplicatonPackageName = args[4];
payload.ApplicatonPackageVersion = args[5];
}
// Cleanup folders
foreach (string dir in directories)
{
if (Directory.Exists(dir))
{
Directory.Delete(dir, true);
}
Directory.CreateDirectory(dir);
}
string sqlPackageBacpacFile = Path.Combine(dataDirectory, payload.DatabaseName + ".bacpac");
string sqlPackageLogPath = payload.DatabaseName + ".log";
var targetDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AppPackagePrefix + "_" + payload.ApplicatonPackageName + "#" + payload.ApplicatonPackageVersion);
var workingDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.TaskWorkingDir);
string taskId = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AzBatchTaskId);
string jobContainerUrl = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.JobContainerUrl);
// Build the import/export command
var cmdBuilder = new StringBuilder();
cmdBuilder.Append($"/Action:{payload.Action}");
cmdBuilder.Append(" /MaxParallelism:16");
cmdBuilder.Append(String.Format(" /DiagnosticsFile:{0}", sqlPackageLogPath));
cmdBuilder.Append(" /p:CommandTimeout=604800");
switch (payload.Action)
{
case ActionType.Export:
cmdBuilder.Append($" /SourceServerName:{payload.LogicalServerName}");
cmdBuilder.Append($" /SourceDatabaseName:{payload.DatabaseName}");
cmdBuilder.Append($" /AccessToken:{payload.AccessToken}");
cmdBuilder.Append($" /TargetFile:{sqlPackageBacpacFile}");
cmdBuilder.Append($" /SourceTimeout:30");
cmdBuilder.Append(String.Format(" /p:TempDirectoryForTableData=\"{0}\"", tempDirectory));
cmdBuilder.Append(" /p:VerifyFullTextDocumentTypesSupported=false");
break;
case ActionType.Import:
cmdBuilder.Append($" /TargetServerName:{payload.LogicalServerName}");
cmdBuilder.Append($" /TargetDatabaseName:{payload.DatabaseName}");
cmdBuilder.Append($" /AccessToken:{payload.AccessToken}");
cmdBuilder.Append($" /TargetTimeout:30");
cmdBuilder.Append($" /SourceFile:{sqlPackageBacpacFile}");
break;
default:
throw new ArgumentException($"Invalid action type: {payload.Action}");
}
if (payload.Action == ActionType.Import)
{
WriteLine(string.Format("Downloading {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
CloudBlobContainer container = new CloudBlobContainer(new Uri(jobContainerUrl));
CloudBlockBlob blob = container.GetBlockBlobReference(String.Format("$JobOutput/{0}.bacpac", payload.DatabaseName));
blob.DownloadToFile(sqlPackageBacpacFile, FileMode.CreateNew);
if (File.Exists(sqlPackageBacpacFile))
{
WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
}
else
{
throw new Exception(string.Format("{0} didn't download", sqlPackageBacpacFile));
}
}
// Perform the import/export process
var startTime = DateTimeOffset.UtcNow;
var process = new Process
{
StartInfo = new ProcessStartInfo
{
WorkingDirectory = workingDir,
FileName = Path.Combine(targetDir, "sqlpackage.exe"),
Arguments = cmdBuilder.ToString(),
CreateNoWindow = true,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true
}
};
process.OutputDataReceived += (s, e) => WriteLine(e.Data);
process.ErrorDataReceived += (s, e) => WriteErrorLine(e.Data);
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
process.WaitForExit();
WriteLine(String.Format("SqlPackage.exe exited with code: {0}", process.ExitCode));
if (payload.Action == ActionType.Export)
{
if (File.Exists(sqlPackageBacpacFile))
{
WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
}
else
{
throw new Exception(string.Format("{0} didn't downloaded", sqlPackageBacpacFile));
}
// Persist the Job Output
JobOutputStorage jobOutputStorage = new JobOutputStorage(new Uri(jobContainerUrl));
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageLogPath);
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageLogPath));
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageBacpacFile, payload.DatabaseName + ".bacpac");
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageBacpacFile));
}
// We are tracking the disk file to save our standard output, but the node agent may take
// up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
await Task.Delay(stdoutFlushDelay);
// Cleanup folders
foreach (string dir in directories)
{
if (Directory.Exists(dir))
{
Directory.Delete(dir, true);
}
}
return process.ExitCode;
}
}
}

10
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/packages.config
View File

@@ -1,10 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Azure.Batch" version="13.0.0" targetFramework="net472" />
<package id="Microsoft.Azure.Batch.Conventions.Files" version="3.5.1" targetFramework="net472" />
<package id="Microsoft.Azure.KeyVault.Core" version="3.0.5" targetFramework="net472" />
<package id="Microsoft.Rest.ClientRuntime" version="2.3.21" targetFramework="net472" />
<package id="Microsoft.Rest.ClientRuntime.Azure" version="3.3.19" targetFramework="net472" />
<package id="Newtonsoft.Json" version="12.0.3" targetFramework="net472" />
<package id="WindowsAzure.Storage" version="9.3.3" targetFramework="net472" />
</packages>

27
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/.vscode/launch.json vendored
View File

@@ -1,27 +0,0 @@
{
// Use IntelliSense to find out which attributes exist for C# debugging
// Use hover for the description of the existing attributes
// For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
"version": "0.2.0",
"configurations": [
{
"name": ".NET Core Launch (console)",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
// If you have changed target frameworks, make sure to update the program path.
"program": "${workspaceFolder}/bin/Debug/netcoreapp3.1/SqlPackageWrapper.dll",
"args": [],
"cwd": "${workspaceFolder}",
// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
"console": "internalConsole",
"stopAtEntry": false
},
{
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach",
"processId": "${command:pickProcess}"
}
]
}

42
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/.vscode/tasks.json vendored
View File

@@ -1,42 +0,0 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "build",
"command": "dotnet",
"type": "process",
"args": [
"build",
"${workspaceFolder}/SqlPackageWrapper.csproj",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"problemMatcher": "$msCompile"
},
{
"label": "publish",
"command": "dotnet",
"type": "process",
"args": [
"publish",
"${workspaceFolder}/SqlPackageWrapper.csproj",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"problemMatcher": "$msCompile"
},
{
"label": "watch",
"command": "dotnet",
"type": "process",
"args": [
"watch",
"run",
"${workspaceFolder}/SqlPackageWrapper.csproj",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"problemMatcher": "$msCompile"
}
]
}

12
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/ActionType.cs
View File

@@ -1,12 +0,0 @@
namespace SqlPackageWrapper
{
/// <summary>
/// The type of sqlpackage action to perform.
/// </summary>
public enum ActionType
{
DefaultInvalid = -1,
Export,
Import
}
}

39
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Constants.cs
View File

@@ -1,39 +0,0 @@
namespace SqlPackageWrapper
{
/// <summary>
/// Constants for the batch wrapper.
/// </summary>
public static class Constants
{
/// <summary>
/// Environment variable names present or needed during the batch task execution.
/// </summary>
public static class EnvironmentVariableNames
{
/// <summary>
/// Path to the directory containing the batch wrapper exe.
/// </summary>
public const string WrapperLocation = "AZ_BATCH_APP_PACKAGE_BATCHWRAPPER";
/// <summary>
/// Path to the directory containing the sqlpackage exe.
/// </summary>
internal const string SqlPackageLocation = "AZ_BATCH_APP_PACKAGE_SQLPACKAGE";
/// <summary>
/// Path to the working directory assigned to the batch task.
/// </summary>
internal const string TaskWorkingDir = "AZ_BATCH_TASK_WORKING_DIR";
/// <summary>
/// Path to the working directory assigned to the batch task.
/// </summary>
internal const string AzBatchTaskId = "AZ_BATCH_TASK_ID";
/// <summary>
/// Path to the working directory assigned to the batch task.
/// </summary>
internal const string JobContainerUrl = "JOB_CONTAINER_URL";
}
}
}

38
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Payload.cs
View File

@@ -1,38 +0,0 @@
namespace SqlPackageWrapper
{
/// <summary>
/// The top-level object stored in the key vault for an import/export operation.
/// </summary>
public sealed class Payload
{
/// <summary>
/// The version of sqlpackage to use for performing the import/export operation.
/// </summary>
public string SqlPackageVersion { get; set; }
/// <summary>
/// The type of sqlpackage action to perform.
/// </summary>
public ActionType Action { get; set; }
/// <summary>
/// The logical server name to export from or import to.
/// </summary>
public string LogicalServerName { get; set; }
/// <summary>
/// The database name to export from or import to.
/// </summary>
public string DatabaseName { get; set; }
/// <summary>
/// The server admin username.
/// </summary>
public string Username { get; set; }
/// <summary>
/// The server admin password.
/// </summary>
public string Password { get; set; }
}
}

162
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/Program.cs
View File

@@ -1,162 +0,0 @@
using Microsoft.Azure.Batch.Conventions.Files;
using Microsoft.WindowsAzure.Storage.Blob;
using System;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
namespace SqlPackageWrapper
{
public static class Program
{
private static string dataDirectory = "F:\\data";
private static string tempDirectory = "F:\\temp";
private static string[] directories = { dataDirectory, tempDirectory };
private static readonly TimeSpan stdoutFlushDelay = TimeSpan.FromSeconds(3);
private static void WriteLine(string message) => WriteLineInternal(Console.Out, message);
private static void WriteErrorLine(string message) => WriteLineInternal(Console.Error, message);
private static void WriteLineInternal(TextWriter writer, string message)
{
var lines = message?.Split('\n') ?? new string[0];
foreach (var line in lines)
{
writer.WriteLine($"[{DateTime.UtcNow:u}] {line?.TrimEnd()}");
}
}
public static async Task<int> Main(string[] args)
{
var assembly = typeof(Program).Assembly;
WriteLine($"{assembly.ManifestModule.Name} v{assembly.GetName().Version.ToString(3)}");
// Get the command payload
var payload = new Payload();
if (args.Length > 0)
{
payload.Action = (ActionType)Enum.Parse(typeof(ActionType), args[0]);
payload.LogicalServerName = args[1] + ".database.windows.net";
payload.DatabaseName = args[2];
payload.Username = args[3];
payload.Password = args[4];
payload.SqlPackageVersion = args[5];
}
// Cleanup folders
foreach (string dir in directories)
{
if (Directory.Exists(dir))
{
Directory.Delete(dir, true);
}
Directory.CreateDirectory(dir);
}
string sqlPackageDataPath = Path.Combine(dataDirectory, payload.DatabaseName + ".bacpac");
string sqlPackageLogPath = Path.Combine(dataDirectory, payload.DatabaseName + ".log");
var targetDir = Environment.GetEnvironmentVariable($"{Constants.EnvironmentVariableNames.SqlPackageLocation}#{payload.SqlPackageVersion}");
var workingDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.TaskWorkingDir);
string taskId = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AzBatchTaskId);
string jobContainerUrl = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.JobContainerUrl);
// Build the import/export command
var cmdBuilder = new StringBuilder();
cmdBuilder.Append($"/Action:{payload.Action}");
cmdBuilder.Append(" /MaxParallelism:16");
cmdBuilder.Append(String.Format(" /DiagnosticsFile:{0}", sqlPackageLogPath));
cmdBuilder.Append(" /p:CommandTimeout=86400");
switch (payload.Action)
{
case ActionType.Export:
cmdBuilder.Append($" /SourceServerName:{payload.LogicalServerName}");
cmdBuilder.Append($" /SourceDatabaseName:{payload.DatabaseName}");
cmdBuilder.Append($" /SourceUser:{payload.Username}");
cmdBuilder.Append($" /SourcePassword:{payload.Password}");
cmdBuilder.Append($" /TargetFile:{sqlPackageDataPath}");
cmdBuilder.Append(String.Format(" /p:TempDirectoryForTableData=\"{0}\"", tempDirectory));
cmdBuilder.Append(" /p:VerifyFullTextDocumentTypesSupported=false");
break;
case ActionType.Import:
cmdBuilder.Append($" /TargetServerName:{payload.LogicalServerName}");
cmdBuilder.Append($" /TargetDatabaseName:{payload.DatabaseName}");
cmdBuilder.Append($" /TargetUser:{payload.Username}");
cmdBuilder.Append($" /TargetPassword:{payload.Password}");
cmdBuilder.Append($" /SourceFile:{sqlPackageDataPath}");
break;
default:
throw new ArgumentException($"Invalid action type: {payload.Action}");
}
if (payload.Action == ActionType.Import)
{
WriteLine(string.Format("Downloading {0} bacpac file to {1}", payload.DatabaseName, sqlPackageDataPath));
CloudBlobContainer container = new CloudBlobContainer(new Uri(jobContainerUrl));
CloudBlockBlob blob = container.GetBlockBlobReference(String.Format("$JobOutput/{0}.bacpac", payload.DatabaseName));
await blob.DownloadToFileAsync(sqlPackageDataPath, FileMode.CreateNew);
WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageDataPath));
await Task.Delay(stdoutFlushDelay);
}
// Perform the import/export process
var startTime = DateTimeOffset.UtcNow;
var process = new Process
{
StartInfo = new ProcessStartInfo
{
WorkingDirectory = workingDir,
FileName = Path.Combine(targetDir, "sqlpackage.exe"),
Arguments = cmdBuilder.ToString(),
CreateNoWindow = true,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true
}
};
process.OutputDataReceived += (s, e) => WriteLine(e.Data);
process.ErrorDataReceived += (s, e) => WriteErrorLine(e.Data);
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
process.WaitForExit();
WriteLine(String.Format("SqlPackage.exe exited with code: {0}", process.ExitCode));
if (payload.Action == ActionType.Export)
{
// Persist the Job Output
JobOutputStorage jobOutputStorage = new JobOutputStorage(new Uri(jobContainerUrl));
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageLogPath, payload.DatabaseName + ".log");
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageLogPath));
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageDataPath, payload.DatabaseName + ".bacpac");
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageDataPath));
// We are tracking the disk file to save our standard output, but the node agent may take
// up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
await Task.Delay(stdoutFlushDelay);
}
// Cleanup folders
foreach (string dir in directories)
{
if (Directory.Exists(dir))
{
Directory.Delete(dir, true);
}
}
return process.ExitCode;
}
}
}

14
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/SqlPackageWrapper/SqlPackageWrapper.csproj
View File

@@ -1,14 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>netcoreapp2.1;net452</TargetFrameworks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.Batch" Version="13.0.0" />
<PackageReference Include="Microsoft.Azure.Batch.Conventions.Files" Version="3.5.1" />
<PackageReference Include="Microsoft.Azure.Management.Batch" Version="11.0.0" />
</ItemGroup>
</Project>

1
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/readme.md
View File

@@ -1 +0,0 @@
Folder for helper components used by notebooks in the Hybrid Cloud Toolkit

7
extensions/azurehybridtoolkit/notebooks/hybridbook/_data/toc.yml
View File

@@ -8,6 +8,13 @@
- title: Search
search: true
- title: ADP
url: /ADP/readme
not_numbered: true
expand_sections: false
sections:
- title: ADP Control
url: ADP/ADPControl
- title: Assessment
url: /Assessments/readme
not_numbered: true

0
extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/ADPControl.ipynb → extensions/azurehybridtoolkit/notebooks/hybridbook/content/ADP/ADPControl.ipynb
View File

8
extensions/azurehybridtoolkit/notebooks/hybridbook/content/ADP/readme.md Normal file
View File

@@ -0,0 +1,8 @@
# ADP Control
[Home](../readme.md)
Notebook to manage ADP Orchestrator import/export requests onto an SQL Server.
## Notebooks in this Chapter
- [ADP Control](ADPControl.ipynb)

12
extensions/azurehybridtoolkit/notebooks/hybridbook/content/readme.md
View File

@@ -2,13 +2,15 @@
## Chapters
* [Prerequisites and Initial Setup](prereqs.ipynb) - Notebook installation of required modules.
* [Assessments](Assessments/readme.md) - Notebooks that contain examples to determine whether a given database or SQL Server instance is ready to migrate by utilizing SQL Assessments. SQL instances are scanned based on a "best practices" set of rules.
* [ADP](ADP/readme.md) - manage ADP Orchestrator import/export requests onto an SQL Server.
* [Assessments](Assessments/readme.md) - Notebooks that contain examples to determine whether a given database or SQL Server instance is ready to migrate by utilizing SQL Assessments. SQL instances are scanned based on a "best practices" set of rules.
* [Networking](networking/readme.md) - Setup secure Point-to-Site (P2S) or Site-to-Site (S2S) network connectivity to Microsoft Azure using a Virtual Private Network (VPN). This notebook serves as a building block for other notebooks as communicating securely between on-premise and Azure is essential for many tasks.
* [Provisioning](provisioning/readme.md) - Creating and communicating with SQL Resources in Microsoft Azure. Includes common tasks such as creating SQL Virtual Machines or SQL Managed Instances in the cloud.
* [Data Portability](data-portability/readme.md) - Install a custom Azure function to facilitate importing and exporting cloud resources. The solution uses parallel tasks in Azure Batch to perform data storage work. Azure Batch is a process that runs large-scale parallel and high-performance computing jobs efficiently in Azure.
* [Data Portability](data-portability/readme.md) - Install a custom Azure function to facilitate importing and exporting cloud resources. The solution uses parallel tasks in Azure Batch to perform data storage work. Azure Batch is a process that runs large-scale parallel and high-performance computing jobs efficiently in Azure.
* [High Availability and Disaster Recovery](hadr/readme.md) - Notebooks to leverage Azure SQL for business continuity in a hybrid cloud environment.
@@ -20,16 +22,16 @@
## About
The **Azure SQL Hybrid Cloud Toolkit** is a [Jupyter Book](https://jupyterbook.org/intro.html) extension of [Azure Data Studio](https://docs.microsoft.com/en-us/sql/azure-data-studio/download-azure-data-studio) (ADS) designed to help [Azure SQL Database](https://azure.microsoft.com/en-us/services/sql-database/) and ADS users deploy, migrate and configure for a hybrid cloud environment. The toolkit was designed with and intended to be executed within ADS. This is to ensure the best possible user experience for those without vast knowledge of Azure services while adhering closely to the software _best practices_ standards required by experienced cloud users.
The **Azure SQL Hybrid Cloud Toolkit** is a [Jupyter Book](https://jupyterbook.org/intro.html) extension of [Azure Data Studio](https://docs.microsoft.com/en-us/sql/azure-data-studio/download-azure-data-studio) (ADS) designed to help [Azure SQL Database](https://azure.microsoft.com/en-us/services/sql-database/) and ADS users deploy, migrate and configure for a hybrid cloud environment. The toolkit was designed with and intended to be executed within ADS. This is to ensure the best possible user experience for those without vast knowledge of Azure services while adhering closely to the software _best practices_ standards required by experienced cloud users.
## Goals and Methodology
The toolkit better positions a customer with regards to planning, migrating, and thriving in a hybrid cloud environment by:
* Providing SQL'zure users with reliable free software and content that is well-written and executable
* Greatly simplifying the integration of Azure Data services into an existing environment
* Positioning Azure to be the natural cloud services choice with a low-friction experience
* Positioning Azure to be the natural cloud services choice with a low-friction experience
* Notebooks are executable by a normal user (unless otherwise specificed) on minimal hardware
* Most notebooks require some configuration. If so, the proper configurations should be clearly located towards the top of the notebook or cell, whichever is most appropriate
* By design, Notebooks are written to be executed from top-to-bottom. Therefore, each notebook has a specific task to perform and should focus only on that task. It may contain several cells to execute but it will adhere to the one-task per notebook paradigm
**NOTE:** Executing notebooks could potentially create new Azure Resources which may incur charges to the Azure Subscription. Make sure the repercussions of executing any cells are understood.
**NOTE:** Executing notebooks could potentially create new Azure Resources which may incur charges to the Azure Subscription. Make sure the repercussions of executing any cells are understood.