mirror of
https://github.com/ckaczor/azuredatastudio.git
synced 2026-03-30 08:40:29 -04:00
Azure SQL Hybrid Cloud Toolkit Notebooks Extension Command (#13286)
* added extension folder incomplete * WIP extension progress * notebook finally opens in side panel * notebook now opens via notebook extension * html file spaces restored * package json fixed * fixed vscode import issue * more cleanup * remove git stuff * placeholder icon logos added * fixed gulpfile * cleanup changes * vscode import fixed * fixed main and yarn.lock * added provided notebooks view * formatting for package.json * removed first command as its not necessary * fixed notebook typo * readded spaces
This commit is contained in:
Alex Ma
@@ -0,0 +1,26 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<AzureFunctionsVersion>v3</AzureFunctionsVersion>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Azure.Batch" Version="13.0.0" />
|
||||
<PackageReference Include="Microsoft.Azure.Management.ResourceManager" Version="3.7.0-preview" />
|
||||
<PackageReference Include="Microsoft.Azure.Management.Storage" Version="17.0.0" />
|
||||
<PackageReference Include="Microsoft.Azure.Services.AppAuthentication" Version="1.5.0" />
|
||||
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="2.2.2" />
|
||||
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.7" />
|
||||
<PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.21" />
|
||||
<PackageReference Include="Microsoft.Rest.ClientRuntime.Azure" Version="3.3.19" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Update="host.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="local.settings.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
<CopyToPublishDirectory>Never</CopyToPublishDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,244 @@
|
||||
using Microsoft.Azure.Management.ResourceManager;
|
||||
using Microsoft.Azure.Management.ResourceManager.Models;
|
||||
using Microsoft.Azure.Management.Storage;
|
||||
using Microsoft.Azure.Management.Storage.Models;
|
||||
using Microsoft.Azure.Services.AppAuthentication;
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Rest;
|
||||
using Microsoft.WindowsAzure.Storage;
|
||||
using Microsoft.WindowsAzure.Storage.Auth;
|
||||
using Microsoft.WindowsAzure.Storage.Blob;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using static ADPControl.HttpSurface;
|
||||
|
||||
namespace ADPControl
|
||||
{
|
||||
public static class AzureResourceManagerActivity
|
||||
{
|
||||
private const string ArmTemplateFileName = "template.json";
|
||||
|
||||
private static string[] AllowedImportSubServerResourceTypes = new string[] {
|
||||
"Microsoft.Sql/servers/firewallRules",
|
||||
"Microsoft.Sql/servers/databases",
|
||||
"Microsoft.Sql/servers/elasticPools",
|
||||
//"Microsoft.Sql/servers/keys",
|
||||
//"Microsoft.Sql/servers/databases/transparentDataEncryption",
|
||||
"Microsoft.Sql/servers/databases/backupShortTermRetentionPolicies",
|
||||
"Microsoft.Sql/servers/administrators"
|
||||
};
|
||||
|
||||
// Deploy the ARM template
|
||||
[FunctionName(nameof(BeginDeployArmTemplateForImport))]
|
||||
public static async Task<string> BeginDeployArmTemplateForImport([ActivityTrigger] ImportRequest request, ILogger log)
|
||||
{
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
||||
|
||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
||||
StorageManagementClient storageMgmtClient = new StorageManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
||||
|
||||
// Get the storage account keys for a given account and resource group
|
||||
IList<StorageAccountKey> acctKeys = storageMgmtClient.StorageAccounts.ListKeys(request.ResourceGroupName, request.StorageAccountName).Keys;
|
||||
|
||||
// Get a Storage account using account creds:
|
||||
StorageCredentials storageCred = new StorageCredentials(request.StorageAccountName, acctKeys.FirstOrDefault().Value);
|
||||
CloudStorageAccount linkedStorageAccount = new CloudStorageAccount(storageCred, true);
|
||||
CloudBlobContainer container = linkedStorageAccount
|
||||
.CreateCloudBlobClient()
|
||||
.GetContainerReference(request.ContainerName);
|
||||
|
||||
CloudBlockBlob blob = container.GetBlockBlobReference(ArmTemplateFileName);
|
||||
string json = await blob.DownloadTextAsync();
|
||||
|
||||
JObject originalTemplate = JObject.Parse(json);
|
||||
JObject importTemplate = UpdateArmTemplateForImport(originalTemplate, request);
|
||||
|
||||
var deployParams = new Deployment
|
||||
{
|
||||
Properties = new DeploymentProperties
|
||||
{
|
||||
Mode = DeploymentMode.Incremental,
|
||||
Template = importTemplate
|
||||
}
|
||||
};
|
||||
|
||||
string deploymentName = request.TargetSqlServerName + "_" + DateTime.UtcNow.ToFileTimeUtc();
|
||||
|
||||
try
|
||||
{
|
||||
await resourcesClient.Deployments.BeginCreateOrUpdateAsync(request.TargetSqlServerResourceGroupName, deploymentName, deployParams);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
log.LogError(ex.ToString());
|
||||
throw ex;
|
||||
}
|
||||
|
||||
return deploymentName;
|
||||
}
|
||||
|
||||
// Get the ARM deployment status
|
||||
[FunctionName(nameof(GetArmDeploymentForImport))]
|
||||
public static async Task<string> GetArmDeploymentForImport([ActivityTrigger] (Guid, string, string) input)
|
||||
{
|
||||
Guid subscriptionId = input.Item1;
|
||||
string resourceGroupName = input.Item2;
|
||||
string deploymentName = input.Item3;
|
||||
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = subscriptionId.ToString() };
|
||||
|
||||
DeploymentExtended result = await resourcesClient.Deployments.GetAsync(resourceGroupName, deploymentName);
|
||||
return result.Properties.ProvisioningState;
|
||||
}
|
||||
|
||||
// Get the ARM template without the parameter of the resource name
|
||||
[FunctionName(nameof(GetArmTemplateForExportSkipParameterization))]
|
||||
public static async Task<dynamic> GetArmTemplateForExportSkipParameterization([ActivityTrigger] ExportRequest request, ILogger log)
|
||||
{
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: entering");
|
||||
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
||||
if (tokenArmCredential != null)
|
||||
{
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: acquired access token");
|
||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
||||
|
||||
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.SourceSqlServerResourceGroupName, request.SourceSqlServerName);
|
||||
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.SourceSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "SkipResourceNameParameterization"));
|
||||
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: server template exported. Size: {0} bytes", exportedTemplate.Template.ToString().Length);
|
||||
dynamic template = (dynamic)exportedTemplate.Template;
|
||||
|
||||
// Filtering the list of databases
|
||||
dynamic databases = template.resources.SelectTokens("$.[?(@.type == 'Microsoft.Sql/servers/databases')]");
|
||||
int numberOfDatabases = 0;
|
||||
foreach (var db in databases)
|
||||
{
|
||||
numberOfDatabases++;
|
||||
}
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with database list. Databases count: {0}", numberOfDatabases);
|
||||
|
||||
return databases;
|
||||
}
|
||||
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with empty database list");
|
||||
return null;
|
||||
}
|
||||
|
||||
// Get the ARM template without the parameter of the resource name
|
||||
[FunctionName(nameof(GetArmTemplateForImportSkipParameterization))]
|
||||
public static async Task<dynamic> GetArmTemplateForImportSkipParameterization([ActivityTrigger] ImportRequest request, ILogger log)
|
||||
{
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
||||
if (tokenArmCredential != null)
|
||||
{
|
||||
log.LogInformation("GetArmTemplateForImportSkipParameterization: acquired access token");
|
||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
||||
|
||||
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.TargetSqlServerResourceGroupName, request.TargetSqlServerName);
|
||||
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.TargetSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "SkipResourceNameParameterization"));
|
||||
|
||||
log.LogInformation("GetArmTemplateForImportSkipParameterization: server template exported. Size: {0} bytes", exportedTemplate.Template.ToString().Length);
|
||||
dynamic template = (dynamic)exportedTemplate.Template;
|
||||
|
||||
// Filtering the list of databases
|
||||
dynamic databases = template.resources.SelectTokens("$.[?(@.type == 'Microsoft.Sql/servers/databases')]");
|
||||
|
||||
int numberOfDatabases = 0;
|
||||
foreach (var db in databases)
|
||||
{
|
||||
numberOfDatabases++;
|
||||
}
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with database list. Databases count: {0}", numberOfDatabases);
|
||||
return databases;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
[FunctionName(nameof(GetArmTemplateForExport))]
|
||||
public static async Task<dynamic> GetArmTemplateForExport([ActivityTrigger] ExportRequest request)
|
||||
{
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
||||
|
||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
||||
|
||||
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.SourceSqlServerResourceGroupName, request.SourceSqlServerName);
|
||||
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.SourceSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "IncludeParameterDefaultValue"));
|
||||
return exportedTemplate.Template;
|
||||
}
|
||||
|
||||
private static JObject UpdateArmTemplateForImport(JObject originalTemplate, ImportRequest request)
|
||||
{
|
||||
string serverNameParameterName = null;
|
||||
|
||||
// Go through every parameter to find the property name is like 'server_%_name'
|
||||
using (JsonTextReader reader = new JsonTextReader(new StringReader(originalTemplate["parameters"].ToString())))
|
||||
{
|
||||
while (reader.Read())
|
||||
{
|
||||
if (reader.TokenType.ToString().Equals("PropertyName")
|
||||
&& reader.ValueType.ToString().Equals("System.String")
|
||||
&& reader.Value.ToString().StartsWith("servers_")
|
||||
&& reader.Value.ToString().EndsWith("_name"))
|
||||
{
|
||||
serverNameParameterName = reader.Value.ToString();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Replacing the default value to the target server name, appending to the new template
|
||||
originalTemplate["parameters"][serverNameParameterName]["defaultValue"] = request.TargetSqlServerName;
|
||||
JObject serverNameParameterValue = (JObject)originalTemplate["parameters"][serverNameParameterName];
|
||||
|
||||
// 2. Cleanup all the parameters except the updated server name
|
||||
((JObject)originalTemplate["parameters"]).RemoveAll();
|
||||
((JObject)originalTemplate["parameters"]).Add(serverNameParameterName, serverNameParameterValue);
|
||||
|
||||
// 3. Adjust the servers resource by adding password after the login
|
||||
JObject server = (JObject)originalTemplate["resources"]
|
||||
.SelectToken("$.[?(@.type == 'Microsoft.Sql/servers')]");
|
||||
|
||||
server.Remove("identity");
|
||||
|
||||
JObject serverProperties = (JObject)server["properties"];
|
||||
serverProperties.Property("administratorLogin")
|
||||
.AddAfterSelf(new JProperty("administratorLoginPassword", request.SqlAdminPassword));
|
||||
|
||||
JArray newResources = new JArray();
|
||||
|
||||
// 4. Getting the whitelisted resources and adding them to the new template later.
|
||||
foreach (string resourceType in AllowedImportSubServerResourceTypes)
|
||||
{
|
||||
List<JToken> resources = originalTemplate["resources"]
|
||||
.SelectTokens(string.Format("$.[?(@.type == '{0}')]", resourceType)).ToList();
|
||||
newResources.Add(resources);
|
||||
}
|
||||
|
||||
// 5. Clean up all the resources excepted the new server and whitelisted resource type.
|
||||
((JArray)originalTemplate["resources"]).Clear();
|
||||
((JArray)originalTemplate["resources"]).Add(server);
|
||||
|
||||
foreach (var resource in newResources)
|
||||
{
|
||||
((JArray)originalTemplate["resources"]).Add(resource);
|
||||
}
|
||||
|
||||
return originalTemplate;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
using Microsoft.Azure.Batch;
|
||||
using Microsoft.Azure.Batch.Auth;
|
||||
using Microsoft.Azure.Batch.Common;
|
||||
using Microsoft.Azure.Services.AppAuthentication;
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using static ADPControl.HttpSurface;
|
||||
|
||||
namespace ADPControl
|
||||
{
|
||||
public static class BatchActivity
|
||||
{
|
||||
// Batch resource settings
|
||||
private const string PoolVMSize = "Standard_D8s_v3";
|
||||
private const string PoolId = PoolVMSize;
|
||||
private const int PoolNodeCount = 2;
|
||||
private const string AppPackageName = "SqlPackageWrapper";
|
||||
public const string AppPackageVersion = "1";
|
||||
|
||||
[FunctionName(nameof(CreateBatchPoolAndExportJob))]
|
||||
public static async Task<string> CreateBatchPoolAndExportJob([ActivityTrigger] ExportRequest request, ILogger log)
|
||||
{
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
|
||||
// Get a Batch client using function identity
|
||||
BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));
|
||||
|
||||
string jobId = request.SourceSqlServerName + "-Export-" + DateTime.UtcNow.ToString("MMddHHmmss");
|
||||
using (BatchClient batchClient = BatchClient.Open(batchCred))
|
||||
{
|
||||
ImageReference imageReference = CreateImageReference();
|
||||
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
|
||||
|
||||
await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
|
||||
await CreateBatchJob(batchClient, jobId, log);
|
||||
}
|
||||
|
||||
return jobId;
|
||||
}
|
||||
|
||||
[FunctionName(nameof(CreateBatchPoolAndImportJob))]
|
||||
public static async Task<string> CreateBatchPoolAndImportJob([ActivityTrigger] ImportRequest request, ILogger log)
|
||||
{
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
|
||||
// Get a Batch client using function identity
|
||||
BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));
|
||||
|
||||
string jobId = request.TargetSqlServerName + "-Import-" + DateTime.UtcNow.ToString("MMddHHmmss");
|
||||
using (BatchClient batchClient = BatchClient.Open(batchCred))
|
||||
{
|
||||
ImageReference imageReference = CreateImageReference();
|
||||
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
|
||||
|
||||
await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
|
||||
await CreateBatchJob(batchClient, jobId, log);
|
||||
}
|
||||
|
||||
return jobId;
|
||||
}
|
||||
|
||||
public static async Task<CloudJob> CreateBatchJob(BatchClient batchClient, string jobId, ILogger log)
|
||||
{
|
||||
// Create a Batch job
|
||||
log.LogInformation("Creating job [{0}]...", jobId);
|
||||
CloudJob job = null;
|
||||
|
||||
try
|
||||
{
|
||||
job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = PoolId });
|
||||
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
|
||||
|
||||
// Commit the job to the Batch service
|
||||
await job.CommitAsync();
|
||||
|
||||
log.LogInformation($"Created job {jobId}");
|
||||
|
||||
// Obtain the bound job from the Batch service
|
||||
await job.RefreshAsync();
|
||||
}
|
||||
catch (BatchException be)
|
||||
{
|
||||
// Accept the specific error code JobExists as that is expected if the job already exists
|
||||
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
|
||||
{
|
||||
log.LogWarning("The job {0} already existed when we tried to create it", jobId);
|
||||
}
|
||||
else
|
||||
{
|
||||
log.LogError("Exception creating job: {0}", be.Message);
|
||||
throw be; // Any other exception is unexpected
|
||||
}
|
||||
}
|
||||
|
||||
return job;
|
||||
}
|
||||
|
||||
// Create the Compute Pool of the Batch Account
|
||||
public static async Task CreateBatchPoolIfNotExist(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration, string vnetSubnetId)
|
||||
{
|
||||
Console.WriteLine("Creating pool [{0}]...", PoolId);
|
||||
|
||||
try
|
||||
{
|
||||
CloudPool pool = batchClient.PoolOperations.CreatePool(
|
||||
poolId: PoolId,
|
||||
targetDedicatedComputeNodes: PoolNodeCount,
|
||||
virtualMachineSize: PoolVMSize,
|
||||
virtualMachineConfiguration: vmConfiguration);
|
||||
|
||||
// Specify the application and version to install on the compute nodes
|
||||
pool.ApplicationPackageReferences = new List<ApplicationPackageReference>
|
||||
{
|
||||
new ApplicationPackageReference {
|
||||
ApplicationId = AppPackageName,
|
||||
Version = AppPackageVersion }
|
||||
};
|
||||
|
||||
// Initial the first data disk for each VM in the pool
|
||||
StartTask startTask = new StartTask("cmd /c Powershell -command \"Get-Disk | Where partitionstyle -eq 'raw' | sort number | Select-Object -first 1 |" +
|
||||
" Initialize-Disk -PartitionStyle MBR -PassThru | New-Partition -UseMaximumSize -DriveLetter F |" +
|
||||
" Format-Volume -FileSystem NTFS -NewFileSystemLabel data1 -Confirm:$false -Force\"");
|
||||
|
||||
startTask.MaxTaskRetryCount = 1;
|
||||
startTask.UserIdentity = new UserIdentity(new AutoUserSpecification(AutoUserScope.Pool, ElevationLevel.Admin));
|
||||
startTask.WaitForSuccess = true;
|
||||
|
||||
pool.StartTask = startTask;
|
||||
|
||||
// Create the Pool within the vnet subnet if it's specified.
|
||||
if (vnetSubnetId != null)
|
||||
{
|
||||
pool.NetworkConfiguration = new NetworkConfiguration();
|
||||
pool.NetworkConfiguration.SubnetId = vnetSubnetId;
|
||||
}
|
||||
|
||||
await pool.CommitAsync();
|
||||
await pool.RefreshAsync();
|
||||
}
|
||||
catch (BatchException be)
|
||||
{
|
||||
// Accept the specific error code PoolExists as that is expected if the pool already exists
|
||||
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
|
||||
{
|
||||
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw; // Any other exception is unexpected
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static VirtualMachineConfiguration CreateVirtualMachineConfiguration(ImageReference imageReference)
|
||||
{
|
||||
VirtualMachineConfiguration config = new VirtualMachineConfiguration(
|
||||
imageReference: imageReference,
|
||||
nodeAgentSkuId: "batch.node.windows amd64");
|
||||
|
||||
config.DataDisks = new List<DataDisk>();
|
||||
config.DataDisks.Add(new DataDisk(0, 2048, CachingType.ReadOnly, StorageAccountType.PremiumLrs));
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
public static ImageReference CreateImageReference()
|
||||
{
|
||||
return new ImageReference(
|
||||
publisher: "MicrosoftWindowsServer",
|
||||
offer: "WindowsServer",
|
||||
sku: "2019-datacenter-smalldisk",
|
||||
version: "latest");
|
||||
}
|
||||
|
||||
public static void CreateBatchTasks(string action, string jobId, string containerUrl, string batchAccountUrl, string sqlServerName, string accessToken, dynamic databases, ILogger log)
|
||||
{
|
||||
// Get a Batch client using function identity
|
||||
log.LogInformation("CreateBatchTasks: entering");
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
BatchTokenCredentials batchCred = new BatchTokenCredentials(batchAccountUrl, azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/").Result);
|
||||
using (BatchClient batchClient = BatchClient.Open(batchCred))
|
||||
{
|
||||
// For each database, submit the Exporting job to Azure Batch Compute Pool.
|
||||
log.LogInformation("CreateBatchTasks: enumerating databases");
|
||||
List<CloudTask> tasks = new List<CloudTask>();
|
||||
foreach (var db in databases)
|
||||
{
|
||||
string serverDatabaseName = db.name.ToString();
|
||||
string logicalDatabase = serverDatabaseName.Remove(0, sqlServerName.Length + 1);
|
||||
|
||||
log.LogInformation("CreateBatchTasks: creating task for database {0}", logicalDatabase);
|
||||
string taskId = sqlServerName + "_" + logicalDatabase;
|
||||
string command = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_{0}#{1}%\\BatchWrapper {2}", AppPackageName.ToUpper(), AppPackageVersion, action);
|
||||
command += string.Format(" {0} {1} {2} {3} {4}", sqlServerName, logicalDatabase, accessToken, AppPackageName.ToUpper(), AppPackageVersion);
|
||||
string taskCommandLine = string.Format(command);
|
||||
|
||||
CloudTask singleTask = new CloudTask(taskId, taskCommandLine);
|
||||
singleTask.EnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
|
||||
|
||||
Console.WriteLine(string.Format("Adding task {0} to job ...", taskId));
|
||||
tasks.Add(singleTask);
|
||||
}
|
||||
|
||||
// Add all tasks to the job.
|
||||
batchClient.JobOperations.AddTask(jobId, tasks);
|
||||
}
|
||||
log.LogInformation("CreateBatchTasks: exiting");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Azure.WebJobs.Extensions.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace ADPControl
|
||||
{
|
||||
public static class HttpSurface
|
||||
{
|
||||
public class ExportRequest
|
||||
{
|
||||
public Guid SubscriptionId { get; set; }
|
||||
|
||||
public string ResourceGroupName { get; set; }
|
||||
|
||||
public string SourceSqlServerResourceGroupName { get; set; }
|
||||
|
||||
public string SourceSqlServerName { get; set; }
|
||||
|
||||
public string BatchAccountUrl { get; set; }
|
||||
|
||||
public string StorageAccountName { get; set; }
|
||||
|
||||
public string AccessToken { get; set; }
|
||||
|
||||
public string VNetSubnetId { get; set; }
|
||||
}
|
||||
|
||||
public class ImportRequest
|
||||
{
|
||||
public Guid SubscriptionId { get; set; }
|
||||
|
||||
public string ResourceGroupName { get; set; }
|
||||
|
||||
public string TargetSqlServerResourceGroupName { get; set; }
|
||||
|
||||
public string TargetSqlServerName { get; set; }
|
||||
|
||||
public string TargetAccessToken { get; set; }
|
||||
|
||||
public string BatchAccountUrl { get; set; }
|
||||
|
||||
public string StorageAccountName { get; set; }
|
||||
|
||||
public string ContainerName { get; set; }
|
||||
|
||||
public string SqlAdminPassword { get; set; }
|
||||
|
||||
public string VNetSubnetId { get; set; }
|
||||
}
|
||||
|
||||
[FunctionName("Export")]
|
||||
public static async Task<HttpResponseMessage> PostExport(
|
||||
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/Export")]
|
||||
HttpRequestMessage req,
|
||||
[DurableClient] IDurableOrchestrationClient starter,
|
||||
ILogger log,
|
||||
Guid subscriptionId,
|
||||
string resourceGroupName)
|
||||
{
|
||||
log.LogInformation("C# HTTP trigger function processed an Export request.");
|
||||
ExportRequest request = await req.Content.ReadAsAsync<ExportRequest>();
|
||||
|
||||
request.SubscriptionId = subscriptionId;
|
||||
request.ResourceGroupName = resourceGroupName;
|
||||
|
||||
if (request.SourceSqlServerResourceGroupName == null)
|
||||
request.SourceSqlServerResourceGroupName = resourceGroupName;
|
||||
|
||||
string instanceId = await starter.StartNewAsync(nameof(Orchestrator.RunExportOrchestrator), request);
|
||||
|
||||
log.LogInformation($"Started orchestration with ID = '{instanceId}'.");
|
||||
return starter.CreateCheckStatusResponse(req, instanceId);
|
||||
}
|
||||
|
||||
[FunctionName("Import")]
|
||||
public static async Task<HttpResponseMessage> PostImport(
|
||||
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/Import")]
|
||||
HttpRequestMessage req,
|
||||
[DurableClient] IDurableOrchestrationClient starter,
|
||||
ILogger log,
|
||||
Guid subscriptionId,
|
||||
string resourceGroupName)
|
||||
{
|
||||
log.LogInformation("C# HTTP trigger function processed an Import request.");
|
||||
ImportRequest request = await req.Content.ReadAsAsync<ImportRequest>();
|
||||
|
||||
request.SubscriptionId = subscriptionId;
|
||||
request.ResourceGroupName = resourceGroupName;
|
||||
|
||||
if (request.TargetSqlServerResourceGroupName == null)
|
||||
request.TargetSqlServerResourceGroupName = resourceGroupName;
|
||||
|
||||
string instanceId = await starter.StartNewAsync(nameof(Orchestrator.RunImportOrchestrator), request);
|
||||
|
||||
log.LogInformation($"Started orchestration with ID = '{instanceId}'.");
|
||||
return starter.CreateCheckStatusResponse(req, instanceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using static ADPControl.HttpSurface;
|
||||
|
||||
namespace ADPControl
|
||||
{
|
||||
public static class Orchestrator
|
||||
{
|
||||
// The Import Orchestrator
|
||||
[FunctionName(nameof(RunImportOrchestrator))]
|
||||
public static async Task RunImportOrchestrator(
|
||||
[OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log)
|
||||
{
|
||||
log.LogInformation("RunImportOrchestrator: entering");
|
||||
|
||||
try {
|
||||
|
||||
ImportRequest importRequest = context.GetInput<ImportRequest>();
|
||||
// Deploy the ARM template to Create empty SQL resource
|
||||
string deploymentName = await context.CallActivityAsync<string>(nameof(AzureResourceManagerActivity.BeginDeployArmTemplateForImport), importRequest);
|
||||
while (true)
|
||||
{
|
||||
log.LogInformation("RunImportOrchestrator: starting ARM deployment");
|
||||
string status = await context.CallActivityAsync<string>(nameof(AzureResourceManagerActivity.GetArmDeploymentForImport), (importRequest.SubscriptionId, importRequest.TargetSqlServerResourceGroupName, deploymentName));
|
||||
if (status == "Succeeded")
|
||||
{
|
||||
log.LogInformation("RunImportOrchestrator: ARM deployment succeeded");
|
||||
break;
|
||||
}
|
||||
else if (status == "Failed")
|
||||
{
|
||||
log.LogInformation("RunImportOrchestrator: ARM deployment failed");
|
||||
throw new Exception("Failed ARM Deployment");
|
||||
}
|
||||
|
||||
// Orchestration sleeps until this time.
|
||||
var nextCheck = context.CurrentUtcDateTime.AddSeconds(10);
|
||||
|
||||
if (!context.IsReplaying) { log.LogInformation($"RunImportOrchestrator: Replaying ARM deployment, next check at {nextCheck}."); }
|
||||
await context.CreateTimer(nextCheck, CancellationToken.None);
|
||||
}
|
||||
|
||||
log.LogInformation("RunImportOrchestrator: Enumerating databases");
|
||||
var databases = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForImportSkipParameterization), importRequest);
|
||||
|
||||
// Create BatchPool And Job
|
||||
log.LogInformation("RunImportOrchestrator: Creating batch pool and import job");
|
||||
string jobId = await context.CallActivityAsync<string>(nameof(BatchActivity.CreateBatchPoolAndImportJob), importRequest);
|
||||
|
||||
string containerUrl = await context.CallActivityAsync<string>(nameof(StorageActivity.GettingJobContainerUrl), (importRequest.SubscriptionId, importRequest.ResourceGroupName, importRequest.StorageAccountName, importRequest.ContainerName));
|
||||
|
||||
log.LogInformation("RunImportOrchestrator: Creating import database tasks");
|
||||
BatchActivity.CreateBatchTasks("Import", jobId, containerUrl, importRequest.BatchAccountUrl, importRequest.TargetSqlServerName, importRequest.TargetAccessToken, databases, log);
|
||||
|
||||
// create output values
|
||||
Tuple<string, string>[] outputValues = {
|
||||
Tuple.Create("Orchestration progress:", "Complete"),
|
||||
Tuple.Create("deploymentName", deploymentName),
|
||||
Tuple.Create("jobId", jobId),
|
||||
Tuple.Create("containerUrl", containerUrl)
|
||||
};
|
||||
context.SetOutput(outputValues);
|
||||
}
|
||||
finally {
|
||||
log.LogInformation("RunImportOrchestrator: exiting");
|
||||
}
|
||||
}
|
||||
|
||||
// The Export Orchestrator
|
||||
[FunctionName(nameof(RunExportOrchestrator))]
|
||||
public static async Task RunExportOrchestrator(
|
||||
[OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log)
|
||||
{
|
||||
ExportRequest exportRequest = context.GetInput<ExportRequest>();
|
||||
|
||||
// Getting the ARM template Skip ResourceName Parameterization.
|
||||
var databases = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForExportSkipParameterization), exportRequest);
|
||||
|
||||
// Getting the ARM template.
|
||||
dynamic Template = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForExport), exportRequest);
|
||||
string json = JsonConvert.SerializeObject(Template);
|
||||
|
||||
// Create BatchPool And Job
|
||||
string jobId = await context.CallActivityAsync<string>(nameof(BatchActivity.CreateBatchPoolAndExportJob), exportRequest);
|
||||
|
||||
string containerUrl = await context.CallActivityAsync<string>(nameof(StorageActivity.GettingJobContainerUrl), (exportRequest.SubscriptionId, exportRequest.ResourceGroupName, exportRequest.StorageAccountName, jobId));
|
||||
await context.CallActivityAsync<string>(nameof(StorageActivity.UploadingArmTemplate), (containerUrl, json));
|
||||
|
||||
BatchActivity.CreateBatchTasks("Export", jobId, containerUrl, exportRequest.BatchAccountUrl, exportRequest.SourceSqlServerName, exportRequest.AccessToken, databases, log);
|
||||
|
||||
// create output values
|
||||
Tuple<string, string>[] outputValues = {
|
||||
Tuple.Create("jobId", jobId),
|
||||
Tuple.Create("containerUrl", containerUrl)
|
||||
};
|
||||
context.SetOutput(outputValues);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
using Microsoft.Azure.Management.Storage;
|
||||
using Microsoft.Azure.Management.Storage.Models;
|
||||
using Microsoft.Azure.Services.AppAuthentication;
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Rest;
|
||||
using Microsoft.WindowsAzure.Storage;
|
||||
using Microsoft.WindowsAzure.Storage.Auth;
|
||||
using Microsoft.WindowsAzure.Storage.Blob;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace ADPControl
|
||||
{
|
||||
public static class StorageActivity
|
||||
{
|
||||
private const string ArmTemplateFileName = "template.json";
|
||||
|
||||
[FunctionName(nameof(GettingJobContainerUrl))]
|
||||
public static string GettingJobContainerUrl([ActivityTrigger] (Guid, string, string, string) input, ILogger log)
|
||||
{
|
||||
Guid SubscriptionId = input.Item1;
|
||||
String ResourceGroupName = input.Item2;
|
||||
String StorageAccountName = input.Item3;
|
||||
String ContainerName = input.Item4;
|
||||
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/").Result);
|
||||
StorageManagementClient storageMgmtClient = new StorageManagementClient(tokenArmCredential) { SubscriptionId = SubscriptionId.ToString() };
|
||||
|
||||
// Get the storage account keys for a given account and resource group
|
||||
IList<StorageAccountKey> acctKeys = storageMgmtClient.StorageAccounts.ListKeys(ResourceGroupName, StorageAccountName).Keys;
|
||||
|
||||
// Get a Storage account using account creds:
|
||||
StorageCredentials storageCred = new StorageCredentials(StorageAccountName, acctKeys.FirstOrDefault().Value);
|
||||
CloudStorageAccount linkedStorageAccount = new CloudStorageAccount(storageCred, true);
|
||||
|
||||
bool createContainer = false;
|
||||
// Normalize the container name for the Export action.
|
||||
if (ContainerName.Contains("-Export-"))
|
||||
{
|
||||
ContainerName = ContainerName.Replace("Export-", "");
|
||||
createContainer = true;
|
||||
}
|
||||
|
||||
CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(ContainerName);
|
||||
|
||||
if(createContainer)
|
||||
container.CreateIfNotExistsAsync().Wait();
|
||||
|
||||
string containerUrl = container.Uri.ToString() +
|
||||
container.GetSharedAccessSignature(new SharedAccessBlobPolicy()
|
||||
{
|
||||
Permissions = SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List,
|
||||
SharedAccessExpiryTime = DateTime.UtcNow.AddDays(7)
|
||||
});
|
||||
return containerUrl;
|
||||
}
|
||||
|
||||
[FunctionName(nameof(UploadingArmTemplate))]
|
||||
public static void UploadingArmTemplate([ActivityTrigger] (string, string) input, ILogger log)
|
||||
{
|
||||
string containerUrl = input.Item1;
|
||||
string json = input.Item2;
|
||||
|
||||
CloudBlobContainer container = new CloudBlobContainer(new Uri(containerUrl));
|
||||
CloudBlockBlob blob = container.GetBlockBlobReference(ArmTemplateFileName);
|
||||
blob.Properties.ContentType = "application/json";
|
||||
using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(json)))
|
||||
{
|
||||
blob.UploadFromStreamAsync(stream).Wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"version": "2.0",
|
||||
"extensions": {
|
||||
"durableTask": {
|
||||
"hubName": "adp"
|
||||
}
|
||||
},
|
||||
"logging": {
|
||||
"applicationInsights": {
|
||||
"samplingExcludedTypes": "Request",
|
||||
"samplingSettings": {
|
||||
"isEnabled": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user