mirror of
https://github.com/ckaczor/azuredatastudio.git
synced 2026-02-16 18:46:40 -05:00
Removal of Components folder, moving ADP notebook to its own notebook. (#13848)
* adp folder removed, notebook moved. * moved ADP notebook to own folder.
This commit is contained in:
Alex Ma
@@ -1,37 +0,0 @@
|
|||||||
|
|
||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
|
||||||
# Visual Studio Version 16
|
|
||||||
VisualStudioVersion = 16.0.29920.165
|
|
||||||
MinimumVisualStudioVersion = 10.0.40219.1
|
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ADPControl", "ADPControl\ADPControl.csproj", "{6309D4C5-F118-4C89-A67E-E557CA41ABA2}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BatchWrapper", "BatchWrapper\BatchWrapper.csproj", "{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}"
|
|
||||||
EndProject
|
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlPackageWrapper", "SqlPackageWrapper\SqlPackageWrapper.csproj", "{A19335D3-9D80-43BE-9351-C3C3704689B0}"
|
|
||||||
EndProject
|
|
||||||
Global
|
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
|
||||||
Debug|Any CPU = Debug|Any CPU
|
|
||||||
Release|Any CPU = Release|Any CPU
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
|
||||||
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Release|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Release|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Release|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Release|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
|
||||||
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Release|Any CPU.Build.0 = Release|Any CPU
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
|
||||||
HideSolutionNode = FALSE
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
|
||||||
SolutionGuid = {3DB5F0CB-1D26-4E33-801D-7BBAE823C39D}
|
|
||||||
EndGlobalSection
|
|
||||||
EndGlobal
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
|
||||||
<AzureFunctionsVersion>v3</AzureFunctionsVersion>
|
|
||||||
</PropertyGroup>
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="Microsoft.Azure.Batch" Version="13.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Azure.Management.ResourceManager" Version="3.7.0-preview" />
|
|
||||||
<PackageReference Include="Microsoft.Azure.Management.Storage" Version="17.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Azure.Services.AppAuthentication" Version="1.5.0" />
|
|
||||||
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="2.2.2" />
|
|
||||||
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.7" />
|
|
||||||
<PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.21" />
|
|
||||||
<PackageReference Include="Microsoft.Rest.ClientRuntime.Azure" Version="3.3.19" />
|
|
||||||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
|
||||||
</ItemGroup>
|
|
||||||
<ItemGroup>
|
|
||||||
<None Update="host.json">
|
|
||||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
|
||||||
</None>
|
|
||||||
<None Update="local.settings.json">
|
|
||||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
|
||||||
<CopyToPublishDirectory>Never</CopyToPublishDirectory>
|
|
||||||
</None>
|
|
||||||
</ItemGroup>
|
|
||||||
</Project>
|
|
||||||
@@ -1,244 +0,0 @@
|
|||||||
using Microsoft.Azure.Management.ResourceManager;
|
|
||||||
using Microsoft.Azure.Management.ResourceManager.Models;
|
|
||||||
using Microsoft.Azure.Management.Storage;
|
|
||||||
using Microsoft.Azure.Management.Storage.Models;
|
|
||||||
using Microsoft.Azure.Services.AppAuthentication;
|
|
||||||
using Microsoft.Azure.WebJobs;
|
|
||||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using Microsoft.Rest;
|
|
||||||
using Microsoft.WindowsAzure.Storage;
|
|
||||||
using Microsoft.WindowsAzure.Storage.Auth;
|
|
||||||
using Microsoft.WindowsAzure.Storage.Blob;
|
|
||||||
using Newtonsoft.Json;
|
|
||||||
using Newtonsoft.Json.Linq;
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using static ADPControl.HttpSurface;
|
|
||||||
|
|
||||||
namespace ADPControl
|
|
||||||
{
|
|
||||||
public static class AzureResourceManagerActivity
|
|
||||||
{
|
|
||||||
private const string ArmTemplateFileName = "template.json";
|
|
||||||
|
|
||||||
private static string[] AllowedImportSubServerResourceTypes = new string[] {
|
|
||||||
"Microsoft.Sql/servers/firewallRules",
|
|
||||||
"Microsoft.Sql/servers/databases",
|
|
||||||
"Microsoft.Sql/servers/elasticPools",
|
|
||||||
//"Microsoft.Sql/servers/keys",
|
|
||||||
//"Microsoft.Sql/servers/databases/transparentDataEncryption",
|
|
||||||
"Microsoft.Sql/servers/databases/backupShortTermRetentionPolicies",
|
|
||||||
"Microsoft.Sql/servers/administrators"
|
|
||||||
};
|
|
||||||
|
|
||||||
// Deploy the ARM template
|
|
||||||
[FunctionName(nameof(BeginDeployArmTemplateForImport))]
|
|
||||||
public static async Task<string> BeginDeployArmTemplateForImport([ActivityTrigger] ImportRequest request, ILogger log)
|
|
||||||
{
|
|
||||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
|
||||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
|
||||||
|
|
||||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
|
||||||
StorageManagementClient storageMgmtClient = new StorageManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
|
||||||
|
|
||||||
// Get the storage account keys for a given account and resource group
|
|
||||||
IList<StorageAccountKey> acctKeys = storageMgmtClient.StorageAccounts.ListKeys(request.ResourceGroupName, request.StorageAccountName).Keys;
|
|
||||||
|
|
||||||
// Get a Storage account using account creds:
|
|
||||||
StorageCredentials storageCred = new StorageCredentials(request.StorageAccountName, acctKeys.FirstOrDefault().Value);
|
|
||||||
CloudStorageAccount linkedStorageAccount = new CloudStorageAccount(storageCred, true);
|
|
||||||
CloudBlobContainer container = linkedStorageAccount
|
|
||||||
.CreateCloudBlobClient()
|
|
||||||
.GetContainerReference(request.ContainerName);
|
|
||||||
|
|
||||||
CloudBlockBlob blob = container.GetBlockBlobReference(ArmTemplateFileName);
|
|
||||||
string json = await blob.DownloadTextAsync();
|
|
||||||
|
|
||||||
JObject originalTemplate = JObject.Parse(json);
|
|
||||||
JObject importTemplate = UpdateArmTemplateForImport(originalTemplate, request);
|
|
||||||
|
|
||||||
var deployParams = new Deployment
|
|
||||||
{
|
|
||||||
Properties = new DeploymentProperties
|
|
||||||
{
|
|
||||||
Mode = DeploymentMode.Incremental,
|
|
||||||
Template = importTemplate
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
string deploymentName = request.TargetSqlServerName + "_" + DateTime.UtcNow.ToFileTimeUtc();
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await resourcesClient.Deployments.BeginCreateOrUpdateAsync(request.TargetSqlServerResourceGroupName, deploymentName, deployParams);
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
log.LogError(ex.ToString());
|
|
||||||
throw ex;
|
|
||||||
}
|
|
||||||
|
|
||||||
return deploymentName;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the ARM deployment status
|
|
||||||
[FunctionName(nameof(GetArmDeploymentForImport))]
|
|
||||||
public static async Task<string> GetArmDeploymentForImport([ActivityTrigger] (Guid, string, string) input)
|
|
||||||
{
|
|
||||||
Guid subscriptionId = input.Item1;
|
|
||||||
string resourceGroupName = input.Item2;
|
|
||||||
string deploymentName = input.Item3;
|
|
||||||
|
|
||||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
|
||||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
|
||||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = subscriptionId.ToString() };
|
|
||||||
|
|
||||||
DeploymentExtended result = await resourcesClient.Deployments.GetAsync(resourceGroupName, deploymentName);
|
|
||||||
return result.Properties.ProvisioningState;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the ARM template without the parameter of the resource name
|
|
||||||
[FunctionName(nameof(GetArmTemplateForExportSkipParameterization))]
|
|
||||||
public static async Task<dynamic> GetArmTemplateForExportSkipParameterization([ActivityTrigger] ExportRequest request, ILogger log)
|
|
||||||
{
|
|
||||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: entering");
|
|
||||||
|
|
||||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
|
||||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
|
||||||
if (tokenArmCredential != null)
|
|
||||||
{
|
|
||||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: acquired access token");
|
|
||||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
|
||||||
|
|
||||||
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.SourceSqlServerResourceGroupName, request.SourceSqlServerName);
|
|
||||||
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.SourceSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "SkipResourceNameParameterization"));
|
|
||||||
|
|
||||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: server template exported. Size: {0} bytes", exportedTemplate.Template.ToString().Length);
|
|
||||||
dynamic template = (dynamic)exportedTemplate.Template;
|
|
||||||
|
|
||||||
// Filtering the list of databases
|
|
||||||
dynamic databases = template.resources.SelectTokens("$.[?(@.type == 'Microsoft.Sql/servers/databases')]");
|
|
||||||
int numberOfDatabases = 0;
|
|
||||||
foreach (var db in databases)
|
|
||||||
{
|
|
||||||
numberOfDatabases++;
|
|
||||||
}
|
|
||||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with database list. Databases count: {0}", numberOfDatabases);
|
|
||||||
|
|
||||||
return databases;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with empty database list");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the ARM template without the parameter of the resource name
|
|
||||||
[FunctionName(nameof(GetArmTemplateForImportSkipParameterization))]
|
|
||||||
public static async Task<dynamic> GetArmTemplateForImportSkipParameterization([ActivityTrigger] ImportRequest request, ILogger log)
|
|
||||||
{
|
|
||||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
|
||||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
|
||||||
if (tokenArmCredential != null)
|
|
||||||
{
|
|
||||||
log.LogInformation("GetArmTemplateForImportSkipParameterization: acquired access token");
|
|
||||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
|
||||||
|
|
||||||
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.TargetSqlServerResourceGroupName, request.TargetSqlServerName);
|
|
||||||
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.TargetSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "SkipResourceNameParameterization"));
|
|
||||||
|
|
||||||
log.LogInformation("GetArmTemplateForImportSkipParameterization: server template exported. Size: {0} bytes", exportedTemplate.Template.ToString().Length);
|
|
||||||
dynamic template = (dynamic)exportedTemplate.Template;
|
|
||||||
|
|
||||||
// Filtering the list of databases
|
|
||||||
dynamic databases = template.resources.SelectTokens("$.[?(@.type == 'Microsoft.Sql/servers/databases')]");
|
|
||||||
|
|
||||||
int numberOfDatabases = 0;
|
|
||||||
foreach (var db in databases)
|
|
||||||
{
|
|
||||||
numberOfDatabases++;
|
|
||||||
}
|
|
||||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with database list. Databases count: {0}", numberOfDatabases);
|
|
||||||
return databases;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
[FunctionName(nameof(GetArmTemplateForExport))]
|
|
||||||
public static async Task<dynamic> GetArmTemplateForExport([ActivityTrigger] ExportRequest request)
|
|
||||||
{
|
|
||||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
|
||||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
|
||||||
|
|
||||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
|
||||||
|
|
||||||
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.SourceSqlServerResourceGroupName, request.SourceSqlServerName);
|
|
||||||
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.SourceSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "IncludeParameterDefaultValue"));
|
|
||||||
return exportedTemplate.Template;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static JObject UpdateArmTemplateForImport(JObject originalTemplate, ImportRequest request)
|
|
||||||
{
|
|
||||||
string serverNameParameterName = null;
|
|
||||||
|
|
||||||
// Go through every parameter to find the property name is like 'server_%_name'
|
|
||||||
using (JsonTextReader reader = new JsonTextReader(new StringReader(originalTemplate["parameters"].ToString())))
|
|
||||||
{
|
|
||||||
while (reader.Read())
|
|
||||||
{
|
|
||||||
if (reader.TokenType.ToString().Equals("PropertyName")
|
|
||||||
&& reader.ValueType.ToString().Equals("System.String")
|
|
||||||
&& reader.Value.ToString().StartsWith("servers_")
|
|
||||||
&& reader.Value.ToString().EndsWith("_name"))
|
|
||||||
{
|
|
||||||
serverNameParameterName = reader.Value.ToString();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. Replacing the default value to the target server name, appending to the new template
|
|
||||||
originalTemplate["parameters"][serverNameParameterName]["defaultValue"] = request.TargetSqlServerName;
|
|
||||||
JObject serverNameParameterValue = (JObject)originalTemplate["parameters"][serverNameParameterName];
|
|
||||||
|
|
||||||
// 2. Cleanup all the parameters except the updated server name
|
|
||||||
((JObject)originalTemplate["parameters"]).RemoveAll();
|
|
||||||
((JObject)originalTemplate["parameters"]).Add(serverNameParameterName, serverNameParameterValue);
|
|
||||||
|
|
||||||
// 3. Adjust the servers resource by adding password after the login
|
|
||||||
JObject server = (JObject)originalTemplate["resources"]
|
|
||||||
.SelectToken("$.[?(@.type == 'Microsoft.Sql/servers')]");
|
|
||||||
|
|
||||||
server.Remove("identity");
|
|
||||||
|
|
||||||
JObject serverProperties = (JObject)server["properties"];
|
|
||||||
serverProperties.Property("administratorLogin")
|
|
||||||
.AddAfterSelf(new JProperty("administratorLoginPassword", request.SqlAdminPassword));
|
|
||||||
|
|
||||||
JArray newResources = new JArray();
|
|
||||||
|
|
||||||
// 4. Getting the whitelisted resources and adding them to the new template later.
|
|
||||||
foreach (string resourceType in AllowedImportSubServerResourceTypes)
|
|
||||||
{
|
|
||||||
List<JToken> resources = originalTemplate["resources"]
|
|
||||||
.SelectTokens(string.Format("$.[?(@.type == '{0}')]", resourceType)).ToList();
|
|
||||||
newResources.Add(resources);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 5. Clean up all the resources excepted the new server and whitelisted resource type.
|
|
||||||
((JArray)originalTemplate["resources"]).Clear();
|
|
||||||
((JArray)originalTemplate["resources"]).Add(server);
|
|
||||||
|
|
||||||
foreach (var resource in newResources)
|
|
||||||
{
|
|
||||||
((JArray)originalTemplate["resources"]).Add(resource);
|
|
||||||
}
|
|
||||||
|
|
||||||
return originalTemplate;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,214 +0,0 @@
|
|||||||
using Microsoft.Azure.Batch;
|
|
||||||
using Microsoft.Azure.Batch.Auth;
|
|
||||||
using Microsoft.Azure.Batch.Common;
|
|
||||||
using Microsoft.Azure.Services.AppAuthentication;
|
|
||||||
using Microsoft.Azure.WebJobs;
|
|
||||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using static ADPControl.HttpSurface;
|
|
||||||
|
|
||||||
namespace ADPControl
|
|
||||||
{
|
|
||||||
public static class BatchActivity
|
|
||||||
{
|
|
||||||
// Batch resource settings
|
|
||||||
private const string PoolVMSize = "Standard_D8s_v3";
|
|
||||||
private const string PoolId = PoolVMSize;
|
|
||||||
private const int PoolNodeCount = 2;
|
|
||||||
private const string AppPackageName = "SqlPackageWrapper";
|
|
||||||
public const string AppPackageVersion = "1";
|
|
||||||
|
|
||||||
[FunctionName(nameof(CreateBatchPoolAndExportJob))]
|
|
||||||
public static async Task<string> CreateBatchPoolAndExportJob([ActivityTrigger] ExportRequest request, ILogger log)
|
|
||||||
{
|
|
||||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
|
||||||
|
|
||||||
// Get a Batch client using function identity
|
|
||||||
BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));
|
|
||||||
|
|
||||||
string jobId = request.SourceSqlServerName + "-Export-" + DateTime.UtcNow.ToString("MMddHHmmss");
|
|
||||||
using (BatchClient batchClient = BatchClient.Open(batchCred))
|
|
||||||
{
|
|
||||||
ImageReference imageReference = CreateImageReference();
|
|
||||||
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
|
|
||||||
|
|
||||||
await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
|
|
||||||
await CreateBatchJob(batchClient, jobId, log);
|
|
||||||
}
|
|
||||||
|
|
||||||
return jobId;
|
|
||||||
}
|
|
||||||
|
|
||||||
[FunctionName(nameof(CreateBatchPoolAndImportJob))]
|
|
||||||
public static async Task<string> CreateBatchPoolAndImportJob([ActivityTrigger] ImportRequest request, ILogger log)
|
|
||||||
{
|
|
||||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
|
||||||
|
|
||||||
// Get a Batch client using function identity
|
|
||||||
BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));
|
|
||||||
|
|
||||||
string jobId = request.TargetSqlServerName + "-Import-" + DateTime.UtcNow.ToString("MMddHHmmss");
|
|
||||||
using (BatchClient batchClient = BatchClient.Open(batchCred))
|
|
||||||
{
|
|
||||||
ImageReference imageReference = CreateImageReference();
|
|
||||||
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
|
|
||||||
|
|
||||||
await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
|
|
||||||
await CreateBatchJob(batchClient, jobId, log);
|
|
||||||
}
|
|
||||||
|
|
||||||
return jobId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async Task<CloudJob> CreateBatchJob(BatchClient batchClient, string jobId, ILogger log)
|
|
||||||
{
|
|
||||||
// Create a Batch job
|
|
||||||
log.LogInformation("Creating job [{0}]...", jobId);
|
|
||||||
CloudJob job = null;
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = PoolId });
|
|
||||||
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
|
|
||||||
|
|
||||||
// Commit the job to the Batch service
|
|
||||||
await job.CommitAsync();
|
|
||||||
|
|
||||||
log.LogInformation($"Created job {jobId}");
|
|
||||||
|
|
||||||
// Obtain the bound job from the Batch service
|
|
||||||
await job.RefreshAsync();
|
|
||||||
}
|
|
||||||
catch (BatchException be)
|
|
||||||
{
|
|
||||||
// Accept the specific error code JobExists as that is expected if the job already exists
|
|
||||||
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
|
|
||||||
{
|
|
||||||
log.LogWarning("The job {0} already existed when we tried to create it", jobId);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
log.LogError("Exception creating job: {0}", be.Message);
|
|
||||||
throw be; // Any other exception is unexpected
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return job;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create the Compute Pool of the Batch Account
|
|
||||||
public static async Task CreateBatchPoolIfNotExist(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration, string vnetSubnetId)
|
|
||||||
{
|
|
||||||
Console.WriteLine("Creating pool [{0}]...", PoolId);
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
CloudPool pool = batchClient.PoolOperations.CreatePool(
|
|
||||||
poolId: PoolId,
|
|
||||||
targetDedicatedComputeNodes: PoolNodeCount,
|
|
||||||
virtualMachineSize: PoolVMSize,
|
|
||||||
virtualMachineConfiguration: vmConfiguration);
|
|
||||||
|
|
||||||
// Specify the application and version to install on the compute nodes
|
|
||||||
pool.ApplicationPackageReferences = new List<ApplicationPackageReference>
|
|
||||||
{
|
|
||||||
new ApplicationPackageReference {
|
|
||||||
ApplicationId = AppPackageName,
|
|
||||||
Version = AppPackageVersion }
|
|
||||||
};
|
|
||||||
|
|
||||||
// Initial the first data disk for each VM in the pool
|
|
||||||
StartTask startTask = new StartTask("cmd /c Powershell -command \"Get-Disk | Where partitionstyle -eq 'raw' | sort number | Select-Object -first 1 |" +
|
|
||||||
" Initialize-Disk -PartitionStyle MBR -PassThru | New-Partition -UseMaximumSize -DriveLetter F |" +
|
|
||||||
" Format-Volume -FileSystem NTFS -NewFileSystemLabel data1 -Confirm:$false -Force\"");
|
|
||||||
|
|
||||||
startTask.MaxTaskRetryCount = 1;
|
|
||||||
startTask.UserIdentity = new UserIdentity(new AutoUserSpecification(AutoUserScope.Pool, ElevationLevel.Admin));
|
|
||||||
startTask.WaitForSuccess = true;
|
|
||||||
|
|
||||||
pool.StartTask = startTask;
|
|
||||||
|
|
||||||
// Create the Pool within the vnet subnet if it's specified.
|
|
||||||
if (vnetSubnetId != null)
|
|
||||||
{
|
|
||||||
pool.NetworkConfiguration = new NetworkConfiguration();
|
|
||||||
pool.NetworkConfiguration.SubnetId = vnetSubnetId;
|
|
||||||
}
|
|
||||||
|
|
||||||
await pool.CommitAsync();
|
|
||||||
await pool.RefreshAsync();
|
|
||||||
}
|
|
||||||
catch (BatchException be)
|
|
||||||
{
|
|
||||||
// Accept the specific error code PoolExists as that is expected if the pool already exists
|
|
||||||
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
|
|
||||||
{
|
|
||||||
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
throw; // Any other exception is unexpected
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static VirtualMachineConfiguration CreateVirtualMachineConfiguration(ImageReference imageReference)
|
|
||||||
{
|
|
||||||
VirtualMachineConfiguration config = new VirtualMachineConfiguration(
|
|
||||||
imageReference: imageReference,
|
|
||||||
nodeAgentSkuId: "batch.node.windows amd64");
|
|
||||||
|
|
||||||
config.DataDisks = new List<DataDisk>();
|
|
||||||
config.DataDisks.Add(new DataDisk(0, 2048, CachingType.ReadOnly, StorageAccountType.PremiumLrs));
|
|
||||||
|
|
||||||
return config;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ImageReference CreateImageReference()
|
|
||||||
{
|
|
||||||
return new ImageReference(
|
|
||||||
publisher: "MicrosoftWindowsServer",
|
|
||||||
offer: "WindowsServer",
|
|
||||||
sku: "2019-datacenter-smalldisk",
|
|
||||||
version: "latest");
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void CreateBatchTasks(string action, string jobId, string containerUrl, string batchAccountUrl, string sqlServerName, string accessToken, dynamic databases, ILogger log)
|
|
||||||
{
|
|
||||||
// Get a Batch client using function identity
|
|
||||||
log.LogInformation("CreateBatchTasks: entering");
|
|
||||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
|
||||||
BatchTokenCredentials batchCred = new BatchTokenCredentials(batchAccountUrl, azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/").Result);
|
|
||||||
using (BatchClient batchClient = BatchClient.Open(batchCred))
|
|
||||||
{
|
|
||||||
// For each database, submit the Exporting job to Azure Batch Compute Pool.
|
|
||||||
log.LogInformation("CreateBatchTasks: enumerating databases");
|
|
||||||
List<CloudTask> tasks = new List<CloudTask>();
|
|
||||||
foreach (var db in databases)
|
|
||||||
{
|
|
||||||
string serverDatabaseName = db.name.ToString();
|
|
||||||
string logicalDatabase = serverDatabaseName.Remove(0, sqlServerName.Length + 1);
|
|
||||||
|
|
||||||
log.LogInformation("CreateBatchTasks: creating task for database {0}", logicalDatabase);
|
|
||||||
string taskId = sqlServerName + "_" + logicalDatabase;
|
|
||||||
string command = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_{0}#{1}%\\BatchWrapper {2}", AppPackageName.ToUpper(), AppPackageVersion, action);
|
|
||||||
command += string.Format(" {0} {1} {2} {3} {4}", sqlServerName, logicalDatabase, accessToken, AppPackageName.ToUpper(), AppPackageVersion);
|
|
||||||
string taskCommandLine = string.Format(command);
|
|
||||||
|
|
||||||
CloudTask singleTask = new CloudTask(taskId, taskCommandLine);
|
|
||||||
singleTask.EnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
|
|
||||||
|
|
||||||
Console.WriteLine(string.Format("Adding task {0} to job ...", taskId));
|
|
||||||
tasks.Add(singleTask);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add all tasks to the job.
|
|
||||||
batchClient.JobOperations.AddTask(jobId, tasks);
|
|
||||||
}
|
|
||||||
log.LogInformation("CreateBatchTasks: exiting");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,103 +0,0 @@
|
|||||||
using Microsoft.Azure.WebJobs;
|
|
||||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
|
||||||
using Microsoft.Azure.WebJobs.Extensions.Http;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using System;
|
|
||||||
using System.Net.Http;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace ADPControl
|
|
||||||
{
|
|
||||||
public static class HttpSurface
|
|
||||||
{
|
|
||||||
public class ExportRequest
|
|
||||||
{
|
|
||||||
public Guid SubscriptionId { get; set; }
|
|
||||||
|
|
||||||
public string ResourceGroupName { get; set; }
|
|
||||||
|
|
||||||
public string SourceSqlServerResourceGroupName { get; set; }
|
|
||||||
|
|
||||||
public string SourceSqlServerName { get; set; }
|
|
||||||
|
|
||||||
public string BatchAccountUrl { get; set; }
|
|
||||||
|
|
||||||
public string StorageAccountName { get; set; }
|
|
||||||
|
|
||||||
public string AccessToken { get; set; }
|
|
||||||
|
|
||||||
public string VNetSubnetId { get; set; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public class ImportRequest
|
|
||||||
{
|
|
||||||
public Guid SubscriptionId { get; set; }
|
|
||||||
|
|
||||||
public string ResourceGroupName { get; set; }
|
|
||||||
|
|
||||||
public string TargetSqlServerResourceGroupName { get; set; }
|
|
||||||
|
|
||||||
public string TargetSqlServerName { get; set; }
|
|
||||||
|
|
||||||
public string TargetAccessToken { get; set; }
|
|
||||||
|
|
||||||
public string BatchAccountUrl { get; set; }
|
|
||||||
|
|
||||||
public string StorageAccountName { get; set; }
|
|
||||||
|
|
||||||
public string ContainerName { get; set; }
|
|
||||||
|
|
||||||
public string SqlAdminPassword { get; set; }
|
|
||||||
|
|
||||||
public string VNetSubnetId { get; set; }
|
|
||||||
}
|
|
||||||
|
|
||||||
[FunctionName("Export")]
|
|
||||||
public static async Task<HttpResponseMessage> PostExport(
|
|
||||||
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/Export")]
|
|
||||||
HttpRequestMessage req,
|
|
||||||
[DurableClient] IDurableOrchestrationClient starter,
|
|
||||||
ILogger log,
|
|
||||||
Guid subscriptionId,
|
|
||||||
string resourceGroupName)
|
|
||||||
{
|
|
||||||
log.LogInformation("C# HTTP trigger function processed an Export request.");
|
|
||||||
ExportRequest request = await req.Content.ReadAsAsync<ExportRequest>();
|
|
||||||
|
|
||||||
request.SubscriptionId = subscriptionId;
|
|
||||||
request.ResourceGroupName = resourceGroupName;
|
|
||||||
|
|
||||||
if (request.SourceSqlServerResourceGroupName == null)
|
|
||||||
request.SourceSqlServerResourceGroupName = resourceGroupName;
|
|
||||||
|
|
||||||
string instanceId = await starter.StartNewAsync(nameof(Orchestrator.RunExportOrchestrator), request);
|
|
||||||
|
|
||||||
log.LogInformation($"Started orchestration with ID = '{instanceId}'.");
|
|
||||||
return starter.CreateCheckStatusResponse(req, instanceId);
|
|
||||||
}
|
|
||||||
|
|
||||||
[FunctionName("Import")]
|
|
||||||
public static async Task<HttpResponseMessage> PostImport(
|
|
||||||
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/Import")]
|
|
||||||
HttpRequestMessage req,
|
|
||||||
[DurableClient] IDurableOrchestrationClient starter,
|
|
||||||
ILogger log,
|
|
||||||
Guid subscriptionId,
|
|
||||||
string resourceGroupName)
|
|
||||||
{
|
|
||||||
log.LogInformation("C# HTTP trigger function processed an Import request.");
|
|
||||||
ImportRequest request = await req.Content.ReadAsAsync<ImportRequest>();
|
|
||||||
|
|
||||||
request.SubscriptionId = subscriptionId;
|
|
||||||
request.ResourceGroupName = resourceGroupName;
|
|
||||||
|
|
||||||
if (request.TargetSqlServerResourceGroupName == null)
|
|
||||||
request.TargetSqlServerResourceGroupName = resourceGroupName;
|
|
||||||
|
|
||||||
string instanceId = await starter.StartNewAsync(nameof(Orchestrator.RunImportOrchestrator), request);
|
|
||||||
|
|
||||||
log.LogInformation($"Started orchestration with ID = '{instanceId}'.");
|
|
||||||
return starter.CreateCheckStatusResponse(req, instanceId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,104 +0,0 @@
|
|||||||
using Microsoft.Azure.WebJobs;
|
|
||||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using Newtonsoft.Json;
|
|
||||||
using System;
|
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using static ADPControl.HttpSurface;
|
|
||||||
|
|
||||||
namespace ADPControl
|
|
||||||
{
|
|
||||||
public static class Orchestrator
|
|
||||||
{
|
|
||||||
// The Import Orchestrator
|
|
||||||
[FunctionName(nameof(RunImportOrchestrator))]
|
|
||||||
public static async Task RunImportOrchestrator(
|
|
||||||
[OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log)
|
|
||||||
{
|
|
||||||
log.LogInformation("RunImportOrchestrator: entering");
|
|
||||||
|
|
||||||
try {
|
|
||||||
|
|
||||||
ImportRequest importRequest = context.GetInput<ImportRequest>();
|
|
||||||
// Deploy the ARM template to Create empty SQL resource
|
|
||||||
string deploymentName = await context.CallActivityAsync<string>(nameof(AzureResourceManagerActivity.BeginDeployArmTemplateForImport), importRequest);
|
|
||||||
while (true)
|
|
||||||
{
|
|
||||||
log.LogInformation("RunImportOrchestrator: starting ARM deployment");
|
|
||||||
string status = await context.CallActivityAsync<string>(nameof(AzureResourceManagerActivity.GetArmDeploymentForImport), (importRequest.SubscriptionId, importRequest.TargetSqlServerResourceGroupName, deploymentName));
|
|
||||||
if (status == "Succeeded")
|
|
||||||
{
|
|
||||||
log.LogInformation("RunImportOrchestrator: ARM deployment succeeded");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
else if (status == "Failed")
|
|
||||||
{
|
|
||||||
log.LogInformation("RunImportOrchestrator: ARM deployment failed");
|
|
||||||
throw new Exception("Failed ARM Deployment");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Orchestration sleeps until this time.
|
|
||||||
var nextCheck = context.CurrentUtcDateTime.AddSeconds(10);
|
|
||||||
|
|
||||||
if (!context.IsReplaying) { log.LogInformation($"RunImportOrchestrator: Replaying ARM deployment, next check at {nextCheck}."); }
|
|
||||||
await context.CreateTimer(nextCheck, CancellationToken.None);
|
|
||||||
}
|
|
||||||
|
|
||||||
log.LogInformation("RunImportOrchestrator: Enumerating databases");
|
|
||||||
var databases = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForImportSkipParameterization), importRequest);
|
|
||||||
|
|
||||||
// Create BatchPool And Job
|
|
||||||
log.LogInformation("RunImportOrchestrator: Creating batch pool and import job");
|
|
||||||
string jobId = await context.CallActivityAsync<string>(nameof(BatchActivity.CreateBatchPoolAndImportJob), importRequest);
|
|
||||||
|
|
||||||
string containerUrl = await context.CallActivityAsync<string>(nameof(StorageActivity.GettingJobContainerUrl), (importRequest.SubscriptionId, importRequest.ResourceGroupName, importRequest.StorageAccountName, importRequest.ContainerName));
|
|
||||||
|
|
||||||
log.LogInformation("RunImportOrchestrator: Creating import database tasks");
|
|
||||||
BatchActivity.CreateBatchTasks("Import", jobId, containerUrl, importRequest.BatchAccountUrl, importRequest.TargetSqlServerName, importRequest.TargetAccessToken, databases, log);
|
|
||||||
|
|
||||||
// create output values
|
|
||||||
Tuple<string, string>[] outputValues = {
|
|
||||||
Tuple.Create("Orchestration progress:", "Complete"),
|
|
||||||
Tuple.Create("deploymentName", deploymentName),
|
|
||||||
Tuple.Create("jobId", jobId),
|
|
||||||
Tuple.Create("containerUrl", containerUrl)
|
|
||||||
};
|
|
||||||
context.SetOutput(outputValues);
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
log.LogInformation("RunImportOrchestrator: exiting");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The Export Orchestrator
|
|
||||||
[FunctionName(nameof(RunExportOrchestrator))]
|
|
||||||
public static async Task RunExportOrchestrator(
|
|
||||||
[OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log)
|
|
||||||
{
|
|
||||||
ExportRequest exportRequest = context.GetInput<ExportRequest>();
|
|
||||||
|
|
||||||
// Getting the ARM template Skip ResourceName Parameterization.
|
|
||||||
var databases = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForExportSkipParameterization), exportRequest);
|
|
||||||
|
|
||||||
// Getting the ARM template.
|
|
||||||
dynamic Template = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForExport), exportRequest);
|
|
||||||
string json = JsonConvert.SerializeObject(Template);
|
|
||||||
|
|
||||||
// Create BatchPool And Job
|
|
||||||
string jobId = await context.CallActivityAsync<string>(nameof(BatchActivity.CreateBatchPoolAndExportJob), exportRequest);
|
|
||||||
|
|
||||||
string containerUrl = await context.CallActivityAsync<string>(nameof(StorageActivity.GettingJobContainerUrl), (exportRequest.SubscriptionId, exportRequest.ResourceGroupName, exportRequest.StorageAccountName, jobId));
|
|
||||||
await context.CallActivityAsync<string>(nameof(StorageActivity.UploadingArmTemplate), (containerUrl, json));
|
|
||||||
|
|
||||||
BatchActivity.CreateBatchTasks("Export", jobId, containerUrl, exportRequest.BatchAccountUrl, exportRequest.SourceSqlServerName, exportRequest.AccessToken, databases, log);
|
|
||||||
|
|
||||||
// create output values
|
|
||||||
Tuple<string, string>[] outputValues = {
|
|
||||||
Tuple.Create("jobId", jobId),
|
|
||||||
Tuple.Create("containerUrl", containerUrl)
|
|
||||||
};
|
|
||||||
context.SetOutput(outputValues);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
using Microsoft.Azure.Management.Storage;
|
|
||||||
using Microsoft.Azure.Management.Storage.Models;
|
|
||||||
using Microsoft.Azure.Services.AppAuthentication;
|
|
||||||
using Microsoft.Azure.WebJobs;
|
|
||||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using Microsoft.Rest;
|
|
||||||
using Microsoft.WindowsAzure.Storage;
|
|
||||||
using Microsoft.WindowsAzure.Storage.Auth;
|
|
||||||
using Microsoft.WindowsAzure.Storage.Blob;
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
namespace ADPControl
|
|
||||||
{
|
|
||||||
public static class StorageActivity
|
|
||||||
{
|
|
||||||
private const string ArmTemplateFileName = "template.json";
|
|
||||||
|
|
||||||
[FunctionName(nameof(GettingJobContainerUrl))]
|
|
||||||
public static string GettingJobContainerUrl([ActivityTrigger] (Guid, string, string, string) input, ILogger log)
|
|
||||||
{
|
|
||||||
Guid SubscriptionId = input.Item1;
|
|
||||||
String ResourceGroupName = input.Item2;
|
|
||||||
String StorageAccountName = input.Item3;
|
|
||||||
String ContainerName = input.Item4;
|
|
||||||
|
|
||||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
|
||||||
TokenCredentials tokenArmCredential = new TokenCredentials(azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/").Result);
|
|
||||||
StorageManagementClient storageMgmtClient = new StorageManagementClient(tokenArmCredential) { SubscriptionId = SubscriptionId.ToString() };
|
|
||||||
|
|
||||||
// Get the storage account keys for a given account and resource group
|
|
||||||
IList<StorageAccountKey> acctKeys = storageMgmtClient.StorageAccounts.ListKeys(ResourceGroupName, StorageAccountName).Keys;
|
|
||||||
|
|
||||||
// Get a Storage account using account creds:
|
|
||||||
StorageCredentials storageCred = new StorageCredentials(StorageAccountName, acctKeys.FirstOrDefault().Value);
|
|
||||||
CloudStorageAccount linkedStorageAccount = new CloudStorageAccount(storageCred, true);
|
|
||||||
|
|
||||||
bool createContainer = false;
|
|
||||||
// Normalize the container name for the Export action.
|
|
||||||
if (ContainerName.Contains("-Export-"))
|
|
||||||
{
|
|
||||||
ContainerName = ContainerName.Replace("Export-", "");
|
|
||||||
createContainer = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(ContainerName);
|
|
||||||
|
|
||||||
if(createContainer)
|
|
||||||
container.CreateIfNotExistsAsync().Wait();
|
|
||||||
|
|
||||||
string containerUrl = container.Uri.ToString() +
|
|
||||||
container.GetSharedAccessSignature(new SharedAccessBlobPolicy()
|
|
||||||
{
|
|
||||||
Permissions = SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List,
|
|
||||||
SharedAccessExpiryTime = DateTime.UtcNow.AddDays(7)
|
|
||||||
});
|
|
||||||
return containerUrl;
|
|
||||||
}
|
|
||||||
|
|
||||||
[FunctionName(nameof(UploadingArmTemplate))]
|
|
||||||
public static void UploadingArmTemplate([ActivityTrigger] (string, string) input, ILogger log)
|
|
||||||
{
|
|
||||||
string containerUrl = input.Item1;
|
|
||||||
string json = input.Item2;
|
|
||||||
|
|
||||||
CloudBlobContainer container = new CloudBlobContainer(new Uri(containerUrl));
|
|
||||||
CloudBlockBlob blob = container.GetBlockBlobReference(ArmTemplateFileName);
|
|
||||||
blob.Properties.ContentType = "application/json";
|
|
||||||
using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(json)))
|
|
||||||
{
|
|
||||||
blob.UploadFromStreamAsync(stream).Wait();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
{
|
|
||||||
"version": "2.0",
|
|
||||||
"extensions": {
|
|
||||||
"durableTask": {
|
|
||||||
"hubName": "adp"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"logging": {
|
|
||||||
"applicationInsights": {
|
|
||||||
"samplingExcludedTypes": "Request",
|
|
||||||
"samplingSettings": {
|
|
||||||
"isEnabled": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
namespace BatchWrapper
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The type of sqlpackage action to perform.
|
|
||||||
/// </summary>
|
|
||||||
public enum ActionType
|
|
||||||
{
|
|
||||||
DefaultInvalid = -1,
|
|
||||||
Export,
|
|
||||||
Import
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
|
||||||
<configuration>
|
|
||||||
<startup>
|
|
||||||
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
|
|
||||||
</startup>
|
|
||||||
<runtime>
|
|
||||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
|
||||||
<dependentAssembly>
|
|
||||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
|
||||||
<bindingRedirect oldVersion="0.0.0.0-12.0.0.0" newVersion="12.0.0.0" />
|
|
||||||
</dependentAssembly>
|
|
||||||
<dependentAssembly>
|
|
||||||
<assemblyIdentity name="Microsoft.Azure.Batch" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
|
||||||
<bindingRedirect oldVersion="0.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
|
|
||||||
</dependentAssembly>
|
|
||||||
<dependentAssembly>
|
|
||||||
<assemblyIdentity name="Microsoft.Azure.KeyVault.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
|
||||||
<bindingRedirect oldVersion="0.0.0.0-3.0.5.0" newVersion="3.0.5.0" />
|
|
||||||
</dependentAssembly>
|
|
||||||
</assemblyBinding>
|
|
||||||
</runtime>
|
|
||||||
</configuration>
|
|
||||||
@@ -1,98 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
|
||||||
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
|
||||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
|
||||||
<PropertyGroup>
|
|
||||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
|
||||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
|
||||||
<ProjectGuid>{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}</ProjectGuid>
|
|
||||||
<OutputType>Exe</OutputType>
|
|
||||||
<RootNamespace>BatchWrapper</RootNamespace>
|
|
||||||
<AssemblyName>BatchWrapper</AssemblyName>
|
|
||||||
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
|
|
||||||
<LangVersion>8.0</LangVersion>
|
|
||||||
<FileAlignment>512</FileAlignment>
|
|
||||||
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
|
|
||||||
<Deterministic>true</Deterministic>
|
|
||||||
<IsWebBootstrapper>false</IsWebBootstrapper>
|
|
||||||
<PublishUrl>publish\</PublishUrl>
|
|
||||||
<Install>true</Install>
|
|
||||||
<InstallFrom>Disk</InstallFrom>
|
|
||||||
<UpdateEnabled>false</UpdateEnabled>
|
|
||||||
<UpdateMode>Foreground</UpdateMode>
|
|
||||||
<UpdateInterval>7</UpdateInterval>
|
|
||||||
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
|
|
||||||
<UpdatePeriodically>false</UpdatePeriodically>
|
|
||||||
<UpdateRequired>false</UpdateRequired>
|
|
||||||
<MapFileExtensions>true</MapFileExtensions>
|
|
||||||
<ApplicationRevision>0</ApplicationRevision>
|
|
||||||
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
|
|
||||||
<UseApplicationTrust>false</UseApplicationTrust>
|
|
||||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
|
||||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
|
||||||
<DebugSymbols>true</DebugSymbols>
|
|
||||||
<DebugType>full</DebugType>
|
|
||||||
<Optimize>false</Optimize>
|
|
||||||
<OutputPath>bin\Debug\</OutputPath>
|
|
||||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
|
||||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
|
||||||
<DebugType>pdbonly</DebugType>
|
|
||||||
<Optimize>true</Optimize>
|
|
||||||
<OutputPath>bin\Release\</OutputPath>
|
|
||||||
<DefineConstants>TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
</PropertyGroup>
|
|
||||||
<ItemGroup>
|
|
||||||
<Reference Include="Microsoft.Azure.Batch.Conventions.Files, Version=3.5.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
|
||||||
<HintPath>..\packages\Microsoft.Azure.Batch.Conventions.Files.3.5.1\lib\net461\Microsoft.Azure.Batch.Conventions.Files.dll</HintPath>
|
|
||||||
</Reference>
|
|
||||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=9.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
|
||||||
<HintPath>..\packages\WindowsAzure.Storage.9.3.3\lib\net45\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
|
||||||
</Reference>
|
|
||||||
<Reference Include="System" />
|
|
||||||
<Reference Include="System.Core" />
|
|
||||||
<Reference Include="System.Net" />
|
|
||||||
<Reference Include="System.Net.Http.WebRequest" />
|
|
||||||
<Reference Include="System.Runtime" />
|
|
||||||
<Reference Include="System.Runtime.Serialization" />
|
|
||||||
<Reference Include="System.Web" />
|
|
||||||
<Reference Include="System.Xml.Linq" />
|
|
||||||
<Reference Include="System.Data.DataSetExtensions" />
|
|
||||||
<Reference Include="Microsoft.CSharp" />
|
|
||||||
<Reference Include="System.Data" />
|
|
||||||
<Reference Include="System.Net.Http" />
|
|
||||||
<Reference Include="System.Xml" />
|
|
||||||
</ItemGroup>
|
|
||||||
<ItemGroup>
|
|
||||||
<Compile Include="ActionType.cs" />
|
|
||||||
<Compile Include="Constants.cs" />
|
|
||||||
<Compile Include="Payload.cs" />
|
|
||||||
<Compile Include="Program.cs" />
|
|
||||||
</ItemGroup>
|
|
||||||
<ItemGroup>
|
|
||||||
<None Include="App.config" />
|
|
||||||
<None Include="packages.config" />
|
|
||||||
</ItemGroup>
|
|
||||||
<ItemGroup>
|
|
||||||
<BootstrapperPackage Include=".NETFramework,Version=v4.7.2">
|
|
||||||
<Visible>False</Visible>
|
|
||||||
<ProductName>Microsoft .NET Framework 4.7.2 %28x86 and x64%29</ProductName>
|
|
||||||
<Install>true</Install>
|
|
||||||
</BootstrapperPackage>
|
|
||||||
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
|
|
||||||
<Visible>False</Visible>
|
|
||||||
<ProductName>.NET Framework 3.5 SP1</ProductName>
|
|
||||||
<Install>false</Install>
|
|
||||||
</BootstrapperPackage>
|
|
||||||
</ItemGroup>
|
|
||||||
<ItemGroup>
|
|
||||||
<Folder Include="Properties\" />
|
|
||||||
</ItemGroup>
|
|
||||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
|
||||||
</Project>
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
namespace BatchWrapper
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Constants for the batch wrapper.
|
|
||||||
/// </summary>
|
|
||||||
public static class Constants
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Environment variable names present or needed during the batch task execution.
|
|
||||||
/// </summary>
|
|
||||||
public static class EnvironmentVariableNames
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Path to the directory containing the sqlpackage exe.
|
|
||||||
/// </summary>
|
|
||||||
internal const string AppPackagePrefix = "AZ_BATCH_APP_PACKAGE";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Path to the working directory assigned to the batch task.
|
|
||||||
/// </summary>
|
|
||||||
internal const string TaskWorkingDir = "AZ_BATCH_TASK_WORKING_DIR";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Path to the working directory assigned to the batch task.
|
|
||||||
/// </summary>
|
|
||||||
internal const string AzBatchTaskId = "AZ_BATCH_TASK_ID";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Path to the working directory assigned to the batch task.
|
|
||||||
/// </summary>
|
|
||||||
internal const string JobContainerUrl = "JOB_CONTAINER_URL";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
namespace BatchWrapper
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The top-level object stored in the key vault for an import/export operation.
|
|
||||||
/// </summary>
|
|
||||||
public sealed class Payload
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The Name of sqlpackage to use for performing the import/export operation.
|
|
||||||
/// </summary>
|
|
||||||
public string ApplicatonPackageName{ get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The Version of sqlpackage to use for performing the import/export operation.
|
|
||||||
/// </summary>
|
|
||||||
public string ApplicatonPackageVersion { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The type of sqlpackage action to perform.
|
|
||||||
/// </summary>
|
|
||||||
public ActionType Action { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The logical server name to export from or import to.
|
|
||||||
/// </summary>
|
|
||||||
public string LogicalServerName { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The database name to export from or import to.
|
|
||||||
/// </summary>
|
|
||||||
public string DatabaseName { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The server admin username.
|
|
||||||
/// </summary>
|
|
||||||
public string AccessToken { get; set; }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,178 +0,0 @@
|
|||||||
using Microsoft.Azure.Batch.Conventions.Files;
|
|
||||||
using Microsoft.WindowsAzure.Storage.Blob;
|
|
||||||
using System;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace BatchWrapper
|
|
||||||
{
|
|
||||||
public static class Program
|
|
||||||
{
|
|
||||||
private static string dataDirectory = "F:\\data";
|
|
||||||
private static string tempDirectory = "F:\\temp";
|
|
||||||
private static string[] directories = { dataDirectory, tempDirectory };
|
|
||||||
|
|
||||||
private static readonly TimeSpan stdoutFlushDelay = TimeSpan.FromSeconds(3);
|
|
||||||
|
|
||||||
private static void WriteLine(string message) => WriteLineInternal(Console.Out, message);
|
|
||||||
private static void WriteErrorLine(string message) => WriteLineInternal(Console.Error, message);
|
|
||||||
private static void WriteLineInternal(TextWriter writer, string message)
|
|
||||||
{
|
|
||||||
var lines = message?.Split('\n') ?? new string[0];
|
|
||||||
foreach (var line in lines)
|
|
||||||
{
|
|
||||||
writer.WriteLine($"[{DateTime.UtcNow:u}] {line?.TrimEnd()}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async Task<int> Main(string[] args)
|
|
||||||
{
|
|
||||||
var assembly = typeof(Program).Assembly;
|
|
||||||
WriteLine($"{assembly.ManifestModule.Name} v{assembly.GetName().Version.ToString(3)}");
|
|
||||||
|
|
||||||
// Get the command payload
|
|
||||||
var payload = new Payload();
|
|
||||||
|
|
||||||
if (args.Length > 0)
|
|
||||||
{
|
|
||||||
payload.Action = (ActionType)Enum.Parse(typeof(ActionType), args[0]);
|
|
||||||
payload.LogicalServerName = args[1] + ".database.windows.net";
|
|
||||||
payload.DatabaseName = args[2];
|
|
||||||
payload.AccessToken = args[3];
|
|
||||||
payload.ApplicatonPackageName = args[4];
|
|
||||||
payload.ApplicatonPackageVersion = args[5];
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cleanup folders
|
|
||||||
foreach (string dir in directories)
|
|
||||||
{
|
|
||||||
if (Directory.Exists(dir))
|
|
||||||
{
|
|
||||||
Directory.Delete(dir, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
Directory.CreateDirectory(dir);
|
|
||||||
}
|
|
||||||
|
|
||||||
string sqlPackageBacpacFile = Path.Combine(dataDirectory, payload.DatabaseName + ".bacpac");
|
|
||||||
string sqlPackageLogPath = payload.DatabaseName + ".log";
|
|
||||||
|
|
||||||
var targetDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AppPackagePrefix + "_" + payload.ApplicatonPackageName + "#" + payload.ApplicatonPackageVersion);
|
|
||||||
var workingDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.TaskWorkingDir);
|
|
||||||
|
|
||||||
string taskId = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AzBatchTaskId);
|
|
||||||
string jobContainerUrl = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.JobContainerUrl);
|
|
||||||
|
|
||||||
// Build the import/export command
|
|
||||||
var cmdBuilder = new StringBuilder();
|
|
||||||
cmdBuilder.Append($"/Action:{payload.Action}");
|
|
||||||
cmdBuilder.Append(" /MaxParallelism:16");
|
|
||||||
cmdBuilder.Append(String.Format(" /DiagnosticsFile:{0}", sqlPackageLogPath));
|
|
||||||
cmdBuilder.Append(" /p:CommandTimeout=604800");
|
|
||||||
|
|
||||||
switch (payload.Action)
|
|
||||||
{
|
|
||||||
case ActionType.Export:
|
|
||||||
cmdBuilder.Append($" /SourceServerName:{payload.LogicalServerName}");
|
|
||||||
cmdBuilder.Append($" /SourceDatabaseName:{payload.DatabaseName}");
|
|
||||||
cmdBuilder.Append($" /AccessToken:{payload.AccessToken}");
|
|
||||||
cmdBuilder.Append($" /TargetFile:{sqlPackageBacpacFile}");
|
|
||||||
cmdBuilder.Append($" /SourceTimeout:30");
|
|
||||||
cmdBuilder.Append(String.Format(" /p:TempDirectoryForTableData=\"{0}\"", tempDirectory));
|
|
||||||
cmdBuilder.Append(" /p:VerifyFullTextDocumentTypesSupported=false");
|
|
||||||
break;
|
|
||||||
|
|
||||||
case ActionType.Import:
|
|
||||||
cmdBuilder.Append($" /TargetServerName:{payload.LogicalServerName}");
|
|
||||||
cmdBuilder.Append($" /TargetDatabaseName:{payload.DatabaseName}");
|
|
||||||
cmdBuilder.Append($" /AccessToken:{payload.AccessToken}");
|
|
||||||
cmdBuilder.Append($" /TargetTimeout:30");
|
|
||||||
cmdBuilder.Append($" /SourceFile:{sqlPackageBacpacFile}");
|
|
||||||
break;
|
|
||||||
|
|
||||||
default:
|
|
||||||
throw new ArgumentException($"Invalid action type: {payload.Action}");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (payload.Action == ActionType.Import)
|
|
||||||
{
|
|
||||||
WriteLine(string.Format("Downloading {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
|
|
||||||
CloudBlobContainer container = new CloudBlobContainer(new Uri(jobContainerUrl));
|
|
||||||
CloudBlockBlob blob = container.GetBlockBlobReference(String.Format("$JobOutput/{0}.bacpac", payload.DatabaseName));
|
|
||||||
blob.DownloadToFile(sqlPackageBacpacFile, FileMode.CreateNew);
|
|
||||||
|
|
||||||
if (File.Exists(sqlPackageBacpacFile))
|
|
||||||
{
|
|
||||||
WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
throw new Exception(string.Format("{0} didn't download", sqlPackageBacpacFile));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Perform the import/export process
|
|
||||||
var startTime = DateTimeOffset.UtcNow;
|
|
||||||
var process = new Process
|
|
||||||
{
|
|
||||||
StartInfo = new ProcessStartInfo
|
|
||||||
{
|
|
||||||
WorkingDirectory = workingDir,
|
|
||||||
FileName = Path.Combine(targetDir, "sqlpackage.exe"),
|
|
||||||
Arguments = cmdBuilder.ToString(),
|
|
||||||
CreateNoWindow = true,
|
|
||||||
UseShellExecute = false,
|
|
||||||
RedirectStandardOutput = true,
|
|
||||||
RedirectStandardError = true
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
process.OutputDataReceived += (s, e) => WriteLine(e.Data);
|
|
||||||
process.ErrorDataReceived += (s, e) => WriteErrorLine(e.Data);
|
|
||||||
process.Start();
|
|
||||||
process.BeginOutputReadLine();
|
|
||||||
process.BeginErrorReadLine();
|
|
||||||
process.WaitForExit();
|
|
||||||
|
|
||||||
WriteLine(String.Format("SqlPackage.exe exited with code: {0}", process.ExitCode));
|
|
||||||
|
|
||||||
if (payload.Action == ActionType.Export)
|
|
||||||
{
|
|
||||||
if (File.Exists(sqlPackageBacpacFile))
|
|
||||||
{
|
|
||||||
WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
throw new Exception(string.Format("{0} didn't downloaded", sqlPackageBacpacFile));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Persist the Job Output
|
|
||||||
JobOutputStorage jobOutputStorage = new JobOutputStorage(new Uri(jobContainerUrl));
|
|
||||||
|
|
||||||
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageLogPath);
|
|
||||||
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageLogPath));
|
|
||||||
|
|
||||||
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageBacpacFile, payload.DatabaseName + ".bacpac");
|
|
||||||
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageBacpacFile));
|
|
||||||
}
|
|
||||||
|
|
||||||
// We are tracking the disk file to save our standard output, but the node agent may take
|
|
||||||
// up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
|
|
||||||
await Task.Delay(stdoutFlushDelay);
|
|
||||||
|
|
||||||
// Cleanup folders
|
|
||||||
foreach (string dir in directories)
|
|
||||||
{
|
|
||||||
if (Directory.Exists(dir))
|
|
||||||
{
|
|
||||||
Directory.Delete(dir, true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return process.ExitCode;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
|
||||||
<packages>
|
|
||||||
<package id="Microsoft.Azure.Batch" version="13.0.0" targetFramework="net472" />
|
|
||||||
<package id="Microsoft.Azure.Batch.Conventions.Files" version="3.5.1" targetFramework="net472" />
|
|
||||||
<package id="Microsoft.Azure.KeyVault.Core" version="3.0.5" targetFramework="net472" />
|
|
||||||
<package id="Microsoft.Rest.ClientRuntime" version="2.3.21" targetFramework="net472" />
|
|
||||||
<package id="Microsoft.Rest.ClientRuntime.Azure" version="3.3.19" targetFramework="net472" />
|
|
||||||
<package id="Newtonsoft.Json" version="12.0.3" targetFramework="net472" />
|
|
||||||
<package id="WindowsAzure.Storage" version="9.3.3" targetFramework="net472" />
|
|
||||||
</packages>
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
{
|
|
||||||
// Use IntelliSense to find out which attributes exist for C# debugging
|
|
||||||
// Use hover for the description of the existing attributes
|
|
||||||
// For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
|
|
||||||
"version": "0.2.0",
|
|
||||||
"configurations": [
|
|
||||||
{
|
|
||||||
"name": ".NET Core Launch (console)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"preLaunchTask": "build",
|
|
||||||
// If you have changed target frameworks, make sure to update the program path.
|
|
||||||
"program": "${workspaceFolder}/bin/Debug/netcoreapp3.1/SqlPackageWrapper.dll",
|
|
||||||
"args": [],
|
|
||||||
"cwd": "${workspaceFolder}",
|
|
||||||
// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
|
|
||||||
"console": "internalConsole",
|
|
||||||
"stopAtEntry": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": ".NET Core Attach",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "attach",
|
|
||||||
"processId": "${command:pickProcess}"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
{
|
|
||||||
"version": "2.0.0",
|
|
||||||
"tasks": [
|
|
||||||
{
|
|
||||||
"label": "build",
|
|
||||||
"command": "dotnet",
|
|
||||||
"type": "process",
|
|
||||||
"args": [
|
|
||||||
"build",
|
|
||||||
"${workspaceFolder}/SqlPackageWrapper.csproj",
|
|
||||||
"/property:GenerateFullPaths=true",
|
|
||||||
"/consoleloggerparameters:NoSummary"
|
|
||||||
],
|
|
||||||
"problemMatcher": "$msCompile"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "publish",
|
|
||||||
"command": "dotnet",
|
|
||||||
"type": "process",
|
|
||||||
"args": [
|
|
||||||
"publish",
|
|
||||||
"${workspaceFolder}/SqlPackageWrapper.csproj",
|
|
||||||
"/property:GenerateFullPaths=true",
|
|
||||||
"/consoleloggerparameters:NoSummary"
|
|
||||||
],
|
|
||||||
"problemMatcher": "$msCompile"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "watch",
|
|
||||||
"command": "dotnet",
|
|
||||||
"type": "process",
|
|
||||||
"args": [
|
|
||||||
"watch",
|
|
||||||
"run",
|
|
||||||
"${workspaceFolder}/SqlPackageWrapper.csproj",
|
|
||||||
"/property:GenerateFullPaths=true",
|
|
||||||
"/consoleloggerparameters:NoSummary"
|
|
||||||
],
|
|
||||||
"problemMatcher": "$msCompile"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
namespace SqlPackageWrapper
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The type of sqlpackage action to perform.
|
|
||||||
/// </summary>
|
|
||||||
public enum ActionType
|
|
||||||
{
|
|
||||||
DefaultInvalid = -1,
|
|
||||||
Export,
|
|
||||||
Import
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
namespace SqlPackageWrapper
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Constants for the batch wrapper.
|
|
||||||
/// </summary>
|
|
||||||
public static class Constants
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Environment variable names present or needed during the batch task execution.
|
|
||||||
/// </summary>
|
|
||||||
public static class EnvironmentVariableNames
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Path to the directory containing the batch wrapper exe.
|
|
||||||
/// </summary>
|
|
||||||
public const string WrapperLocation = "AZ_BATCH_APP_PACKAGE_BATCHWRAPPER";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Path to the directory containing the sqlpackage exe.
|
|
||||||
/// </summary>
|
|
||||||
internal const string SqlPackageLocation = "AZ_BATCH_APP_PACKAGE_SQLPACKAGE";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Path to the working directory assigned to the batch task.
|
|
||||||
/// </summary>
|
|
||||||
internal const string TaskWorkingDir = "AZ_BATCH_TASK_WORKING_DIR";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Path to the working directory assigned to the batch task.
|
|
||||||
/// </summary>
|
|
||||||
internal const string AzBatchTaskId = "AZ_BATCH_TASK_ID";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Path to the working directory assigned to the batch task.
|
|
||||||
/// </summary>
|
|
||||||
internal const string JobContainerUrl = "JOB_CONTAINER_URL";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
namespace SqlPackageWrapper
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The top-level object stored in the key vault for an import/export operation.
|
|
||||||
/// </summary>
|
|
||||||
public sealed class Payload
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// The version of sqlpackage to use for performing the import/export operation.
|
|
||||||
/// </summary>
|
|
||||||
public string SqlPackageVersion { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The type of sqlpackage action to perform.
|
|
||||||
/// </summary>
|
|
||||||
public ActionType Action { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The logical server name to export from or import to.
|
|
||||||
/// </summary>
|
|
||||||
public string LogicalServerName { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The database name to export from or import to.
|
|
||||||
/// </summary>
|
|
||||||
public string DatabaseName { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The server admin username.
|
|
||||||
/// </summary>
|
|
||||||
public string Username { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The server admin password.
|
|
||||||
/// </summary>
|
|
||||||
public string Password { get; set; }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,162 +0,0 @@
|
|||||||
using Microsoft.Azure.Batch.Conventions.Files;
|
|
||||||
using Microsoft.WindowsAzure.Storage.Blob;
|
|
||||||
using System;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace SqlPackageWrapper
|
|
||||||
{
|
|
||||||
public static class Program
|
|
||||||
{
|
|
||||||
private static string dataDirectory = "F:\\data";
|
|
||||||
private static string tempDirectory = "F:\\temp";
|
|
||||||
private static string[] directories = { dataDirectory, tempDirectory };
|
|
||||||
|
|
||||||
private static readonly TimeSpan stdoutFlushDelay = TimeSpan.FromSeconds(3);
|
|
||||||
|
|
||||||
private static void WriteLine(string message) => WriteLineInternal(Console.Out, message);
|
|
||||||
private static void WriteErrorLine(string message) => WriteLineInternal(Console.Error, message);
|
|
||||||
private static void WriteLineInternal(TextWriter writer, string message)
|
|
||||||
{
|
|
||||||
var lines = message?.Split('\n') ?? new string[0];
|
|
||||||
foreach (var line in lines)
|
|
||||||
{
|
|
||||||
writer.WriteLine($"[{DateTime.UtcNow:u}] {line?.TrimEnd()}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async Task<int> Main(string[] args)
|
|
||||||
{
|
|
||||||
var assembly = typeof(Program).Assembly;
|
|
||||||
WriteLine($"{assembly.ManifestModule.Name} v{assembly.GetName().Version.ToString(3)}");
|
|
||||||
|
|
||||||
// Get the command payload
|
|
||||||
var payload = new Payload();
|
|
||||||
|
|
||||||
if (args.Length > 0)
|
|
||||||
{
|
|
||||||
payload.Action = (ActionType)Enum.Parse(typeof(ActionType), args[0]);
|
|
||||||
payload.LogicalServerName = args[1] + ".database.windows.net";
|
|
||||||
payload.DatabaseName = args[2];
|
|
||||||
payload.Username = args[3];
|
|
||||||
payload.Password = args[4];
|
|
||||||
payload.SqlPackageVersion = args[5];
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cleanup folders
|
|
||||||
foreach (string dir in directories)
|
|
||||||
{
|
|
||||||
if (Directory.Exists(dir))
|
|
||||||
{
|
|
||||||
Directory.Delete(dir, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
Directory.CreateDirectory(dir);
|
|
||||||
}
|
|
||||||
|
|
||||||
string sqlPackageDataPath = Path.Combine(dataDirectory, payload.DatabaseName + ".bacpac");
|
|
||||||
string sqlPackageLogPath = Path.Combine(dataDirectory, payload.DatabaseName + ".log");
|
|
||||||
|
|
||||||
var targetDir = Environment.GetEnvironmentVariable($"{Constants.EnvironmentVariableNames.SqlPackageLocation}#{payload.SqlPackageVersion}");
|
|
||||||
var workingDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.TaskWorkingDir);
|
|
||||||
|
|
||||||
string taskId = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AzBatchTaskId);
|
|
||||||
string jobContainerUrl = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.JobContainerUrl);
|
|
||||||
|
|
||||||
// Build the import/export command
|
|
||||||
var cmdBuilder = new StringBuilder();
|
|
||||||
cmdBuilder.Append($"/Action:{payload.Action}");
|
|
||||||
cmdBuilder.Append(" /MaxParallelism:16");
|
|
||||||
cmdBuilder.Append(String.Format(" /DiagnosticsFile:{0}", sqlPackageLogPath));
|
|
||||||
cmdBuilder.Append(" /p:CommandTimeout=86400");
|
|
||||||
|
|
||||||
switch (payload.Action)
|
|
||||||
{
|
|
||||||
case ActionType.Export:
|
|
||||||
cmdBuilder.Append($" /SourceServerName:{payload.LogicalServerName}");
|
|
||||||
cmdBuilder.Append($" /SourceDatabaseName:{payload.DatabaseName}");
|
|
||||||
cmdBuilder.Append($" /SourceUser:{payload.Username}");
|
|
||||||
cmdBuilder.Append($" /SourcePassword:{payload.Password}");
|
|
||||||
cmdBuilder.Append($" /TargetFile:{sqlPackageDataPath}");
|
|
||||||
cmdBuilder.Append(String.Format(" /p:TempDirectoryForTableData=\"{0}\"", tempDirectory));
|
|
||||||
cmdBuilder.Append(" /p:VerifyFullTextDocumentTypesSupported=false");
|
|
||||||
break;
|
|
||||||
|
|
||||||
case ActionType.Import:
|
|
||||||
cmdBuilder.Append($" /TargetServerName:{payload.LogicalServerName}");
|
|
||||||
cmdBuilder.Append($" /TargetDatabaseName:{payload.DatabaseName}");
|
|
||||||
cmdBuilder.Append($" /TargetUser:{payload.Username}");
|
|
||||||
cmdBuilder.Append($" /TargetPassword:{payload.Password}");
|
|
||||||
cmdBuilder.Append($" /SourceFile:{sqlPackageDataPath}");
|
|
||||||
break;
|
|
||||||
|
|
||||||
default:
|
|
||||||
throw new ArgumentException($"Invalid action type: {payload.Action}");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (payload.Action == ActionType.Import)
|
|
||||||
{
|
|
||||||
WriteLine(string.Format("Downloading {0} bacpac file to {1}", payload.DatabaseName, sqlPackageDataPath));
|
|
||||||
CloudBlobContainer container = new CloudBlobContainer(new Uri(jobContainerUrl));
|
|
||||||
CloudBlockBlob blob = container.GetBlockBlobReference(String.Format("$JobOutput/{0}.bacpac", payload.DatabaseName));
|
|
||||||
await blob.DownloadToFileAsync(sqlPackageDataPath, FileMode.CreateNew);
|
|
||||||
WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageDataPath));
|
|
||||||
|
|
||||||
await Task.Delay(stdoutFlushDelay);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Perform the import/export process
|
|
||||||
var startTime = DateTimeOffset.UtcNow;
|
|
||||||
var process = new Process
|
|
||||||
{
|
|
||||||
StartInfo = new ProcessStartInfo
|
|
||||||
{
|
|
||||||
WorkingDirectory = workingDir,
|
|
||||||
FileName = Path.Combine(targetDir, "sqlpackage.exe"),
|
|
||||||
Arguments = cmdBuilder.ToString(),
|
|
||||||
CreateNoWindow = true,
|
|
||||||
UseShellExecute = false,
|
|
||||||
RedirectStandardOutput = true,
|
|
||||||
RedirectStandardError = true
|
|
||||||
}
|
|
||||||
};
|
|
||||||
process.OutputDataReceived += (s, e) => WriteLine(e.Data);
|
|
||||||
process.ErrorDataReceived += (s, e) => WriteErrorLine(e.Data);
|
|
||||||
process.Start();
|
|
||||||
process.BeginOutputReadLine();
|
|
||||||
process.BeginErrorReadLine();
|
|
||||||
process.WaitForExit();
|
|
||||||
|
|
||||||
WriteLine(String.Format("SqlPackage.exe exited with code: {0}", process.ExitCode));
|
|
||||||
|
|
||||||
if (payload.Action == ActionType.Export)
|
|
||||||
{
|
|
||||||
// Persist the Job Output
|
|
||||||
JobOutputStorage jobOutputStorage = new JobOutputStorage(new Uri(jobContainerUrl));
|
|
||||||
|
|
||||||
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageLogPath, payload.DatabaseName + ".log");
|
|
||||||
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageLogPath));
|
|
||||||
|
|
||||||
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageDataPath, payload.DatabaseName + ".bacpac");
|
|
||||||
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageDataPath));
|
|
||||||
|
|
||||||
// We are tracking the disk file to save our standard output, but the node agent may take
|
|
||||||
// up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
|
|
||||||
await Task.Delay(stdoutFlushDelay);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cleanup folders
|
|
||||||
foreach (string dir in directories)
|
|
||||||
{
|
|
||||||
if (Directory.Exists(dir))
|
|
||||||
{
|
|
||||||
Directory.Delete(dir, true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return process.ExitCode;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<OutputType>Exe</OutputType>
|
|
||||||
<TargetFrameworks>netcoreapp2.1;net452</TargetFrameworks>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="Microsoft.Azure.Batch" Version="13.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Azure.Batch.Conventions.Files" Version="3.5.1" />
|
|
||||||
<PackageReference Include="Microsoft.Azure.Management.Batch" Version="11.0.0" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
Folder for helper components used by notebooks in the Hybrid Cloud Toolkit
|
|
||||||
@@ -8,6 +8,13 @@
|
|||||||
- title: Search
|
- title: Search
|
||||||
search: true
|
search: true
|
||||||
|
|
||||||
|
- title: ADP
|
||||||
|
url: /ADP/readme
|
||||||
|
not_numbered: true
|
||||||
|
expand_sections: false
|
||||||
|
sections:
|
||||||
|
- title: ADP Control
|
||||||
|
url: ADP/ADPControl
|
||||||
- title: Assessment
|
- title: Assessment
|
||||||
url: /Assessments/readme
|
url: /Assessments/readme
|
||||||
not_numbered: true
|
not_numbered: true
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# ADP Control
|
||||||
|
|
||||||
|
[Home](../readme.md)
|
||||||
|
|
||||||
|
Notebook to manage ADP Orchestrator import/export requests onto an SQL Server.
|
||||||
|
|
||||||
|
## Notebooks in this Chapter
|
||||||
|
- [ADP Control](ADPControl.ipynb)
|
||||||
@@ -2,6 +2,8 @@
|
|||||||
## Chapters
|
## Chapters
|
||||||
* [Prerequisites and Initial Setup](prereqs.ipynb) - Notebook installation of required modules.
|
* [Prerequisites and Initial Setup](prereqs.ipynb) - Notebook installation of required modules.
|
||||||
|
|
||||||
|
* [ADP](ADP/readme.md) - manage ADP Orchestrator import/export requests onto an SQL Server.
|
||||||
|
|
||||||
* [Assessments](Assessments/readme.md) - Notebooks that contain examples to determine whether a given database or SQL Server instance is ready to migrate by utilizing SQL Assessments. SQL instances are scanned based on a "best practices" set of rules.
|
* [Assessments](Assessments/readme.md) - Notebooks that contain examples to determine whether a given database or SQL Server instance is ready to migrate by utilizing SQL Assessments. SQL instances are scanned based on a "best practices" set of rules.
|
||||||
|
|
||||||
* [Networking](networking/readme.md) - Setup secure Point-to-Site (P2S) or Site-to-Site (S2S) network connectivity to Microsoft Azure using a Virtual Private Network (VPN). This notebook serves as a building block for other notebooks as communicating securely between on-premise and Azure is essential for many tasks.
|
* [Networking](networking/readme.md) - Setup secure Point-to-Site (P2S) or Site-to-Site (S2S) network connectivity to Microsoft Azure using a Virtual Private Network (VPN). This notebook serves as a building block for other notebooks as communicating securely between on-premise and Azure is essential for many tasks.
|
||||||
|
|||||||
Reference in New Issue
Block a user