mirror of
https://github.com/ckaczor/azuredatastudio.git
synced 2026-02-16 10:58:30 -05:00
Removal of Components folder, moving ADP notebook to its own notebook. (#13848)
* adp folder removed, notebook moved. * moved ADP notebook to own folder.
This commit is contained in:
Alex Ma
@@ -1,37 +0,0 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio Version 16
|
||||
VisualStudioVersion = 16.0.29920.165
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ADPControl", "ADPControl\ADPControl.csproj", "{6309D4C5-F118-4C89-A67E-E557CA41ABA2}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BatchWrapper", "BatchWrapper\BatchWrapper.csproj", "{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlPackageWrapper", "SqlPackageWrapper\SqlPackageWrapper.csproj", "{A19335D3-9D80-43BE-9351-C3C3704689B0}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Release|Any CPU = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Release|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{6309D4C5-F118-4C89-A67E-E557CA41ABA2}.Release|Any CPU.Build.0 = Debug|Any CPU
|
||||
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Release|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}.Release|Any CPU.Build.0 = Debug|Any CPU
|
||||
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{A19335D3-9D80-43BE-9351-C3C3704689B0}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {3DB5F0CB-1D26-4E33-801D-7BBAE823C39D}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
@@ -1,26 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<AzureFunctionsVersion>v3</AzureFunctionsVersion>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Azure.Batch" Version="13.0.0" />
|
||||
<PackageReference Include="Microsoft.Azure.Management.ResourceManager" Version="3.7.0-preview" />
|
||||
<PackageReference Include="Microsoft.Azure.Management.Storage" Version="17.0.0" />
|
||||
<PackageReference Include="Microsoft.Azure.Services.AppAuthentication" Version="1.5.0" />
|
||||
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="2.2.2" />
|
||||
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.7" />
|
||||
<PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.21" />
|
||||
<PackageReference Include="Microsoft.Rest.ClientRuntime.Azure" Version="3.3.19" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Update="host.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="local.settings.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
<CopyToPublishDirectory>Never</CopyToPublishDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,244 +0,0 @@
|
||||
using Microsoft.Azure.Management.ResourceManager;
|
||||
using Microsoft.Azure.Management.ResourceManager.Models;
|
||||
using Microsoft.Azure.Management.Storage;
|
||||
using Microsoft.Azure.Management.Storage.Models;
|
||||
using Microsoft.Azure.Services.AppAuthentication;
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Rest;
|
||||
using Microsoft.WindowsAzure.Storage;
|
||||
using Microsoft.WindowsAzure.Storage.Auth;
|
||||
using Microsoft.WindowsAzure.Storage.Blob;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using static ADPControl.HttpSurface;
|
||||
|
||||
namespace ADPControl
|
||||
{
|
||||
public static class AzureResourceManagerActivity
|
||||
{
|
||||
private const string ArmTemplateFileName = "template.json";
|
||||
|
||||
private static string[] AllowedImportSubServerResourceTypes = new string[] {
|
||||
"Microsoft.Sql/servers/firewallRules",
|
||||
"Microsoft.Sql/servers/databases",
|
||||
"Microsoft.Sql/servers/elasticPools",
|
||||
//"Microsoft.Sql/servers/keys",
|
||||
//"Microsoft.Sql/servers/databases/transparentDataEncryption",
|
||||
"Microsoft.Sql/servers/databases/backupShortTermRetentionPolicies",
|
||||
"Microsoft.Sql/servers/administrators"
|
||||
};
|
||||
|
||||
// Deploy the ARM template
|
||||
[FunctionName(nameof(BeginDeployArmTemplateForImport))]
|
||||
public static async Task<string> BeginDeployArmTemplateForImport([ActivityTrigger] ImportRequest request, ILogger log)
|
||||
{
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
||||
|
||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
||||
StorageManagementClient storageMgmtClient = new StorageManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
||||
|
||||
// Get the storage account keys for a given account and resource group
|
||||
IList<StorageAccountKey> acctKeys = storageMgmtClient.StorageAccounts.ListKeys(request.ResourceGroupName, request.StorageAccountName).Keys;
|
||||
|
||||
// Get a Storage account using account creds:
|
||||
StorageCredentials storageCred = new StorageCredentials(request.StorageAccountName, acctKeys.FirstOrDefault().Value);
|
||||
CloudStorageAccount linkedStorageAccount = new CloudStorageAccount(storageCred, true);
|
||||
CloudBlobContainer container = linkedStorageAccount
|
||||
.CreateCloudBlobClient()
|
||||
.GetContainerReference(request.ContainerName);
|
||||
|
||||
CloudBlockBlob blob = container.GetBlockBlobReference(ArmTemplateFileName);
|
||||
string json = await blob.DownloadTextAsync();
|
||||
|
||||
JObject originalTemplate = JObject.Parse(json);
|
||||
JObject importTemplate = UpdateArmTemplateForImport(originalTemplate, request);
|
||||
|
||||
var deployParams = new Deployment
|
||||
{
|
||||
Properties = new DeploymentProperties
|
||||
{
|
||||
Mode = DeploymentMode.Incremental,
|
||||
Template = importTemplate
|
||||
}
|
||||
};
|
||||
|
||||
string deploymentName = request.TargetSqlServerName + "_" + DateTime.UtcNow.ToFileTimeUtc();
|
||||
|
||||
try
|
||||
{
|
||||
await resourcesClient.Deployments.BeginCreateOrUpdateAsync(request.TargetSqlServerResourceGroupName, deploymentName, deployParams);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
log.LogError(ex.ToString());
|
||||
throw ex;
|
||||
}
|
||||
|
||||
return deploymentName;
|
||||
}
|
||||
|
||||
// Get the ARM deployment status
|
||||
[FunctionName(nameof(GetArmDeploymentForImport))]
|
||||
public static async Task<string> GetArmDeploymentForImport([ActivityTrigger] (Guid, string, string) input)
|
||||
{
|
||||
Guid subscriptionId = input.Item1;
|
||||
string resourceGroupName = input.Item2;
|
||||
string deploymentName = input.Item3;
|
||||
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = subscriptionId.ToString() };
|
||||
|
||||
DeploymentExtended result = await resourcesClient.Deployments.GetAsync(resourceGroupName, deploymentName);
|
||||
return result.Properties.ProvisioningState;
|
||||
}
|
||||
|
||||
// Get the ARM template without the parameter of the resource name
|
||||
[FunctionName(nameof(GetArmTemplateForExportSkipParameterization))]
|
||||
public static async Task<dynamic> GetArmTemplateForExportSkipParameterization([ActivityTrigger] ExportRequest request, ILogger log)
|
||||
{
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: entering");
|
||||
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
||||
if (tokenArmCredential != null)
|
||||
{
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: acquired access token");
|
||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
||||
|
||||
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.SourceSqlServerResourceGroupName, request.SourceSqlServerName);
|
||||
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.SourceSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "SkipResourceNameParameterization"));
|
||||
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: server template exported. Size: {0} bytes", exportedTemplate.Template.ToString().Length);
|
||||
dynamic template = (dynamic)exportedTemplate.Template;
|
||||
|
||||
// Filtering the list of databases
|
||||
dynamic databases = template.resources.SelectTokens("$.[?(@.type == 'Microsoft.Sql/servers/databases')]");
|
||||
int numberOfDatabases = 0;
|
||||
foreach (var db in databases)
|
||||
{
|
||||
numberOfDatabases++;
|
||||
}
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with database list. Databases count: {0}", numberOfDatabases);
|
||||
|
||||
return databases;
|
||||
}
|
||||
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with empty database list");
|
||||
return null;
|
||||
}
|
||||
|
||||
// Get the ARM template without the parameter of the resource name
|
||||
[FunctionName(nameof(GetArmTemplateForImportSkipParameterization))]
|
||||
public static async Task<dynamic> GetArmTemplateForImportSkipParameterization([ActivityTrigger] ImportRequest request, ILogger log)
|
||||
{
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
||||
if (tokenArmCredential != null)
|
||||
{
|
||||
log.LogInformation("GetArmTemplateForImportSkipParameterization: acquired access token");
|
||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
||||
|
||||
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.TargetSqlServerResourceGroupName, request.TargetSqlServerName);
|
||||
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.TargetSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "SkipResourceNameParameterization"));
|
||||
|
||||
log.LogInformation("GetArmTemplateForImportSkipParameterization: server template exported. Size: {0} bytes", exportedTemplate.Template.ToString().Length);
|
||||
dynamic template = (dynamic)exportedTemplate.Template;
|
||||
|
||||
// Filtering the list of databases
|
||||
dynamic databases = template.resources.SelectTokens("$.[?(@.type == 'Microsoft.Sql/servers/databases')]");
|
||||
|
||||
int numberOfDatabases = 0;
|
||||
foreach (var db in databases)
|
||||
{
|
||||
numberOfDatabases++;
|
||||
}
|
||||
log.LogInformation("GetArmTemplateForExportSkipParameterization: exiting with database list. Databases count: {0}", numberOfDatabases);
|
||||
return databases;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
[FunctionName(nameof(GetArmTemplateForExport))]
|
||||
public static async Task<dynamic> GetArmTemplateForExport([ActivityTrigger] ExportRequest request)
|
||||
{
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/"));
|
||||
|
||||
ResourceManagementClient resourcesClient = new ResourceManagementClient(tokenArmCredential) { SubscriptionId = request.SubscriptionId.ToString() };
|
||||
|
||||
string sourceSqlServerResourceId = string.Format("/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Sql/servers/{2}", request.SubscriptionId, request.SourceSqlServerResourceGroupName, request.SourceSqlServerName);
|
||||
ResourceGroupExportResult exportedTemplate = resourcesClient.ResourceGroups.ExportTemplate(request.SourceSqlServerResourceGroupName, new ExportTemplateRequest(new List<string> { sourceSqlServerResourceId }, "IncludeParameterDefaultValue"));
|
||||
return exportedTemplate.Template;
|
||||
}
|
||||
|
||||
private static JObject UpdateArmTemplateForImport(JObject originalTemplate, ImportRequest request)
|
||||
{
|
||||
string serverNameParameterName = null;
|
||||
|
||||
// Go through every parameter to find the property name is like 'server_%_name'
|
||||
using (JsonTextReader reader = new JsonTextReader(new StringReader(originalTemplate["parameters"].ToString())))
|
||||
{
|
||||
while (reader.Read())
|
||||
{
|
||||
if (reader.TokenType.ToString().Equals("PropertyName")
|
||||
&& reader.ValueType.ToString().Equals("System.String")
|
||||
&& reader.Value.ToString().StartsWith("servers_")
|
||||
&& reader.Value.ToString().EndsWith("_name"))
|
||||
{
|
||||
serverNameParameterName = reader.Value.ToString();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Replacing the default value to the target server name, appending to the new template
|
||||
originalTemplate["parameters"][serverNameParameterName]["defaultValue"] = request.TargetSqlServerName;
|
||||
JObject serverNameParameterValue = (JObject)originalTemplate["parameters"][serverNameParameterName];
|
||||
|
||||
// 2. Cleanup all the parameters except the updated server name
|
||||
((JObject)originalTemplate["parameters"]).RemoveAll();
|
||||
((JObject)originalTemplate["parameters"]).Add(serverNameParameterName, serverNameParameterValue);
|
||||
|
||||
// 3. Adjust the servers resource by adding password after the login
|
||||
JObject server = (JObject)originalTemplate["resources"]
|
||||
.SelectToken("$.[?(@.type == 'Microsoft.Sql/servers')]");
|
||||
|
||||
server.Remove("identity");
|
||||
|
||||
JObject serverProperties = (JObject)server["properties"];
|
||||
serverProperties.Property("administratorLogin")
|
||||
.AddAfterSelf(new JProperty("administratorLoginPassword", request.SqlAdminPassword));
|
||||
|
||||
JArray newResources = new JArray();
|
||||
|
||||
// 4. Getting the whitelisted resources and adding them to the new template later.
|
||||
foreach (string resourceType in AllowedImportSubServerResourceTypes)
|
||||
{
|
||||
List<JToken> resources = originalTemplate["resources"]
|
||||
.SelectTokens(string.Format("$.[?(@.type == '{0}')]", resourceType)).ToList();
|
||||
newResources.Add(resources);
|
||||
}
|
||||
|
||||
// 5. Clean up all the resources excepted the new server and whitelisted resource type.
|
||||
((JArray)originalTemplate["resources"]).Clear();
|
||||
((JArray)originalTemplate["resources"]).Add(server);
|
||||
|
||||
foreach (var resource in newResources)
|
||||
{
|
||||
((JArray)originalTemplate["resources"]).Add(resource);
|
||||
}
|
||||
|
||||
return originalTemplate;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
using Microsoft.Azure.Batch;
|
||||
using Microsoft.Azure.Batch.Auth;
|
||||
using Microsoft.Azure.Batch.Common;
|
||||
using Microsoft.Azure.Services.AppAuthentication;
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using static ADPControl.HttpSurface;
|
||||
|
||||
namespace ADPControl
|
||||
{
|
||||
public static class BatchActivity
|
||||
{
|
||||
// Batch resource settings
|
||||
private const string PoolVMSize = "Standard_D8s_v3";
|
||||
private const string PoolId = PoolVMSize;
|
||||
private const int PoolNodeCount = 2;
|
||||
private const string AppPackageName = "SqlPackageWrapper";
|
||||
public const string AppPackageVersion = "1";
|
||||
|
||||
[FunctionName(nameof(CreateBatchPoolAndExportJob))]
|
||||
public static async Task<string> CreateBatchPoolAndExportJob([ActivityTrigger] ExportRequest request, ILogger log)
|
||||
{
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
|
||||
// Get a Batch client using function identity
|
||||
BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));
|
||||
|
||||
string jobId = request.SourceSqlServerName + "-Export-" + DateTime.UtcNow.ToString("MMddHHmmss");
|
||||
using (BatchClient batchClient = BatchClient.Open(batchCred))
|
||||
{
|
||||
ImageReference imageReference = CreateImageReference();
|
||||
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
|
||||
|
||||
await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
|
||||
await CreateBatchJob(batchClient, jobId, log);
|
||||
}
|
||||
|
||||
return jobId;
|
||||
}
|
||||
|
||||
[FunctionName(nameof(CreateBatchPoolAndImportJob))]
|
||||
public static async Task<string> CreateBatchPoolAndImportJob([ActivityTrigger] ImportRequest request, ILogger log)
|
||||
{
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
|
||||
// Get a Batch client using function identity
|
||||
BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));
|
||||
|
||||
string jobId = request.TargetSqlServerName + "-Import-" + DateTime.UtcNow.ToString("MMddHHmmss");
|
||||
using (BatchClient batchClient = BatchClient.Open(batchCred))
|
||||
{
|
||||
ImageReference imageReference = CreateImageReference();
|
||||
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
|
||||
|
||||
await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
|
||||
await CreateBatchJob(batchClient, jobId, log);
|
||||
}
|
||||
|
||||
return jobId;
|
||||
}
|
||||
|
||||
public static async Task<CloudJob> CreateBatchJob(BatchClient batchClient, string jobId, ILogger log)
|
||||
{
|
||||
// Create a Batch job
|
||||
log.LogInformation("Creating job [{0}]...", jobId);
|
||||
CloudJob job = null;
|
||||
|
||||
try
|
||||
{
|
||||
job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = PoolId });
|
||||
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
|
||||
|
||||
// Commit the job to the Batch service
|
||||
await job.CommitAsync();
|
||||
|
||||
log.LogInformation($"Created job {jobId}");
|
||||
|
||||
// Obtain the bound job from the Batch service
|
||||
await job.RefreshAsync();
|
||||
}
|
||||
catch (BatchException be)
|
||||
{
|
||||
// Accept the specific error code JobExists as that is expected if the job already exists
|
||||
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
|
||||
{
|
||||
log.LogWarning("The job {0} already existed when we tried to create it", jobId);
|
||||
}
|
||||
else
|
||||
{
|
||||
log.LogError("Exception creating job: {0}", be.Message);
|
||||
throw be; // Any other exception is unexpected
|
||||
}
|
||||
}
|
||||
|
||||
return job;
|
||||
}
|
||||
|
||||
// Create the Compute Pool of the Batch Account
|
||||
public static async Task CreateBatchPoolIfNotExist(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration, string vnetSubnetId)
|
||||
{
|
||||
Console.WriteLine("Creating pool [{0}]...", PoolId);
|
||||
|
||||
try
|
||||
{
|
||||
CloudPool pool = batchClient.PoolOperations.CreatePool(
|
||||
poolId: PoolId,
|
||||
targetDedicatedComputeNodes: PoolNodeCount,
|
||||
virtualMachineSize: PoolVMSize,
|
||||
virtualMachineConfiguration: vmConfiguration);
|
||||
|
||||
// Specify the application and version to install on the compute nodes
|
||||
pool.ApplicationPackageReferences = new List<ApplicationPackageReference>
|
||||
{
|
||||
new ApplicationPackageReference {
|
||||
ApplicationId = AppPackageName,
|
||||
Version = AppPackageVersion }
|
||||
};
|
||||
|
||||
// Initial the first data disk for each VM in the pool
|
||||
StartTask startTask = new StartTask("cmd /c Powershell -command \"Get-Disk | Where partitionstyle -eq 'raw' | sort number | Select-Object -first 1 |" +
|
||||
" Initialize-Disk -PartitionStyle MBR -PassThru | New-Partition -UseMaximumSize -DriveLetter F |" +
|
||||
" Format-Volume -FileSystem NTFS -NewFileSystemLabel data1 -Confirm:$false -Force\"");
|
||||
|
||||
startTask.MaxTaskRetryCount = 1;
|
||||
startTask.UserIdentity = new UserIdentity(new AutoUserSpecification(AutoUserScope.Pool, ElevationLevel.Admin));
|
||||
startTask.WaitForSuccess = true;
|
||||
|
||||
pool.StartTask = startTask;
|
||||
|
||||
// Create the Pool within the vnet subnet if it's specified.
|
||||
if (vnetSubnetId != null)
|
||||
{
|
||||
pool.NetworkConfiguration = new NetworkConfiguration();
|
||||
pool.NetworkConfiguration.SubnetId = vnetSubnetId;
|
||||
}
|
||||
|
||||
await pool.CommitAsync();
|
||||
await pool.RefreshAsync();
|
||||
}
|
||||
catch (BatchException be)
|
||||
{
|
||||
// Accept the specific error code PoolExists as that is expected if the pool already exists
|
||||
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
|
||||
{
|
||||
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw; // Any other exception is unexpected
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static VirtualMachineConfiguration CreateVirtualMachineConfiguration(ImageReference imageReference)
|
||||
{
|
||||
VirtualMachineConfiguration config = new VirtualMachineConfiguration(
|
||||
imageReference: imageReference,
|
||||
nodeAgentSkuId: "batch.node.windows amd64");
|
||||
|
||||
config.DataDisks = new List<DataDisk>();
|
||||
config.DataDisks.Add(new DataDisk(0, 2048, CachingType.ReadOnly, StorageAccountType.PremiumLrs));
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
public static ImageReference CreateImageReference()
|
||||
{
|
||||
return new ImageReference(
|
||||
publisher: "MicrosoftWindowsServer",
|
||||
offer: "WindowsServer",
|
||||
sku: "2019-datacenter-smalldisk",
|
||||
version: "latest");
|
||||
}
|
||||
|
||||
public static void CreateBatchTasks(string action, string jobId, string containerUrl, string batchAccountUrl, string sqlServerName, string accessToken, dynamic databases, ILogger log)
|
||||
{
|
||||
// Get a Batch client using function identity
|
||||
log.LogInformation("CreateBatchTasks: entering");
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
BatchTokenCredentials batchCred = new BatchTokenCredentials(batchAccountUrl, azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/").Result);
|
||||
using (BatchClient batchClient = BatchClient.Open(batchCred))
|
||||
{
|
||||
// For each database, submit the Exporting job to Azure Batch Compute Pool.
|
||||
log.LogInformation("CreateBatchTasks: enumerating databases");
|
||||
List<CloudTask> tasks = new List<CloudTask>();
|
||||
foreach (var db in databases)
|
||||
{
|
||||
string serverDatabaseName = db.name.ToString();
|
||||
string logicalDatabase = serverDatabaseName.Remove(0, sqlServerName.Length + 1);
|
||||
|
||||
log.LogInformation("CreateBatchTasks: creating task for database {0}", logicalDatabase);
|
||||
string taskId = sqlServerName + "_" + logicalDatabase;
|
||||
string command = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_{0}#{1}%\\BatchWrapper {2}", AppPackageName.ToUpper(), AppPackageVersion, action);
|
||||
command += string.Format(" {0} {1} {2} {3} {4}", sqlServerName, logicalDatabase, accessToken, AppPackageName.ToUpper(), AppPackageVersion);
|
||||
string taskCommandLine = string.Format(command);
|
||||
|
||||
CloudTask singleTask = new CloudTask(taskId, taskCommandLine);
|
||||
singleTask.EnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
|
||||
|
||||
Console.WriteLine(string.Format("Adding task {0} to job ...", taskId));
|
||||
tasks.Add(singleTask);
|
||||
}
|
||||
|
||||
// Add all tasks to the job.
|
||||
batchClient.JobOperations.AddTask(jobId, tasks);
|
||||
}
|
||||
log.LogInformation("CreateBatchTasks: exiting");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Azure.WebJobs.Extensions.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace ADPControl
|
||||
{
|
||||
public static class HttpSurface
|
||||
{
|
||||
public class ExportRequest
|
||||
{
|
||||
public Guid SubscriptionId { get; set; }
|
||||
|
||||
public string ResourceGroupName { get; set; }
|
||||
|
||||
public string SourceSqlServerResourceGroupName { get; set; }
|
||||
|
||||
public string SourceSqlServerName { get; set; }
|
||||
|
||||
public string BatchAccountUrl { get; set; }
|
||||
|
||||
public string StorageAccountName { get; set; }
|
||||
|
||||
public string AccessToken { get; set; }
|
||||
|
||||
public string VNetSubnetId { get; set; }
|
||||
}
|
||||
|
||||
public class ImportRequest
|
||||
{
|
||||
public Guid SubscriptionId { get; set; }
|
||||
|
||||
public string ResourceGroupName { get; set; }
|
||||
|
||||
public string TargetSqlServerResourceGroupName { get; set; }
|
||||
|
||||
public string TargetSqlServerName { get; set; }
|
||||
|
||||
public string TargetAccessToken { get; set; }
|
||||
|
||||
public string BatchAccountUrl { get; set; }
|
||||
|
||||
public string StorageAccountName { get; set; }
|
||||
|
||||
public string ContainerName { get; set; }
|
||||
|
||||
public string SqlAdminPassword { get; set; }
|
||||
|
||||
public string VNetSubnetId { get; set; }
|
||||
}
|
||||
|
||||
[FunctionName("Export")]
|
||||
public static async Task<HttpResponseMessage> PostExport(
|
||||
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/Export")]
|
||||
HttpRequestMessage req,
|
||||
[DurableClient] IDurableOrchestrationClient starter,
|
||||
ILogger log,
|
||||
Guid subscriptionId,
|
||||
string resourceGroupName)
|
||||
{
|
||||
log.LogInformation("C# HTTP trigger function processed an Export request.");
|
||||
ExportRequest request = await req.Content.ReadAsAsync<ExportRequest>();
|
||||
|
||||
request.SubscriptionId = subscriptionId;
|
||||
request.ResourceGroupName = resourceGroupName;
|
||||
|
||||
if (request.SourceSqlServerResourceGroupName == null)
|
||||
request.SourceSqlServerResourceGroupName = resourceGroupName;
|
||||
|
||||
string instanceId = await starter.StartNewAsync(nameof(Orchestrator.RunExportOrchestrator), request);
|
||||
|
||||
log.LogInformation($"Started orchestration with ID = '{instanceId}'.");
|
||||
return starter.CreateCheckStatusResponse(req, instanceId);
|
||||
}
|
||||
|
||||
[FunctionName("Import")]
|
||||
public static async Task<HttpResponseMessage> PostImport(
|
||||
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/Import")]
|
||||
HttpRequestMessage req,
|
||||
[DurableClient] IDurableOrchestrationClient starter,
|
||||
ILogger log,
|
||||
Guid subscriptionId,
|
||||
string resourceGroupName)
|
||||
{
|
||||
log.LogInformation("C# HTTP trigger function processed an Import request.");
|
||||
ImportRequest request = await req.Content.ReadAsAsync<ImportRequest>();
|
||||
|
||||
request.SubscriptionId = subscriptionId;
|
||||
request.ResourceGroupName = resourceGroupName;
|
||||
|
||||
if (request.TargetSqlServerResourceGroupName == null)
|
||||
request.TargetSqlServerResourceGroupName = resourceGroupName;
|
||||
|
||||
string instanceId = await starter.StartNewAsync(nameof(Orchestrator.RunImportOrchestrator), request);
|
||||
|
||||
log.LogInformation($"Started orchestration with ID = '{instanceId}'.");
|
||||
return starter.CreateCheckStatusResponse(req, instanceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,104 +0,0 @@
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using static ADPControl.HttpSurface;
|
||||
|
||||
namespace ADPControl
|
||||
{
|
||||
public static class Orchestrator
|
||||
{
|
||||
// The Import Orchestrator
|
||||
[FunctionName(nameof(RunImportOrchestrator))]
|
||||
public static async Task RunImportOrchestrator(
|
||||
[OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log)
|
||||
{
|
||||
log.LogInformation("RunImportOrchestrator: entering");
|
||||
|
||||
try {
|
||||
|
||||
ImportRequest importRequest = context.GetInput<ImportRequest>();
|
||||
// Deploy the ARM template to Create empty SQL resource
|
||||
string deploymentName = await context.CallActivityAsync<string>(nameof(AzureResourceManagerActivity.BeginDeployArmTemplateForImport), importRequest);
|
||||
while (true)
|
||||
{
|
||||
log.LogInformation("RunImportOrchestrator: starting ARM deployment");
|
||||
string status = await context.CallActivityAsync<string>(nameof(AzureResourceManagerActivity.GetArmDeploymentForImport), (importRequest.SubscriptionId, importRequest.TargetSqlServerResourceGroupName, deploymentName));
|
||||
if (status == "Succeeded")
|
||||
{
|
||||
log.LogInformation("RunImportOrchestrator: ARM deployment succeeded");
|
||||
break;
|
||||
}
|
||||
else if (status == "Failed")
|
||||
{
|
||||
log.LogInformation("RunImportOrchestrator: ARM deployment failed");
|
||||
throw new Exception("Failed ARM Deployment");
|
||||
}
|
||||
|
||||
// Orchestration sleeps until this time.
|
||||
var nextCheck = context.CurrentUtcDateTime.AddSeconds(10);
|
||||
|
||||
if (!context.IsReplaying) { log.LogInformation($"RunImportOrchestrator: Replaying ARM deployment, next check at {nextCheck}."); }
|
||||
await context.CreateTimer(nextCheck, CancellationToken.None);
|
||||
}
|
||||
|
||||
log.LogInformation("RunImportOrchestrator: Enumerating databases");
|
||||
var databases = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForImportSkipParameterization), importRequest);
|
||||
|
||||
// Create BatchPool And Job
|
||||
log.LogInformation("RunImportOrchestrator: Creating batch pool and import job");
|
||||
string jobId = await context.CallActivityAsync<string>(nameof(BatchActivity.CreateBatchPoolAndImportJob), importRequest);
|
||||
|
||||
string containerUrl = await context.CallActivityAsync<string>(nameof(StorageActivity.GettingJobContainerUrl), (importRequest.SubscriptionId, importRequest.ResourceGroupName, importRequest.StorageAccountName, importRequest.ContainerName));
|
||||
|
||||
log.LogInformation("RunImportOrchestrator: Creating import database tasks");
|
||||
BatchActivity.CreateBatchTasks("Import", jobId, containerUrl, importRequest.BatchAccountUrl, importRequest.TargetSqlServerName, importRequest.TargetAccessToken, databases, log);
|
||||
|
||||
// create output values
|
||||
Tuple<string, string>[] outputValues = {
|
||||
Tuple.Create("Orchestration progress:", "Complete"),
|
||||
Tuple.Create("deploymentName", deploymentName),
|
||||
Tuple.Create("jobId", jobId),
|
||||
Tuple.Create("containerUrl", containerUrl)
|
||||
};
|
||||
context.SetOutput(outputValues);
|
||||
}
|
||||
finally {
|
||||
log.LogInformation("RunImportOrchestrator: exiting");
|
||||
}
|
||||
}
|
||||
|
||||
// The Export Orchestrator
|
||||
[FunctionName(nameof(RunExportOrchestrator))]
|
||||
public static async Task RunExportOrchestrator(
|
||||
[OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log)
|
||||
{
|
||||
ExportRequest exportRequest = context.GetInput<ExportRequest>();
|
||||
|
||||
// Getting the ARM template Skip ResourceName Parameterization.
|
||||
var databases = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForExportSkipParameterization), exportRequest);
|
||||
|
||||
// Getting the ARM template.
|
||||
dynamic Template = await context.CallActivityAsync<dynamic>(nameof(AzureResourceManagerActivity.GetArmTemplateForExport), exportRequest);
|
||||
string json = JsonConvert.SerializeObject(Template);
|
||||
|
||||
// Create BatchPool And Job
|
||||
string jobId = await context.CallActivityAsync<string>(nameof(BatchActivity.CreateBatchPoolAndExportJob), exportRequest);
|
||||
|
||||
string containerUrl = await context.CallActivityAsync<string>(nameof(StorageActivity.GettingJobContainerUrl), (exportRequest.SubscriptionId, exportRequest.ResourceGroupName, exportRequest.StorageAccountName, jobId));
|
||||
await context.CallActivityAsync<string>(nameof(StorageActivity.UploadingArmTemplate), (containerUrl, json));
|
||||
|
||||
BatchActivity.CreateBatchTasks("Export", jobId, containerUrl, exportRequest.BatchAccountUrl, exportRequest.SourceSqlServerName, exportRequest.AccessToken, databases, log);
|
||||
|
||||
// create output values
|
||||
Tuple<string, string>[] outputValues = {
|
||||
Tuple.Create("jobId", jobId),
|
||||
Tuple.Create("containerUrl", containerUrl)
|
||||
};
|
||||
context.SetOutput(outputValues);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
using Microsoft.Azure.Management.Storage;
|
||||
using Microsoft.Azure.Management.Storage.Models;
|
||||
using Microsoft.Azure.Services.AppAuthentication;
|
||||
using Microsoft.Azure.WebJobs;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Rest;
|
||||
using Microsoft.WindowsAzure.Storage;
|
||||
using Microsoft.WindowsAzure.Storage.Auth;
|
||||
using Microsoft.WindowsAzure.Storage.Blob;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace ADPControl
|
||||
{
|
||||
public static class StorageActivity
|
||||
{
|
||||
private const string ArmTemplateFileName = "template.json";
|
||||
|
||||
[FunctionName(nameof(GettingJobContainerUrl))]
|
||||
public static string GettingJobContainerUrl([ActivityTrigger] (Guid, string, string, string) input, ILogger log)
|
||||
{
|
||||
Guid SubscriptionId = input.Item1;
|
||||
String ResourceGroupName = input.Item2;
|
||||
String StorageAccountName = input.Item3;
|
||||
String ContainerName = input.Item4;
|
||||
|
||||
var azureServiceTokenProvider = new AzureServiceTokenProvider();
|
||||
TokenCredentials tokenArmCredential = new TokenCredentials(azureServiceTokenProvider.GetAccessTokenAsync("https://management.core.windows.net/").Result);
|
||||
StorageManagementClient storageMgmtClient = new StorageManagementClient(tokenArmCredential) { SubscriptionId = SubscriptionId.ToString() };
|
||||
|
||||
// Get the storage account keys for a given account and resource group
|
||||
IList<StorageAccountKey> acctKeys = storageMgmtClient.StorageAccounts.ListKeys(ResourceGroupName, StorageAccountName).Keys;
|
||||
|
||||
// Get a Storage account using account creds:
|
||||
StorageCredentials storageCred = new StorageCredentials(StorageAccountName, acctKeys.FirstOrDefault().Value);
|
||||
CloudStorageAccount linkedStorageAccount = new CloudStorageAccount(storageCred, true);
|
||||
|
||||
bool createContainer = false;
|
||||
// Normalize the container name for the Export action.
|
||||
if (ContainerName.Contains("-Export-"))
|
||||
{
|
||||
ContainerName = ContainerName.Replace("Export-", "");
|
||||
createContainer = true;
|
||||
}
|
||||
|
||||
CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(ContainerName);
|
||||
|
||||
if(createContainer)
|
||||
container.CreateIfNotExistsAsync().Wait();
|
||||
|
||||
string containerUrl = container.Uri.ToString() +
|
||||
container.GetSharedAccessSignature(new SharedAccessBlobPolicy()
|
||||
{
|
||||
Permissions = SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List,
|
||||
SharedAccessExpiryTime = DateTime.UtcNow.AddDays(7)
|
||||
});
|
||||
return containerUrl;
|
||||
}
|
||||
|
||||
[FunctionName(nameof(UploadingArmTemplate))]
|
||||
public static void UploadingArmTemplate([ActivityTrigger] (string, string) input, ILogger log)
|
||||
{
|
||||
string containerUrl = input.Item1;
|
||||
string json = input.Item2;
|
||||
|
||||
CloudBlobContainer container = new CloudBlobContainer(new Uri(containerUrl));
|
||||
CloudBlockBlob blob = container.GetBlockBlobReference(ArmTemplateFileName);
|
||||
blob.Properties.ContentType = "application/json";
|
||||
using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(json)))
|
||||
{
|
||||
blob.UploadFromStreamAsync(stream).Wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"version": "2.0",
|
||||
"extensions": {
|
||||
"durableTask": {
|
||||
"hubName": "adp"
|
||||
}
|
||||
},
|
||||
"logging": {
|
||||
"applicationInsights": {
|
||||
"samplingExcludedTypes": "Request",
|
||||
"samplingSettings": {
|
||||
"isEnabled": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
namespace BatchWrapper
|
||||
{
|
||||
/// <summary>
|
||||
/// The type of sqlpackage action to perform.
|
||||
/// </summary>
|
||||
public enum ActionType
|
||||
{
|
||||
DefaultInvalid = -1,
|
||||
Export,
|
||||
Import
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<configuration>
|
||||
<startup>
|
||||
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
|
||||
</startup>
|
||||
<runtime>
|
||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-12.0.0.0" newVersion="12.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Azure.Batch" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Azure.KeyVault.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.0.5.0" newVersion="3.0.5.0" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
@@ -1,98 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<RootNamespace>BatchWrapper</RootNamespace>
|
||||
<AssemblyName>BatchWrapper</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
|
||||
<LangVersion>8.0</LangVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
|
||||
<Deterministic>true</Deterministic>
|
||||
<IsWebBootstrapper>false</IsWebBootstrapper>
|
||||
<PublishUrl>publish\</PublishUrl>
|
||||
<Install>true</Install>
|
||||
<InstallFrom>Disk</InstallFrom>
|
||||
<UpdateEnabled>false</UpdateEnabled>
|
||||
<UpdateMode>Foreground</UpdateMode>
|
||||
<UpdateInterval>7</UpdateInterval>
|
||||
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
|
||||
<UpdatePeriodically>false</UpdatePeriodically>
|
||||
<UpdateRequired>false</UpdateRequired>
|
||||
<MapFileExtensions>true</MapFileExtensions>
|
||||
<ApplicationRevision>0</ApplicationRevision>
|
||||
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
|
||||
<UseApplicationTrust>false</UseApplicationTrust>
|
||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Azure.Batch.Conventions.Files, Version=3.5.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Microsoft.Azure.Batch.Conventions.Files.3.5.1\lib\net461\Microsoft.Azure.Batch.Conventions.Files.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=9.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\WindowsAzure.Storage.9.3.3\lib\net45\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Runtime" />
|
||||
<Reference Include="System.Runtime.Serialization" />
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
<Reference Include="Microsoft.CSharp" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="ActionType.cs" />
|
||||
<Compile Include="Constants.cs" />
|
||||
<Compile Include="Payload.cs" />
|
||||
<Compile Include="Program.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="App.config" />
|
||||
<None Include="packages.config" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<BootstrapperPackage Include=".NETFramework,Version=v4.7.2">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>Microsoft .NET Framework 4.7.2 %28x86 and x64%29</ProductName>
|
||||
<Install>true</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.5 SP1</ProductName>
|
||||
<Install>false</Install>
|
||||
</BootstrapperPackage>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Folder Include="Properties\" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
</Project>
|
||||
@@ -1,34 +0,0 @@
|
||||
namespace BatchWrapper
|
||||
{
|
||||
/// <summary>
|
||||
/// Constants for the batch wrapper.
|
||||
/// </summary>
|
||||
public static class Constants
|
||||
{
|
||||
/// <summary>
|
||||
/// Environment variable names present or needed during the batch task execution.
|
||||
/// </summary>
|
||||
public static class EnvironmentVariableNames
|
||||
{
|
||||
/// <summary>
|
||||
/// Path to the directory containing the sqlpackage exe.
|
||||
/// </summary>
|
||||
internal const string AppPackagePrefix = "AZ_BATCH_APP_PACKAGE";
|
||||
|
||||
/// <summary>
|
||||
/// Path to the working directory assigned to the batch task.
|
||||
/// </summary>
|
||||
internal const string TaskWorkingDir = "AZ_BATCH_TASK_WORKING_DIR";
|
||||
|
||||
/// <summary>
|
||||
/// Path to the working directory assigned to the batch task.
|
||||
/// </summary>
|
||||
internal const string AzBatchTaskId = "AZ_BATCH_TASK_ID";
|
||||
|
||||
/// <summary>
|
||||
/// Path to the working directory assigned to the batch task.
|
||||
/// </summary>
|
||||
internal const string JobContainerUrl = "JOB_CONTAINER_URL";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
namespace BatchWrapper
|
||||
{
|
||||
/// <summary>
|
||||
/// The top-level object stored in the key vault for an import/export operation.
|
||||
/// </summary>
|
||||
public sealed class Payload
|
||||
{
|
||||
/// <summary>
|
||||
/// The Name of sqlpackage to use for performing the import/export operation.
|
||||
/// </summary>
|
||||
public string ApplicatonPackageName{ get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The Version of sqlpackage to use for performing the import/export operation.
|
||||
/// </summary>
|
||||
public string ApplicatonPackageVersion { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The type of sqlpackage action to perform.
|
||||
/// </summary>
|
||||
public ActionType Action { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The logical server name to export from or import to.
|
||||
/// </summary>
|
||||
public string LogicalServerName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The database name to export from or import to.
|
||||
/// </summary>
|
||||
public string DatabaseName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The server admin username.
|
||||
/// </summary>
|
||||
public string AccessToken { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -1,178 +0,0 @@
|
||||
using Microsoft.Azure.Batch.Conventions.Files;
|
||||
using Microsoft.WindowsAzure.Storage.Blob;
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace BatchWrapper
|
||||
{
|
||||
public static class Program
|
||||
{
|
||||
private static string dataDirectory = "F:\\data";
|
||||
private static string tempDirectory = "F:\\temp";
|
||||
private static string[] directories = { dataDirectory, tempDirectory };
|
||||
|
||||
private static readonly TimeSpan stdoutFlushDelay = TimeSpan.FromSeconds(3);
|
||||
|
||||
private static void WriteLine(string message) => WriteLineInternal(Console.Out, message);
|
||||
private static void WriteErrorLine(string message) => WriteLineInternal(Console.Error, message);
|
||||
private static void WriteLineInternal(TextWriter writer, string message)
|
||||
{
|
||||
var lines = message?.Split('\n') ?? new string[0];
|
||||
foreach (var line in lines)
|
||||
{
|
||||
writer.WriteLine($"[{DateTime.UtcNow:u}] {line?.TrimEnd()}");
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
var assembly = typeof(Program).Assembly;
|
||||
WriteLine($"{assembly.ManifestModule.Name} v{assembly.GetName().Version.ToString(3)}");
|
||||
|
||||
// Get the command payload
|
||||
var payload = new Payload();
|
||||
|
||||
if (args.Length > 0)
|
||||
{
|
||||
payload.Action = (ActionType)Enum.Parse(typeof(ActionType), args[0]);
|
||||
payload.LogicalServerName = args[1] + ".database.windows.net";
|
||||
payload.DatabaseName = args[2];
|
||||
payload.AccessToken = args[3];
|
||||
payload.ApplicatonPackageName = args[4];
|
||||
payload.ApplicatonPackageVersion = args[5];
|
||||
}
|
||||
|
||||
// Cleanup folders
|
||||
foreach (string dir in directories)
|
||||
{
|
||||
if (Directory.Exists(dir))
|
||||
{
|
||||
Directory.Delete(dir, true);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(dir);
|
||||
}
|
||||
|
||||
string sqlPackageBacpacFile = Path.Combine(dataDirectory, payload.DatabaseName + ".bacpac");
|
||||
string sqlPackageLogPath = payload.DatabaseName + ".log";
|
||||
|
||||
var targetDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AppPackagePrefix + "_" + payload.ApplicatonPackageName + "#" + payload.ApplicatonPackageVersion);
|
||||
var workingDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.TaskWorkingDir);
|
||||
|
||||
string taskId = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AzBatchTaskId);
|
||||
string jobContainerUrl = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.JobContainerUrl);
|
||||
|
||||
// Build the import/export command
|
||||
var cmdBuilder = new StringBuilder();
|
||||
cmdBuilder.Append($"/Action:{payload.Action}");
|
||||
cmdBuilder.Append(" /MaxParallelism:16");
|
||||
cmdBuilder.Append(String.Format(" /DiagnosticsFile:{0}", sqlPackageLogPath));
|
||||
cmdBuilder.Append(" /p:CommandTimeout=604800");
|
||||
|
||||
switch (payload.Action)
|
||||
{
|
||||
case ActionType.Export:
|
||||
cmdBuilder.Append($" /SourceServerName:{payload.LogicalServerName}");
|
||||
cmdBuilder.Append($" /SourceDatabaseName:{payload.DatabaseName}");
|
||||
cmdBuilder.Append($" /AccessToken:{payload.AccessToken}");
|
||||
cmdBuilder.Append($" /TargetFile:{sqlPackageBacpacFile}");
|
||||
cmdBuilder.Append($" /SourceTimeout:30");
|
||||
cmdBuilder.Append(String.Format(" /p:TempDirectoryForTableData=\"{0}\"", tempDirectory));
|
||||
cmdBuilder.Append(" /p:VerifyFullTextDocumentTypesSupported=false");
|
||||
break;
|
||||
|
||||
case ActionType.Import:
|
||||
cmdBuilder.Append($" /TargetServerName:{payload.LogicalServerName}");
|
||||
cmdBuilder.Append($" /TargetDatabaseName:{payload.DatabaseName}");
|
||||
cmdBuilder.Append($" /AccessToken:{payload.AccessToken}");
|
||||
cmdBuilder.Append($" /TargetTimeout:30");
|
||||
cmdBuilder.Append($" /SourceFile:{sqlPackageBacpacFile}");
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new ArgumentException($"Invalid action type: {payload.Action}");
|
||||
}
|
||||
|
||||
if (payload.Action == ActionType.Import)
|
||||
{
|
||||
WriteLine(string.Format("Downloading {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
|
||||
CloudBlobContainer container = new CloudBlobContainer(new Uri(jobContainerUrl));
|
||||
CloudBlockBlob blob = container.GetBlockBlobReference(String.Format("$JobOutput/{0}.bacpac", payload.DatabaseName));
|
||||
blob.DownloadToFile(sqlPackageBacpacFile, FileMode.CreateNew);
|
||||
|
||||
if (File.Exists(sqlPackageBacpacFile))
|
||||
{
|
||||
WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception(string.Format("{0} didn't download", sqlPackageBacpacFile));
|
||||
}
|
||||
}
|
||||
|
||||
// Perform the import/export process
|
||||
var startTime = DateTimeOffset.UtcNow;
|
||||
var process = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
{
|
||||
WorkingDirectory = workingDir,
|
||||
FileName = Path.Combine(targetDir, "sqlpackage.exe"),
|
||||
Arguments = cmdBuilder.ToString(),
|
||||
CreateNoWindow = true,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true
|
||||
}
|
||||
};
|
||||
|
||||
process.OutputDataReceived += (s, e) => WriteLine(e.Data);
|
||||
process.ErrorDataReceived += (s, e) => WriteErrorLine(e.Data);
|
||||
process.Start();
|
||||
process.BeginOutputReadLine();
|
||||
process.BeginErrorReadLine();
|
||||
process.WaitForExit();
|
||||
|
||||
WriteLine(String.Format("SqlPackage.exe exited with code: {0}", process.ExitCode));
|
||||
|
||||
if (payload.Action == ActionType.Export)
|
||||
{
|
||||
if (File.Exists(sqlPackageBacpacFile))
|
||||
{
|
||||
WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception(string.Format("{0} didn't downloaded", sqlPackageBacpacFile));
|
||||
}
|
||||
|
||||
// Persist the Job Output
|
||||
JobOutputStorage jobOutputStorage = new JobOutputStorage(new Uri(jobContainerUrl));
|
||||
|
||||
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageLogPath);
|
||||
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageLogPath));
|
||||
|
||||
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageBacpacFile, payload.DatabaseName + ".bacpac");
|
||||
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageBacpacFile));
|
||||
}
|
||||
|
||||
// We are tracking the disk file to save our standard output, but the node agent may take
|
||||
// up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
|
||||
await Task.Delay(stdoutFlushDelay);
|
||||
|
||||
// Cleanup folders
|
||||
foreach (string dir in directories)
|
||||
{
|
||||
if (Directory.Exists(dir))
|
||||
{
|
||||
Directory.Delete(dir, true);
|
||||
}
|
||||
}
|
||||
|
||||
return process.ExitCode;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Azure.Batch" version="13.0.0" targetFramework="net472" />
|
||||
<package id="Microsoft.Azure.Batch.Conventions.Files" version="3.5.1" targetFramework="net472" />
|
||||
<package id="Microsoft.Azure.KeyVault.Core" version="3.0.5" targetFramework="net472" />
|
||||
<package id="Microsoft.Rest.ClientRuntime" version="2.3.21" targetFramework="net472" />
|
||||
<package id="Microsoft.Rest.ClientRuntime.Azure" version="3.3.19" targetFramework="net472" />
|
||||
<package id="Newtonsoft.Json" version="12.0.3" targetFramework="net472" />
|
||||
<package id="WindowsAzure.Storage" version="9.3.3" targetFramework="net472" />
|
||||
</packages>
|
||||
@@ -1,27 +0,0 @@
|
||||
{
|
||||
// Use IntelliSense to find out which attributes exist for C# debugging
|
||||
// Use hover for the description of the existing attributes
|
||||
// For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": ".NET Core Launch (console)",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"preLaunchTask": "build",
|
||||
// If you have changed target frameworks, make sure to update the program path.
|
||||
"program": "${workspaceFolder}/bin/Debug/netcoreapp3.1/SqlPackageWrapper.dll",
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}",
|
||||
// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
|
||||
"console": "internalConsole",
|
||||
"stopAtEntry": false
|
||||
},
|
||||
{
|
||||
"name": ".NET Core Attach",
|
||||
"type": "coreclr",
|
||||
"request": "attach",
|
||||
"processId": "${command:pickProcess}"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "build",
|
||||
"command": "dotnet",
|
||||
"type": "process",
|
||||
"args": [
|
||||
"build",
|
||||
"${workspaceFolder}/SqlPackageWrapper.csproj",
|
||||
"/property:GenerateFullPaths=true",
|
||||
"/consoleloggerparameters:NoSummary"
|
||||
],
|
||||
"problemMatcher": "$msCompile"
|
||||
},
|
||||
{
|
||||
"label": "publish",
|
||||
"command": "dotnet",
|
||||
"type": "process",
|
||||
"args": [
|
||||
"publish",
|
||||
"${workspaceFolder}/SqlPackageWrapper.csproj",
|
||||
"/property:GenerateFullPaths=true",
|
||||
"/consoleloggerparameters:NoSummary"
|
||||
],
|
||||
"problemMatcher": "$msCompile"
|
||||
},
|
||||
{
|
||||
"label": "watch",
|
||||
"command": "dotnet",
|
||||
"type": "process",
|
||||
"args": [
|
||||
"watch",
|
||||
"run",
|
||||
"${workspaceFolder}/SqlPackageWrapper.csproj",
|
||||
"/property:GenerateFullPaths=true",
|
||||
"/consoleloggerparameters:NoSummary"
|
||||
],
|
||||
"problemMatcher": "$msCompile"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
namespace SqlPackageWrapper
|
||||
{
|
||||
/// <summary>
|
||||
/// The type of sqlpackage action to perform.
|
||||
/// </summary>
|
||||
public enum ActionType
|
||||
{
|
||||
DefaultInvalid = -1,
|
||||
Export,
|
||||
Import
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
namespace SqlPackageWrapper
|
||||
{
|
||||
/// <summary>
|
||||
/// Constants for the batch wrapper.
|
||||
/// </summary>
|
||||
public static class Constants
|
||||
{
|
||||
/// <summary>
|
||||
/// Environment variable names present or needed during the batch task execution.
|
||||
/// </summary>
|
||||
public static class EnvironmentVariableNames
|
||||
{
|
||||
/// <summary>
|
||||
/// Path to the directory containing the batch wrapper exe.
|
||||
/// </summary>
|
||||
public const string WrapperLocation = "AZ_BATCH_APP_PACKAGE_BATCHWRAPPER";
|
||||
|
||||
/// <summary>
|
||||
/// Path to the directory containing the sqlpackage exe.
|
||||
/// </summary>
|
||||
internal const string SqlPackageLocation = "AZ_BATCH_APP_PACKAGE_SQLPACKAGE";
|
||||
|
||||
/// <summary>
|
||||
/// Path to the working directory assigned to the batch task.
|
||||
/// </summary>
|
||||
internal const string TaskWorkingDir = "AZ_BATCH_TASK_WORKING_DIR";
|
||||
|
||||
/// <summary>
|
||||
/// Path to the working directory assigned to the batch task.
|
||||
/// </summary>
|
||||
internal const string AzBatchTaskId = "AZ_BATCH_TASK_ID";
|
||||
|
||||
/// <summary>
|
||||
/// Path to the working directory assigned to the batch task.
|
||||
/// </summary>
|
||||
internal const string JobContainerUrl = "JOB_CONTAINER_URL";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
namespace SqlPackageWrapper
|
||||
{
|
||||
/// <summary>
|
||||
/// The top-level object stored in the key vault for an import/export operation.
|
||||
/// </summary>
|
||||
public sealed class Payload
|
||||
{
|
||||
/// <summary>
|
||||
/// The version of sqlpackage to use for performing the import/export operation.
|
||||
/// </summary>
|
||||
public string SqlPackageVersion { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The type of sqlpackage action to perform.
|
||||
/// </summary>
|
||||
public ActionType Action { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The logical server name to export from or import to.
|
||||
/// </summary>
|
||||
public string LogicalServerName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The database name to export from or import to.
|
||||
/// </summary>
|
||||
public string DatabaseName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The server admin username.
|
||||
/// </summary>
|
||||
public string Username { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The server admin password.
|
||||
/// </summary>
|
||||
public string Password { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
using Microsoft.Azure.Batch.Conventions.Files;
|
||||
using Microsoft.WindowsAzure.Storage.Blob;
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace SqlPackageWrapper
|
||||
{
|
||||
public static class Program
|
||||
{
|
||||
private static string dataDirectory = "F:\\data";
|
||||
private static string tempDirectory = "F:\\temp";
|
||||
private static string[] directories = { dataDirectory, tempDirectory };
|
||||
|
||||
private static readonly TimeSpan stdoutFlushDelay = TimeSpan.FromSeconds(3);
|
||||
|
||||
private static void WriteLine(string message) => WriteLineInternal(Console.Out, message);
|
||||
private static void WriteErrorLine(string message) => WriteLineInternal(Console.Error, message);
|
||||
private static void WriteLineInternal(TextWriter writer, string message)
|
||||
{
|
||||
var lines = message?.Split('\n') ?? new string[0];
|
||||
foreach (var line in lines)
|
||||
{
|
||||
writer.WriteLine($"[{DateTime.UtcNow:u}] {line?.TrimEnd()}");
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
var assembly = typeof(Program).Assembly;
|
||||
WriteLine($"{assembly.ManifestModule.Name} v{assembly.GetName().Version.ToString(3)}");
|
||||
|
||||
// Get the command payload
|
||||
var payload = new Payload();
|
||||
|
||||
if (args.Length > 0)
|
||||
{
|
||||
payload.Action = (ActionType)Enum.Parse(typeof(ActionType), args[0]);
|
||||
payload.LogicalServerName = args[1] + ".database.windows.net";
|
||||
payload.DatabaseName = args[2];
|
||||
payload.Username = args[3];
|
||||
payload.Password = args[4];
|
||||
payload.SqlPackageVersion = args[5];
|
||||
}
|
||||
|
||||
// Cleanup folders
|
||||
foreach (string dir in directories)
|
||||
{
|
||||
if (Directory.Exists(dir))
|
||||
{
|
||||
Directory.Delete(dir, true);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(dir);
|
||||
}
|
||||
|
||||
string sqlPackageDataPath = Path.Combine(dataDirectory, payload.DatabaseName + ".bacpac");
|
||||
string sqlPackageLogPath = Path.Combine(dataDirectory, payload.DatabaseName + ".log");
|
||||
|
||||
var targetDir = Environment.GetEnvironmentVariable($"{Constants.EnvironmentVariableNames.SqlPackageLocation}#{payload.SqlPackageVersion}");
|
||||
var workingDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.TaskWorkingDir);
|
||||
|
||||
string taskId = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AzBatchTaskId);
|
||||
string jobContainerUrl = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.JobContainerUrl);
|
||||
|
||||
// Build the import/export command
|
||||
var cmdBuilder = new StringBuilder();
|
||||
cmdBuilder.Append($"/Action:{payload.Action}");
|
||||
cmdBuilder.Append(" /MaxParallelism:16");
|
||||
cmdBuilder.Append(String.Format(" /DiagnosticsFile:{0}", sqlPackageLogPath));
|
||||
cmdBuilder.Append(" /p:CommandTimeout=86400");
|
||||
|
||||
switch (payload.Action)
|
||||
{
|
||||
case ActionType.Export:
|
||||
cmdBuilder.Append($" /SourceServerName:{payload.LogicalServerName}");
|
||||
cmdBuilder.Append($" /SourceDatabaseName:{payload.DatabaseName}");
|
||||
cmdBuilder.Append($" /SourceUser:{payload.Username}");
|
||||
cmdBuilder.Append($" /SourcePassword:{payload.Password}");
|
||||
cmdBuilder.Append($" /TargetFile:{sqlPackageDataPath}");
|
||||
cmdBuilder.Append(String.Format(" /p:TempDirectoryForTableData=\"{0}\"", tempDirectory));
|
||||
cmdBuilder.Append(" /p:VerifyFullTextDocumentTypesSupported=false");
|
||||
break;
|
||||
|
||||
case ActionType.Import:
|
||||
cmdBuilder.Append($" /TargetServerName:{payload.LogicalServerName}");
|
||||
cmdBuilder.Append($" /TargetDatabaseName:{payload.DatabaseName}");
|
||||
cmdBuilder.Append($" /TargetUser:{payload.Username}");
|
||||
cmdBuilder.Append($" /TargetPassword:{payload.Password}");
|
||||
cmdBuilder.Append($" /SourceFile:{sqlPackageDataPath}");
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new ArgumentException($"Invalid action type: {payload.Action}");
|
||||
}
|
||||
|
||||
if (payload.Action == ActionType.Import)
|
||||
{
|
||||
WriteLine(string.Format("Downloading {0} bacpac file to {1}", payload.DatabaseName, sqlPackageDataPath));
|
||||
CloudBlobContainer container = new CloudBlobContainer(new Uri(jobContainerUrl));
|
||||
CloudBlockBlob blob = container.GetBlockBlobReference(String.Format("$JobOutput/{0}.bacpac", payload.DatabaseName));
|
||||
await blob.DownloadToFileAsync(sqlPackageDataPath, FileMode.CreateNew);
|
||||
WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageDataPath));
|
||||
|
||||
await Task.Delay(stdoutFlushDelay);
|
||||
}
|
||||
|
||||
// Perform the import/export process
|
||||
var startTime = DateTimeOffset.UtcNow;
|
||||
var process = new Process
|
||||
{
|
||||
StartInfo = new ProcessStartInfo
|
||||
{
|
||||
WorkingDirectory = workingDir,
|
||||
FileName = Path.Combine(targetDir, "sqlpackage.exe"),
|
||||
Arguments = cmdBuilder.ToString(),
|
||||
CreateNoWindow = true,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true
|
||||
}
|
||||
};
|
||||
process.OutputDataReceived += (s, e) => WriteLine(e.Data);
|
||||
process.ErrorDataReceived += (s, e) => WriteErrorLine(e.Data);
|
||||
process.Start();
|
||||
process.BeginOutputReadLine();
|
||||
process.BeginErrorReadLine();
|
||||
process.WaitForExit();
|
||||
|
||||
WriteLine(String.Format("SqlPackage.exe exited with code: {0}", process.ExitCode));
|
||||
|
||||
if (payload.Action == ActionType.Export)
|
||||
{
|
||||
// Persist the Job Output
|
||||
JobOutputStorage jobOutputStorage = new JobOutputStorage(new Uri(jobContainerUrl));
|
||||
|
||||
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageLogPath, payload.DatabaseName + ".log");
|
||||
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageLogPath));
|
||||
|
||||
await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageDataPath, payload.DatabaseName + ".bacpac");
|
||||
WriteLine(String.Format("Uploaded {0} to job account", sqlPackageDataPath));
|
||||
|
||||
// We are tracking the disk file to save our standard output, but the node agent may take
|
||||
// up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
|
||||
await Task.Delay(stdoutFlushDelay);
|
||||
}
|
||||
|
||||
// Cleanup folders
|
||||
foreach (string dir in directories)
|
||||
{
|
||||
if (Directory.Exists(dir))
|
||||
{
|
||||
Directory.Delete(dir, true);
|
||||
}
|
||||
}
|
||||
|
||||
return process.ExitCode;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>netcoreapp2.1;net452</TargetFrameworks>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Azure.Batch" Version="13.0.0" />
|
||||
<PackageReference Include="Microsoft.Azure.Batch.Conventions.Files" Version="3.5.1" />
|
||||
<PackageReference Include="Microsoft.Azure.Management.Batch" Version="11.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -1 +0,0 @@
|
||||
Folder for helper components used by notebooks in the Hybrid Cloud Toolkit
|
||||
@@ -8,6 +8,13 @@
|
||||
- title: Search
|
||||
search: true
|
||||
|
||||
- title: ADP
|
||||
url: /ADP/readme
|
||||
not_numbered: true
|
||||
expand_sections: false
|
||||
sections:
|
||||
- title: ADP Control
|
||||
url: ADP/ADPControl
|
||||
- title: Assessment
|
||||
url: /Assessments/readme
|
||||
not_numbered: true
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
# ADP Control
|
||||
|
||||
[Home](../readme.md)
|
||||
|
||||
Notebook to manage ADP Orchestrator import/export requests onto an SQL Server.
|
||||
|
||||
## Notebooks in this Chapter
|
||||
- [ADP Control](ADPControl.ipynb)
|
||||
@@ -2,6 +2,8 @@
|
||||
## Chapters
|
||||
* [Prerequisites and Initial Setup](prereqs.ipynb) - Notebook installation of required modules.
|
||||
|
||||
* [ADP](ADP/readme.md) - manage ADP Orchestrator import/export requests onto an SQL Server.
|
||||
|
||||
* [Assessments](Assessments/readme.md) - Notebooks that contain examples to determine whether a given database or SQL Server instance is ready to migrate by utilizing SQL Assessments. SQL instances are scanned based on a "best practices" set of rules.
|
||||
|
||||
* [Networking](networking/readme.md) - Setup secure Point-to-Site (P2S) or Site-to-Site (S2S) network connectivity to Microsoft Azure using a Virtual Private Network (VPN). This notebook serves as a building block for other notebooks as communicating securely between on-premise and Azure is essential for many tasks.
|
||||
|
||||
Reference in New Issue
Block a user