Backup/Restore Managed Instance (#1428)

* Enabled backup to and restore from URL

* Created RPC, but when process tries to load Microsoft.Azure.Storage.Blob.dll, it crashes

* Added create shared access token

* Code refactor

* Minor changes

* Changed RPC path

* Moved createSas RPC to the newly created BlobService, fixed PR comments

* Added sas expiration date parameter to the RPC

* Added copyright headers

* Removed ConnectionInstance property from BlobService

* Removed unhelpful comment

* Removed unused using statements

* Changed copy/paste comments

* Disposable objects fix

* Small formatting fix

* Changed backup to/restore from url supported device types

* Added backup to url integration test

* Created restore integration test. Test are now getting azure blob params from env variables instead of file.

* Culture invariant epiration date param, fixed comment, and typo

* Updated headers

* PR comments fix

* Changed supported device type logic

* string localization fix

* String formatting fix

* build failure fix

* Typo

* Updated supported restore device types
This commit is contained in:
Nemanja Milovančević
2022-04-20 23:01:13 +02:00
committed by GitHub
parent 35e1782a3f
commit 881c335cdf
21 changed files with 828 additions and 7 deletions

View File

@@ -0,0 +1,81 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Threading.Tasks;
using Microsoft.Data.SqlClient;
using Microsoft.SqlServer.Management.Common;
using Microsoft.SqlServer.Management.Smo;
using Microsoft.SqlTools.Hosting.Protocol;
using Microsoft.SqlTools.ServiceLayer.AzureBlob.Contracts;
using Microsoft.SqlTools.ServiceLayer.Connection;
namespace Microsoft.SqlTools.ServiceLayer.AzureBlob
{
public class BlobService
{
private static readonly Lazy<BlobService> instance = new Lazy<BlobService>(() => new BlobService());
/// <summary>
/// Default, parameterless constructor.
/// </summary>
internal BlobService()
{
}
/// <summary>
/// Gets the singleton instance object
/// </summary>
public static BlobService Instance
{
get { return instance.Value; }
}
public void InitializeService(IProtocolEndpoint serviceHost)
{
serviceHost.SetRequestHandler(CreateSasRequest.Type, HandleCreateSasRequest);
}
internal async Task HandleCreateSasRequest(
CreateSasParams optionsParams,
RequestContext<CreateSasResponse> requestContext)
{
try
{
ConnectionInfo connInfo;
ConnectionService.Instance.TryFindConnection(
optionsParams.OwnerUri,
out connInfo);
var response = new CreateSasResponse();
if (connInfo == null)
{
await requestContext.SendError(SR.ConnectionServiceListDbErrorNotConnected(optionsParams.OwnerUri));
return;
}
if (connInfo.IsCloud)
{
await requestContext.SendError(SR.NotSupportedCloudCreateSas);
return;
}
using (SqlConnection sqlConn = ConnectionService.OpenSqlConnection(connInfo, "AzureBlob"))
{
// Connection gets disconnected when backup is done
ServerConnection serverConnection = new ServerConnection(sqlConn);
Server sqlServer = new Server(serverConnection);
SharedAccessSignatureCreator sharedAccessSignatureCreator = new SharedAccessSignatureCreator(sqlServer);
string sharedAccessSignature = sharedAccessSignatureCreator.CreateSqlSASCredential(optionsParams.StorageAccountName, optionsParams.BlobContainerKey, optionsParams.BlobContainerUri, optionsParams.ExpirationDate);
response.SharedAccessSignature = sharedAccessSignature;
await requestContext.SendResult(response);
}
}
catch (Exception ex)
{
await requestContext.SendError(ex);
}
}
}
}

View File

@@ -0,0 +1,37 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
namespace Microsoft.SqlTools.ServiceLayer.AzureBlob.Contracts
{
public static class BlobSasResource
{
/*
* Specify "c" if the shared resource is a blob container.
* This grants access to the content and metadata of any blob in the container,
* and to the list of blobs in the container.
*/
public const string BLOB_CONTAINER = "c";
/*
* Specify "b" if the shared resource is a blob.
* This grants access to the content and metadata of the blob.
*/
public const string BLOB = "b";
/*
* Beginning in version 2018-11-09, specify "bs" if the shared resource is a blob snapshot.
* This grants access to the content and metadata of the specific snapshot,
* but not the corresponding root blob.
*/
public const string BLOB_SNAPSHOT = "bs";
/*
* Beginning in version 2019-12-12, specify "bv" if the shared resource is a blob version.
* This grants access to the content and metadata of the specific version,
* but not the corresponding root blob.
*/
public const string BLOB_VERSION = "bv";
}
}

View File

@@ -0,0 +1,55 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using Microsoft.SqlTools.Hosting.Protocol.Contracts;
namespace Microsoft.SqlTools.ServiceLayer.AzureBlob.Contracts
{
/// <summary>
/// Parameters passed for creating shared access signature
/// </summary>
public class CreateSasParams
{
/// <summary>
/// Connection URI
/// </summary>
public string OwnerUri { get; set; }
/// <summary>
/// Blob container URI
/// </summary>
public string BlobContainerUri { get; set; }
/// <summary>
/// Blob container key
/// </summary>
public string BlobContainerKey { get; set; }
/// <summary>
/// Storage account name
/// </summary>
public string StorageAccountName { get; set; }
/// <summary>
/// Shared access signature expiration date
/// </summary>
public string ExpirationDate { get; set; }
}
/// <summary>
/// Response class for creating shared access signature
/// </summary>
public class CreateSasResponse
{
public string SharedAccessSignature { get; set; }
}
/// <summary>
/// Request class for creating shared access signature
/// </summary>
public class CreateSasRequest
{
public static readonly
RequestType<CreateSasParams, CreateSasResponse> Type =
RequestType<CreateSasParams, CreateSasResponse>.Create("blob/createSas");
}
}

View File

@@ -0,0 +1,116 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using Microsoft.SqlServer.Management.Smo;
using Azure.Storage.Blobs;
using Azure.Storage;
using Azure.Storage.Sas;
using Microsoft.SqlTools.ServiceLayer.AzureBlob.Contracts;
using System.Globalization;
namespace Microsoft.SqlTools.ServiceLayer.AzureBlob
{
class SharedAccessSignatureCreator
{
private Server sqlServer;
public SharedAccessSignatureCreator(Server sqlServer)
{
this.sqlServer = sqlServer;
}
public string CreateSqlSASCredential(string accountName, string accountKey, string containerUri, string expirationDateString)
{
DateTimeOffset? expirationDate = null;
if (!String.IsNullOrEmpty(expirationDateString))
{
expirationDate = DateTimeOffset.Parse(expirationDateString, CultureInfo.InvariantCulture);
}
var containerClient = new BlobContainerClient(new Uri(containerUri), new StorageSharedKeyCredential(accountName, accountKey));
Uri secretStringUri = GetServiceSasUriForContainer(containerClient, null, expirationDate);
string secretString = secretStringUri.ToString().Split('?')[1];
string identity = "Shared Access Signature";
WriteSASCredentialToSqlServer(containerUri, identity, secretString);
return secretString;
}
/// <summary>
/// Create sql sas credential with the given credential name
/// </summary>
/// <param name="credentialName">Name of sas credential, here is the same of the full container url.</param>
/// <param name="identity">Identity for credential, here is fixed as "Shared Access Signature"</param>
/// <param name="secretString">Secret of credential, which is sharedAccessSignatureForContainer </param>
/// <returns> The newly created SAS credential</returns>
public Credential WriteSASCredentialToSqlServer(string credentialName, string identity, string secretString)
{
try
{
// Format of Sql SAS credential:
// CREATE CREDENTIAL [https://<StorageAccountName>.blob.core.windows.net/<ContainerName>] WITH IDENTITY = N'Shared Access Signature',
// SECRET = N'sv=2014-02-14&sr=c&sig=lxb2aXr%2Bi0Aeygg%2B0a4REZ%2BqsUxxxxxxsqUybg0tVzg%3D&st=2015-10-15T08%3A00%3A00Z&se=2015-11-15T08%3A00%3A00Z&sp=rwdl'
//
CredentialCollection credentials = sqlServer.Credentials;
Credential azureCredential = new Credential(sqlServer, credentialName);
// Container can have many SAS credentials coexisting, here we'll always drop existing one once customer choose to create new credential
// since sql customer has no way to know its existency and even harder to retrive its secret string.
if (credentials.Contains(credentialName))
{
Credential oldCredential = credentials[credentialName];
oldCredential.Drop();
}
azureCredential.Create(identity, secretString);
return azureCredential;
}
catch (Exception ex)
{
throw new FailedOperationException(SR.WriteSASCredentialToSqlServerFailed, ex);
}
}
/// <summary>
/// Create Shared Access Policy for container
/// Default Accesss permission is Write/List/Read/Delete
/// </summary>
/// <param name="container"></param>
/// <param name="policyName"></param>
/// <param name="selectedSaredAccessExpiryTime"></param>
public Uri GetServiceSasUriForContainer(BlobContainerClient containerClient,
string storedPolicyName = null,
DateTimeOffset? expiringDate = null)
{
// Check whether this BlobContainerClient object has been authorized with Shared Key.
if (containerClient.CanGenerateSasUri)
{
// Create a SAS token
BlobSasBuilder sasBuilder = new BlobSasBuilder()
{
BlobContainerName = containerClient.Name,
Resource = BlobSasResource.BLOB_CONTAINER
};
if (storedPolicyName == null)
{
sasBuilder.ExpiresOn = (DateTimeOffset)(expiringDate == null ? DateTimeOffset.UtcNow.AddYears(1) : expiringDate);
sasBuilder.SetPermissions(BlobContainerSasPermissions.Read | BlobContainerSasPermissions.List | BlobContainerSasPermissions.Write | BlobContainerSasPermissions.Delete);
}
else
{
sasBuilder.Identifier = storedPolicyName;
}
Uri sasUri = containerClient.GenerateSasUri(sasBuilder);
return sasUri;
}
else
{
throw new FailedOperationException(SR.CreateSasForBlobContainerFailed);
}
}
}
}

View File

@@ -211,12 +211,12 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery
{
string destName = Convert.ToString(this.backupInfo.BackupPathList[i], System.Globalization.CultureInfo.InvariantCulture);
int deviceType = (int)(this.backupInfo.BackupPathDevices[destName]);
int backupDeviceType =
GetDeviceType(Convert.ToString(destName,
System.Globalization.CultureInfo.InvariantCulture));
switch (deviceType)
{
case (int)DeviceType.LogicalDevice:
int backupDeviceType =
GetDeviceType(Convert.ToString(destName,
System.Globalization.CultureInfo.InvariantCulture));
if (this.backupDeviceType == BackupDeviceType.Disk && backupDeviceType == constDeviceTypeFile)
{
@@ -229,6 +229,12 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery
this.backup.Devices.AddDevice(destName, DeviceType.File);
}
break;
case (int)DeviceType.Url:
if (this.backupDeviceType == BackupDeviceType.Url)
{
this.backup.Devices.AddDevice(destName, DeviceType.Url);
}
break;
}
}

View File

@@ -51,6 +51,18 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.Contracts
}
}
internal int DeviceType
{
get
{
return GetOptionValue<int>(RestoreOptionsHelper.DeviceType);
}
set
{
SetOptionValue(RestoreOptionsHelper.DeviceType, value);
}
}
/// <summary>
/// Target Database name to restore to
/// </summary>

View File

@@ -77,6 +77,12 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation
private bool? isTailLogBackupWithNoRecoveryPossible = null;
private string backupMediaList = string.Empty;
private Server server;
private static readonly DeviceType[] managedInstanceSupportedDeviceTypes = { DeviceType.Url };
private static readonly DeviceType[] defaultSupportedDeviceTypes = { DeviceType.File, DeviceType.Url };
private static readonly Dictionary<Edition, DeviceType[]> specialEngineEditionSupportedDeviceTypes = new Dictionary<Edition, DeviceType[]>
{
{ Edition.SqlManagedInstance, managedInstanceSupportedDeviceTypes },
};
public RestoreDatabaseTaskDataObject(Server server, String databaseName)
{
@@ -195,8 +201,10 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation
/// Add a backup file to restore plan media list
/// </summary>
/// <param name="filePaths"></param>
public void AddFiles(string filePaths)
/// <param name="deviceType"></deviceType>
public void AddDevices(string filePaths, DeviceType deviceType)
{
ThrowIfUnsupportedDeviceType(this.Server.EngineEdition, deviceType);
backupMediaList = filePaths;
PlanUpdateRequired = true;
if (!string.IsNullOrWhiteSpace(filePaths))
@@ -209,7 +217,7 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation
{
this.RestorePlanner.BackupMediaList.Add(new BackupDeviceItem
{
DeviceType = DeviceType.File,
DeviceType = deviceType,
Name = file
});
}
@@ -223,6 +231,20 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation
}
}
private void ThrowIfUnsupportedDeviceType(Edition engineEdition, DeviceType deviceType)
{
if (!IsSupportedDeviceType(engineEdition, deviceType))
{
throw new UnsupportedDeviceTypeException(engineEdition, deviceType);
}
}
private bool IsSupportedDeviceType(Edition engineEdition, DeviceType deviceType)
{
return (defaultSupportedDeviceTypes.Contains(deviceType) && !specialEngineEditionSupportedDeviceTypes.ContainsKey(engineEdition))
|| (specialEngineEditionSupportedDeviceTypes.ContainsKey(engineEdition) && specialEngineEditionSupportedDeviceTypes[engineEdition].Contains(deviceType));
}
/// <summary>
/// Returns the last backup taken
@@ -1368,7 +1390,7 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation
if (!string.IsNullOrEmpty(RestoreParams.BackupFilePaths) && RestoreParams.ReadHeaderFromMedia)
{
AddFiles(RestoreParams.BackupFilePaths);
AddDevices(RestoreParams.BackupFilePaths, (DeviceType)RestoreParams.DeviceType);
}
else
{

View File

@@ -0,0 +1,17 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using Microsoft.SqlServer.Management.Smo;
namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation
{
public class UnsupportedDeviceTypeException: Exception
{
public UnsupportedDeviceTypeException(Edition engineEdition, DeviceType deviceType) : base(SR.UnsupportedDeviceType(deviceType.ToString(), engineEdition.ToString()))
{
}
}
}

View File

@@ -62,6 +62,9 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery
//The key name to use to set the backup file paths in the request
internal const string BackupFilePaths = "backupFilePaths";
//The key name to use to set the device type
internal const string DeviceType = "deviceType";
//The key name to use to set the target database name in the request
internal const string TargetDatabaseName = "targetDatabaseName";

View File

@@ -36,6 +36,7 @@ using Microsoft.SqlTools.ServiceLayer.Workspace;
using Microsoft.SqlTools.ServiceLayer.NotebookConvert;
using Microsoft.SqlTools.ServiceLayer.ModelManagement;
using Microsoft.SqlTools.ServiceLayer.TableDesigner;
using Microsoft.SqlTools.ServiceLayer.AzureBlob;
using Microsoft.SqlTools.ServiceLayer.ExecutionPlan;
namespace Microsoft.SqlTools.ServiceLayer
@@ -164,6 +165,9 @@ namespace Microsoft.SqlTools.ServiceLayer
TableDesignerService.Instance.InitializeService(serviceHost);
serviceProvider.RegisterSingleService(TableDesignerService.Instance);
BlobService.Instance.InitializeService(serviceHost);
serviceProvider.RegisterSingleService(BlobService.Instance);
InitializeHostedServices(serviceProvider, serviceHost);
serviceHost.ServiceProvider = serviceProvider;

View File

@@ -2589,6 +2589,30 @@ namespace Microsoft.SqlTools.ServiceLayer
}
}
public static string NotSupportedCloudCreateSas
{
get
{
return Keys.GetString(Keys.NotSupportedCloudCreateSas);
}
}
public static string CreateSasForBlobContainerFailed
{
get
{
return Keys.GetString(Keys.CreateSasForBlobContainerFailed);
}
}
public static string WriteSASCredentialToSqlServerFailed
{
get
{
return Keys.GetString(Keys.WriteSASCredentialToSqlServerFailed);
}
}
public static string CategoryLocal
{
get
@@ -9192,6 +9216,11 @@ namespace Microsoft.SqlTools.ServiceLayer
return Keys.GetString(Keys.EditDataIncorrectTable, tableName);
}
public static string UnsupportedDeviceType(String deviceType, String engineEdition)
{
return Keys.GetString(Keys.UnsupportedDeviceType, deviceType, engineEdition);
}
public static string CreateSessionFailed(String error)
{
return Keys.GetString(Keys.CreateSessionFailed, error);
@@ -10388,6 +10417,9 @@ namespace Microsoft.SqlTools.ServiceLayer
public const string NoBackupsetsToRestore = "NoBackupsetsToRestore";
public const string UnsupportedDeviceType = "UnsupportedDeviceType";
public const string ScriptTaskName = "ScriptTaskName";
@@ -10418,6 +10450,15 @@ namespace Microsoft.SqlTools.ServiceLayer
public const string SessionAlreadyExists = "SessionAlreadyExists";
public const string NotSupportedCloudCreateSas = "NotSupportedCloudCreateSas";
public const string CreateSasForBlobContainerFailed = "CreateSasForBlobContainerFailed";
public const string WriteSASCredentialToSqlServerFailed = "WriteSASCredentialToSqlServerFailed";
public const string CategoryLocal = "CategoryLocal";

View File

@@ -1532,6 +1532,11 @@
<value>No backupset selected to be restored</value>
<comment></comment>
</data>
<data name="UnsupportedDeviceType" xml:space="preserve">
<value>Unsupported device type {0} for engine edition {1}.</value>
<comment>.
Parameters: 0 - deviceType (String), 1 - engineEdition (String) </comment>
</data>
<data name="ScriptTaskName" xml:space="preserve">
<value>scripting</value>
<comment></comment>
@@ -1577,6 +1582,18 @@
<comment>.
Parameters: 0 - sessionName (String) </comment>
</data>
<data name="NotSupportedCloudCreateSas" xml:space="preserve">
<value>Create shared access signature is not supported for cloud instances.</value>
<comment></comment>
</data>
<data name="CreateSasForBlobContainerFailed" xml:space="preserve">
<value>Cannot generate SAS URI for blob container.</value>
<comment></comment>
</data>
<data name="WriteSASCredentialToSqlServerFailed" xml:space="preserve">
<value>Failed storing shared access signature token on the SQL Servers.</value>
<comment></comment>
</data>
<data name="CategoryLocal" xml:space="preserve">
<value>[Uncategorized (Local)]</value>
<comment>job categories</comment>

View File

@@ -742,6 +742,7 @@ RestoreBackupSetUserName = User Name
RestoreBackupSetExpiration = Expiration
TheLastBackupTaken = The last backup taken ({0})
NoBackupsetsToRestore = No backupset selected to be restored
UnsupportedDeviceType(String deviceType, String engineEdition) = Unsupported device type {0} for engine edition {1}.
############################################################################
# Generate Script
@@ -762,6 +763,11 @@ StopSessionFailed(String error) = Failed to stop session: {0}
SessionNotFound = Cannot find requested XEvent session
SessionAlreadyExists(String sessionName) = An XEvent session named {0} already exists
############################################################################
# Azure Blob Service
NotSupportedCloudCreateSas = Create shared access signature is not supported for cloud instances.
CreateSasForBlobContainerFailed = Cannot generate SAS URI for blob container.
WriteSASCredentialToSqlServerFailed = Failed storing shared access signature token on the SQL Servers.
;job categories
CategoryLocal = [Uncategorized (Local)]

View File

@@ -5757,6 +5757,26 @@ The Query Processor estimates that implementing the following index could improv
*/</target>
<note>title of missing index details</note>
</trans-unit>
<trans-unit id="CreateSasForBlobContainerFailed">
<source>Cannot generate SAS URI for blob container.</source>
<target state="new">Cannot generate SAS URI for blob container.</target>
<note></note>
</trans-unit>
<trans-unit id="WriteSASCredentialToSqlServerFailed">
<source>Failed storing shared access signature token on the SQL Servers.</source>
<target state="new">Failed storing shared access signature token on the SQL Servers.</target>
<note></note>
</trans-unit>
<trans-unit id="UnsupportedDeviceType">
<source>Unsupported device type {0} for engine edition {1}.</source>
<target state="new">Unsupported device type {0} for engine edition {1}.</target>
<note></note>
</trans-unit>
<trans-unit id="NotSupportedCloudCreateSas">
<source>Create shared access signature is not supported for cloud instances.</source>
<target state="new">Create shared access signature is not supported for cloud instances.</target>
<note></note>
</trans-unit>
<trans-unit id="TableDesignerGraphTableTypeTitle">
<source>Type</source>
<target state="new">Type</target>

View File

@@ -21,6 +21,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.Storage.Blobs" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" />
<PackageReference Include="Microsoft.CodeAnalysis.Workspaces.Common" />
@@ -29,7 +30,7 @@
<PackageReference Include="System.Text.Encoding.CodePages" />
<PackageReference Include="Microsoft.SqlServer.Assessment" />
<PackageReference Include="Microsoft.SqlServer.Migration.Assessment" />
<PackageReference Include="Microsoft.SqlServer.Management.SqlParser"/>
<PackageReference Include="Microsoft.SqlServer.Management.SqlParser" />
<PackageReference Include="System.Text.Encoding.CodePages" />
<PackageReference Include="Microsoft.SqlServer.TransactSql.ScriptDom.NRT">
<Aliases>ASAScriptDom</Aliases>