diff --git a/Packages.props b/Packages.props index ff98c2a2..fbd73da0 100644 --- a/Packages.props +++ b/Packages.props @@ -40,5 +40,6 @@ + \ No newline at end of file diff --git a/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/BlobService.cs b/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/BlobService.cs new file mode 100644 index 00000000..ad25b13f --- /dev/null +++ b/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/BlobService.cs @@ -0,0 +1,81 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient; +using Microsoft.SqlServer.Management.Common; +using Microsoft.SqlServer.Management.Smo; +using Microsoft.SqlTools.Hosting.Protocol; +using Microsoft.SqlTools.ServiceLayer.AzureBlob.Contracts; +using Microsoft.SqlTools.ServiceLayer.Connection; + +namespace Microsoft.SqlTools.ServiceLayer.AzureBlob +{ + public class BlobService + { + private static readonly Lazy instance = new Lazy(() => new BlobService()); + + /// + /// Default, parameterless constructor. + /// + internal BlobService() + { + } + + /// + /// Gets the singleton instance object + /// + public static BlobService Instance + { + get { return instance.Value; } + } + + public void InitializeService(IProtocolEndpoint serviceHost) + { + serviceHost.SetRequestHandler(CreateSasRequest.Type, HandleCreateSasRequest); + } + + internal async Task HandleCreateSasRequest( + CreateSasParams optionsParams, + RequestContext requestContext) + { + try + { + ConnectionInfo connInfo; + ConnectionService.Instance.TryFindConnection( + optionsParams.OwnerUri, + out connInfo); + var response = new CreateSasResponse(); + + if (connInfo == null) + { + await requestContext.SendError(SR.ConnectionServiceListDbErrorNotConnected(optionsParams.OwnerUri)); + return; + } + if (connInfo.IsCloud) + { + await requestContext.SendError(SR.NotSupportedCloudCreateSas); + return; + } + using (SqlConnection sqlConn = ConnectionService.OpenSqlConnection(connInfo, "AzureBlob")) + { + // Connection gets disconnected when backup is done + ServerConnection serverConnection = new ServerConnection(sqlConn); + Server sqlServer = new Server(serverConnection); + + SharedAccessSignatureCreator sharedAccessSignatureCreator = new SharedAccessSignatureCreator(sqlServer); + string sharedAccessSignature = sharedAccessSignatureCreator.CreateSqlSASCredential(optionsParams.StorageAccountName, optionsParams.BlobContainerKey, optionsParams.BlobContainerUri, optionsParams.ExpirationDate); + response.SharedAccessSignature = sharedAccessSignature; + await requestContext.SendResult(response); + } + } + catch (Exception ex) + { + await requestContext.SendError(ex); + } + } + } +} diff --git a/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/Contracts/BlobSasResource.cs b/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/Contracts/BlobSasResource.cs new file mode 100644 index 00000000..f9fce961 --- /dev/null +++ b/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/Contracts/BlobSasResource.cs @@ -0,0 +1,37 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.SqlTools.ServiceLayer.AzureBlob.Contracts +{ + public static class BlobSasResource + { + /* + * Specify "c" if the shared resource is a blob container. + * This grants access to the content and metadata of any blob in the container, + * and to the list of blobs in the container. + */ + public const string BLOB_CONTAINER = "c"; + + /* + * Specify "b" if the shared resource is a blob. + * This grants access to the content and metadata of the blob. + */ + public const string BLOB = "b"; + + /* + * Beginning in version 2018-11-09, specify "bs" if the shared resource is a blob snapshot. + * This grants access to the content and metadata of the specific snapshot, + * but not the corresponding root blob. + */ + public const string BLOB_SNAPSHOT = "bs"; + + /* + * Beginning in version 2019-12-12, specify "bv" if the shared resource is a blob version. + * This grants access to the content and metadata of the specific version, + * but not the corresponding root blob. + */ + public const string BLOB_VERSION = "bv"; + } +} diff --git a/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/Contracts/CreateSasRequest.cs b/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/Contracts/CreateSasRequest.cs new file mode 100644 index 00000000..3f52a76d --- /dev/null +++ b/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/Contracts/CreateSasRequest.cs @@ -0,0 +1,55 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using Microsoft.SqlTools.Hosting.Protocol.Contracts; + +namespace Microsoft.SqlTools.ServiceLayer.AzureBlob.Contracts +{ + /// + /// Parameters passed for creating shared access signature + /// + public class CreateSasParams + { + /// + /// Connection URI + /// + public string OwnerUri { get; set; } + /// + /// Blob container URI + /// + public string BlobContainerUri { get; set; } + /// + /// Blob container key + /// + public string BlobContainerKey { get; set; } + /// + /// Storage account name + /// + public string StorageAccountName { get; set; } + /// + /// Shared access signature expiration date + /// + public string ExpirationDate { get; set; } + } + + /// + /// Response class for creating shared access signature + /// + public class CreateSasResponse + { + public string SharedAccessSignature { get; set; } + + } + + /// + /// Request class for creating shared access signature + /// + public class CreateSasRequest + { + public static readonly + RequestType Type = + RequestType.Create("blob/createSas"); + } +} diff --git a/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/SharedAccessSignatureCreator.cs b/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/SharedAccessSignatureCreator.cs new file mode 100644 index 00000000..78213528 --- /dev/null +++ b/src/Microsoft.SqlTools.ServiceLayer/AzureBlob/SharedAccessSignatureCreator.cs @@ -0,0 +1,116 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using Microsoft.SqlServer.Management.Smo; +using Azure.Storage.Blobs; +using Azure.Storage; +using Azure.Storage.Sas; +using Microsoft.SqlTools.ServiceLayer.AzureBlob.Contracts; +using System.Globalization; + +namespace Microsoft.SqlTools.ServiceLayer.AzureBlob +{ + class SharedAccessSignatureCreator + { + private Server sqlServer; + + public SharedAccessSignatureCreator(Server sqlServer) + { + this.sqlServer = sqlServer; + } + + public string CreateSqlSASCredential(string accountName, string accountKey, string containerUri, string expirationDateString) + { + DateTimeOffset? expirationDate = null; + if (!String.IsNullOrEmpty(expirationDateString)) + { + expirationDate = DateTimeOffset.Parse(expirationDateString, CultureInfo.InvariantCulture); + } + var containerClient = new BlobContainerClient(new Uri(containerUri), new StorageSharedKeyCredential(accountName, accountKey)); + Uri secretStringUri = GetServiceSasUriForContainer(containerClient, null, expirationDate); + string secretString = secretStringUri.ToString().Split('?')[1]; + string identity = "Shared Access Signature"; + WriteSASCredentialToSqlServer(containerUri, identity, secretString); + return secretString; + } + + /// + /// Create sql sas credential with the given credential name + /// + /// Name of sas credential, here is the same of the full container url. + /// Identity for credential, here is fixed as "Shared Access Signature" + /// Secret of credential, which is sharedAccessSignatureForContainer + /// The newly created SAS credential + public Credential WriteSASCredentialToSqlServer(string credentialName, string identity, string secretString) + { + try + { + // Format of Sql SAS credential: + // CREATE CREDENTIAL [https://.blob.core.windows.net/] WITH IDENTITY = N'Shared Access Signature', + // SECRET = N'sv=2014-02-14&sr=c&sig=lxb2aXr%2Bi0Aeygg%2B0a4REZ%2BqsUxxxxxxsqUybg0tVzg%3D&st=2015-10-15T08%3A00%3A00Z&se=2015-11-15T08%3A00%3A00Z&sp=rwdl' + // + CredentialCollection credentials = sqlServer.Credentials; + + Credential azureCredential = new Credential(sqlServer, credentialName); + + // Container can have many SAS credentials coexisting, here we'll always drop existing one once customer choose to create new credential + // since sql customer has no way to know its existency and even harder to retrive its secret string. + if (credentials.Contains(credentialName)) + { + Credential oldCredential = credentials[credentialName]; + oldCredential.Drop(); + } + + azureCredential.Create(identity, secretString); + return azureCredential; + } + catch (Exception ex) + { + throw new FailedOperationException(SR.WriteSASCredentialToSqlServerFailed, ex); + } + } + + /// + /// Create Shared Access Policy for container + /// Default Accesss permission is Write/List/Read/Delete + /// + /// + /// + /// + public Uri GetServiceSasUriForContainer(BlobContainerClient containerClient, + string storedPolicyName = null, + DateTimeOffset? expiringDate = null) + { + // Check whether this BlobContainerClient object has been authorized with Shared Key. + if (containerClient.CanGenerateSasUri) + { + // Create a SAS token + BlobSasBuilder sasBuilder = new BlobSasBuilder() + { + BlobContainerName = containerClient.Name, + Resource = BlobSasResource.BLOB_CONTAINER + }; + + if (storedPolicyName == null) + { + sasBuilder.ExpiresOn = (DateTimeOffset)(expiringDate == null ? DateTimeOffset.UtcNow.AddYears(1) : expiringDate); + sasBuilder.SetPermissions(BlobContainerSasPermissions.Read | BlobContainerSasPermissions.List | BlobContainerSasPermissions.Write | BlobContainerSasPermissions.Delete); + } + else + { + sasBuilder.Identifier = storedPolicyName; + } + Uri sasUri = containerClient.GenerateSasUri(sasBuilder); + + return sasUri; + } + else + { + throw new FailedOperationException(SR.CreateSasForBlobContainerFailed); + } + } + } +} diff --git a/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/BackupOperation/BackupOperation.cs b/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/BackupOperation/BackupOperation.cs index 478ae741..c4544a06 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/BackupOperation/BackupOperation.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/BackupOperation/BackupOperation.cs @@ -211,12 +211,12 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery { string destName = Convert.ToString(this.backupInfo.BackupPathList[i], System.Globalization.CultureInfo.InvariantCulture); int deviceType = (int)(this.backupInfo.BackupPathDevices[destName]); + int backupDeviceType = + GetDeviceType(Convert.ToString(destName, + System.Globalization.CultureInfo.InvariantCulture)); switch (deviceType) { case (int)DeviceType.LogicalDevice: - int backupDeviceType = - GetDeviceType(Convert.ToString(destName, - System.Globalization.CultureInfo.InvariantCulture)); if (this.backupDeviceType == BackupDeviceType.Disk && backupDeviceType == constDeviceTypeFile) { @@ -229,6 +229,12 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery this.backup.Devices.AddDevice(destName, DeviceType.File); } break; + case (int)DeviceType.Url: + if (this.backupDeviceType == BackupDeviceType.Url) + { + this.backup.Devices.AddDevice(destName, DeviceType.Url); + } + break; } } diff --git a/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/Contracts/RestoreRequestParams.cs b/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/Contracts/RestoreRequestParams.cs index cf4692b0..975e8fe0 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/Contracts/RestoreRequestParams.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/Contracts/RestoreRequestParams.cs @@ -51,6 +51,18 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.Contracts } } + internal int DeviceType + { + get + { + return GetOptionValue(RestoreOptionsHelper.DeviceType); + } + set + { + SetOptionValue(RestoreOptionsHelper.DeviceType, value); + } + } + /// /// Target Database name to restore to /// diff --git a/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOperation/RestoreDatabaseTaskDataObject.cs b/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOperation/RestoreDatabaseTaskDataObject.cs index 3b226a95..32b35768 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOperation/RestoreDatabaseTaskDataObject.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOperation/RestoreDatabaseTaskDataObject.cs @@ -77,6 +77,12 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation private bool? isTailLogBackupWithNoRecoveryPossible = null; private string backupMediaList = string.Empty; private Server server; + private static readonly DeviceType[] managedInstanceSupportedDeviceTypes = { DeviceType.Url }; + private static readonly DeviceType[] defaultSupportedDeviceTypes = { DeviceType.File, DeviceType.Url }; + private static readonly Dictionary specialEngineEditionSupportedDeviceTypes = new Dictionary + { + { Edition.SqlManagedInstance, managedInstanceSupportedDeviceTypes }, + }; public RestoreDatabaseTaskDataObject(Server server, String databaseName) { @@ -195,8 +201,10 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation /// Add a backup file to restore plan media list /// /// - public void AddFiles(string filePaths) + /// + public void AddDevices(string filePaths, DeviceType deviceType) { + ThrowIfUnsupportedDeviceType(this.Server.EngineEdition, deviceType); backupMediaList = filePaths; PlanUpdateRequired = true; if (!string.IsNullOrWhiteSpace(filePaths)) @@ -209,7 +217,7 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation { this.RestorePlanner.BackupMediaList.Add(new BackupDeviceItem { - DeviceType = DeviceType.File, + DeviceType = deviceType, Name = file }); } @@ -223,6 +231,20 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation } } + private void ThrowIfUnsupportedDeviceType(Edition engineEdition, DeviceType deviceType) + { + if (!IsSupportedDeviceType(engineEdition, deviceType)) + { + throw new UnsupportedDeviceTypeException(engineEdition, deviceType); + } + } + + private bool IsSupportedDeviceType(Edition engineEdition, DeviceType deviceType) + { + return (defaultSupportedDeviceTypes.Contains(deviceType) && !specialEngineEditionSupportedDeviceTypes.ContainsKey(engineEdition)) + || (specialEngineEditionSupportedDeviceTypes.ContainsKey(engineEdition) && specialEngineEditionSupportedDeviceTypes[engineEdition].Contains(deviceType)); + } + /// /// Returns the last backup taken @@ -1368,7 +1390,7 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation if (!string.IsNullOrEmpty(RestoreParams.BackupFilePaths) && RestoreParams.ReadHeaderFromMedia) { - AddFiles(RestoreParams.BackupFilePaths); + AddDevices(RestoreParams.BackupFilePaths, (DeviceType)RestoreParams.DeviceType); } else { diff --git a/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOperation/UnsupportedDeviceTypeException.cs b/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOperation/UnsupportedDeviceTypeException.cs new file mode 100644 index 00000000..03f55e00 --- /dev/null +++ b/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOperation/UnsupportedDeviceTypeException.cs @@ -0,0 +1,17 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using Microsoft.SqlServer.Management.Smo; + +namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation +{ + public class UnsupportedDeviceTypeException: Exception + { + public UnsupportedDeviceTypeException(Edition engineEdition, DeviceType deviceType) : base(SR.UnsupportedDeviceType(deviceType.ToString(), engineEdition.ToString())) + { + } + } +} diff --git a/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOptionsHelper.cs b/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOptionsHelper.cs index 6ed04126..f52574e4 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOptionsHelper.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/DisasterRecovery/RestoreOptionsHelper.cs @@ -62,6 +62,9 @@ namespace Microsoft.SqlTools.ServiceLayer.DisasterRecovery //The key name to use to set the backup file paths in the request internal const string BackupFilePaths = "backupFilePaths"; + //The key name to use to set the device type + internal const string DeviceType = "deviceType"; + //The key name to use to set the target database name in the request internal const string TargetDatabaseName = "targetDatabaseName"; diff --git a/src/Microsoft.SqlTools.ServiceLayer/HostLoader.cs b/src/Microsoft.SqlTools.ServiceLayer/HostLoader.cs index 24739657..44075086 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/HostLoader.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/HostLoader.cs @@ -36,6 +36,7 @@ using Microsoft.SqlTools.ServiceLayer.Workspace; using Microsoft.SqlTools.ServiceLayer.NotebookConvert; using Microsoft.SqlTools.ServiceLayer.ModelManagement; using Microsoft.SqlTools.ServiceLayer.TableDesigner; +using Microsoft.SqlTools.ServiceLayer.AzureBlob; using Microsoft.SqlTools.ServiceLayer.ExecutionPlan; namespace Microsoft.SqlTools.ServiceLayer @@ -164,6 +165,9 @@ namespace Microsoft.SqlTools.ServiceLayer TableDesignerService.Instance.InitializeService(serviceHost); serviceProvider.RegisterSingleService(TableDesignerService.Instance); + BlobService.Instance.InitializeService(serviceHost); + serviceProvider.RegisterSingleService(BlobService.Instance); + InitializeHostedServices(serviceProvider, serviceHost); serviceHost.ServiceProvider = serviceProvider; diff --git a/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.cs b/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.cs index f31d7669..d78853bb 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.cs @@ -2589,6 +2589,30 @@ namespace Microsoft.SqlTools.ServiceLayer } } + public static string NotSupportedCloudCreateSas + { + get + { + return Keys.GetString(Keys.NotSupportedCloudCreateSas); + } + } + + public static string CreateSasForBlobContainerFailed + { + get + { + return Keys.GetString(Keys.CreateSasForBlobContainerFailed); + } + } + + public static string WriteSASCredentialToSqlServerFailed + { + get + { + return Keys.GetString(Keys.WriteSASCredentialToSqlServerFailed); + } + } + public static string CategoryLocal { get @@ -9192,6 +9216,11 @@ namespace Microsoft.SqlTools.ServiceLayer return Keys.GetString(Keys.EditDataIncorrectTable, tableName); } + public static string UnsupportedDeviceType(String deviceType, String engineEdition) + { + return Keys.GetString(Keys.UnsupportedDeviceType, deviceType, engineEdition); + } + public static string CreateSessionFailed(String error) { return Keys.GetString(Keys.CreateSessionFailed, error); @@ -10388,6 +10417,9 @@ namespace Microsoft.SqlTools.ServiceLayer public const string NoBackupsetsToRestore = "NoBackupsetsToRestore"; + public const string UnsupportedDeviceType = "UnsupportedDeviceType"; + + public const string ScriptTaskName = "ScriptTaskName"; @@ -10418,6 +10450,15 @@ namespace Microsoft.SqlTools.ServiceLayer public const string SessionAlreadyExists = "SessionAlreadyExists"; + public const string NotSupportedCloudCreateSas = "NotSupportedCloudCreateSas"; + + + public const string CreateSasForBlobContainerFailed = "CreateSasForBlobContainerFailed"; + + + public const string WriteSASCredentialToSqlServerFailed = "WriteSASCredentialToSqlServerFailed"; + + public const string CategoryLocal = "CategoryLocal"; diff --git a/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.resx b/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.resx index 1be18845..073366a7 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.resx +++ b/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.resx @@ -1532,6 +1532,11 @@ No backupset selected to be restored + + Unsupported device type {0} for engine edition {1}. + . + Parameters: 0 - deviceType (String), 1 - engineEdition (String) + scripting @@ -1577,6 +1582,18 @@ . Parameters: 0 - sessionName (String) + + Create shared access signature is not supported for cloud instances. + + + + Cannot generate SAS URI for blob container. + + + + Failed storing shared access signature token on the SQL Servers. + + [Uncategorized (Local)] job categories diff --git a/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.strings b/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.strings index eb2983e9..817a7f7c 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.strings +++ b/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.strings @@ -742,6 +742,7 @@ RestoreBackupSetUserName = User Name RestoreBackupSetExpiration = Expiration TheLastBackupTaken = The last backup taken ({0}) NoBackupsetsToRestore = No backupset selected to be restored +UnsupportedDeviceType(String deviceType, String engineEdition) = Unsupported device type {0} for engine edition {1}. ############################################################################ # Generate Script @@ -762,6 +763,11 @@ StopSessionFailed(String error) = Failed to stop session: {0} SessionNotFound = Cannot find requested XEvent session SessionAlreadyExists(String sessionName) = An XEvent session named {0} already exists +############################################################################ +# Azure Blob Service +NotSupportedCloudCreateSas = Create shared access signature is not supported for cloud instances. +CreateSasForBlobContainerFailed = Cannot generate SAS URI for blob container. +WriteSASCredentialToSqlServerFailed = Failed storing shared access signature token on the SQL Servers. ;job categories CategoryLocal = [Uncategorized (Local)] diff --git a/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.xlf b/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.xlf index 10a45d4a..70e4a4ff 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.xlf +++ b/src/Microsoft.SqlTools.ServiceLayer/Localization/sr.xlf @@ -5757,6 +5757,26 @@ The Query Processor estimates that implementing the following index could improv */ title of missing index details + + Cannot generate SAS URI for blob container. + Cannot generate SAS URI for blob container. + + + + Failed storing shared access signature token on the SQL Servers. + Failed storing shared access signature token on the SQL Servers. + + + + Unsupported device type {0} for engine edition {1}. + Unsupported device type {0} for engine edition {1}. + + + + Create shared access signature is not supported for cloud instances. + Create shared access signature is not supported for cloud instances. + + Type Type diff --git a/src/Microsoft.SqlTools.ServiceLayer/Microsoft.SqlTools.ServiceLayer.csproj b/src/Microsoft.SqlTools.ServiceLayer/Microsoft.SqlTools.ServiceLayer.csproj index e9869532..4b026106 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/Microsoft.SqlTools.ServiceLayer.csproj +++ b/src/Microsoft.SqlTools.ServiceLayer/Microsoft.SqlTools.ServiceLayer.csproj @@ -21,6 +21,7 @@ + @@ -29,7 +30,7 @@ - + ASAScriptDom diff --git a/test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/DisasterRecovery/BackupRestoreUrlTests.cs b/test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/DisasterRecovery/BackupRestoreUrlTests.cs new file mode 100644 index 00000000..c0649379 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/DisasterRecovery/BackupRestoreUrlTests.cs @@ -0,0 +1,253 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Azure.Storage; +using Azure.Storage.Blobs; +using Microsoft.Data.SqlClient; +using Microsoft.SqlServer.Management.Common; +using Microsoft.SqlServer.Management.Smo; +using Microsoft.SqlTools.ServiceLayer.Admin; +using Microsoft.SqlTools.ServiceLayer.AzureBlob; +using Microsoft.SqlTools.ServiceLayer.Connection; +using Microsoft.SqlTools.ServiceLayer.DisasterRecovery; +using Microsoft.SqlTools.ServiceLayer.DisasterRecovery.Contracts; +using Microsoft.SqlTools.ServiceLayer.DisasterRecovery.RestoreOperation; +using Microsoft.SqlTools.ServiceLayer.IntegrationTests.Utility; +using Microsoft.SqlTools.ServiceLayer.Management; +using Microsoft.SqlTools.ServiceLayer.TaskServices; +using Microsoft.SqlTools.ServiceLayer.Test.Common; +using NUnit.Framework; +using static Microsoft.SqlTools.ServiceLayer.IntegrationTests.Utility.LiveConnectionHelper; + +namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.DisasterRecovery +{ + class BackupRestoreUrlTests + { + /// + /// Create simple backup test + /// + [Test] + public async Task BackupDatabaseToUrlAndRestoreFromUrlTest() + { + DisasterRecoveryService service = new DisasterRecoveryService(); + string databaseName = "SqlToolsService_TestBackupToUrl_" + new Random().Next(10000000, 99999999); + + using (SqlTestDb testDb = SqlTestDb.CreateNew(TestServerType.OnPrem, false, databaseName)) + { + var liveConnection = LiveConnectionHelper.InitLiveConnectionInfo(databaseName); + using (DatabaseTaskHelper helper = AdminService.CreateDatabaseTaskHelper(liveConnection.ConnectionInfo, databaseExists: true)) + using (SqlConnection sqlConn = ConnectionService.OpenSqlConnection(liveConnection.ConnectionInfo)) + { + ServerConnection serverConn = new ServerConnection(sqlConn); + Server server = new Server(serverConn); + SharedAccessSignatureCreator sasCreator = new SharedAccessSignatureCreator(server); + AzureBlobConnectionSetting azureBlobConnection = TestAzureBlobConnectionService.Instance.Settings; + sasCreator.CreateSqlSASCredential(azureBlobConnection.AccountName, azureBlobConnection.AccountKey, azureBlobConnection.BlobContainerUri, ""); + string backupPath = GetAzureBlobBackupPath(databaseName); + + BackupInfo backupInfo = CreateDefaultBackupInfo(databaseName, + BackupType.Full, + new List() { backupPath }, + new Dictionary() { { backupPath, (int)DeviceType.Url } }); + BackupOperation backupOperation = CreateBackupOperation(service, liveConnection.ConnectionInfo.OwnerUri, backupInfo, helper.DataContainer, sqlConn); + + // Backup the database + service.PerformBackup(backupOperation); + + testDb.Cleanup(); + } + } + + await VerifyRestore(databaseName, true, TaskExecutionModeFlag.Execute, databaseName); + + VerifyAndCleanAzureBlobBackup(databaseName); + } + + private BackupInfo CreateDefaultBackupInfo(string databaseName, BackupType backupType, List backupPathList, Dictionary backupPathDevices) + { + BackupInfo backupInfo = new BackupInfo(); + backupInfo.BackupComponent = (int)BackupComponent.Database; + backupInfo.BackupDeviceType = (int)BackupDeviceType.Url; + backupInfo.BackupPathDevices = backupPathDevices; + backupInfo.BackupPathList = backupPathList; + backupInfo.BackupsetName = "default_backup"; + backupInfo.BackupType = (int)backupType; + backupInfo.DatabaseName = databaseName; + backupInfo.SelectedFileGroup = null; + backupInfo.SelectedFiles = ""; + return backupInfo; + } + + private string GetAzureBlobBackupPath(string databaseName) + { + AzureBlobConnectionSetting azureBlobConnection = TestAzureBlobConnectionService.Instance.Settings; + return azureBlobConnection.BlobContainerUri + "/" + databaseName + ".bak"; + } + + private void VerifyAndCleanAzureBlobBackup(string databaseName) + { + AzureBlobConnectionSetting azureBlobConnection = TestAzureBlobConnectionService.Instance.Settings; + string blobUri = GetAzureBlobBackupPath(databaseName); + string accountKey = azureBlobConnection.AccountKey; + string accountName = azureBlobConnection.AccountName; + bool result = BlobDropIfExists(blobUri, accountName, accountKey); + Assert.True(result, "Backup doesn't exists on Azure blob storage"); + } + + public static bool BlobDropIfExists(string blobUri, string accountName, string accountKey) + { + BlobClient client = new BlobClient(new Uri(blobUri), new StorageSharedKeyCredential(accountName, accountKey)); + return client.DeleteIfExists(); + } + + private BackupOperation CreateBackupOperation(DisasterRecoveryService service, string uri, BackupInfo backupInfo, CDataContainer dataContainer, SqlConnection sqlConn) + { + var backupParams = new BackupParams + { + OwnerUri = uri, + BackupInfo = backupInfo, + }; + + return service.CreateBackupOperation(dataContainer, sqlConn, backupParams.BackupInfo); + } + + private async Task VerifyRestore( + string sourceDbName = null, + bool canRestore = true, + TaskExecutionModeFlag executionMode = TaskExecutionModeFlag.None, + string targetDatabase = null, + string[] selectedBackupSets = null, + Dictionary options = null, + Func verifyDatabase = null, + bool shouldFail = false) + { + string backUpFilePath = GetAzureBlobBackupPath(targetDatabase); + + using (SqlTestDb testDb = SqlTestDb.CreateNew(TestServerType.OnPrem, false, "master")) + { + TestConnectionResult connectionResult = await LiveConnectionHelper.InitLiveConnectionInfoAsync("master", testDb.ConnectionString); + + RestoreDatabaseHelper service = new RestoreDatabaseHelper(); + + // If source database is sepecified verfiy it's part of source db list + if (!string.IsNullOrEmpty(sourceDbName)) + { + RestoreConfigInfoResponse configInfoResponse = service.CreateConfigInfoResponse(new RestoreConfigInfoRequestParams + { + OwnerUri = testDb.ConnectionString + }); + IEnumerable dbNames = configInfoResponse.ConfigInfo[RestoreOptionsHelper.SourceDatabaseNamesWithBackupSets] as IEnumerable; + Assert.True(dbNames.Any(x => x == sourceDbName)); + } + var request = new RestoreParams + { + BackupFilePaths = backUpFilePath, + TargetDatabaseName = targetDatabase, + OwnerUri = testDb.ConnectionString, + SelectedBackupSets = selectedBackupSets, + SourceDatabaseName = sourceDbName, + DeviceType = (int)DeviceType.Url + }; + request.Options[RestoreOptionsHelper.ReadHeaderFromMedia] = string.IsNullOrEmpty(backUpFilePath); + + if (options != null) + { + foreach (var item in options) + { + if (!request.Options.ContainsKey(item.Key)) + { + request.Options.Add(item.Key, item.Value); + } + } + } + + var restoreDataObject = service.CreateRestoreDatabaseTaskDataObject(request, connectionResult.ConnectionInfo); + restoreDataObject.ConnectionInfo = connectionResult.ConnectionInfo; + var response = service.CreateRestorePlanResponse(restoreDataObject); + + Assert.NotNull(response); + Assert.False(string.IsNullOrWhiteSpace(response.SessionId)); + Assert.AreEqual(response.CanRestore, canRestore); + if (canRestore) + { + Assert.True(response.DbFiles.Any()); + if (string.IsNullOrEmpty(targetDatabase)) + { + targetDatabase = response.DatabaseName; + } + Assert.AreEqual(response.DatabaseName, targetDatabase); + Assert.NotNull(response.PlanDetails); + Assert.True(response.PlanDetails.Any()); + Assert.NotNull(response.PlanDetails[RestoreOptionsHelper.BackupTailLog]); + Assert.NotNull(response.PlanDetails[RestoreOptionsHelper.TailLogBackupFile]); + Assert.NotNull(response.PlanDetails[RestoreOptionsHelper.DataFileFolder]); + Assert.NotNull(response.PlanDetails[RestoreOptionsHelper.LogFileFolder]); + Assert.NotNull(response.PlanDetails[RestoreOptionsHelper.StandbyFile]); + Assert.NotNull(response.PlanDetails[RestoreOptionsHelper.StandbyFile]); + + if (executionMode != TaskExecutionModeFlag.None) + { + try + { + request.SessionId = response.SessionId; + restoreDataObject = service.CreateRestoreDatabaseTaskDataObject(request); + Assert.AreEqual(response.SessionId, restoreDataObject.SessionId); + request.RelocateDbFiles = !restoreDataObject.DbFilesLocationAreValid(); + restoreDataObject.Execute((TaskExecutionMode)Enum.Parse(typeof(TaskExecutionMode), executionMode.ToString())); + + if (executionMode.HasFlag(TaskExecutionModeFlag.Execute)) + { + Assert.True(restoreDataObject.Server.Databases.Contains(targetDatabase)); + + if (verifyDatabase != null) + { + Assert.True(verifyDatabase(restoreDataObject.Server.Databases[targetDatabase])); + } + + //To verify the backupset that are restored, verifying the database is a better options. + //Some tests still verify the number of backup sets that are executed which in some cases can be less than the selected list + if (verifyDatabase == null && selectedBackupSets != null) + { + Assert.AreEqual(selectedBackupSets.Count(), restoreDataObject.RestorePlanToExecute.RestoreOperations.Count()); + } + } + if (executionMode.HasFlag(TaskExecutionModeFlag.Script)) + { + Assert.False(string.IsNullOrEmpty(restoreDataObject.ScriptContent)); + } + } + catch (Exception ex) + { + if (!shouldFail) + { + Assert.False(true, ex.Message); + } + } + finally + { + await DropDatabase(targetDatabase); + } + } + } + + return response; + } + } + + private async Task DropDatabase(string databaseName) + { + string dropDatabaseQuery = string.Format(CultureInfo.InvariantCulture, + Scripts.DropDatabaseIfExist, databaseName); + + await TestServiceProvider.Instance.RunQueryAsync(TestServerType.OnPrem, "master", dropDatabaseQuery); + } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/AzureBlobConnectionSetting.cs b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/AzureBlobConnectionSetting.cs new file mode 100644 index 00000000..1c040205 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/AzureBlobConnectionSetting.cs @@ -0,0 +1,17 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.SqlTools.ServiceLayer.Test.Common +{ + public class AzureBlobConnectionSetting + { + + public string BlobContainerUri { get; set; } + + public string AccountKey { get; set; } + + public string AccountName { get; set; } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/Constants.cs b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/Constants.cs index c747ab44..6fa476d1 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/Constants.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/Constants.cs @@ -9,6 +9,12 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common { public const string SqlConectionSettingsEnvironmentVariable = "SettingsFileName"; + public const string AzureStorageAccountKey = "AzureStorageAccountKey"; + + public const string AzureStorageAccountName = "AzureStorageAccountName"; + + public const string AzureBlobContainerUri = "AzureBlobContainerUri"; + /// /// Environment variable used to get the TSDATA source directory root. /// K2 is under it. diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestAzureBlobConnectionService.cs b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestAzureBlobConnectionService.cs new file mode 100644 index 00000000..72927db1 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestAzureBlobConnectionService.cs @@ -0,0 +1,65 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; + +namespace Microsoft.SqlTools.ServiceLayer.Test.Common +{ + public class TestAzureBlobConnectionService + { + private static Lazy instance = new Lazy(() => new TestAzureBlobConnectionService()); + private AzureBlobConnectionSetting settings; + + private TestAzureBlobConnectionService() + { + LoadInstanceSettings(); + } + + public static TestAzureBlobConnectionService Instance + { + get + { + return instance.Value; + } + } + + public AzureBlobConnectionSetting Settings + { + get + { + return settings; + } + } + + internal void LoadInstanceSettings() + { + try + { + this.settings = TestAzureBlobConnectionService.InitAzureBlobConnectionSetting(); + } + catch (Exception ex) + { + throw new Exception("Fail to load the SQL connection instances.", ex); + } + } + + internal static AzureBlobConnectionSetting InitAzureBlobConnectionSetting() + { + try + { + AzureBlobConnectionSetting settings = new AzureBlobConnectionSetting(); + settings.AccountKey = Environment.GetEnvironmentVariable(Constants.AzureStorageAccountKey); + settings.AccountName = Environment.GetEnvironmentVariable(Constants.AzureStorageAccountName); + settings.BlobContainerUri = Environment.GetEnvironmentVariable(Constants.AzureBlobContainerUri); + Console.WriteLine("Azure Blob Connection Settings loaded successfully"); + return settings; + } + catch (Exception ex) + { + throw new Exception("Failed to load the azure blob connection settings.", ex); + } + } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.UnitTests/DisasterRecovery/SharedAccessSignatureCreatorTests.cs b/test/Microsoft.SqlTools.ServiceLayer.UnitTests/DisasterRecovery/SharedAccessSignatureCreatorTests.cs new file mode 100644 index 00000000..32b4d33a --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.UnitTests/DisasterRecovery/SharedAccessSignatureCreatorTests.cs @@ -0,0 +1,41 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using Moq; +using NUnit.Framework; +using Azure.Storage.Blobs; +using Microsoft.SqlTools.ServiceLayer.AzureBlob; +using Microsoft.SqlServer.Management.Smo; +using Azure.Storage.Sas; + +namespace Microsoft.SqlTools.ServiceLayer.UnitTests.DisasterRecovery +{ + class SharedAccessSignatureCreatorTests + { + [Test] + public void GetServiceSasUriForContainerReturnsNullWhenCannotGenerateSasUri() + { + var mockBlobContainerClient = new Mock(); + mockBlobContainerClient.Setup(x => x.CanGenerateSasUri).Returns(false); + var mockServer = new Server(); + SharedAccessSignatureCreator sharedAccessSignatureCreator = new SharedAccessSignatureCreator(mockServer); + Assert.Throws(() => sharedAccessSignatureCreator.GetServiceSasUriForContainer(mockBlobContainerClient.Object)); + } + + [Test] + public void GetServiceSasUriForContainerReturnsSasUri() + { + Uri sharedAccessSignatureUriMock = new Uri("https://azureblob/mocked-shared-access-signature"); + var mockBlobContainerClient = new Mock(); + mockBlobContainerClient.Setup(x => x.CanGenerateSasUri).Returns(true); + mockBlobContainerClient.Setup(x => x.GenerateSasUri(It.IsAny())).Returns(sharedAccessSignatureUriMock); + var mockServer = new Server(); + SharedAccessSignatureCreator sharedAccessSignatureCreator = new SharedAccessSignatureCreator(mockServer); + Uri result = sharedAccessSignatureCreator.GetServiceSasUriForContainer(mockBlobContainerClient.Object); + Assert.AreEqual(result, sharedAccessSignatureUriMock); + } + } +}