From 4867a3747c0a396f89ea61e6edbf2fa44c33fec0 Mon Sep 17 00:00:00 2001 From: Raymond Truong Date: Wed, 29 Mar 2023 12:48:22 -0700 Subject: [PATCH] [SQL Migration] Add storage/MI connectivity validation (#22410) * wip * Add SQL VM POC * Undo azurecore changes * Add warning banner instead of blocking on next * Fix warning banner behavior * Add private endpoint support * Fix navigation issue * Add offline scenario * Address PR comments * Fix merge conflicts --- .../dataModels/azure/networkInterfaceModel.ts | 25 ++++++- extensions/sql-migration/src/api/sqlUtils.ts | 74 ++++++++++++++++++- .../sql-migration/src/constants/strings.ts | 5 ++ .../src/wizard/databaseBackupPage.ts | 71 ++++++++++++++++-- 4 files changed, 164 insertions(+), 11 deletions(-) diff --git a/extensions/sql-migration/src/api/dataModels/azure/networkInterfaceModel.ts b/extensions/sql-migration/src/api/dataModels/azure/networkInterfaceModel.ts index a46e9c0bf8..374818bcfd 100644 --- a/extensions/sql-migration/src/api/dataModels/azure/networkInterfaceModel.ts +++ b/extensions/sql-migration/src/api/dataModels/azure/networkInterfaceModel.ts @@ -32,7 +32,8 @@ export interface NetworkInterfaceIpConfiguration extends NetworkResource { privateIPAddress: string, privateIPAddressVersion: string, provisioningState: string, - publicIPAddress: NetworkResource + publicIPAddress: NetworkResource, + subnet: { id: string } } } @@ -42,6 +43,19 @@ export interface PublicIpAddress extends NetworkResource { } } +export interface PrivateEndpointConnection extends NetworkResource { + properties: { + privateEndpoint: { id: string }, + privateLinkServiceConnectionState: { description: string, status: string } + } +} + +export interface PrivateEndpoint extends NetworkResource { + properties: { + subnet: { id: string } + } +} + export class NetworkInterfaceModel { public static IPv4VersionType = "IPv4".toLocaleLowerCase(); private static NETWORK_API_VERSION = '2022-09-01'; @@ -145,4 +159,13 @@ export class NetworkInterfaceModel { return networkInterfaces; } + + + public static async getPrivateEndpoint(account: azdata.Account, subscription: Subscription, privateEndpointId: string): Promise { + return getAzureResourceGivenId(account, subscription, privateEndpointId, this.NETWORK_API_VERSION); + } + + public static getVirtualNetworkFromSubnet(subnetId: string): string { + return subnetId.replace(RegExp('^(.*?)/virtualNetworks/'), '').replace(RegExp('/subnets/.*'), '').toLowerCase(); + } } diff --git a/extensions/sql-migration/src/api/sqlUtils.ts b/extensions/sql-migration/src/api/sqlUtils.ts index 8a9597ac90..1c869cba12 100644 --- a/extensions/sql-migration/src/api/sqlUtils.ts +++ b/extensions/sql-migration/src/api/sqlUtils.ts @@ -5,11 +5,12 @@ import * as azdata from 'azdata'; import { azureResource } from 'azurecore'; -import { AzureSqlDatabase, AzureSqlDatabaseServer } from './azure'; -import { generateGuid } from './utils'; +import { AzureSqlDatabase, AzureSqlDatabaseServer, SqlManagedInstance, SqlVMServer, StorageAccount, Subscription } from './azure'; +import { generateGuid, MigrationTargetType } from './utils'; import * as utils from '../api/utils'; import { TelemetryAction, TelemetryViews, logError } from '../telemetry'; import * as constants from '../constants/strings'; +import { NetworkInterfaceModel, PrivateEndpointConnection } from './dataModels/azure/networkInterfaceModel'; const query_database_tables_sql = ` SELECT @@ -509,3 +510,72 @@ export async function isSourceConnectionSysAdmin(): Promise { return getSqlBoolean(results.rows[0][0]); } + +export async function canTargetConnectToStorageAccount( + targetType: MigrationTargetType, + targetServer: SqlManagedInstance | SqlVMServer | AzureSqlDatabaseServer, + storageAccount: StorageAccount, + account: azdata.Account, + subscription: Subscription): Promise { + + // additional ARM properties of storage accounts which aren't exposed in azurecore + interface StorageAccountAdditionalProperties { + publicNetworkAccess: string, + networkAcls: NetworkRuleSet, + privateEndpointConnections: PrivateEndpointConnection[] + } + interface NetworkRuleSet { + virtualNetworkRules: VirtualNetworkRule[], + defaultAction: string + } + interface VirtualNetworkRule { + id: string, + state: string, + action: string + } + + const ENABLED = 'Enabled'; + const ALLOW = 'Allow'; + + const storageAccountProperties: StorageAccountAdditionalProperties = (storageAccount as any)['properties']; + const storageAccountPublicAccessEnabled: boolean = storageAccountProperties.publicNetworkAccess ? storageAccountProperties.publicNetworkAccess.toLowerCase() === ENABLED.toLowerCase() : true; + const storageAccountDefaultIsAllow: boolean = storageAccountProperties.networkAcls ? storageAccountProperties.networkAcls.defaultAction.toLowerCase() === ALLOW.toLowerCase() : true; + const storageAccountWhitelistedVNets: string[] = storageAccountProperties.networkAcls ? storageAccountProperties.networkAcls.virtualNetworkRules.filter(rule => rule.action.toLowerCase() === ALLOW.toLowerCase()).map(rule => rule.id) : []; + + var enabledFromAllNetworks: boolean = false; + var enabledFromWhitelistedVNet: boolean = false; + var enabledFromPrivateEndpoint: boolean = false; + + // 1) check for access from all networks + enabledFromAllNetworks = storageAccountPublicAccessEnabled && storageAccountDefaultIsAllow; + + switch (targetType) { + case MigrationTargetType.SQLMI: + const targetManagedInstanceVNet: string = (targetServer.properties as any)['subnetId'] ?? ''; + const targetManagedInstancePrivateEndpointConnections: PrivateEndpointConnection[] = (targetServer.properties as any)['privateEndpointConnections'] ?? []; + const storageAccountPrivateEndpointConnections: PrivateEndpointConnection[] = storageAccountProperties.privateEndpointConnections ?? []; + + // 2) check for access from whitelisted vnet + if (storageAccountWhitelistedVNets.length > 0) { + enabledFromWhitelistedVNet = storageAccountWhitelistedVNets.some(vnet => vnet.toLowerCase() === targetManagedInstanceVNet.toLowerCase()); + } + + // 3) check for access from private endpoint + if (targetManagedInstancePrivateEndpointConnections.length > 0) { + enabledFromPrivateEndpoint = storageAccountPrivateEndpointConnections.some(async privateEndpointConnection => { + const privateEndpoint = await NetworkInterfaceModel.getPrivateEndpoint(account, subscription, privateEndpointConnection.id); + const privateEndpointSubnet = privateEndpoint.properties.subnet ? privateEndpoint.properties.subnet.id : ''; + return NetworkInterfaceModel.getVirtualNetworkFromSubnet(privateEndpointSubnet).toLowerCase() === NetworkInterfaceModel.getVirtualNetworkFromSubnet(targetManagedInstanceVNet).toLowerCase(); + }); + } + + break; + case MigrationTargetType.SQLVM: + // to-do: VM scenario -- get subnet by first checking underlying compute VM, then its network interface + return true; + default: + return true; + } + + return enabledFromAllNetworks || enabledFromWhitelistedVNet || enabledFromPrivateEndpoint; +} diff --git a/extensions/sql-migration/src/constants/strings.ts b/extensions/sql-migration/src/constants/strings.ts index 719a411920..3ce6db0aec 100644 --- a/extensions/sql-migration/src/constants/strings.ts +++ b/extensions/sql-migration/src/constants/strings.ts @@ -612,6 +612,11 @@ export const INVALID_RESOURCE_GROUP_ERROR = localize('sql.migration.invalid.reso export const INVALID_STORAGE_ACCOUNT_ERROR = localize('sql.migration.invalid.storageAccount.error', "To continue, select a valid storage account."); export const MISSING_TARGET_USERNAME_ERROR = localize('sql.migration.missing.targetUserName.error', "To continue, enter a valid target user name."); export const MISSING_TARGET_PASSWORD_ERROR = localize('sql.migration.missing.targetPassword.error', "To continue, enter a valid target password."); +export function STORAGE_ACCOUNT_CONNECTIVITY_WARNING(targetServer: string, databases: string[]): string { + return databases.length === 1 + ? localize('sql.migration.storageAccount.warning.many', "Target instance '{0}' may not be able to access storage account '{1}'. Ensure that the subnet of the target instance is whitelisted on the storage account, and if applicable, that the private endpoint is in the same virtual network as the target server.", targetServer, databases[0]) + : localize('sql.migration.storageAccount.warning.one', "Target instance '{0}' may not be able to access storage accounts '{1}'. Ensure that the subnet of the target instance is whitelisted on the storage accounts, and if applicable, that the private endpoints are on the same virtual network as the target server.", targetServer, databases.join("', '")); +} export const TARGET_TABLE_NOT_EMPTY = localize('sql.migration.target.table.not.empty', "Target table is not empty."); export const TARGET_TABLE_MISSING = localize('sql.migration.target.table.missing', "Target table does not exist"); diff --git a/extensions/sql-migration/src/wizard/databaseBackupPage.ts b/extensions/sql-migration/src/wizard/databaseBackupPage.ts index b4c5e259b4..e933c3b084 100644 --- a/extensions/sql-migration/src/wizard/databaseBackupPage.ts +++ b/extensions/sql-migration/src/wizard/databaseBackupPage.ts @@ -18,7 +18,7 @@ import { logError, TelemetryViews } from '../telemetry'; import * as styles from '../constants/styles'; import { TableMigrationSelectionDialog } from '../dialog/tableMigrationSelection/tableMigrationSelectionDialog'; import { ValidateIrDialog } from '../dialog/validationResults/validateIrDialog'; -import { getSourceConnectionCredentials, getSourceConnectionProfile, getSourceConnectionQueryProvider, getSourceConnectionUri } from '../api/sqlUtils'; +import { canTargetConnectToStorageAccount, getSourceConnectionCredentials, getSourceConnectionProfile, getSourceConnectionQueryProvider, getSourceConnectionUri } from '../api/sqlUtils'; const WIZARD_TABLE_COLUMN_WIDTH = '200px'; const WIZARD_TABLE_COLUMN_WIDTH_SMALL = '170px'; @@ -71,6 +71,7 @@ export class DatabaseBackupPage extends MigrationWizardPage { private _existingDatabases: string[] = []; private _nonPageBlobErrors: string[] = []; + private _inaccessibleStorageAccounts: string[] = []; private _disposables: vscode.Disposable[] = []; // SQL DB table selection @@ -605,7 +606,7 @@ export class DatabaseBackupPage extends MigrationWizardPage { fireOnTextChange: true, }).component(); this._disposables.push( - this._networkShareContainerStorageAccountDropdown.onValueChanged((value) => { + this._networkShareContainerStorageAccountDropdown.onValueChanged(async (value) => { if (value && value !== 'undefined') { const selectedStorageAccount = this.migrationStateModel._storageAccounts.find(sa => sa.name === value); if (selectedStorageAccount) { @@ -613,6 +614,22 @@ export class DatabaseBackupPage extends MigrationWizardPage { this.migrationStateModel._databaseBackup.networkShares[i].storageAccount = selectedStorageAccount; } this.migrationStateModel.resetIrValidationResults(); + + // check for storage account connectivity + if ((this.migrationStateModel.isSqlMiTarget || this.migrationStateModel.isSqlVmTarget)) { + if (!(await canTargetConnectToStorageAccount(this.migrationStateModel._targetType, this.migrationStateModel._targetServerInstance, selectedStorageAccount, this.migrationStateModel._azureAccount, this.migrationStateModel._targetSubscription))) { + this._inaccessibleStorageAccounts = [selectedStorageAccount.name]; + } else { + this._inaccessibleStorageAccounts = []; + } + + this.wizard.message = { + text: this._inaccessibleStorageAccounts.length > 0 + ? constants.STORAGE_ACCOUNT_CONNECTIVITY_WARNING(this.migrationStateModel._targetServerInstance.name, this._inaccessibleStorageAccounts) + : '', + level: azdata.window.MessageLevel.Warning + } + } } } })); @@ -628,7 +645,26 @@ export class DatabaseBackupPage extends MigrationWizardPage { this._disposables.push( this._networkShareContainerStorageAccountRefreshButton.onDidClick( - value => this.loadNetworkShareStorageDropdown())); + async () => { + this.loadNetworkShareStorageDropdown(); + + // check for storage account connectivity + const selectedStorageAccount = this.migrationStateModel._storageAccounts.find(sa => sa.name === (this._networkShareContainerStorageAccountDropdown.value as azdata.CategoryValue).displayName); + if ((this.migrationStateModel.isSqlMiTarget || this.migrationStateModel.isSqlVmTarget) && selectedStorageAccount) { + if (!(await canTargetConnectToStorageAccount(this.migrationStateModel._targetType, this.migrationStateModel._targetServerInstance, selectedStorageAccount, this.migrationStateModel._azureAccount, this.migrationStateModel._targetSubscription))) { + this._inaccessibleStorageAccounts = [selectedStorageAccount.name]; + } else { + this._inaccessibleStorageAccounts = []; + } + + this.wizard.message = { + text: this._inaccessibleStorageAccounts.length > 0 + ? constants.STORAGE_ACCOUNT_CONNECTIVITY_WARNING(this.migrationStateModel._targetServerInstance.name, this._inaccessibleStorageAccounts) + : '', + level: azdata.window.MessageLevel.Warning + } + } + })); const storageAccountContainer = this._view.modelBuilder.flexContainer() .withProps({ CSSStyles: { 'margin-top': '-1em' } }) @@ -825,10 +861,6 @@ export class DatabaseBackupPage extends MigrationWizardPage { } this._blobContainerTargetDatabaseNamesTable.columns[folderColumnIndex].hidden = folderColumnNewHidden; - - - - const connectionProfile = await getSourceConnectionProfile(); const queryProvider = await getSourceConnectionQueryProvider(); let username = ''; @@ -871,6 +903,7 @@ export class DatabaseBackupPage extends MigrationWizardPage { this._blobContainerDropdowns = []; this._blobContainerLastBackupFileDropdowns = []; this._blobContainerFolderDropdowns = []; + this._inaccessibleStorageAccounts = []; if (this.migrationStateModel.isSqlMiTarget) { this._existingDatabases = await this.migrationStateModel.getManagedDatabases(); @@ -1073,7 +1106,28 @@ export class DatabaseBackupPage extends MigrationWizardPage { if (value && value !== 'undefined') { const selectedStorageAccount = this.migrationStateModel._storageAccounts.find(sa => sa.name === value); if (selectedStorageAccount && !blobStorageAccountErrorStrings.includes(value)) { + const oldSelectedStorageAccount = this.migrationStateModel._databaseBackup.blobs[index].storageAccount ? this.migrationStateModel._databaseBackup.blobs[index].storageAccount.name : ''; this.migrationStateModel._databaseBackup.blobs[index].storageAccount = selectedStorageAccount; + + // check for storage account connectivity + if ((this.migrationStateModel.isSqlMiTarget || this.migrationStateModel.isSqlVmTarget)) { + if (this.migrationStateModel._databaseBackup.blobs.filter((e, i) => i !== index).every(blob => blob.storageAccount && blob.storageAccount.name.toLowerCase() !== oldSelectedStorageAccount.toLowerCase())) { + this._inaccessibleStorageAccounts = this._inaccessibleStorageAccounts.filter(storageAccountName => storageAccountName.toLowerCase() !== oldSelectedStorageAccount.toLowerCase()); + } + + if (!(await canTargetConnectToStorageAccount(this.migrationStateModel._targetType, this.migrationStateModel._targetServerInstance, selectedStorageAccount, this.migrationStateModel._azureAccount, this.migrationStateModel._targetSubscription))) { + this._inaccessibleStorageAccounts = this._inaccessibleStorageAccounts.filter(storageAccountName => storageAccountName.toLowerCase() !== selectedStorageAccount.name.toLowerCase()); + this._inaccessibleStorageAccounts.push(selectedStorageAccount.name); + } + + this.wizard.message = { + text: this._inaccessibleStorageAccounts.length > 0 + ? constants.STORAGE_ACCOUNT_CONNECTIVITY_WARNING(this.migrationStateModel._targetServerInstance.name, this._inaccessibleStorageAccounts) + : '', + level: azdata.window.MessageLevel.Warning + } + } + await this.loadBlobContainerDropdown(index); await blobContainerDropdown.updateProperties({ enabled: true }); } else { @@ -1090,6 +1144,7 @@ export class DatabaseBackupPage extends MigrationWizardPage { if (selectedBlobContainer && !blobContainerErrorStrings.includes(value)) { this.migrationStateModel._databaseBackup.blobs[index].blobContainer = selectedBlobContainer; + // check for block blobs for SQL VM <= 2014 if (this.migrationStateModel.isSqlVmTarget && utils.isTargetSqlVm2014OrBelow(this.migrationStateModel._targetServerInstance as SqlVMServer)) { const backups = await utils.getBlobLastBackupFileNames( this.migrationStateModel._azureAccount, @@ -1137,7 +1192,7 @@ export class DatabaseBackupPage extends MigrationWizardPage { this._disposables.push( blobContainerFolderDropdown.onValueChanged(value => { if (value && value !== 'undefined') { - if (this.migrationStateModel._blobContainerFolders.includes(value) && !blobFolderErrorStrings.includes(value)) { + if (this.migrationStateModel._blobContainerFolders && this.migrationStateModel._blobContainerFolders.includes(value) && !blobFolderErrorStrings.includes(value)) { const selectedFolder = value; this.migrationStateModel._databaseBackup.blobs[index].folderName = selectedFolder; }