[Feature] SKU recommendations in SQL migration extension (#1399)

* Initial check in for SQL migration SKU recommendation feature (#1362)

Co-authored-by: Raymond Truong <ratruong@microsoft.com>

* Integration test for Get SKU Recommendation. (#1377)

* Integration test for Get SKU Recommendation.

* Addressing comments -
1) Moving sample files to data folder.
2) Changed Assert for Positive Justification. Ideally for MI we are expecting ~6 justifications but this might change so sticking with 'recommendation should have atleast one positive justification'.

* Implement start/stop perf data collection (#1369)

* Add SqlInstanceRequirements to SKU recommendation output (#1378)

* test for data collection start and stop (#1395)

* improve error handling, add RefreshPerfDataCollection  (#1393)

* WIP - refresh data collection

* Capture messages before logging

* Update Microsoft.SqlServer.Migration.Assessment NuGet version (#1402)

* Update NuGet version to 1.0.20220208.23, update assessment metadata

* Update SKU recommendation metadata

* Include preview SKUs

* Clear message/error queue after refreshing

* Clean up

* Add 'IsCollecting' to RefreshPerfDataCollection (#1405)

Co-authored-by: Neetu Singh <23.neetu@gmail.com>
This commit is contained in:
Raymond Truong
2022-02-10 19:02:27 -08:00
committed by GitHub
parent 4a33da5a18
commit 1693843ab0
35 changed files with 5200 additions and 2973 deletions

View File

@@ -8,6 +8,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Globalization;
using Microsoft.SqlServer.DataCollection.Common;
using Microsoft.SqlServer.Management.Assessment.Checks;
using Microsoft.SqlServer.Management.Assessment;
@@ -15,12 +16,22 @@ using Microsoft.SqlServer.Migration.Assessment.Common.Contracts.Models;
using Microsoft.SqlServer.Migration.Assessment.Common.Engine;
using Microsoft.SqlTools.Hosting.Protocol;
using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
using Microsoft.SqlTools.ServiceLayer.Connection.ReliableConnection;
using Microsoft.SqlTools.ServiceLayer.Connection;
using Microsoft.SqlTools.ServiceLayer.Hosting;
using Microsoft.SqlTools.ServiceLayer.Migration.Contracts;
using Microsoft.SqlTools.ServiceLayer.SqlAssessment;
using Microsoft.SqlTools.Utility;
using Microsoft.SqlServer.Migration.SkuRecommendation.Aggregation;
using Microsoft.SqlServer.Migration.SkuRecommendation.Models.Sql;
using Microsoft.SqlServer.Migration.SkuRecommendation;
using Microsoft.SqlServer.Migration.SkuRecommendation.Contracts.Constants;
using Microsoft.SqlServer.Migration.SkuRecommendation.Billing;
using Microsoft.SqlServer.Migration.SkuRecommendation.Contracts;
using Microsoft.SqlServer.Migration.SkuRecommendation.Contracts.Models.Sku;
using Microsoft.SqlServer.Migration.SkuRecommendation.Contracts.Models;
using Microsoft.SqlServer.Migration.SkuRecommendation.Contracts.Exceptions;
using Newtonsoft.Json;
using System.Reflection;
using Microsoft.SqlServer.Migration.SkuRecommendation.Contracts.Models.Environment;
namespace Microsoft.SqlTools.ServiceLayer.Migration
{
@@ -84,6 +95,15 @@ namespace Microsoft.SqlTools.ServiceLayer.Migration
set;
}
/// <summary>
/// Controller for collecting performance data for SKU recommendation
/// </summary>
internal SqlDataQueryController DataCollectionController
{
get;
set;
}
/// <summary>
/// Initializes the Migration Service instance
/// </summary>
@@ -91,6 +111,10 @@ namespace Microsoft.SqlTools.ServiceLayer.Migration
{
this.ServiceHost = serviceHost;
this.ServiceHost.SetRequestHandler(MigrationAssessmentsRequest.Type, HandleMigrationAssessmentsRequest);
this.ServiceHost.SetRequestHandler(StartPerfDataCollectionRequest.Type, HandleStartPerfDataCollectionRequest);
this.ServiceHost.SetRequestHandler(StopPerfDataCollectionRequest.Type, HandleStopPerfDataCollectionRequest);
this.ServiceHost.SetRequestHandler(RefreshPerfDataCollectionRequest.Type, HandleRefreshPerfDataCollectionRequest);
this.ServiceHost.SetRequestHandler(GetSkuRecommendationsRequest.Type, HandleGetSkuRecommendationsRequest);
}
/// <summary>
@@ -121,7 +145,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Migration
var connection = await ConnectionService.Instance.GetOrOpenConnection(randomUri, ConnectionType.Default);
var connectionStrings = new List<string>();
if (parameters.Databases != null)
if (parameters.Databases != null)
{
foreach (string database in parameters.Databases)
{
@@ -143,6 +167,240 @@ namespace Microsoft.SqlTools.ServiceLayer.Migration
}
}
/// <summary>
/// Handle request to start performance data collection process
/// </summary>
internal async Task HandleStartPerfDataCollectionRequest(
StartPerfDataCollectionParams parameters,
RequestContext<StartPerfDataCollectionResult> requestContext)
{
string randomUri = Guid.NewGuid().ToString();
try
{
// get connection
if (!ConnectionService.TryFindConnection(parameters.OwnerUri, out var connInfo))
{
await requestContext.SendError("Could not find migration connection");
return;
}
ConnectParams connectParams = new ConnectParams
{
OwnerUri = randomUri,
Connection = connInfo.ConnectionDetails,
Type = ConnectionType.Default
};
await ConnectionService.Connect(connectParams);
var connection = await ConnectionService.Instance.GetOrOpenConnection(randomUri, ConnectionType.Default);
var connectionString = ConnectionService.BuildConnectionString(connInfo.ConnectionDetails);
this.DataCollectionController = new SqlDataQueryController(
connectionString,
parameters.DataFolder,
parameters.PerfQueryIntervalInSec,
parameters.NumberOfIterations,
parameters.StaticQueryIntervalInSec,
null);
this.DataCollectionController.Start();
// TO-DO: what should be returned?
await requestContext.SendResult(new StartPerfDataCollectionResult() { DateTimeStarted = DateTime.UtcNow });
}
catch (Exception e)
{
await requestContext.SendError(e.ToString());
}
finally
{
ConnectionService.Disconnect(new DisconnectParams { OwnerUri = randomUri, Type = null });
}
}
/// <summary>
/// Handle request to stop performance data collection process
/// </summary>
internal async Task HandleStopPerfDataCollectionRequest(
StopPerfDataCollectionParams parameters,
RequestContext<StopPerfDataCollectionResult> requestContext)
{
try
{
this.DataCollectionController.Dispose();
// TO-DO: what should be returned?
await requestContext.SendResult(new StopPerfDataCollectionResult() { DateTimeStopped = DateTime.UtcNow });
}
catch (Exception e)
{
await requestContext.SendError(e.ToString());
}
}
/// <summary>
/// Handle request to refresh performance data collection status
/// </summary>
internal async Task HandleRefreshPerfDataCollectionRequest(
RefreshPerfDataCollectionParams parameters,
RequestContext<RefreshPerfDataCollectionResult> requestContext)
{
try
{
bool isCollecting = !(this.DataCollectionController is null) ? this.DataCollectionController.IsRunning() : false;
List<string> messages = !(this.DataCollectionController is null) ? this.DataCollectionController.FetchLatestMessages(parameters.LastRefreshedTime) : new List<string>();
List<string> errors = !(this.DataCollectionController is null) ? this.DataCollectionController.FetchLatestErrors(parameters.LastRefreshedTime) : new List<string>();
RefreshPerfDataCollectionResult result = new RefreshPerfDataCollectionResult()
{
RefreshTime = DateTime.UtcNow,
IsCollecting = isCollecting,
Messages = messages,
Errors = errors,
};
await requestContext.SendResult(result);
}
catch (Exception e)
{
await requestContext.SendError(e.ToString());
}
}
/// <summary>
/// Handle request to generate SKU recommendations
/// </summary>
internal async Task HandleGetSkuRecommendationsRequest(
GetSkuRecommendationsParams parameters,
RequestContext<GetSkuRecommendationsResult> requestContext)
{
try
{
SqlAssessmentConfiguration.EnableLocalLogging = true;
SqlAssessmentConfiguration.ReportsAndLogsRootFolderPath = Path.GetDirectoryName(Logger.LogFileFullPath);
CsvRequirementsAggregator aggregator = new CsvRequirementsAggregator(parameters.DataFolder);
SqlInstanceRequirements req = aggregator.ComputeSqlInstanceRequirements(
agentId: null,
instanceId: parameters.TargetSqlInstance,
targetPercentile: parameters.TargetPercentile,
startTime: DateTime.ParseExact(parameters.StartTime, RecommendationConstants.TimestampDateTimeFormat, CultureInfo.InvariantCulture),
endTime: DateTime.ParseExact(parameters.EndTime, RecommendationConstants.TimestampDateTimeFormat, CultureInfo.InvariantCulture),
collectionInterval: parameters.PerfQueryIntervalInSec,
dbsToInclude: new HashSet<string>(parameters.DatabaseAllowList),
hostRequirements: new SqlServerHostRequirements() { NICCount = 1 });
SkuRecommendationServiceProvider provider = new SkuRecommendationServiceProvider(new AzureSqlSkuBillingServiceProvider());
// generate SQL DB recommendations, if applicable
List<SkuRecommendationResult> sqlDbResults = new List<SkuRecommendationResult>();
if (parameters.TargetPlatforms.Contains("AzureSqlDatabase"))
{
var prefs = new AzurePreferences()
{
EligibleSkuCategories = GetEligibleSkuCategories("AzureSqlDatabase", parameters.IncludePreviewSkus),
ScalingFactor = parameters.ScalingFactor / 100.0
};
sqlDbResults = provider.GetSkuRecommendation(prefs, req);
if (sqlDbResults.Count < parameters.DatabaseAllowList.Count)
{
// if there are fewer recommendations than expected, find which databases didn't have a result generated and create a result with a null SKU
// TO-DO: in the future the NuGet will supply this logic directly so this won't be necessary anymore
List<string> databasesWithRecommendation = sqlDbResults.Select(db => db.DatabaseName).ToList();
foreach (var databaseWithoutRecommendation in parameters.DatabaseAllowList.Where(db => !databasesWithRecommendation.Contains(db)))
{
sqlDbResults.Add(new SkuRecommendationResult()
{
//SqlInstanceName = sqlDbResults.FirstOrDefault().SqlInstanceName,
SqlInstanceName = parameters.TargetSqlInstance,
DatabaseName = databaseWithoutRecommendation,
TargetSku = null,
MonthlyCost = null,
Ranking = -1,
PositiveJustifications = null,
NegativeJustifications = null,
});
}
}
}
// generate SQL MI recommendations, if applicable
List<SkuRecommendationResult> sqlMiResults = new List<SkuRecommendationResult>();
if (parameters.TargetPlatforms.Contains("AzureSqlManagedInstance"))
{
var prefs = new AzurePreferences()
{
EligibleSkuCategories = GetEligibleSkuCategories("AzureSqlManagedInstance", parameters.IncludePreviewSkus),
ScalingFactor = parameters.ScalingFactor / 100.0
};
sqlMiResults = provider.GetSkuRecommendation(prefs, req);
// if no result was generated, create a result with a null SKU
// TO-DO: in the future the NuGet will supply this logic directly so this won't be necessary anymore
if (!sqlMiResults.Any())
{
sqlMiResults.Add(new SkuRecommendationResult()
{
SqlInstanceName = parameters.TargetSqlInstance,
DatabaseName = null,
TargetSku = null,
MonthlyCost = null,
Ranking = -1,
PositiveJustifications = null,
NegativeJustifications = null,
});
}
}
// generate SQL VM recommendations, if applicable
List<SkuRecommendationResult> sqlVmResults = new List<SkuRecommendationResult>();
if (parameters.TargetPlatforms.Contains("AzureSqlVirtualMachine"))
{
var prefs = new AzurePreferences()
{
EligibleSkuCategories = GetEligibleSkuCategories("AzureSqlVirtualMachine", parameters.IncludePreviewSkus),
ScalingFactor = parameters.ScalingFactor / 100.0,
TargetEnvironment = TargetEnvironmentType.Production
};
sqlVmResults = provider.GetSkuRecommendation(prefs, req);
// if no result was generated, create a result with a null SKU
// TO-DO: in the future the NuGet will supply this logic directly so this won't be necessary anymore
if (!sqlVmResults.Any())
{
sqlVmResults.Add(new SkuRecommendationResult()
{
SqlInstanceName = parameters.TargetSqlInstance,
DatabaseName = null,
TargetSku = null,
MonthlyCost = null,
Ranking = -1,
PositiveJustifications = null,
NegativeJustifications = null,
});
}
}
GetSkuRecommendationsResult results = new GetSkuRecommendationsResult
{
SqlDbRecommendationResults = sqlDbResults,
SqlMiRecommendationResults = sqlMiResults,
SqlVmRecommendationResults = sqlVmResults,
InstanceRequirements = req
};
await requestContext.SendResult(results);
}
catch (FailedToQueryCountersException e)
{
await requestContext.SendError($"Unable to read collected performance data from {parameters.DataFolder}. Please specify another folder or start data collection instead.");
}
catch (Exception e)
{
await requestContext.SendError(e.ToString());
}
}
internal class AssessmentRequest : IAssessmentRequest
{
@@ -174,11 +432,11 @@ namespace Microsoft.SqlTools.ServiceLayer.Migration
internal async Task<MigrationAssessmentResult> GetAssessmentItems(string[] connectionStrings)
{
SqlAssessmentConfiguration.EnableLocalLogging = true;
SqlAssessmentConfiguration.AssessmentReportAndLogsRootFolderPath = Path.GetDirectoryName(Logger.LogFileFullPath);
SqlAssessmentConfiguration.ReportsAndLogsRootFolderPath = Path.GetDirectoryName(Logger.LogFileFullPath);
DmaEngine engine = new DmaEngine(connectionStrings);
ISqlMigrationAssessmentModel contextualizedAssessmentResult = await engine.GetTargetAssessmentResultsListWithCheck(System.Threading.CancellationToken.None);
engine.SaveAssessmentResultsToJson(contextualizedAssessmentResult, false);
var server = (contextualizedAssessmentResult.Servers.Count > 0)? ParseServerAssessmentInfo(contextualizedAssessmentResult.Servers[0], engine): null;
var server = (contextualizedAssessmentResult.Servers.Count > 0) ? ParseServerAssessmentInfo(contextualizedAssessmentResult.Servers[0], engine) : null;
return new MigrationAssessmentResult()
{
AssessmentResult = server,
@@ -283,7 +541,118 @@ namespace Microsoft.SqlTools.ServiceLayer.Migration
internal string CreateAssessmentResultKey(ISqlMigrationAssessmentResult assessment)
{
return assessment.ServerName+assessment.DatabaseName+assessment.FeatureId.ToString()+assessment.IssueCategory.ToString()+assessment.Message + assessment.TargetType.ToString() + assessment.AppliesToMigrationTargetPlatform.ToString();
return assessment.ServerName + assessment.DatabaseName + assessment.FeatureId.ToString() + assessment.IssueCategory.ToString() + assessment.Message + assessment.TargetType.ToString() + assessment.AppliesToMigrationTargetPlatform.ToString();
}
// Returns the list of eligible SKUs to consider, depending on the desired target platform
internal static List<AzureSqlSkuCategory> GetEligibleSkuCategories(string targetPlatform, bool includePreviewSkus)
{
List<AzureSqlSkuCategory> eligibleSkuCategories = new List<AzureSqlSkuCategory>();
switch (targetPlatform)
{
case "AzureSqlDatabase":
// Gen5 BC/GP DB
eligibleSkuCategories.Add(new AzureSqlSkuPaaSCategory(
AzureSqlTargetPlatform.AzureSqlDatabase,
AzureSqlPurchasingModel.vCore,
AzureSqlPaaSServiceTier.BusinessCritical,
ComputeTier.Provisioned,
AzureSqlPaaSHardwareType.Gen5));
eligibleSkuCategories.Add(new AzureSqlSkuPaaSCategory(
AzureSqlTargetPlatform.AzureSqlDatabase,
AzureSqlPurchasingModel.vCore,
AzureSqlPaaSServiceTier.GeneralPurpose,
ComputeTier.Provisioned,
AzureSqlPaaSHardwareType.Gen5));
break;
case "AzureSqlManagedInstance":
// Gen5 BC/GP MI
eligibleSkuCategories.Add(new AzureSqlSkuPaaSCategory(
AzureSqlTargetPlatform.AzureSqlManagedInstance,
AzureSqlPurchasingModel.vCore,
AzureSqlPaaSServiceTier.BusinessCritical,
ComputeTier.Provisioned,
AzureSqlPaaSHardwareType.Gen5));
eligibleSkuCategories.Add(new AzureSqlSkuPaaSCategory(
AzureSqlTargetPlatform.AzureSqlManagedInstance,
AzureSqlPurchasingModel.vCore,
AzureSqlPaaSServiceTier.GeneralPurpose,
ComputeTier.Provisioned,
AzureSqlPaaSHardwareType.Gen5));
if (includePreviewSkus)
{
// Premium BC/GP
eligibleSkuCategories.Add(new AzureSqlSkuPaaSCategory(
AzureSqlTargetPlatform.AzureSqlManagedInstance,
AzureSqlPurchasingModel.vCore,
AzureSqlPaaSServiceTier.BusinessCritical,
ComputeTier.Provisioned,
AzureSqlPaaSHardwareType.PremiumSeries));
eligibleSkuCategories.Add(new AzureSqlSkuPaaSCategory(
AzureSqlTargetPlatform.AzureSqlManagedInstance,
AzureSqlPurchasingModel.vCore,
AzureSqlPaaSServiceTier.GeneralPurpose,
ComputeTier.Provisioned,
AzureSqlPaaSHardwareType.PremiumSeries));
// Premium Memory Optimized BC/GP
eligibleSkuCategories.Add(new AzureSqlSkuPaaSCategory(
AzureSqlTargetPlatform.AzureSqlManagedInstance,
AzureSqlPurchasingModel.vCore,
AzureSqlPaaSServiceTier.BusinessCritical,
ComputeTier.Provisioned,
AzureSqlPaaSHardwareType.PremiumSeriesMemoryOptimized));
eligibleSkuCategories.Add(new AzureSqlSkuPaaSCategory(
AzureSqlTargetPlatform.AzureSqlManagedInstance,
AzureSqlPurchasingModel.vCore,
AzureSqlPaaSServiceTier.GeneralPurpose,
ComputeTier.Provisioned,
AzureSqlPaaSHardwareType.PremiumSeriesMemoryOptimized));
}
break;
case "AzureSqlVirtualMachine":
string assemblyPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
// load Azure VM capabilities
string jsonFile = File.ReadAllText(Path.Combine(assemblyPath, RecommendationConstants.DataFolder, RecommendationConstants.SqlVmCapability));
List<AzureSqlIaaSCapability> vmCapabilities = JsonConvert.DeserializeObject<List<AzureSqlIaaSCapability>>(jsonFile);
if (includePreviewSkus)
{
// Eb series (in preview) capabilities stored separately
string computePreviewFilePath = Path.Combine(assemblyPath, RecommendationConstants.DataFolder, RecommendationConstants.SqlVmPreviewCapability);
if (File.Exists(computePreviewFilePath))
{
jsonFile = File.ReadAllText(computePreviewFilePath);
List<AzureSqlIaaSCapability> vmPreviewCapabilities = JsonConvert.DeserializeObject<List<AzureSqlIaaSCapability>>(jsonFile);
vmCapabilities.AddRange(vmPreviewCapabilities);
}
}
foreach (VirtualMachineFamily family in AzureVirtualMachineFamilyGroup.FamilyGroups[VirtualMachineFamilyType.GeneralPurpose]
.Concat(AzureVirtualMachineFamilyGroup.FamilyGroups[VirtualMachineFamilyType.MemoryOptimized]))
{
var skus = vmCapabilities.Where(c => string.Equals(c.Family, family.ToString(), StringComparison.OrdinalIgnoreCase)).Select(c => c.Name);
AzureSqlSkuIaaSCategory category = new AzureSqlSkuIaaSCategory(family);
category.AvailableVmSkus.AddRange(skus);
eligibleSkuCategories.Add(category);
}
break;
default:
break;
}
return eligibleSkuCategories;
}
/// <summary>
@@ -293,6 +662,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Migration
{
if (!disposed)
{
this.DataCollectionController.Dispose();
disposed = true;
}
}