mirror of
https://github.com/ckaczor/sqltoolsservice.git
synced 2026-01-27 01:25:42 -05:00
Added new Kusto ServiceLayer (#1009)
* Copy smoModel some rename * Copy entire service layer * Building copy * Fixing some references * Launch profile * Resolve namespace issues * Compiling tests. Correct manifest. * Fixing localization resources * ReliableKustoClient * Some trimming of extra code and Kusto code * Kusto client creation in bindingContent * Removing Smo and new Kusto classes * More trimming * Kusto schema hookup * Solidying DataSource abstraction * Solidifying further * Latest refatoring * More refactoring * Building and launching Kusto service layer * Working model which enumerates databases * Refactoring to pass IDataSource to all tree nodes * Removing some dependencies on the context * Working with tables and schema * Comment checkin * Refactoring to give out select script * Query created and sent back to ADS * Fix query generation * Fix listing of databases * Tunneling the query through. * Successful query execution * Return only results table * Deleting Cms * Delete DacFx * Delete SchemaCompare and TaskServices * Change build definition to not stop at launch * Fix error after merge * Save Kusto results in different formats (#935) * save results as csv etc * some fixes Co-authored-by: Monica Gupta <mogupt@microsoft.com> * 2407 Added OrderBy clause in KustoDataSource > GetDatabaseMetaData and GetColumnMetadata (#959) * 2405 Defaulted Options when setting ServerInfo in ConnectionService > GetConnectionCompleteParams (#965) * 2747 Fixed IsUnknownType error for Kusto (#989) * 2747 Removed unused directives in Kusto > DbColumnWrapper. Refactored IsUnknownType to handle null DataTypeName * 2747 Reverted IsUnknownType change in DbColumnWrapper. Changed DataTypeName to get calue from ColumnType. Refactored SafeGetValue to type check before hard casting to reduce case exceptions. * Added EmbeddedResourceUseDependentUponConvention to Microsoft.Kusto.ServiceLayer.csproj. Also renamed DACfx to match Microsoft.SqlTools.ServiceLayer. Added to compile Exclude="**/obj/**/*.cs" * Srahman cleanup sql code (#992) * Removed Management and Security Service Code. * Remove FileBrowser service * Comment why we are using SqlServer library * Remove SQL specific type definitions * clean up formatter service (#996) Co-authored-by: Monica Gupta <mogupt@microsoft.com> * Code clean up and Kusto intellisense (#994) * Code clean up and Kusto intellisense * Addressed few comments * Addressed few comments * addressed comments Co-authored-by: Monica Gupta <mogupt@microsoft.com> * Return multiple tables for Kusto * Changes required for Kusto manage dashboard (#1039) * Changes required for manage dashboard * Addressed comments Co-authored-by: Monica Gupta <mogupt@microsoft.com> * 2728 Kusto function support (#1038) * loc update (#914) * loc update * loc updates * 2728 moved ColumnInfo and KustoResultsReader to separate files. Added Folder and Function to TreeNode.cs * 2728 Added FunctionInfo. Added Folder to ColumnInfo. Removed partial class from KustoResultsReader. Set Function.IsAlwaysLeaf=true in TreeNode.cs. In KustoDataSource changed tableMetadata type to TableMetaData. Added folder and function dictionaries. Refactored GetSchema function. Renamed GenerateColumnMetadataKey to GenerateMetadataKey * 2728 Added FunctionInfo. Added Folder to ColumnInfo. Removed partial class from KustoResultsReader. Set Function.IsAlwaysLeaf=true in TreeNode.cs. In KustoDataSource changed tableMetadata type to TableMetaData. Added folder and function dictionaries. Refactored GetSchema function. Renamed GenerateColumnMetadataKey to GenerateMetadataKey * 2728 Created new SqlConnection within using block. Refactored KustoDataSource > columnmetadata to sort on get instead of insert. * 2728 Added GetFunctionInfo function to KustoDataSource. * 2728 Reverted change to Microsoft.Kusto.ServiceLayer.csproj from merge * 2728 Reverted change to SqlTools.ServiceLayer\Localization\transXliff * 2728 Reverted change to sr.de.xlf and sr.zh-hans.xlf * 2728 Refactored KustoDataSource Function folders to support subfolders * 2728 Refactored KustoDataSource to use urn for folders, functions, and tables instead of name. * Merge remote-tracking branch 'origin/main' into feature-ADE # Conflicts: # Packages.props * 2728 Moved metadata files into Metadata subdirectory. Added GenerateAlterFunction to IDataSource and DataSourceBase. * 2728 Added summary information to SafeAdd in SystemExtensions. Renamed local variable in SetTableMetadata * 2728 Moved SafeAdd from SystemExtensions to KustoQueryUtils. Added check when getting database schema to return existing records before querying again. Added AddRange function to KustoQueryUtils. Created SetFolderMetadataForFunctions method. * 2728 Added DatabaseKeyPrefix to only return tables to a database for the dashboard. Added logic to store all database tables within the tableMetadata dictionary for the dashboard. * 2728 Created TableInfo and moved info objects into Models directory. Refactored KustoDataSource to lazy load columns for tables. Refactored logic to load tables using cslschema instead of schema. * 2728 Renamed LoadColumnSchema to GetTableSchema to be consistent. Co-authored-by: khoiph1 <khoiph@microsoft.com> * Addressed comments Co-authored-by: Shafiq Rahman <srahman@microsoft.com> Co-authored-by: Monica Gupta <mogupt@microsoft.com> Co-authored-by: Justin M <63619224+JustinMDotNet@users.noreply.github.com> Co-authored-by: rkselfhost <rkselfhost@outlook.com> Co-authored-by: khoiph1 <khoiph@microsoft.com>
This commit is contained in:
Monica Gupta
@@ -0,0 +1,298 @@
|
||||
//
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
//
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data.SqlClient;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using Microsoft.SqlServer.Management.SqlScriptPublish;
|
||||
using Microsoft.Kusto.ServiceLayer.Scripting.Contracts;
|
||||
using Microsoft.SqlTools.Utility;
|
||||
|
||||
namespace Microsoft.Kusto.ServiceLayer.Scripting
|
||||
{
|
||||
/// <summary>
|
||||
/// Class to represent an in-progress script operation.
|
||||
/// </summary>
|
||||
public sealed class ScriptingScriptOperation : SmoScriptingOperation
|
||||
{
|
||||
|
||||
private int scriptedObjectCount = 0;
|
||||
|
||||
private int totalScriptedObjectCount = 0;
|
||||
|
||||
private int eventSequenceNumber = 1;
|
||||
|
||||
private string azureAccessToken;
|
||||
|
||||
public ScriptingScriptOperation(ScriptingParams parameters, string azureAccessToken): base(parameters)
|
||||
{
|
||||
this.azureAccessToken = azureAccessToken;
|
||||
}
|
||||
|
||||
public override void Execute()
|
||||
{
|
||||
SqlScriptPublishModel publishModel = null;
|
||||
|
||||
try
|
||||
{
|
||||
this.CancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
this.ValidateScriptDatabaseParams();
|
||||
|
||||
publishModel = BuildPublishModel();
|
||||
publishModel.ScriptItemsCollected += this.OnPublishModelScriptItemsCollected;
|
||||
publishModel.ScriptProgress += this.OnPublishModelScriptProgress;
|
||||
publishModel.ScriptError += this.OnPublishModelScriptError;
|
||||
publishModel.AllowSystemObjects = true;
|
||||
|
||||
ScriptDestination destination = !string.IsNullOrWhiteSpace(this.Parameters.ScriptDestination)
|
||||
? (ScriptDestination)Enum.Parse(typeof(ScriptDestination), this.Parameters.ScriptDestination)
|
||||
: ScriptDestination.ToSingleFile;
|
||||
|
||||
// SMO is currently hardcoded to produce UTF-8 encoding when running on dotnet core.
|
||||
ScriptOutputOptions outputOptions = new ScriptOutputOptions
|
||||
{
|
||||
SaveFileMode = ScriptFileMode.Overwrite,
|
||||
SaveFileName = this.Parameters.FilePath,
|
||||
ScriptDestination = destination,
|
||||
};
|
||||
|
||||
this.CancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
publishModel.GenerateScript(outputOptions);
|
||||
|
||||
this.CancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
Logger.Write(
|
||||
TraceEventType.Verbose,
|
||||
string.Format(
|
||||
"Sending script complete notification event for operation {0}, sequence number {1} with total count {2} and scripted count {3}",
|
||||
this.OperationId,
|
||||
this.eventSequenceNumber,
|
||||
this.totalScriptedObjectCount,
|
||||
this.scriptedObjectCount));
|
||||
|
||||
ScriptText = publishModel.RawScript;
|
||||
|
||||
this.SendCompletionNotificationEvent(new ScriptingCompleteParams
|
||||
{
|
||||
Success = true,
|
||||
});
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
if (e.IsOperationCanceledException())
|
||||
{
|
||||
Logger.Write(TraceEventType.Information, string.Format("Scripting operation {0} was canceled", this.OperationId));
|
||||
this.SendCompletionNotificationEvent(new ScriptingCompleteParams
|
||||
{
|
||||
Canceled = true,
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.Write(TraceEventType.Error, string.Format("Scripting operation {0} failed with exception {1}", this.OperationId, e));
|
||||
this.SendCompletionNotificationEvent(new ScriptingCompleteParams
|
||||
{
|
||||
HasError = true,
|
||||
ErrorMessage = e.Message,
|
||||
ErrorDetails = e.ToString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (publishModel != null)
|
||||
{
|
||||
publishModel.ScriptItemsCollected -= this.OnPublishModelScriptItemsCollected;
|
||||
publishModel.ScriptProgress -= this.OnPublishModelScriptProgress;
|
||||
publishModel.ScriptError -= this.OnPublishModelScriptError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected override void SendCompletionNotificationEvent(ScriptingCompleteParams parameters)
|
||||
{
|
||||
base.SendCompletionNotificationEvent(parameters);
|
||||
}
|
||||
|
||||
protected override void SendPlanNotificationEvent(ScriptingPlanNotificationParams parameters)
|
||||
{
|
||||
base.SendPlanNotificationEvent(parameters);
|
||||
}
|
||||
|
||||
protected override void SendProgressNotificationEvent(ScriptingProgressNotificationParams parameters)
|
||||
{
|
||||
base.SendProgressNotificationEvent(parameters);
|
||||
}
|
||||
|
||||
protected override void SetCommonEventProperties(ScriptingEventParams parameters)
|
||||
{
|
||||
base.SetCommonEventProperties(parameters);
|
||||
parameters.SequenceNumber = this.eventSequenceNumber;
|
||||
this.eventSequenceNumber += 1;
|
||||
}
|
||||
|
||||
private SqlScriptPublishModel BuildPublishModel()
|
||||
{
|
||||
SqlScriptPublishModel publishModel = new SqlScriptPublishModel(this.Parameters.ConnectionString);
|
||||
|
||||
// See if any filtering criteria was specified. If not, we're scripting the entire database. Otherwise, the filtering
|
||||
// criteria should include the target objects to script.
|
||||
//
|
||||
bool hasObjectsSpecified = this.Parameters.ScriptingObjects != null && this.Parameters.ScriptingObjects.Any();
|
||||
bool hasCriteriaSpecified =
|
||||
(this.Parameters.IncludeObjectCriteria != null && this.Parameters.IncludeObjectCriteria.Any()) ||
|
||||
(this.Parameters.ExcludeObjectCriteria != null && this.Parameters.ExcludeObjectCriteria.Any()) ||
|
||||
(this.Parameters.IncludeSchemas != null && this.Parameters.IncludeSchemas.Any()) ||
|
||||
(this.Parameters.ExcludeSchemas != null && this.Parameters.ExcludeSchemas.Any()) ||
|
||||
(this.Parameters.IncludeTypes != null && this.Parameters.IncludeTypes.Any()) ||
|
||||
(this.Parameters.ExcludeTypes != null && this.Parameters.ExcludeTypes.Any());
|
||||
bool scriptAllObjects = !hasObjectsSpecified && !hasCriteriaSpecified;
|
||||
|
||||
// In the getter for SqlScriptPublishModel.AdvancedOptions, there is some strange logic which will
|
||||
// cause the SqlScriptPublishModel.AdvancedOptions to get reset and lose all values based the ordering
|
||||
// of when SqlScriptPublishModel.ScriptAllObjects is set.
|
||||
//
|
||||
publishModel.ScriptAllObjects = scriptAllObjects;
|
||||
if (scriptAllObjects)
|
||||
{
|
||||
// Due to the getter logic within publishModel.AdvancedOptions, we explicitly populate the options
|
||||
// after we determine what objects we are scripting.
|
||||
//
|
||||
PopulateAdvancedScriptOptions(this.Parameters.ScriptOptions, publishModel.AdvancedOptions);
|
||||
return publishModel;
|
||||
}
|
||||
|
||||
IEnumerable<ScriptingObject> selectedObjects = new List<ScriptingObject>();
|
||||
|
||||
if (hasCriteriaSpecified)
|
||||
{
|
||||
// This is an expensive remote call to load all objects from the database.
|
||||
//
|
||||
List<ScriptingObject> allObjects = publishModel.GetDatabaseObjects();
|
||||
selectedObjects = ScriptingObjectMatcher.Match(
|
||||
this.Parameters.IncludeObjectCriteria,
|
||||
this.Parameters.ExcludeObjectCriteria,
|
||||
this.Parameters.IncludeSchemas,
|
||||
this.Parameters.ExcludeSchemas,
|
||||
this.Parameters.IncludeTypes,
|
||||
this.Parameters.ExcludeTypes,
|
||||
allObjects);
|
||||
}
|
||||
|
||||
if (hasObjectsSpecified)
|
||||
{
|
||||
selectedObjects = selectedObjects.Union(this.Parameters.ScriptingObjects);
|
||||
}
|
||||
|
||||
// Populating advanced options after we select our objects in question, otherwise we lose all
|
||||
// advanced options. After this call to PopulateAdvancedScriptOptions, DO NOT reference the
|
||||
// publishModel.AdvancedOptions getter as it will reset the options in the model.
|
||||
//
|
||||
PopulateAdvancedScriptOptions(this.Parameters.ScriptOptions, publishModel.AdvancedOptions);
|
||||
|
||||
Logger.Write(
|
||||
TraceEventType.Information,
|
||||
string.Format(
|
||||
"Scripting object count {0}, objects: {1}",
|
||||
selectedObjects.Count(),
|
||||
string.Join(", ", selectedObjects)));
|
||||
|
||||
string server = GetServerNameFromLiveInstance(this.Parameters.ConnectionString, this.azureAccessToken);
|
||||
string database = new SqlConnectionStringBuilder(this.Parameters.ConnectionString).InitialCatalog;
|
||||
|
||||
foreach (ScriptingObject scriptingObject in selectedObjects)
|
||||
{
|
||||
publishModel.SelectedObjects.Add(scriptingObject.ToUrn(server, database));
|
||||
}
|
||||
return publishModel;
|
||||
}
|
||||
|
||||
private void OnPublishModelScriptError(object sender, ScriptEventArgs e)
|
||||
{
|
||||
this.CancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
Logger.Write(
|
||||
TraceEventType.Verbose,
|
||||
string.Format(
|
||||
"Sending scripting error progress event, Urn={0}, OperationId={1}, Sequence={2}, Completed={3}, Error={4}",
|
||||
e.Urn,
|
||||
this.OperationId,
|
||||
this.eventSequenceNumber,
|
||||
e.Completed,
|
||||
e?.Error?.ToString() ?? "null"));
|
||||
|
||||
// Keep scripting...it's a best effort operation.
|
||||
e.ContinueScripting = true;
|
||||
|
||||
this.SendProgressNotificationEvent(new ScriptingProgressNotificationParams
|
||||
{
|
||||
ScriptingObject = e.Urn?.ToScriptingObject(),
|
||||
Status = e.GetStatus(),
|
||||
CompletedCount = this.scriptedObjectCount,
|
||||
TotalCount = this.totalScriptedObjectCount,
|
||||
ErrorMessage = e?.Error?.Message,
|
||||
ErrorDetails = e?.Error?.ToString(),
|
||||
});
|
||||
}
|
||||
|
||||
private void OnPublishModelScriptItemsCollected(object sender, ScriptItemsArgs e)
|
||||
{
|
||||
this.CancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
List<ScriptingObject> scriptingObjects = e.Urns.Select(urn => urn.ToScriptingObject()).ToList();
|
||||
this.totalScriptedObjectCount = scriptingObjects.Count;
|
||||
|
||||
Logger.Write(
|
||||
TraceEventType.Verbose,
|
||||
string.Format(
|
||||
"Sending scripting plan notification event OperationId={0}, Sequence={1}, Count={2}, Objects: {3}",
|
||||
this.OperationId,
|
||||
this.eventSequenceNumber,
|
||||
this.totalScriptedObjectCount,
|
||||
string.Join(", ", e.Urns)));
|
||||
|
||||
this.SendPlanNotificationEvent(new ScriptingPlanNotificationParams
|
||||
{
|
||||
ScriptingObjects = scriptingObjects,
|
||||
Count = scriptingObjects.Count,
|
||||
});
|
||||
}
|
||||
|
||||
private void OnPublishModelScriptProgress(object sender, ScriptEventArgs e)
|
||||
{
|
||||
this.CancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (e.Completed)
|
||||
{
|
||||
this.scriptedObjectCount += 1;
|
||||
}
|
||||
|
||||
Logger.Write(
|
||||
TraceEventType.Verbose,
|
||||
string.Format(
|
||||
"Sending progress event, Urn={0}, OperationId={1}, Sequence={2}, Status={3}, Error={4}",
|
||||
e.Urn,
|
||||
this.OperationId,
|
||||
this.eventSequenceNumber,
|
||||
e.GetStatus(),
|
||||
e?.Error?.ToString() ?? "null"));
|
||||
|
||||
this.SendProgressNotificationEvent(new ScriptingProgressNotificationParams
|
||||
{
|
||||
ScriptingObject = e.Urn.ToScriptingObject(),
|
||||
Status = e.GetStatus(),
|
||||
CompletedCount = this.scriptedObjectCount,
|
||||
TotalCount = this.totalScriptedObjectCount,
|
||||
ErrorMessage = e?.Error?.Message,
|
||||
ErrorDetails = e?.Error?.ToString(),
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user