mirror of
https://github.com/ckaczor/sqltoolsservice.git
synced 2026-01-25 01:25:40 -05:00
Added new Kusto ServiceLayer (#1009)
* Copy smoModel some rename * Copy entire service layer * Building copy * Fixing some references * Launch profile * Resolve namespace issues * Compiling tests. Correct manifest. * Fixing localization resources * ReliableKustoClient * Some trimming of extra code and Kusto code * Kusto client creation in bindingContent * Removing Smo and new Kusto classes * More trimming * Kusto schema hookup * Solidying DataSource abstraction * Solidifying further * Latest refatoring * More refactoring * Building and launching Kusto service layer * Working model which enumerates databases * Refactoring to pass IDataSource to all tree nodes * Removing some dependencies on the context * Working with tables and schema * Comment checkin * Refactoring to give out select script * Query created and sent back to ADS * Fix query generation * Fix listing of databases * Tunneling the query through. * Successful query execution * Return only results table * Deleting Cms * Delete DacFx * Delete SchemaCompare and TaskServices * Change build definition to not stop at launch * Fix error after merge * Save Kusto results in different formats (#935) * save results as csv etc * some fixes Co-authored-by: Monica Gupta <mogupt@microsoft.com> * 2407 Added OrderBy clause in KustoDataSource > GetDatabaseMetaData and GetColumnMetadata (#959) * 2405 Defaulted Options when setting ServerInfo in ConnectionService > GetConnectionCompleteParams (#965) * 2747 Fixed IsUnknownType error for Kusto (#989) * 2747 Removed unused directives in Kusto > DbColumnWrapper. Refactored IsUnknownType to handle null DataTypeName * 2747 Reverted IsUnknownType change in DbColumnWrapper. Changed DataTypeName to get calue from ColumnType. Refactored SafeGetValue to type check before hard casting to reduce case exceptions. * Added EmbeddedResourceUseDependentUponConvention to Microsoft.Kusto.ServiceLayer.csproj. Also renamed DACfx to match Microsoft.SqlTools.ServiceLayer. Added to compile Exclude="**/obj/**/*.cs" * Srahman cleanup sql code (#992) * Removed Management and Security Service Code. * Remove FileBrowser service * Comment why we are using SqlServer library * Remove SQL specific type definitions * clean up formatter service (#996) Co-authored-by: Monica Gupta <mogupt@microsoft.com> * Code clean up and Kusto intellisense (#994) * Code clean up and Kusto intellisense * Addressed few comments * Addressed few comments * addressed comments Co-authored-by: Monica Gupta <mogupt@microsoft.com> * Return multiple tables for Kusto * Changes required for Kusto manage dashboard (#1039) * Changes required for manage dashboard * Addressed comments Co-authored-by: Monica Gupta <mogupt@microsoft.com> * 2728 Kusto function support (#1038) * loc update (#914) * loc update * loc updates * 2728 moved ColumnInfo and KustoResultsReader to separate files. Added Folder and Function to TreeNode.cs * 2728 Added FunctionInfo. Added Folder to ColumnInfo. Removed partial class from KustoResultsReader. Set Function.IsAlwaysLeaf=true in TreeNode.cs. In KustoDataSource changed tableMetadata type to TableMetaData. Added folder and function dictionaries. Refactored GetSchema function. Renamed GenerateColumnMetadataKey to GenerateMetadataKey * 2728 Added FunctionInfo. Added Folder to ColumnInfo. Removed partial class from KustoResultsReader. Set Function.IsAlwaysLeaf=true in TreeNode.cs. In KustoDataSource changed tableMetadata type to TableMetaData. Added folder and function dictionaries. Refactored GetSchema function. Renamed GenerateColumnMetadataKey to GenerateMetadataKey * 2728 Created new SqlConnection within using block. Refactored KustoDataSource > columnmetadata to sort on get instead of insert. * 2728 Added GetFunctionInfo function to KustoDataSource. * 2728 Reverted change to Microsoft.Kusto.ServiceLayer.csproj from merge * 2728 Reverted change to SqlTools.ServiceLayer\Localization\transXliff * 2728 Reverted change to sr.de.xlf and sr.zh-hans.xlf * 2728 Refactored KustoDataSource Function folders to support subfolders * 2728 Refactored KustoDataSource to use urn for folders, functions, and tables instead of name. * Merge remote-tracking branch 'origin/main' into feature-ADE # Conflicts: # Packages.props * 2728 Moved metadata files into Metadata subdirectory. Added GenerateAlterFunction to IDataSource and DataSourceBase. * 2728 Added summary information to SafeAdd in SystemExtensions. Renamed local variable in SetTableMetadata * 2728 Moved SafeAdd from SystemExtensions to KustoQueryUtils. Added check when getting database schema to return existing records before querying again. Added AddRange function to KustoQueryUtils. Created SetFolderMetadataForFunctions method. * 2728 Added DatabaseKeyPrefix to only return tables to a database for the dashboard. Added logic to store all database tables within the tableMetadata dictionary for the dashboard. * 2728 Created TableInfo and moved info objects into Models directory. Refactored KustoDataSource to lazy load columns for tables. Refactored logic to load tables using cslschema instead of schema. * 2728 Renamed LoadColumnSchema to GetTableSchema to be consistent. Co-authored-by: khoiph1 <khoiph@microsoft.com> * Addressed comments Co-authored-by: Shafiq Rahman <srahman@microsoft.com> Co-authored-by: Monica Gupta <mogupt@microsoft.com> Co-authored-by: Justin M <63619224+JustinMDotNet@users.noreply.github.com> Co-authored-by: rkselfhost <rkselfhost@outlook.com> Co-authored-by: khoiph1 <khoiph@microsoft.com>
This commit is contained in:
Monica Gupta
@@ -0,0 +1,318 @@
|
||||
//
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
//
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Composition;
|
||||
using System.Diagnostics;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.SqlTools.Extensibility;
|
||||
using Microsoft.SqlTools.Hosting;
|
||||
using Microsoft.SqlTools.Hosting.Protocol;
|
||||
using Microsoft.Kusto.ServiceLayer.QueryExecution.Contracts;
|
||||
using Microsoft.Kusto.ServiceLayer.QueryExecution.DataStorage;
|
||||
using Microsoft.Kusto.ServiceLayer.Utility;
|
||||
using Microsoft.SqlTools.Utility;
|
||||
|
||||
|
||||
namespace Microsoft.Kusto.ServiceLayer.QueryExecution
|
||||
{
|
||||
|
||||
[Export(typeof(IHostedService))]
|
||||
public class SerializationService : HostedService<SerializationService>, IComposableService
|
||||
{
|
||||
private ConcurrentDictionary<string, DataSerializer> inProgressSerializations;
|
||||
|
||||
public SerializationService()
|
||||
{
|
||||
inProgressSerializations = new ConcurrentDictionary<string, DataSerializer>();
|
||||
}
|
||||
|
||||
public override void InitializeService(IProtocolEndpoint serviceHost)
|
||||
{
|
||||
Logger.Write(TraceEventType.Verbose, "SerializationService initialized");
|
||||
serviceHost.SetRequestHandler(SerializeStartRequest.Type, HandleSerializeStartRequest);
|
||||
serviceHost.SetRequestHandler(SerializeContinueRequest.Type, HandleSerializeContinueRequest);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begin to process request to save a resultSet to a file in CSV format
|
||||
/// </summary>
|
||||
internal Task HandleSerializeStartRequest(SerializeDataStartRequestParams serializeParams,
|
||||
RequestContext<SerializeDataResult> requestContext)
|
||||
{
|
||||
// Run in separate thread so that message thread isn't held up by a potentially time consuming file write
|
||||
Task.Run(async () => {
|
||||
await RunSerializeStartRequest(serializeParams, requestContext);
|
||||
}).ContinueWithOnFaulted(async t => await SendErrorAndCleanup(serializeParams?.FilePath, requestContext, t.Exception));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
internal async Task RunSerializeStartRequest(SerializeDataStartRequestParams serializeParams, RequestContext<SerializeDataResult> requestContext)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Verify we have sensible inputs and there isn't a task running for this file already
|
||||
Validate.IsNotNull(nameof(serializeParams), serializeParams);
|
||||
Validate.IsNotNullOrWhitespaceString("FilePath", serializeParams.FilePath);
|
||||
|
||||
DataSerializer serializer = null;
|
||||
if (inProgressSerializations.TryGetValue(serializeParams.FilePath, out serializer))
|
||||
{
|
||||
// Cannot proceed as there is an in progress serialization happening
|
||||
throw new Exception(SR.SerializationServiceRequestInProgress(serializeParams.FilePath));
|
||||
}
|
||||
|
||||
// Create a new serializer, save for future calls if needed, and write the request out
|
||||
serializer = new DataSerializer(serializeParams);
|
||||
if (!serializeParams.IsLastBatch)
|
||||
{
|
||||
inProgressSerializations.AddOrUpdate(serializer.FilePath, serializer, (key, old) => serializer);
|
||||
}
|
||||
|
||||
Logger.Write(TraceEventType.Verbose, "HandleSerializeStartRequest");
|
||||
SerializeDataResult result = serializer.ProcessRequest(serializeParams);
|
||||
await requestContext.SendResult(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await SendErrorAndCleanup(serializeParams.FilePath, requestContext, ex);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SendErrorAndCleanup(string filePath, RequestContext<SerializeDataResult> requestContext, Exception ex)
|
||||
{
|
||||
if (filePath != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
DataSerializer removed;
|
||||
inProgressSerializations.TryRemove(filePath, out removed);
|
||||
if (removed != null)
|
||||
{
|
||||
// Flush any contents to disk and remove the writer
|
||||
removed.CloseStreams();
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Do not care if there was an error removing this, must always delete if something failed
|
||||
}
|
||||
}
|
||||
await requestContext.SendError(ex.Message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process request to save a resultSet to a file in CSV format
|
||||
/// </summary>
|
||||
internal Task HandleSerializeContinueRequest(SerializeDataContinueRequestParams serializeParams,
|
||||
RequestContext<SerializeDataResult> requestContext)
|
||||
{
|
||||
// Run in separate thread so that message thread isn't held up by a potentially time consuming file write
|
||||
Task.Run(async () =>
|
||||
{
|
||||
await RunSerializeContinueRequest(serializeParams, requestContext);
|
||||
}).ContinueWithOnFaulted(async t => await SendErrorAndCleanup(serializeParams?.FilePath, requestContext, t.Exception));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
internal async Task RunSerializeContinueRequest(SerializeDataContinueRequestParams serializeParams, RequestContext<SerializeDataResult> requestContext)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Verify we have sensible inputs and some data has already been sent for the file
|
||||
Validate.IsNotNull(nameof(serializeParams), serializeParams);
|
||||
Validate.IsNotNullOrWhitespaceString("FilePath", serializeParams.FilePath);
|
||||
|
||||
DataSerializer serializer = null;
|
||||
if (!inProgressSerializations.TryGetValue(serializeParams.FilePath, out serializer))
|
||||
{
|
||||
throw new Exception(SR.SerializationServiceRequestNotFound(serializeParams.FilePath));
|
||||
}
|
||||
|
||||
// Write to file and cleanup if needed
|
||||
Logger.Write(TraceEventType.Verbose, "HandleSerializeContinueRequest");
|
||||
SerializeDataResult result = serializer.ProcessRequest(serializeParams);
|
||||
if (serializeParams.IsLastBatch)
|
||||
{
|
||||
// Cleanup the serializer
|
||||
this.inProgressSerializations.TryRemove(serializer.FilePath, out serializer);
|
||||
}
|
||||
await requestContext.SendResult(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await SendErrorAndCleanup(serializeParams.FilePath, requestContext, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
class DataSerializer
|
||||
{
|
||||
private IFileStreamWriter writer;
|
||||
private SerializeDataStartRequestParams requestParams;
|
||||
private IList<DbColumnWrapper> columns;
|
||||
|
||||
public string FilePath { get; private set; }
|
||||
|
||||
public DataSerializer(SerializeDataStartRequestParams requestParams)
|
||||
{
|
||||
this.requestParams = requestParams;
|
||||
this.columns = this.MapColumns(requestParams.Columns);
|
||||
this.FilePath = requestParams.FilePath;
|
||||
}
|
||||
|
||||
private IList<DbColumnWrapper> MapColumns(ColumnInfo[] columns)
|
||||
{
|
||||
List<DbColumnWrapper> columnWrappers = new List<DbColumnWrapper>();
|
||||
foreach (ColumnInfo column in columns)
|
||||
{
|
||||
DbColumnWrapper wrapper = new DbColumnWrapper(column);
|
||||
columnWrappers.Add(wrapper);
|
||||
}
|
||||
return columnWrappers;
|
||||
}
|
||||
|
||||
|
||||
public SerializeDataResult ProcessRequest(ISerializationParams serializeParams)
|
||||
{
|
||||
SerializeDataResult result = new SerializeDataResult();
|
||||
try
|
||||
{
|
||||
this.WriteData(serializeParams.Rows, serializeParams.IsLastBatch);
|
||||
if (serializeParams.IsLastBatch)
|
||||
{
|
||||
this.CloseStreams();
|
||||
}
|
||||
result.Succeeded = true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.Messages = ex.Message;
|
||||
result.Succeeded = false;
|
||||
this.CloseStreams();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public void WriteData(DbCellValue[][] rows, bool isComplete)
|
||||
{
|
||||
this.EnsureWriterCreated();
|
||||
foreach (var row in rows)
|
||||
{
|
||||
SetRawObjects(row);
|
||||
writer.WriteRow(row, this.columns);
|
||||
}
|
||||
}
|
||||
|
||||
private void SetRawObjects(DbCellValue[] row)
|
||||
{
|
||||
for (int i = 0; i < row.Length; i++)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Try to set as the "correct" type
|
||||
var value = Convert.ChangeType(row[i].DisplayValue, columns[i].DataType);
|
||||
row[i].RawObject = value;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
row[i].RawObject = row[i].DisplayValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void EnsureWriterCreated()
|
||||
{
|
||||
if (this.writer == null)
|
||||
{
|
||||
IFileStreamFactory factory;
|
||||
switch (this.requestParams.SaveFormat.ToLowerInvariant())
|
||||
{
|
||||
case "json":
|
||||
factory = new SaveAsJsonFileStreamFactory()
|
||||
{
|
||||
SaveRequestParams = CreateJsonRequestParams()
|
||||
};
|
||||
break;
|
||||
case "csv":
|
||||
factory = new SaveAsCsvFileStreamFactory()
|
||||
{
|
||||
SaveRequestParams = CreateCsvRequestParams()
|
||||
};
|
||||
break;
|
||||
case "xml":
|
||||
factory = new SaveAsXmlFileStreamFactory()
|
||||
{
|
||||
SaveRequestParams = CreateXmlRequestParams()
|
||||
};
|
||||
break;
|
||||
case "excel":
|
||||
factory = new SaveAsExcelFileStreamFactory()
|
||||
{
|
||||
SaveRequestParams = CreateExcelRequestParams()
|
||||
};
|
||||
break;
|
||||
default:
|
||||
throw new Exception(SR.SerializationServiceUnsupportedFormat(this.requestParams.SaveFormat));
|
||||
}
|
||||
this.writer = factory.GetWriter(requestParams.FilePath);
|
||||
}
|
||||
}
|
||||
public void CloseStreams()
|
||||
{
|
||||
if (this.writer != null)
|
||||
{
|
||||
this.writer.Dispose();
|
||||
this.writer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private SaveResultsAsJsonRequestParams CreateJsonRequestParams()
|
||||
{
|
||||
return new SaveResultsAsJsonRequestParams
|
||||
{
|
||||
FilePath = this.requestParams.FilePath,
|
||||
BatchIndex = 0,
|
||||
ResultSetIndex = 0
|
||||
};
|
||||
}
|
||||
private SaveResultsAsExcelRequestParams CreateExcelRequestParams()
|
||||
{
|
||||
return new SaveResultsAsExcelRequestParams
|
||||
{
|
||||
FilePath = this.requestParams.FilePath,
|
||||
BatchIndex = 0,
|
||||
ResultSetIndex = 0,
|
||||
IncludeHeaders = this.requestParams.IncludeHeaders
|
||||
};
|
||||
}
|
||||
private SaveResultsAsCsvRequestParams CreateCsvRequestParams()
|
||||
{
|
||||
return new SaveResultsAsCsvRequestParams
|
||||
{
|
||||
FilePath = this.requestParams.FilePath,
|
||||
BatchIndex = 0,
|
||||
ResultSetIndex = 0,
|
||||
IncludeHeaders = this.requestParams.IncludeHeaders,
|
||||
Delimiter = this.requestParams.Delimiter,
|
||||
LineSeperator = this.requestParams.LineSeparator,
|
||||
TextIdentifier = this.requestParams.TextIdentifier,
|
||||
Encoding = this.requestParams.Encoding
|
||||
};
|
||||
}
|
||||
private SaveResultsAsXmlRequestParams CreateXmlRequestParams()
|
||||
{
|
||||
return new SaveResultsAsXmlRequestParams
|
||||
{
|
||||
FilePath = this.requestParams.FilePath,
|
||||
BatchIndex = 0,
|
||||
ResultSetIndex = 0,
|
||||
Formatted = this.requestParams.Formatted,
|
||||
Encoding = this.requestParams.Encoding
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user