mirror of
https://github.com/ckaczor/sqltoolsservice.git
synced 2026-01-14 01:25:40 -05:00
Added new Kusto ServiceLayer (#1009)
* Copy smoModel some rename * Copy entire service layer * Building copy * Fixing some references * Launch profile * Resolve namespace issues * Compiling tests. Correct manifest. * Fixing localization resources * ReliableKustoClient * Some trimming of extra code and Kusto code * Kusto client creation in bindingContent * Removing Smo and new Kusto classes * More trimming * Kusto schema hookup * Solidying DataSource abstraction * Solidifying further * Latest refatoring * More refactoring * Building and launching Kusto service layer * Working model which enumerates databases * Refactoring to pass IDataSource to all tree nodes * Removing some dependencies on the context * Working with tables and schema * Comment checkin * Refactoring to give out select script * Query created and sent back to ADS * Fix query generation * Fix listing of databases * Tunneling the query through. * Successful query execution * Return only results table * Deleting Cms * Delete DacFx * Delete SchemaCompare and TaskServices * Change build definition to not stop at launch * Fix error after merge * Save Kusto results in different formats (#935) * save results as csv etc * some fixes Co-authored-by: Monica Gupta <mogupt@microsoft.com> * 2407 Added OrderBy clause in KustoDataSource > GetDatabaseMetaData and GetColumnMetadata (#959) * 2405 Defaulted Options when setting ServerInfo in ConnectionService > GetConnectionCompleteParams (#965) * 2747 Fixed IsUnknownType error for Kusto (#989) * 2747 Removed unused directives in Kusto > DbColumnWrapper. Refactored IsUnknownType to handle null DataTypeName * 2747 Reverted IsUnknownType change in DbColumnWrapper. Changed DataTypeName to get calue from ColumnType. Refactored SafeGetValue to type check before hard casting to reduce case exceptions. * Added EmbeddedResourceUseDependentUponConvention to Microsoft.Kusto.ServiceLayer.csproj. Also renamed DACfx to match Microsoft.SqlTools.ServiceLayer. Added to compile Exclude="**/obj/**/*.cs" * Srahman cleanup sql code (#992) * Removed Management and Security Service Code. * Remove FileBrowser service * Comment why we are using SqlServer library * Remove SQL specific type definitions * clean up formatter service (#996) Co-authored-by: Monica Gupta <mogupt@microsoft.com> * Code clean up and Kusto intellisense (#994) * Code clean up and Kusto intellisense * Addressed few comments * Addressed few comments * addressed comments Co-authored-by: Monica Gupta <mogupt@microsoft.com> * Return multiple tables for Kusto * Changes required for Kusto manage dashboard (#1039) * Changes required for manage dashboard * Addressed comments Co-authored-by: Monica Gupta <mogupt@microsoft.com> * 2728 Kusto function support (#1038) * loc update (#914) * loc update * loc updates * 2728 moved ColumnInfo and KustoResultsReader to separate files. Added Folder and Function to TreeNode.cs * 2728 Added FunctionInfo. Added Folder to ColumnInfo. Removed partial class from KustoResultsReader. Set Function.IsAlwaysLeaf=true in TreeNode.cs. In KustoDataSource changed tableMetadata type to TableMetaData. Added folder and function dictionaries. Refactored GetSchema function. Renamed GenerateColumnMetadataKey to GenerateMetadataKey * 2728 Added FunctionInfo. Added Folder to ColumnInfo. Removed partial class from KustoResultsReader. Set Function.IsAlwaysLeaf=true in TreeNode.cs. In KustoDataSource changed tableMetadata type to TableMetaData. Added folder and function dictionaries. Refactored GetSchema function. Renamed GenerateColumnMetadataKey to GenerateMetadataKey * 2728 Created new SqlConnection within using block. Refactored KustoDataSource > columnmetadata to sort on get instead of insert. * 2728 Added GetFunctionInfo function to KustoDataSource. * 2728 Reverted change to Microsoft.Kusto.ServiceLayer.csproj from merge * 2728 Reverted change to SqlTools.ServiceLayer\Localization\transXliff * 2728 Reverted change to sr.de.xlf and sr.zh-hans.xlf * 2728 Refactored KustoDataSource Function folders to support subfolders * 2728 Refactored KustoDataSource to use urn for folders, functions, and tables instead of name. * Merge remote-tracking branch 'origin/main' into feature-ADE # Conflicts: # Packages.props * 2728 Moved metadata files into Metadata subdirectory. Added GenerateAlterFunction to IDataSource and DataSourceBase. * 2728 Added summary information to SafeAdd in SystemExtensions. Renamed local variable in SetTableMetadata * 2728 Moved SafeAdd from SystemExtensions to KustoQueryUtils. Added check when getting database schema to return existing records before querying again. Added AddRange function to KustoQueryUtils. Created SetFolderMetadataForFunctions method. * 2728 Added DatabaseKeyPrefix to only return tables to a database for the dashboard. Added logic to store all database tables within the tableMetadata dictionary for the dashboard. * 2728 Created TableInfo and moved info objects into Models directory. Refactored KustoDataSource to lazy load columns for tables. Refactored logic to load tables using cslschema instead of schema. * 2728 Renamed LoadColumnSchema to GetTableSchema to be consistent. Co-authored-by: khoiph1 <khoiph@microsoft.com> * Addressed comments Co-authored-by: Shafiq Rahman <srahman@microsoft.com> Co-authored-by: Monica Gupta <mogupt@microsoft.com> Co-authored-by: Justin M <63619224+JustinMDotNet@users.noreply.github.com> Co-authored-by: rkselfhost <rkselfhost@outlook.com> Co-authored-by: khoiph1 <khoiph@microsoft.com>
This commit is contained in:
Monica Gupta
@@ -0,0 +1,309 @@
|
||||
//
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
//
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Data.Common;
|
||||
using System.Data.SqlClient;
|
||||
using System.Data.SqlTypes;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml;
|
||||
using Microsoft.Kusto.ServiceLayer.QueryExecution.Contracts;
|
||||
using Microsoft.SqlTools.Utility;
|
||||
|
||||
namespace Microsoft.Kusto.ServiceLayer.QueryExecution.DataStorage
|
||||
{
|
||||
/// <summary>
|
||||
/// Wrapper around a DbData reader to perform some special operations more simply
|
||||
/// </summary>
|
||||
public class StorageDataReader
|
||||
{
|
||||
/// <summary>
|
||||
/// Constructs a new wrapper around the provided reader
|
||||
/// </summary>
|
||||
/// <param name="reader">The reader to wrap around</param>
|
||||
public StorageDataReader(IDataReader reader)
|
||||
{
|
||||
// Sanity check to make sure there is a data reader
|
||||
Validate.IsNotNull(nameof(reader), reader);
|
||||
|
||||
DataReader = reader;
|
||||
|
||||
// Read the columns into a set of wrappers
|
||||
List<DbColumnWrapper> columnList = new List<DbColumnWrapper>();
|
||||
var rows = DataReader.GetSchemaTable().Rows;
|
||||
|
||||
foreach (DataRow row in rows)
|
||||
{
|
||||
columnList.Add(new DbColumnWrapper(row));
|
||||
}
|
||||
|
||||
Columns = columnList.ToArray();
|
||||
HasLongColumns = Columns.Any(column => column.IsLong.HasValue && column.IsLong.Value);
|
||||
}
|
||||
|
||||
#region Properties
|
||||
|
||||
/// <summary>
|
||||
/// All the columns that this reader currently contains
|
||||
/// </summary>
|
||||
public DbColumnWrapper[] Columns { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="DataReader"/> that will be read from
|
||||
/// </summary>
|
||||
public IDataReader DataReader { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether or not any of the columns of this reader are 'long', such as nvarchar(max)
|
||||
/// </summary>
|
||||
public bool HasLongColumns { get; private set; }
|
||||
|
||||
#endregion
|
||||
|
||||
#region DbDataReader Methods
|
||||
|
||||
/// <summary>
|
||||
/// Pass-through to DbDataReader.ReadAsync()
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">The cancellation token to use for cancelling a query</param>
|
||||
/// <returns></returns>
|
||||
public Task<bool> ReadAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.Run(() => DataReader.Read());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves a value
|
||||
/// </summary>
|
||||
/// <param name="i">Column ordinal</param>
|
||||
/// <returns>The value of the given column</returns>
|
||||
public object GetValue(int i)
|
||||
{
|
||||
return DataReader.GetValue(i);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stores all values of the current row into the provided object array
|
||||
/// </summary>
|
||||
/// <param name="values">Where to store the values from this row</param>
|
||||
public void GetValues(object[] values)
|
||||
{
|
||||
DataReader.GetValues(values);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether or not the cell of the given column at the current row is a DBNull
|
||||
/// </summary>
|
||||
/// <param name="i">Column ordinal</param>
|
||||
/// <returns>True if the cell is DBNull, false otherwise</returns>
|
||||
public bool IsDBNull(int i)
|
||||
{
|
||||
return DataReader.IsDBNull(i);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Public Methods
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves bytes with a maximum number of bytes to return
|
||||
/// </summary>
|
||||
/// <param name="iCol">Column ordinal</param>
|
||||
/// <param name="maxNumBytesToReturn">Number of bytes to return at maximum</param>
|
||||
/// <returns>Byte array</returns>
|
||||
public byte[] GetBytesWithMaxCapacity(int iCol, int maxNumBytesToReturn)
|
||||
{
|
||||
if (maxNumBytesToReturn <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(maxNumBytesToReturn), SR.QueryServiceDataReaderByteCountInvalid);
|
||||
}
|
||||
|
||||
//first, ask provider how much data it has and calculate the final # of bytes
|
||||
//NOTE: -1 means that it doesn't know how much data it has
|
||||
long neededLength;
|
||||
long origLength = neededLength = GetBytes(iCol, 0, null, 0, 0);
|
||||
if (neededLength == -1 || neededLength > maxNumBytesToReturn)
|
||||
{
|
||||
neededLength = maxNumBytesToReturn;
|
||||
}
|
||||
|
||||
//get the data up to the maxNumBytesToReturn
|
||||
byte[] bytesBuffer = new byte[neededLength];
|
||||
GetBytes(iCol, 0, bytesBuffer, 0, (int)neededLength);
|
||||
|
||||
//see if server sent back more data than we should return
|
||||
if (origLength == -1 || origLength > neededLength)
|
||||
{
|
||||
//pump the rest of data from the reader and discard it right away
|
||||
long dataIndex = neededLength;
|
||||
const int tmpBufSize = 100000;
|
||||
byte[] tmpBuf = new byte[tmpBufSize];
|
||||
while (GetBytes(iCol, dataIndex, tmpBuf, 0, tmpBufSize) == tmpBufSize)
|
||||
{
|
||||
dataIndex += tmpBufSize;
|
||||
}
|
||||
}
|
||||
|
||||
return bytesBuffer;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves characters with a maximum number of charss to return
|
||||
/// </summary>
|
||||
/// <param name="iCol">Column ordinal</param>
|
||||
/// <param name="maxCharsToReturn">Number of chars to return at maximum</param>
|
||||
/// <returns>String</returns>
|
||||
public string GetCharsWithMaxCapacity(int iCol, int maxCharsToReturn)
|
||||
{
|
||||
if (maxCharsToReturn <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(maxCharsToReturn), SR.QueryServiceDataReaderCharCountInvalid);
|
||||
}
|
||||
|
||||
//first, ask provider how much data it has and calculate the final # of chars
|
||||
//NOTE: -1 means that it doesn't know how much data it has
|
||||
long neededLength;
|
||||
long origLength = neededLength = GetChars(iCol, 0, null, 0, 0);
|
||||
if (neededLength == -1 || neededLength > maxCharsToReturn)
|
||||
{
|
||||
neededLength = maxCharsToReturn;
|
||||
}
|
||||
Debug.Assert(neededLength < int.MaxValue);
|
||||
|
||||
//get the data up to maxCharsToReturn
|
||||
char[] buffer = new char[neededLength];
|
||||
if (neededLength > 0)
|
||||
{
|
||||
GetChars(iCol, 0, buffer, 0, (int)neededLength);
|
||||
}
|
||||
|
||||
//see if server sent back more data than we should return
|
||||
if (origLength == -1 || origLength > neededLength)
|
||||
{
|
||||
//pump the rest of data from the reader and discard it right away
|
||||
long dataIndex = neededLength;
|
||||
const int tmpBufSize = 100000;
|
||||
char[] tmpBuf = new char[tmpBufSize];
|
||||
while (GetChars(iCol, dataIndex, tmpBuf, 0, tmpBufSize) == tmpBufSize)
|
||||
{
|
||||
dataIndex += tmpBufSize;
|
||||
}
|
||||
}
|
||||
string res = new string(buffer);
|
||||
return res;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves xml with a maximum number of bytes to return
|
||||
/// </summary>
|
||||
/// <param name="iCol">Column ordinal</param>
|
||||
/// <param name="maxCharsToReturn">Number of chars to return at maximum</param>
|
||||
/// <returns>String</returns>
|
||||
public string GetXmlWithMaxCapacity(int iCol, int maxCharsToReturn)
|
||||
{
|
||||
if (maxCharsToReturn <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(maxCharsToReturn), SR.QueryServiceDataReaderXmlCountInvalid);
|
||||
}
|
||||
|
||||
object o = GetValue(iCol);
|
||||
return o?.ToString();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Private Helpers
|
||||
|
||||
private long GetBytes(int i, long dataIndex, byte[] buffer, int bufferIndex, int length)
|
||||
{
|
||||
return DataReader.GetBytes(i, dataIndex, buffer, bufferIndex, length);
|
||||
}
|
||||
|
||||
private long GetChars(int i, long dataIndex, char[] buffer, int bufferIndex, int length)
|
||||
{
|
||||
return DataReader.GetChars(i, dataIndex, buffer, bufferIndex, length);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
/// <summary>
|
||||
/// Internal class for writing strings with a maximum capacity
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This code is take almost verbatim from Microsoft.SqlServer.Management.UI.Grid, SSMS
|
||||
/// DataStorage, StorageDataReader class.
|
||||
/// </remarks>
|
||||
internal class StringWriterWithMaxCapacity : StringWriter
|
||||
{
|
||||
private bool stopWriting;
|
||||
|
||||
private int CurrentLength
|
||||
{
|
||||
get { return GetStringBuilder().Length; }
|
||||
}
|
||||
|
||||
public StringWriterWithMaxCapacity(IFormatProvider formatProvider, int capacity) : base(formatProvider)
|
||||
{
|
||||
MaximumCapacity = capacity;
|
||||
}
|
||||
|
||||
private int MaximumCapacity { get; set; }
|
||||
|
||||
public override void Write(char value)
|
||||
{
|
||||
if (stopWriting) { return; }
|
||||
|
||||
if (CurrentLength < MaximumCapacity)
|
||||
{
|
||||
base.Write(value);
|
||||
}
|
||||
else
|
||||
{
|
||||
stopWriting = true;
|
||||
}
|
||||
}
|
||||
|
||||
public override void Write(char[] buffer, int index, int count)
|
||||
{
|
||||
if (stopWriting) { return; }
|
||||
|
||||
int curLen = CurrentLength;
|
||||
if (curLen + (count - index) > MaximumCapacity)
|
||||
{
|
||||
stopWriting = true;
|
||||
|
||||
count = MaximumCapacity - curLen + index;
|
||||
if (count < 0)
|
||||
{
|
||||
count = 0;
|
||||
}
|
||||
}
|
||||
base.Write(buffer, index, count);
|
||||
}
|
||||
|
||||
public override void Write(string value)
|
||||
{
|
||||
if (stopWriting) { return; }
|
||||
|
||||
int curLen = CurrentLength;
|
||||
if (value.Length + curLen > MaximumCapacity)
|
||||
{
|
||||
stopWriting = true;
|
||||
base.Write(value.Substring(0, MaximumCapacity - curLen));
|
||||
}
|
||||
else
|
||||
{
|
||||
base.Write(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user