mirror of
https://github.com/ckaczor/sqltoolsservice.git
synced 2026-02-03 01:25:44 -05:00
Added new Kusto ServiceLayer (#1009)
* Copy smoModel some rename * Copy entire service layer * Building copy * Fixing some references * Launch profile * Resolve namespace issues * Compiling tests. Correct manifest. * Fixing localization resources * ReliableKustoClient * Some trimming of extra code and Kusto code * Kusto client creation in bindingContent * Removing Smo and new Kusto classes * More trimming * Kusto schema hookup * Solidying DataSource abstraction * Solidifying further * Latest refatoring * More refactoring * Building and launching Kusto service layer * Working model which enumerates databases * Refactoring to pass IDataSource to all tree nodes * Removing some dependencies on the context * Working with tables and schema * Comment checkin * Refactoring to give out select script * Query created and sent back to ADS * Fix query generation * Fix listing of databases * Tunneling the query through. * Successful query execution * Return only results table * Deleting Cms * Delete DacFx * Delete SchemaCompare and TaskServices * Change build definition to not stop at launch * Fix error after merge * Save Kusto results in different formats (#935) * save results as csv etc * some fixes Co-authored-by: Monica Gupta <mogupt@microsoft.com> * 2407 Added OrderBy clause in KustoDataSource > GetDatabaseMetaData and GetColumnMetadata (#959) * 2405 Defaulted Options when setting ServerInfo in ConnectionService > GetConnectionCompleteParams (#965) * 2747 Fixed IsUnknownType error for Kusto (#989) * 2747 Removed unused directives in Kusto > DbColumnWrapper. Refactored IsUnknownType to handle null DataTypeName * 2747 Reverted IsUnknownType change in DbColumnWrapper. Changed DataTypeName to get calue from ColumnType. Refactored SafeGetValue to type check before hard casting to reduce case exceptions. * Added EmbeddedResourceUseDependentUponConvention to Microsoft.Kusto.ServiceLayer.csproj. Also renamed DACfx to match Microsoft.SqlTools.ServiceLayer. Added to compile Exclude="**/obj/**/*.cs" * Srahman cleanup sql code (#992) * Removed Management and Security Service Code. * Remove FileBrowser service * Comment why we are using SqlServer library * Remove SQL specific type definitions * clean up formatter service (#996) Co-authored-by: Monica Gupta <mogupt@microsoft.com> * Code clean up and Kusto intellisense (#994) * Code clean up and Kusto intellisense * Addressed few comments * Addressed few comments * addressed comments Co-authored-by: Monica Gupta <mogupt@microsoft.com> * Return multiple tables for Kusto * Changes required for Kusto manage dashboard (#1039) * Changes required for manage dashboard * Addressed comments Co-authored-by: Monica Gupta <mogupt@microsoft.com> * 2728 Kusto function support (#1038) * loc update (#914) * loc update * loc updates * 2728 moved ColumnInfo and KustoResultsReader to separate files. Added Folder and Function to TreeNode.cs * 2728 Added FunctionInfo. Added Folder to ColumnInfo. Removed partial class from KustoResultsReader. Set Function.IsAlwaysLeaf=true in TreeNode.cs. In KustoDataSource changed tableMetadata type to TableMetaData. Added folder and function dictionaries. Refactored GetSchema function. Renamed GenerateColumnMetadataKey to GenerateMetadataKey * 2728 Added FunctionInfo. Added Folder to ColumnInfo. Removed partial class from KustoResultsReader. Set Function.IsAlwaysLeaf=true in TreeNode.cs. In KustoDataSource changed tableMetadata type to TableMetaData. Added folder and function dictionaries. Refactored GetSchema function. Renamed GenerateColumnMetadataKey to GenerateMetadataKey * 2728 Created new SqlConnection within using block. Refactored KustoDataSource > columnmetadata to sort on get instead of insert. * 2728 Added GetFunctionInfo function to KustoDataSource. * 2728 Reverted change to Microsoft.Kusto.ServiceLayer.csproj from merge * 2728 Reverted change to SqlTools.ServiceLayer\Localization\transXliff * 2728 Reverted change to sr.de.xlf and sr.zh-hans.xlf * 2728 Refactored KustoDataSource Function folders to support subfolders * 2728 Refactored KustoDataSource to use urn for folders, functions, and tables instead of name. * Merge remote-tracking branch 'origin/main' into feature-ADE # Conflicts: # Packages.props * 2728 Moved metadata files into Metadata subdirectory. Added GenerateAlterFunction to IDataSource and DataSourceBase. * 2728 Added summary information to SafeAdd in SystemExtensions. Renamed local variable in SetTableMetadata * 2728 Moved SafeAdd from SystemExtensions to KustoQueryUtils. Added check when getting database schema to return existing records before querying again. Added AddRange function to KustoQueryUtils. Created SetFolderMetadataForFunctions method. * 2728 Added DatabaseKeyPrefix to only return tables to a database for the dashboard. Added logic to store all database tables within the tableMetadata dictionary for the dashboard. * 2728 Created TableInfo and moved info objects into Models directory. Refactored KustoDataSource to lazy load columns for tables. Refactored logic to load tables using cslschema instead of schema. * 2728 Renamed LoadColumnSchema to GetTableSchema to be consistent. Co-authored-by: khoiph1 <khoiph@microsoft.com> * Addressed comments Co-authored-by: Shafiq Rahman <srahman@microsoft.com> Co-authored-by: Monica Gupta <mogupt@microsoft.com> Co-authored-by: Justin M <63619224+JustinMDotNet@users.noreply.github.com> Co-authored-by: rkselfhost <rkselfhost@outlook.com> Co-authored-by: khoiph1 <khoiph@microsoft.com>
This commit is contained in:
Monica Gupta
@@ -0,0 +1,502 @@
|
||||
//
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
//
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data.SqlClient;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Net.Sockets;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Kusto.ServiceLayer.Utility;
|
||||
using Microsoft.SqlTools.Utility;
|
||||
|
||||
|
||||
namespace Microsoft.Kusto.ServiceLayer.LanguageServices
|
||||
{
|
||||
/// <summary>
|
||||
/// Main class for the Binding Queue
|
||||
/// </summary>
|
||||
public class BindingQueue<T> : IDisposable where T : IBindingContext, new()
|
||||
{
|
||||
internal const int QueueThreadStackSize = 5 * 1024 * 1024;
|
||||
|
||||
private CancellationTokenSource processQueueCancelToken = null;
|
||||
|
||||
private ManualResetEvent itemQueuedEvent = new ManualResetEvent(initialState: false);
|
||||
|
||||
private object bindingQueueLock = new object();
|
||||
|
||||
private LinkedList<QueueItem> bindingQueue = new LinkedList<QueueItem>();
|
||||
|
||||
private object bindingContextLock = new object();
|
||||
|
||||
private Task queueProcessorTask;
|
||||
|
||||
public delegate void UnhandledExceptionDelegate(string connectionKey, Exception ex);
|
||||
|
||||
public event UnhandledExceptionDelegate OnUnhandledException;
|
||||
|
||||
/// <summary>
|
||||
/// Map from context keys to binding context instances
|
||||
/// Internal for testing purposes only
|
||||
/// </summary>
|
||||
internal Dictionary<string, IBindingContext> BindingContextMap { get; set; }
|
||||
|
||||
internal Dictionary<IBindingContext, Task> BindingContextTasks { get; set; } = new Dictionary<IBindingContext, Task>();
|
||||
|
||||
/// <summary>
|
||||
/// Constructor for a binding queue instance
|
||||
/// </summary>
|
||||
public BindingQueue()
|
||||
{
|
||||
this.BindingContextMap = new Dictionary<string, IBindingContext>();
|
||||
this.StartQueueProcessor();
|
||||
}
|
||||
|
||||
public void StartQueueProcessor()
|
||||
{
|
||||
this.queueProcessorTask = StartQueueProcessorAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops the binding queue by sending cancellation request
|
||||
/// </summary>
|
||||
/// <param name="timeout"></param>
|
||||
public bool StopQueueProcessor(int timeout)
|
||||
{
|
||||
this.processQueueCancelToken.Cancel();
|
||||
return this.queueProcessorTask.Wait(timeout);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if cancellation is requested
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public bool IsCancelRequested
|
||||
{
|
||||
get
|
||||
{
|
||||
return this.processQueueCancelToken.IsCancellationRequested;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queue a binding request item
|
||||
/// </summary>
|
||||
public virtual QueueItem QueueBindingOperation(
|
||||
string key,
|
||||
Func<IBindingContext, CancellationToken, object> bindOperation,
|
||||
Func<IBindingContext, object> timeoutOperation = null,
|
||||
Func<Exception, object> errorHandler = null,
|
||||
int? bindingTimeout = null,
|
||||
int? waitForLockTimeout = null)
|
||||
{
|
||||
// don't add null operations to the binding queue
|
||||
if (bindOperation == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
QueueItem queueItem = new QueueItem()
|
||||
{
|
||||
Key = key,
|
||||
BindOperation = bindOperation,
|
||||
TimeoutOperation = timeoutOperation,
|
||||
ErrorHandler = errorHandler,
|
||||
BindingTimeout = bindingTimeout,
|
||||
WaitForLockTimeout = waitForLockTimeout
|
||||
};
|
||||
|
||||
lock (this.bindingQueueLock)
|
||||
{
|
||||
this.bindingQueue.AddLast(queueItem);
|
||||
}
|
||||
|
||||
this.itemQueuedEvent.Set();
|
||||
|
||||
return queueItem;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a particular binding context is connected or not
|
||||
/// </summary>
|
||||
/// <param name="key"></param>
|
||||
public bool IsBindingContextConnected(string key)
|
||||
{
|
||||
lock (this.bindingContextLock)
|
||||
{
|
||||
IBindingContext context;
|
||||
if (this.BindingContextMap.TryGetValue(key, out context))
|
||||
{
|
||||
return context.IsConnected;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or creates a binding context for the provided context key
|
||||
/// </summary>
|
||||
/// <param name="key"></param>
|
||||
protected IBindingContext GetOrCreateBindingContext(string key)
|
||||
{
|
||||
// use a default binding context for disconnected requests
|
||||
if (string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
key = "disconnected_binding_context";
|
||||
}
|
||||
|
||||
lock (this.bindingContextLock)
|
||||
{
|
||||
if (!this.BindingContextMap.ContainsKey(key))
|
||||
{
|
||||
var bindingContext = new T();
|
||||
this.BindingContextMap.Add(key, bindingContext);
|
||||
this.BindingContextTasks.Add(bindingContext, Task.Run(() => null));
|
||||
}
|
||||
|
||||
return this.BindingContextMap[key];
|
||||
}
|
||||
}
|
||||
|
||||
protected IEnumerable<IBindingContext> GetBindingContexts(string keyPrefix)
|
||||
{
|
||||
// use a default binding context for disconnected requests
|
||||
if (string.IsNullOrWhiteSpace(keyPrefix))
|
||||
{
|
||||
keyPrefix = "disconnected_binding_context";
|
||||
}
|
||||
|
||||
lock (this.bindingContextLock)
|
||||
{
|
||||
return this.BindingContextMap.Where(x => x.Key.StartsWith(keyPrefix)).Select(v => v.Value);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a binding context already exists for the provided context key
|
||||
/// </summary>
|
||||
protected bool BindingContextExists(string key)
|
||||
{
|
||||
lock (this.bindingContextLock)
|
||||
{
|
||||
return this.BindingContextMap.ContainsKey(key);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove the binding queue entry
|
||||
/// </summary>
|
||||
protected void RemoveBindingContext(string key)
|
||||
{
|
||||
lock (this.bindingContextLock)
|
||||
{
|
||||
if (this.BindingContextMap.ContainsKey(key))
|
||||
{
|
||||
// disconnect existing connection
|
||||
var bindingContext = this.BindingContextMap[key];
|
||||
if (bindingContext.ServerConnection != null && bindingContext.ServerConnection.IsOpen)
|
||||
{
|
||||
// Disconnecting can take some time so run it in a separate task so that it doesn't block removal
|
||||
Task.Run(() =>
|
||||
{
|
||||
bindingContext.ServerConnection.Cancel();
|
||||
bindingContext.ServerConnection.Disconnect();
|
||||
});
|
||||
}
|
||||
|
||||
// remove key from the map
|
||||
this.BindingContextMap.Remove(key);
|
||||
this.BindingContextTasks.Remove(bindingContext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public bool HasPendingQueueItems
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (this.bindingQueueLock)
|
||||
{
|
||||
return this.bindingQueue.Count > 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the next pending queue item
|
||||
/// </summary>
|
||||
private QueueItem GetNextQueueItem()
|
||||
{
|
||||
lock (this.bindingQueueLock)
|
||||
{
|
||||
if (this.bindingQueue.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
QueueItem queueItem = this.bindingQueue.First.Value;
|
||||
this.bindingQueue.RemoveFirst();
|
||||
return queueItem;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts the queue processing thread
|
||||
/// </summary>
|
||||
private Task StartQueueProcessorAsync()
|
||||
{
|
||||
if (this.processQueueCancelToken != null)
|
||||
{
|
||||
this.processQueueCancelToken.Dispose();
|
||||
}
|
||||
this.processQueueCancelToken = new CancellationTokenSource();
|
||||
|
||||
return Task.Factory.StartNew(
|
||||
ProcessQueue,
|
||||
this.processQueueCancelToken.Token,
|
||||
TaskCreationOptions.LongRunning,
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The core queue processing method
|
||||
/// </summary>
|
||||
/// <param name="state"></param>
|
||||
private void ProcessQueue()
|
||||
{
|
||||
CancellationToken token = this.processQueueCancelToken.Token;
|
||||
WaitHandle[] waitHandles = new WaitHandle[2]
|
||||
{
|
||||
this.itemQueuedEvent,
|
||||
token.WaitHandle
|
||||
};
|
||||
|
||||
while (true)
|
||||
{
|
||||
// wait for with an item to be queued or the a cancellation request
|
||||
WaitHandle.WaitAny(waitHandles);
|
||||
if (token.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// dispatch all pending queue items
|
||||
while (this.HasPendingQueueItems)
|
||||
{
|
||||
QueueItem queueItem = GetNextQueueItem();
|
||||
if (queueItem == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
IBindingContext bindingContext = GetOrCreateBindingContext(queueItem.Key);
|
||||
if (bindingContext == null)
|
||||
{
|
||||
queueItem.ItemProcessed.Set();
|
||||
continue;
|
||||
}
|
||||
|
||||
var bindingContextTask = this.BindingContextTasks[bindingContext];
|
||||
|
||||
// Run in the binding context task in case this task has to wait for a previous binding operation
|
||||
this.BindingContextTasks[bindingContext] = bindingContextTask.ContinueWith((task) =>
|
||||
{
|
||||
bool lockTaken = false;
|
||||
try
|
||||
{
|
||||
// prefer the queue item binding item, otherwise use the context default timeout
|
||||
int bindTimeout = queueItem.BindingTimeout ?? bindingContext.BindingTimeout;
|
||||
|
||||
// handle the case a previous binding operation is still running
|
||||
if (!bindingContext.BindingLock.WaitOne(queueItem.WaitForLockTimeout ?? 0))
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.Write(TraceEventType.Warning, "Binding queue operation timed out waiting for previous operation to finish");
|
||||
queueItem.Result = queueItem.TimeoutOperation != null
|
||||
? queueItem.TimeoutOperation(bindingContext)
|
||||
: null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Write(TraceEventType.Error, "Exception running binding queue lock timeout handler: " + ex.ToString());
|
||||
}
|
||||
finally
|
||||
{
|
||||
queueItem.ItemProcessed.Set();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
bindingContext.BindingLock.Reset();
|
||||
|
||||
lockTaken = true;
|
||||
|
||||
// execute the binding operation
|
||||
object result = null;
|
||||
CancellationTokenSource cancelToken = new CancellationTokenSource();
|
||||
|
||||
// run the operation in a separate thread
|
||||
var bindTask = Task.Run(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
result = queueItem.BindOperation(
|
||||
bindingContext,
|
||||
cancelToken.Token);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Write(TraceEventType.Error, "Unexpected exception on the binding queue: " + ex.ToString());
|
||||
if (queueItem.ErrorHandler != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
result = queueItem.ErrorHandler(ex);
|
||||
}
|
||||
catch (Exception ex2)
|
||||
{
|
||||
Logger.Write(TraceEventType.Error, "Unexpected exception in binding queue error handler: " + ex2.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
if (IsExceptionOfType(ex, typeof(SqlException)) || IsExceptionOfType(ex, typeof(SocketException)))
|
||||
{
|
||||
if (this.OnUnhandledException != null)
|
||||
{
|
||||
this.OnUnhandledException(queueItem.Key, ex);
|
||||
}
|
||||
|
||||
RemoveBindingContext(queueItem.Key);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Task.Run(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
// check if the binding tasks completed within the binding timeout
|
||||
if (bindTask.Wait(bindTimeout))
|
||||
{
|
||||
queueItem.Result = result;
|
||||
}
|
||||
else
|
||||
{
|
||||
cancelToken.Cancel();
|
||||
|
||||
// if the task didn't complete then call the timeout callback
|
||||
if (queueItem.TimeoutOperation != null)
|
||||
{
|
||||
queueItem.Result = queueItem.TimeoutOperation(bindingContext);
|
||||
}
|
||||
|
||||
bindTask.ContinueWithOnFaulted(t => Logger.Write(TraceEventType.Error, "Binding queue threw exception " + t.Exception.ToString()));
|
||||
|
||||
// Give the task a chance to complete before moving on to the next operation
|
||||
bindTask.Wait();
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Write(TraceEventType.Error, "Binding queue task completion threw exception " + ex.ToString());
|
||||
}
|
||||
finally
|
||||
{
|
||||
// set item processed to avoid deadlocks
|
||||
if (lockTaken)
|
||||
{
|
||||
bindingContext.BindingLock.Set();
|
||||
}
|
||||
queueItem.ItemProcessed.Set();
|
||||
}
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// catch and log any exceptions raised in the binding calls
|
||||
// set item processed to avoid deadlocks
|
||||
Logger.Write(TraceEventType.Error, "Binding queue threw exception " + ex.ToString());
|
||||
// set item processed to avoid deadlocks
|
||||
if (lockTaken)
|
||||
{
|
||||
bindingContext.BindingLock.Set();
|
||||
}
|
||||
queueItem.ItemProcessed.Set();
|
||||
}
|
||||
}, TaskContinuationOptions.None);
|
||||
|
||||
// if a queue processing cancellation was requested then exit the loop
|
||||
if (token.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
lock (this.bindingQueueLock)
|
||||
{
|
||||
// verify the binding queue is still empty
|
||||
if (this.bindingQueue.Count == 0)
|
||||
{
|
||||
// reset the item queued event since we've processed all the pending items
|
||||
this.itemQueuedEvent.Reset();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clear queued items
|
||||
/// </summary>
|
||||
public void ClearQueuedItems()
|
||||
{
|
||||
lock (this.bindingQueueLock)
|
||||
{
|
||||
if (this.bindingQueue.Count > 0)
|
||||
{
|
||||
this.bindingQueue.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (this.processQueueCancelToken != null)
|
||||
{
|
||||
this.processQueueCancelToken.Dispose();
|
||||
}
|
||||
|
||||
if (itemQueuedEvent != null)
|
||||
{
|
||||
itemQueuedEvent.Dispose();
|
||||
}
|
||||
|
||||
if (this.BindingContextMap != null)
|
||||
{
|
||||
foreach (var item in this.BindingContextMap)
|
||||
{
|
||||
if (item.Value != null && item.Value.ServerConnection != null && item.Value.ServerConnection.SqlConnectionObject != null)
|
||||
{
|
||||
item.Value.ServerConnection.SqlConnectionObject.Close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsExceptionOfType(Exception ex, Type t)
|
||||
{
|
||||
return ex.GetType() == t || (ex.InnerException != null && ex.InnerException.GetType() == t);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user