Added AzureMonitor to Microsoft.Kusto.ServiceLayer (#1208)

* Added AzureMonitor to Microsoft.Kusto.ServiceLayer.

* Added Intellisense for AzureMonitor. Moved Intellisense logic from KustoIntellisenseClient to IntellisenseClientBase.

* Added ServiceName as a command parameter for starting Kusto.

* Added check to return null if connectionInfo is not in the connectionService.

* Added support for Dashboard in MetadataService and AdminService.

* Removed workspace id from databaseName for Monitor. Added logic for MetadataService and AdminService to return different information for AzureMonitor.

* Moved providerName and providerDescription to DataSourceFactory.

* Changed DatabaseName to include Name and Id. Changed ProviderName to LOGANALYTICS in DataSourceFactory

* Fixed unit tests

* Changed logic to use ServiceName instead of server to determine DataSourceType

* Code review feedback and reverted changes to ObjectExplorerService.

* Removed unused reference from HostLoader

* Changed Parallel.Invoke to await Task.Run

* Moved Kusto datasource and supporting classes to separate directory.

* Removed unused datasourceFactory from ConnectionService. Added GetDatabases and GetDatabaseInfo to IDataSource and child classes

* Renamed Instance variables in ObjectExplorerService. Removed unused attribute on TSqlFormatterService. Removed invalid comment in ConnectionService.

* Fixed warnings in build.

* Moved SizeInMB to DatabaseMetadata. Refactored ConvertToDatabaseInfo

* Fixed unit test
This commit is contained in:
Justin M
2021-06-25 21:40:45 -07:00
committed by GitHub
parent 9877af54b9
commit 1f7da97e6c
49 changed files with 1070 additions and 603 deletions

View File

@@ -7,6 +7,8 @@ using System.Collections.Generic;
using System.Threading;
using System.Data;
using System.Threading.Tasks;
using Microsoft.Kusto.ServiceLayer.Admin.Contracts;
using Microsoft.Kusto.ServiceLayer.Connection.Contracts;
using Microsoft.Kusto.ServiceLayer.DataSource.Intellisense;
using Microsoft.Kusto.ServiceLayer.Utility;
using Microsoft.Kusto.ServiceLayer.DataSource.Metadata;
@@ -100,13 +102,16 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
public abstract CompletionItem[] GetAutoCompleteSuggestions(ScriptDocumentInfo scriptDocumentInfo, Position textPosition,
bool throwOnError = false);
public abstract ListDatabasesResponse GetDatabases(string serverName, bool includeDetails);
public abstract DatabaseInfo GetDatabaseInfo(string serverName, string databaseName);
/// <inheritdoc/>
public DataSourceType DataSourceType { get; protected set; }
/// <inheritdoc/>
public abstract string ClusterName { get; }
public abstract string DatabaseName { get; }
public abstract string DatabaseName { get; set; }
#endregion
}

View File

@@ -6,6 +6,8 @@ using Microsoft.Kusto.ServiceLayer.Connection.Contracts;
using Microsoft.Kusto.ServiceLayer.DataSource.Contracts;
using Microsoft.SqlTools.ServiceLayer.Connection.ReliableConnection;
using Microsoft.Kusto.ServiceLayer.DataSource.Intellisense;
using Microsoft.Kusto.ServiceLayer.DataSource.Kusto;
using Microsoft.Kusto.ServiceLayer.DataSource.Monitor;
using Microsoft.Kusto.ServiceLayer.LanguageServices;
using Microsoft.Kusto.ServiceLayer.LanguageServices.Contracts;
using Microsoft.Kusto.ServiceLayer.Utility;
@@ -16,8 +18,14 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
[Export(typeof(IDataSourceFactory))]
public class DataSourceFactory : IDataSourceFactory
{
public IDataSource Create(DataSourceType dataSourceType, ConnectionDetails connectionDetails, string ownerUri)
private const string KustoProviderName = "KUSTO";
private const string LogAnalyticsProviderName = "LOGANALYTICS";
private const string KustoProviderDescription = "Microsoft Azure Data Explorer";
private const string LogAnalyticsProviderDescription = "Microsoft Azure Monitor Explorer";
public IDataSource Create(ConnectionDetails connectionDetails, string ownerUri)
{
var dataSourceType = GetDataSourceType();
switch (dataSourceType)
{
case DataSourceType.Kusto:
@@ -27,7 +35,12 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
var intellisenseClient = new KustoIntellisenseClient(kustoClient);
return new KustoDataSource(kustoClient, intellisenseClient);
}
case DataSourceType.LogAnalytics:
{
var httpClient = new MonitorClient(connectionDetails.ServerName, connectionDetails.AccountToken);
var intellisenseClient = new MonitorIntellisenseClient(httpClient);
return new MonitorDataSource(httpClient, intellisenseClient);
}
default:
throw new ArgumentException($@"Unsupported data source type ""{dataSourceType}""",
@@ -35,6 +48,11 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
}
}
private DataSourceType GetDataSourceType()
{
return Program.ServiceName.Contains("Kusto") ? DataSourceType.Kusto : DataSourceType.LogAnalytics;
}
private DataSourceConnectionDetails MapKustoConnectionDetails(ConnectionDetails connectionDetails)
{
if (connectionDetails.AuthenticationType == "dstsAuth" || connectionDetails.AuthenticationType == "AzureMFA")
@@ -95,7 +113,7 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
}
default:
throw new ArgumentException($"Unsupported data source type \"{dataSourceType}\"", nameof(dataSourceType));
throw new ArgumentException($@"Unsupported data source type ""{dataSourceType}""", nameof(dataSourceType));
}
}
@@ -109,7 +127,7 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
}
default:
throw new ArgumentException($"Unsupported data source type \"{dataSourceType}\"", nameof(dataSourceType));
throw new ArgumentException($@"Unsupported data source type ""{dataSourceType}""", nameof(dataSourceType));
}
}
@@ -126,8 +144,18 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
}
default:
throw new ArgumentException($"Unsupported data source type \"{dataSourceType}\"", nameof(dataSourceType));
throw new ArgumentException($@"Unsupported data source type ""{dataSourceType}""", nameof(dataSourceType));
}
}
public static string GetProviderName()
{
return Program.ServiceName.Contains("Kusto") ? KustoProviderName : LogAnalyticsProviderName;
}
public static string GetProviderDescription()
{
return Program.ServiceName.Contains("Kusto") ? KustoProviderDescription : LogAnalyticsProviderDescription;
}
}
}

View File

@@ -23,6 +23,6 @@
/// <summary>
/// An Operations Management Suite (OMS) Log Analytics workspace.
/// </summary>
OmsLogAnalytics
LogAnalytics
}
}

View File

@@ -3,6 +3,8 @@ using System.Collections.Generic;
using System.Data;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Kusto.ServiceLayer.Admin.Contracts;
using Microsoft.Kusto.ServiceLayer.Connection.Contracts;
using Microsoft.Kusto.ServiceLayer.DataSource.Intellisense;
using Microsoft.Kusto.ServiceLayer.DataSource.Metadata;
using Microsoft.Kusto.ServiceLayer.LanguageServices;
@@ -117,5 +119,7 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
DefinitionResult GetDefinition(string queryText, int index, int startLine, int startColumn, bool throwOnError = false);
Hover GetHoverHelp(ScriptDocumentInfo scriptDocumentInfo, Position textPosition, bool throwOnError = false);
CompletionItem[] GetAutoCompleteSuggestions(ScriptDocumentInfo scriptDocumentInfo, Position textPosition, bool throwOnError = false);
ListDatabasesResponse GetDatabases(string serverName, bool includeDetails);
DatabaseInfo GetDatabaseInfo(string serverName, string databaseName);
}
}

View File

@@ -4,6 +4,6 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
{
public interface IDataSourceFactory
{
IDataSource Create(DataSourceType dataSourceType, ConnectionDetails connectionDetails, string ownerUri);
IDataSource Create(ConnectionDetails connectionDetails, string ownerUri);
}
}

View File

@@ -1,15 +0,0 @@
using Microsoft.Kusto.ServiceLayer.LanguageServices;
using Microsoft.Kusto.ServiceLayer.LanguageServices.Contracts;
using Microsoft.Kusto.ServiceLayer.Workspace.Contracts;
namespace Microsoft.Kusto.ServiceLayer.DataSource.Intellisense
{
public interface IIntellisenseClient
{
void UpdateDatabase(string databaseName);
ScriptFileMarker[] GetSemanticMarkers(ScriptParseInfo parseInfo, ScriptFile scriptFile, string queryText);
DefinitionResult GetDefinition(string queryText, int index, int startLine, int startColumn, bool throwOnError = false);
Hover GetHoverHelp(ScriptDocumentInfo scriptDocumentInfo, Position textPosition, bool throwOnError = false);
CompletionItem[] GetAutoCompleteSuggestions(ScriptDocumentInfo scriptDocumentInfo, Position textPosition, bool throwOnError = false);
}
}

View File

@@ -0,0 +1,141 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Kusto.Language;
using Kusto.Language.Editor;
using Microsoft.Kusto.ServiceLayer.LanguageServices;
using Microsoft.Kusto.ServiceLayer.LanguageServices.Contracts;
using Microsoft.Kusto.ServiceLayer.Workspace.Contracts;
using CompletionItem = Microsoft.Kusto.ServiceLayer.LanguageServices.Contracts.CompletionItem;
using Diagnostic = Kusto.Language.Diagnostic;
namespace Microsoft.Kusto.ServiceLayer.DataSource.Intellisense
{
public abstract class IntellisenseClientBase
{
protected GlobalState schemaState;
public abstract void UpdateDatabase(string databaseName);
public ScriptFileMarker[] GetSemanticMarkers(ScriptParseInfo parseInfo, ScriptFile scriptFile, string queryText)
{
var kustoCodeService = new KustoCodeService(queryText, schemaState);
var script = CodeScript.From(queryText, schemaState);
var parseResult = new List<Diagnostic>();
foreach (var codeBlock in script.Blocks)
{
parseResult.AddRange(codeBlock.Service.GetDiagnostics());
}
parseInfo.ParseResult = parseResult;
if (!parseResult.Any())
{
return Array.Empty<ScriptFileMarker>();
}
// build a list of Kusto script file markers from the errors.
var markers = new List<ScriptFileMarker>();
foreach (var error in parseResult)
{
script.TryGetLineAndOffset(error.Start, out var startLine, out var startOffset);
script.TryGetLineAndOffset(error.End, out var endLine, out var endOffset);
// vscode specific format for error markers.
markers.Add(new ScriptFileMarker
{
Message = error.Message,
Level = ScriptFileMarkerLevel.Error,
ScriptRegion = new ScriptRegion
{
File = scriptFile.FilePath,
StartLineNumber = startLine,
StartColumnNumber = startOffset,
StartOffset = 0,
EndLineNumber = endLine,
EndColumnNumber = endOffset,
EndOffset = 0
}
});
}
return markers.ToArray();
}
public DefinitionResult GetDefinition(string queryText, int index, int startLine, int startColumn,
bool throwOnError = false)
{
//TODOKusto: API wasnt working properly, need to check that part.
return null;
}
public Hover GetHoverHelp(ScriptDocumentInfo scriptDocumentInfo, Position textPosition, bool throwOnError = false)
{
var script = CodeScript.From(scriptDocumentInfo.Contents, schemaState);
script.TryGetTextPosition(textPosition.Line + 1, textPosition.Character + 1, out int position);
var codeBlock = script.GetBlockAtPosition(position);
var quickInfo = codeBlock.Service.GetQuickInfo(position);
return AutoCompleteHelper.ConvertQuickInfoToHover(
quickInfo.Text,
"kusto",
scriptDocumentInfo.StartLine,
scriptDocumentInfo.StartColumn,
textPosition.Character);
}
public CompletionItem[] GetAutoCompleteSuggestions(ScriptDocumentInfo scriptDocumentInfo, Position textPosition,
bool throwOnError = false)
{
var script = CodeScript.From(scriptDocumentInfo.Contents, schemaState);
script.TryGetTextPosition(textPosition.Line + 1, textPosition.Character + 1, out int position); // Gets the actual offset based on line and local offset
var codeBlock = script.GetBlockAtPosition(position);
var completion = codeBlock.Service.GetCompletionItems(position);
scriptDocumentInfo.ScriptParseInfo.CurrentSuggestions = completion.Items; // this is declaration item so removed for now, but keep the info when api gets updated
var completions = new List<CompletionItem>();
foreach (var autoCompleteItem in completion.Items)
{
var label = autoCompleteItem.DisplayText;
var insertText = autoCompleteItem.Kind == CompletionKind.Table || autoCompleteItem.Kind == CompletionKind.Database
? KustoQueryUtils.EscapeName(label)
: label;
var completionKind = CreateCompletionItemKind(autoCompleteItem.Kind);
completions.Add(AutoCompleteHelper.CreateCompletionItem(label, autoCompleteItem.Kind.ToString(),
insertText, completionKind, scriptDocumentInfo.StartLine, scriptDocumentInfo.StartColumn,
textPosition.Character));
}
return completions.ToArray();
}
private CompletionItemKind CreateCompletionItemKind(CompletionKind kustoKind)
{
switch (kustoKind)
{
case CompletionKind.Syntax:
return CompletionItemKind.Module;
case CompletionKind.Column:
return CompletionItemKind.Field;
case CompletionKind.Variable:
return CompletionItemKind.Variable;
case CompletionKind.Table:
return CompletionItemKind.File;
case CompletionKind.Database:
return CompletionItemKind.Method;
case CompletionKind.LocalFunction:
case CompletionKind.DatabaseFunction:
case CompletionKind.BuiltInFunction:
case CompletionKind.AggregateFunction:
return CompletionItemKind.Function;
default:
return CompletionItemKind.Keyword;
}
}
}
}

View File

@@ -3,7 +3,7 @@ using System.Data;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.Kusto.ServiceLayer.DataSource
namespace Microsoft.Kusto.ServiceLayer.DataSource.Kusto
{
public interface IKustoClient
{

View File

@@ -17,7 +17,7 @@ using Microsoft.Kusto.ServiceLayer.Connection;
using Microsoft.Kusto.ServiceLayer.DataSource.Contracts;
using Microsoft.Kusto.ServiceLayer.Utility;
namespace Microsoft.Kusto.ServiceLayer.DataSource
namespace Microsoft.Kusto.ServiceLayer.DataSource.Kusto
{
public class KustoClient : IKustoClient
{

View File

@@ -1,19 +1,21 @@
// <copyright file="KustoDataSource.cs" company="Microsoft">
// Copyright (c) Microsoft. All Rights Reserved.
// </copyright>
using System;
using System.Collections.Generic;
using System.Threading;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Data;
using System.Globalization;
using System.Linq;
using System.Text;
using Newtonsoft.Json;
using System.Threading;
using System.Threading.Tasks;
using Kusto.Cloud.Platform.Data;
using Kusto.Data;
using Kusto.Data.Data;
using Microsoft.Kusto.ServiceLayer.Admin.Contracts;
using Microsoft.Kusto.ServiceLayer.Connection.Contracts;
using Microsoft.Kusto.ServiceLayer.DataSource.Intellisense;
using Microsoft.Kusto.ServiceLayer.DataSource.Metadata;
using Microsoft.Kusto.ServiceLayer.DataSource.Models;
@@ -21,8 +23,9 @@ using Microsoft.Kusto.ServiceLayer.LanguageServices;
using Microsoft.Kusto.ServiceLayer.LanguageServices.Contracts;
using Microsoft.Kusto.ServiceLayer.Utility;
using Microsoft.Kusto.ServiceLayer.Workspace.Contracts;
using Newtonsoft.Json;
namespace Microsoft.Kusto.ServiceLayer.DataSource
namespace Microsoft.Kusto.ServiceLayer.DataSource.Kusto
{
/// <summary>
/// Represents Kusto utilities.
@@ -30,7 +33,7 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
public class KustoDataSource : DataSourceBase
{
private readonly IKustoClient _kustoClient;
private readonly IIntellisenseClient _intellisenseClient;
private readonly IntellisenseClientBase _intellisenseClient;
/// <summary>
/// List of databases.
@@ -57,7 +60,11 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
/// </summary>
private ConcurrentDictionary<string, IEnumerable<FunctionMetadata>> _functionMetadata = new ConcurrentDictionary<string, IEnumerable<FunctionMetadata>>();
public override string DatabaseName => _kustoClient.DatabaseName;
public override string DatabaseName
{
get => _kustoClient.DatabaseName;
set => throw new NotImplementedException();
}
public override string ClusterName => _kustoClient.ClusterName;
@@ -79,7 +86,7 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
/// <summary>
/// Prevents a default instance of the <see cref="IDataSource"/> class from being created.
/// </summary>
public KustoDataSource(IKustoClient kustoClient, IIntellisenseClient intellisenseClient)
public KustoDataSource(IKustoClient kustoClient, IntellisenseClientBase intellisenseClient)
{
_kustoClient = kustoClient;
_intellisenseClient = intellisenseClient;
@@ -835,6 +842,44 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
return _intellisenseClient.GetAutoCompleteSuggestions(scriptDocumentInfo, textPosition, throwOnError);
}
public override ListDatabasesResponse GetDatabases(string serverName, bool includeDetails)
{
DataSourceObjectMetadata objectMetadata = MetadataFactory.CreateClusterMetadata(serverName);
// Mainly used by "manage" dashboard
if (includeDetails)
{
IEnumerable<DataSourceObjectMetadata> databaseMetadataInfo = GetChildObjects(objectMetadata, true);
List<DatabaseInfo> metadata = MetadataFactory.ConvertToDatabaseInfo(databaseMetadataInfo);
return new ListDatabasesResponse
{
Databases = metadata.ToArray()
};
}
IEnumerable<DataSourceObjectMetadata> databaseMetadata = GetChildObjects(objectMetadata);
if (databaseMetadata != null)
{
return new ListDatabasesResponse
{
DatabaseNames = databaseMetadata
.Select(objMeta => objMeta.PrettyName == objMeta.Name ? objMeta.PrettyName : $"{objMeta.PrettyName} ({objMeta.Name})")
.ToArray()
};
}
return new ListDatabasesResponse();;
}
public override DatabaseInfo GetDatabaseInfo(string serverName, string databaseName)
{
DataSourceObjectMetadata objectMetadata = MetadataFactory.CreateClusterMetadata(serverName);
var metadata = GetChildObjects(objectMetadata, true).Where(o => o.Name == databaseName).ToList();
List<DatabaseInfo> databaseInfo = MetadataFactory.ConvertToDatabaseInfo(metadata);
return databaseInfo.ElementAtOrDefault(0);
}
#endregion
}
}

View File

@@ -4,34 +4,25 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Kusto.Language;
using Kusto.Language.Editor;
using Kusto.Language.Symbols;
using Kusto.Language.Syntax;
using Microsoft.Kusto.ServiceLayer.LanguageServices;
using Microsoft.Kusto.ServiceLayer.LanguageServices.Contracts;
using Microsoft.Kusto.ServiceLayer.Workspace.Contracts;
using Diagnostic = Kusto.Language.Diagnostic;
using Microsoft.Kusto.ServiceLayer.DataSource.Intellisense;
namespace Microsoft.Kusto.ServiceLayer.DataSource.Intellisense
namespace Microsoft.Kusto.ServiceLayer.DataSource.Kusto
{
public class KustoIntellisenseClient : IIntellisenseClient
public class KustoIntellisenseClient : IntellisenseClientBase
{
private readonly IKustoClient _kustoClient;
/// <summary>
/// SchemaState used for getting intellisense info.
/// </summary>
private GlobalState _schemaState;
public KustoIntellisenseClient(IKustoClient kustoClient)
{
_kustoClient = kustoClient;
_schemaState = LoadSchemaState(kustoClient.DatabaseName, kustoClient.ClusterName);
schemaState = LoadSchemaState(kustoClient.DatabaseName, kustoClient.ClusterName);
}
public void UpdateDatabase(string databaseName)
public override void UpdateDatabase(string databaseName)
{
_schemaState = LoadSchemaState(databaseName, _kustoClient.ClusterName);
schemaState = LoadSchemaState(databaseName, _kustoClient.ClusterName);
}
private GlobalState LoadSchemaState(string databaseName, string clusterName)
@@ -237,135 +228,5 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource.Intellisense
return function.Signatures[0].Parameters;
}
public ScriptFileMarker[] GetSemanticMarkers(ScriptParseInfo parseInfo, ScriptFile scriptFile, string queryText)
{
var kustoCodeService = new KustoCodeService(queryText, _schemaState);
var script = CodeScript.From(queryText, _schemaState);
var parseResult = new List<Diagnostic>();
foreach (var codeBlock in script.Blocks)
{
parseResult.AddRange(codeBlock.Service.GetDiagnostics());
}
parseInfo.ParseResult = parseResult;
if (!parseResult.Any())
{
return Array.Empty<ScriptFileMarker>();
}
// build a list of Kusto script file markers from the errors.
var markers = new List<ScriptFileMarker>();
foreach (var error in parseResult)
{
script.TryGetLineAndOffset(error.Start, out var startLine, out var startOffset);
script.TryGetLineAndOffset(error.End, out var endLine, out var endOffset);
// vscode specific format for error markers.
markers.Add(new ScriptFileMarker
{
Message = error.Message,
Level = ScriptFileMarkerLevel.Error,
ScriptRegion = new ScriptRegion
{
File = scriptFile.FilePath,
StartLineNumber = startLine,
StartColumnNumber = startOffset,
StartOffset = 0,
EndLineNumber = endLine,
EndColumnNumber = endOffset,
EndOffset = 0
}
});
}
return markers.ToArray();
}
public DefinitionResult GetDefinition(string queryText, int index, int startLine, int startColumn, bool throwOnError = false)
{
//TODOKusto: API wasnt working properly, need to check that part.
var abc = KustoCode.ParseAndAnalyze(queryText, _schemaState);
var kustoCodeService = new KustoCodeService(abc);
//var kustoCodeService = new KustoCodeService(queryText, globals);
var relatedInfo = kustoCodeService.GetRelatedElements(index);
if (relatedInfo != null && relatedInfo.Elements.Count > 1)
{
}
return null;
}
public Hover GetHoverHelp(ScriptDocumentInfo scriptDocumentInfo, Position textPosition, bool throwOnError = false)
{
var script = CodeScript.From(scriptDocumentInfo.Contents, _schemaState);
script.TryGetTextPosition(textPosition.Line + 1, textPosition.Character + 1, out int position);
var codeBlock = script.GetBlockAtPosition(position);
var quickInfo = codeBlock.Service.GetQuickInfo(position);
return AutoCompleteHelper.ConvertQuickInfoToHover(
quickInfo.Text,
"kusto",
scriptDocumentInfo.StartLine,
scriptDocumentInfo.StartColumn,
textPosition.Character);
}
public LanguageServices.Contracts.CompletionItem[] GetAutoCompleteSuggestions(ScriptDocumentInfo scriptDocumentInfo, Position textPosition, bool throwOnError = false)
{
var script = CodeScript.From(scriptDocumentInfo.Contents, _schemaState);
script.TryGetTextPosition(textPosition.Line + 1, textPosition.Character + 1,
out int position); // Gets the actual offset based on line and local offset
var codeBlock = script.GetBlockAtPosition(position);
var completion = codeBlock.Service.GetCompletionItems(position);
scriptDocumentInfo.ScriptParseInfo.CurrentSuggestions =
completion.Items; // this is declaration item so removed for now, but keep the info when api gets updated
var completions = new List<LanguageServices.Contracts.CompletionItem>();
foreach (var autoCompleteItem in completion.Items)
{
var label = autoCompleteItem.DisplayText;
var insertText = autoCompleteItem.Kind == CompletionKind.Table || autoCompleteItem.Kind == CompletionKind.Database
? KustoQueryUtils.EscapeName(label)
: label;
var completionKind = CreateCompletionItemKind(autoCompleteItem.Kind);
completions.Add(AutoCompleteHelper.CreateCompletionItem(label, autoCompleteItem.Kind.ToString(),
insertText, completionKind, scriptDocumentInfo.StartLine, scriptDocumentInfo.StartColumn,
textPosition.Character));
}
return completions.ToArray();
}
private CompletionItemKind CreateCompletionItemKind(CompletionKind kustoKind)
{
switch (kustoKind)
{
case CompletionKind.Syntax:
return CompletionItemKind.Module;
case CompletionKind.Column:
return CompletionItemKind.Field;
case CompletionKind.Variable:
return CompletionItemKind.Variable;
case CompletionKind.Table:
return CompletionItemKind.File;
case CompletionKind.Database:
return CompletionItemKind.Method;
case CompletionKind.LocalFunction:
case CompletionKind.DatabaseFunction:
case CompletionKind.BuiltInFunction:
case CompletionKind.AggregateFunction:
return CompletionItemKind.Function;
default:
return CompletionItemKind.Keyword;
}
}
}
}

View File

@@ -7,11 +7,12 @@ using System.Collections.Generic;
using System.Linq;
using Kusto.Language;
using Kusto.Language.Editor;
using Microsoft.Kusto.ServiceLayer.DataSource.Intellisense;
using Microsoft.Kusto.ServiceLayer.LanguageServices;
using Microsoft.Kusto.ServiceLayer.LanguageServices.Contracts;
using Microsoft.Kusto.ServiceLayer.Workspace.Contracts;
namespace Microsoft.Kusto.ServiceLayer.DataSource.Intellisense
namespace Microsoft.Kusto.ServiceLayer.DataSource.Kusto
{
/// <summary>
/// Kusto specific class for intellisense helper functions.

View File

@@ -1,7 +1,7 @@
using System.Collections.Generic;
using System.Data;
namespace Microsoft.Kusto.ServiceLayer.DataSource
namespace Microsoft.Kusto.ServiceLayer.DataSource.Kusto
{
internal class KustoResultsReader : DataReaderWrapper
{

View File

@@ -92,6 +92,18 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
}
}
public static void SafeAdd(this Dictionary<string, List<DataSourceObjectMetadata>> dictionary, string key, DataSourceObjectMetadata node)
{
if (dictionary.ContainsKey(key))
{
dictionary[key].Add(node);
}
else
{
dictionary[key] = new List<DataSourceObjectMetadata> {node};
}
}
/// <summary>
/// Add a range to a dictionary of ConcurrentDictionary. Adds range to existing IEnumerable within dictionary
/// at the same key.

View File

@@ -15,6 +15,6 @@
public string Urn { get; set; }
public string SizeInMB { get; set; }
}
}

View File

@@ -6,5 +6,7 @@
public class DatabaseMetadata : DataSourceObjectMetadata
{
public string ClusterName { get; set; }
public string SizeInMB { get; set; }
}
}

View File

@@ -59,24 +59,34 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource.Metadata
/// <summary>
/// Converts database details shown on cluster manage dashboard to DatabaseInfo type. Add DataSourceType as param if required to show different properties
/// </summary>
/// <param name="clusterDBDetails"></param>
/// <param name="clusterDbDetails"></param>
/// <returns></returns>
public static List<DatabaseInfo> ConvertToDatabaseInfo(IEnumerable<DataSourceObjectMetadata> clusterDBDetails)
public static List<DatabaseInfo> ConvertToDatabaseInfo(IEnumerable<DataSourceObjectMetadata> clusterDbDetails)
{
var databaseDetails = new List<DatabaseInfo>();
if (clusterDBDetails.FirstOrDefault() is DatabaseMetadata)
if (clusterDbDetails.FirstOrDefault() is not DatabaseMetadata)
{
foreach (var dbDetail in clusterDBDetails)
{
DatabaseInfo databaseInfo = new DatabaseInfo();
long.TryParse(dbDetail.SizeInMB, out long sum_OriginalSize);
databaseInfo.Options["name"] = dbDetail.Name;
databaseInfo.Options["sizeInMB"] = (sum_OriginalSize / (1024 * 1024)).ToString();
databaseDetails.Add(databaseInfo);
}
return new List<DatabaseInfo>();
}
var databaseDetails = new List<DatabaseInfo>();
foreach (var dataSourceObjectMetadata in clusterDbDetails)
{
var dbDetail = (DatabaseMetadata) dataSourceObjectMetadata;
long.TryParse(dbDetail.SizeInMB, out long sizeInMb);
var databaseInfo = new DatabaseInfo
{
Options =
{
["name"] = dbDetail.Name,
["sizeInMB"] = (sizeInMb / (1024 * 1024)).ToString()
}
};
databaseDetails.Add(databaseInfo);
}
return databaseDetails;
}
@@ -100,5 +110,17 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource.Metadata
return databaseChildDetails;
}
public static DataSourceObjectMetadata CreateDataSourceObjectMetadata(DataSourceMetadataType datatype, string name, string urn)
{
return new DataSourceObjectMetadata
{
MetadataType = datatype,
MetadataTypeName = datatype.ToString(),
Name = name,
PrettyName = name,
Urn = $"{urn}",
};
}
}
}

View File

@@ -0,0 +1,78 @@
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.OperationalInsights;
using Microsoft.Azure.OperationalInsights.Models;
using Microsoft.Kusto.ServiceLayer.DataSource.Monitor.Responses;
using Microsoft.Rest;
namespace Microsoft.Kusto.ServiceLayer.DataSource.Monitor
{
public class MonitorClient
{
private readonly OperationalInsightsDataClient _queryClient;
private readonly HttpClient _httpClient;
private readonly string _workspaceId;
private readonly string _token;
private const string BaseUri = "https://api.loganalytics.io/v1/workspaces";
private WorkspaceResponse _metadata;
public string WorkspaceId => _workspaceId;
public MonitorClient(string workspaceId, string token)
{
_workspaceId = workspaceId;
_token = token;
_httpClient = GetHttpClient(token);
_queryClient = new OperationalInsightsDataClient(new TokenCredentials(token))
{
WorkspaceId = workspaceId
};
}
public WorkspaceResponse LoadMetadata(bool refresh = false)
{
if (_metadata != null && !refresh)
{
return _metadata;
}
var url = $"{BaseUri}/{_workspaceId}/metadata";
var httpResponseMessage = _httpClient.GetAsync(url).Result;
var results = httpResponseMessage.Content.ReadAsStringAsync().Result;
var options = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true
};
_metadata = JsonSerializer.Deserialize<WorkspaceResponse>(results, options);
return _metadata;
}
private HttpClient GetHttpClient(string token)
{
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
return httpClient;
}
public async Task<QueryResults> QueryAsync(string query, CancellationToken cancellationToken)
{
return await _queryClient.QueryAsync(query, cancellationToken: cancellationToken);
}
public QueryResults Query(string query)
{
return _queryClient.Query(query);
}
~MonitorClient()
{
_httpClient.Dispose();
_queryClient.Dispose();
}
}
}

View File

@@ -0,0 +1,223 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Kusto.ServiceLayer.Admin.Contracts;
using Microsoft.Kusto.ServiceLayer.Connection.Contracts;
using Microsoft.Kusto.ServiceLayer.DataSource.Intellisense;
using Microsoft.Kusto.ServiceLayer.DataSource.Metadata;
using Microsoft.Kusto.ServiceLayer.DataSource.Monitor.Responses;
using Microsoft.Kusto.ServiceLayer.DataSource.Monitor.Responses.Models;
using Microsoft.Kusto.ServiceLayer.LanguageServices;
using Microsoft.Kusto.ServiceLayer.LanguageServices.Contracts;
using Microsoft.Kusto.ServiceLayer.Workspace.Contracts;
namespace Microsoft.Kusto.ServiceLayer.DataSource.Monitor
{
public class MonitorDataSource : DataSourceBase
{
private readonly MonitorClient _monitorClient;
private readonly IntellisenseClientBase _intellisenseClient;
private WorkspaceResponse _metadata;
private Dictionary<string, List<DataSourceObjectMetadata>> _nodes;
public override string ClusterName => _monitorClient.WorkspaceId;
public override string DatabaseName { get; set; }
public MonitorDataSource(MonitorClient monitorClient, IntellisenseClientBase intellisenseClient)
{
_monitorClient = monitorClient;
_intellisenseClient = intellisenseClient;
_nodes = new Dictionary<string, List<DataSourceObjectMetadata>>();
_metadata = _monitorClient.LoadMetadata();
DataSourceType = DataSourceType.LogAnalytics;
SetupTableGroups(monitorClient.WorkspaceId);
}
private void SetupTableGroups(string workspaceId)
{
var workspace = _metadata.Workspaces.First(x => x.Id == workspaceId);
DatabaseName = $"{workspace.Name} ({workspace.Id})";
var metadataTableGroups = _metadata.TableGroups.ToDictionary(x => x.Id);
foreach (string workspaceTableGroup in workspace.TableGroups)
{
var tableGroup = metadataTableGroups[workspaceTableGroup];
var tableGroupNodeInfo =
MetadataFactory.CreateDataSourceObjectMetadata(DataSourceMetadataType.Folder, tableGroup.Name, $"{workspace.Id}.{tableGroup.Name}");
_nodes.SafeAdd($"{workspace.Id}", tableGroupNodeInfo);
SetupTables(tableGroupNodeInfo);
}
}
private void SetupTables(DataSourceObjectMetadata tableGroupNodeInfo)
{
var tables = GetNonEmptyTableNames();
var metadataTables = _metadata.Tables.ToDictionary(x => x.Name);
foreach (string tableName in tables)
{
var table = metadataTables[tableName];
var tableNodeInfo = MetadataFactory.CreateDataSourceObjectMetadata(DataSourceMetadataType.Table, table.Name,
$"{tableGroupNodeInfo.Urn}.{table.Name}");
_nodes.SafeAdd(tableGroupNodeInfo.Urn, tableNodeInfo);
SetupColumns(table, tableNodeInfo);
}
}
private IEnumerable<string> GetNonEmptyTableNames()
{
string query = "union * | summarize count() by Type";
var results = _monitorClient.Query(query);
return results.Tables[0].Rows.Select(x => x[0]).OrderBy(x => x);
}
private void SetupColumns(TablesModel table, DataSourceObjectMetadata tableNodeInfo)
{
foreach (var column in table.Columns)
{
var columnNodeInfo = MetadataFactory.CreateDataSourceObjectMetadata(DataSourceMetadataType.Column, column.Name,
$"{tableNodeInfo.Urn}.{column.Name}");
_nodes.SafeAdd(tableNodeInfo.Urn, columnNodeInfo);
}
}
public override async Task<IDataReader> ExecuteQueryAsync(string query, CancellationToken cancellationToken, string databaseName = null)
{
var results = await _monitorClient.QueryAsync(query, cancellationToken);
return results.ToDataReader();
}
public override Task<IEnumerable<T>> ExecuteControlCommandAsync<T>(string command, bool throwOnError, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public override DiagnosticsInfo GetDiagnostics(DataSourceObjectMetadata parentMetadata)
{
return new DiagnosticsInfo();
}
public override IEnumerable<DataSourceObjectMetadata> GetChildObjects(DataSourceObjectMetadata parentMetadata, bool includeSizeDetails = false)
{
// columns are always leaf nodes
if (parentMetadata.MetadataType == DataSourceMetadataType.Column)
{
return Enumerable.Empty<DataSourceObjectMetadata>();
}
if (parentMetadata.MetadataType == DataSourceMetadataType.Cluster && includeSizeDetails)
{
var child = _nodes[parentMetadata.Urn].FirstOrDefault();
return child == null ? Enumerable.Empty<DataSourceObjectMetadata>() : _nodes[child.Urn];
}
return _nodes[parentMetadata.Urn].OrderBy(x => x.PrettyName, StringComparer.OrdinalIgnoreCase);
}
public override void Refresh(bool includeDatabase)
{
// reset the data source
_nodes = new Dictionary<string, List<DataSourceObjectMetadata>>();
_metadata = _monitorClient.LoadMetadata();
SetupTableGroups(_monitorClient.WorkspaceId);
}
public override void Refresh(DataSourceObjectMetadata objectMetadata)
{
Refresh(false);
}
public override void UpdateDatabase(string databaseName)
{
// LogAnalytics is treating the workspace name as the database name
var workspaceId = ParseWorkspaceId(databaseName);
_metadata = _monitorClient.LoadMetadata(true);
var workspace = _metadata.Workspaces.First(x => x.Id == workspaceId);
DatabaseName = $"{workspace.Name} ({workspace.Id})";
_intellisenseClient.UpdateDatabase(databaseName);
}
private string ParseWorkspaceId(string workspace)
{
var regex = new Regex(@"(?<=\().+?(?=\))");
return regex.IsMatch(workspace)
? regex.Match(workspace).Value
: workspace;
}
public override Task<bool> Exists()
{
return Task.FromResult(true);
}
public override bool Exists(DataSourceObjectMetadata objectMetadata)
{
return true;
}
public override string GenerateAlterFunctionScript(string functionName)
{
throw new NotImplementedException();
}
public override string GenerateExecuteFunctionScript(string functionName)
{
throw new NotImplementedException();
}
public override ScriptFileMarker[] GetSemanticMarkers(ScriptParseInfo parseInfo, ScriptFile scriptFile, string queryText)
{
return _intellisenseClient.GetSemanticMarkers(parseInfo, scriptFile, queryText);
}
public override DefinitionResult GetDefinition(string queryText, int index, int startLine, int startColumn, bool throwOnError = false)
{
return _intellisenseClient.GetDefinition(queryText, index, startLine, startColumn, throwOnError);
}
public override Hover GetHoverHelp(ScriptDocumentInfo scriptDocumentInfo, Position textPosition, bool throwOnError = false)
{
return _intellisenseClient.GetHoverHelp(scriptDocumentInfo, textPosition, throwOnError);
}
public override CompletionItem[] GetAutoCompleteSuggestions(ScriptDocumentInfo scriptDocumentInfo, Position textPosition, bool throwOnError = false)
{
return _intellisenseClient.GetAutoCompleteSuggestions(scriptDocumentInfo, textPosition, throwOnError);
}
public override ListDatabasesResponse GetDatabases(string serverName, bool includeDetails)
{
return new ListDatabasesResponse
{
DatabaseNames = new[]
{
DatabaseName
}
};
}
public override DatabaseInfo GetDatabaseInfo(string serverName, string databaseName)
{
return new DatabaseInfo
{
Options = new Dictionary<string, object>
{
{"id", ClusterName},
{"name", DatabaseName}
}
};
}
}
}

View File

@@ -0,0 +1,91 @@
using System;
using System.Data;
using System.Linq;
using Kusto.Language.Symbols;
using Microsoft.Azure.OperationalInsights.Models;
namespace Microsoft.Kusto.ServiceLayer.DataSource.Monitor
{
public static class MonitorExtensions
{
/// <summary>
/// Converts QueryResults object into an IDataReader
/// </summary>
/// <param name="queryResults"></param>
/// <returns></returns>
public static IDataReader ToDataReader(this QueryResults queryResults)
{
var resultTable = queryResults.Tables.FirstOrDefault();
if (resultTable == null)
{
return new DataTableReader(new DataTable());
}
var dataTable = new DataTable(resultTable.Name);
foreach (var column in resultTable.Columns)
{
dataTable.Columns.Add(column.Name, MapType(column.Type));
}
foreach (var row in resultTable.Rows)
{
var dataRow = dataTable.NewRow();
for (int i = 0; i < row.Count; i++)
{
dataRow[i] = row[i] ?? DBNull.Value as object;
}
dataTable.Rows.Add(dataRow);
}
return new DataTableReader(dataTable);
}
/// <summary>
/// Map Kusto type to .NET Type equivalent using scalar data types
/// </summary>
/// <seealso href="https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/scalar-data-types/">Here</seealso>
/// <param name="type">Kusto Type</param>
/// <returns>.NET Equivalent Type</returns>
private static Type MapType(string type)
{
switch (type)
{
case "bool": return Type.GetType("System.Boolean");
case "datetime": return Type.GetType("System.DateTime");
case "dynamic": return Type.GetType("System.Object");
case "guid": return Type.GetType("System.Guid");
case "int": return Type.GetType("System.Int32");
case "long": return Type.GetType("System.Int64");
case "real": return Type.GetType("System.Double");
case "string": return Type.GetType("System.String");
case "timespan": return Type.GetType("System.TimeSpan");
case "decimal": return Type.GetType("System.Data.SqlTypes.SqlDecimal");
default: return typeof(string);
}
}
public static ScalarSymbol ToSymbolType(this string type)
{
switch (type)
{
case "bool": return ScalarTypes.Bool;
case "datetime": return ScalarTypes.DateTime;
case "dynamic": return ScalarTypes.Dynamic;
case "guid": return ScalarTypes.Guid;
case "int": return ScalarTypes.Int;
case "long": return ScalarTypes.Long;
case "real": return ScalarTypes.Real;
case "string": return ScalarTypes.String;
case "timespan": return ScalarTypes.TimeSpan;
case "decimal": return ScalarTypes.Decimal;
default: return ScalarTypes.String;
}
}
}
}

View File

@@ -0,0 +1,43 @@
using System.Collections.Generic;
using System.Linq;
using Kusto.Language;
using Kusto.Language.Symbols;
using Microsoft.Kusto.ServiceLayer.DataSource.Intellisense;
using Microsoft.Kusto.ServiceLayer.DataSource.Monitor.Responses;
namespace Microsoft.Kusto.ServiceLayer.DataSource.Monitor
{
public class MonitorIntellisenseClient : IntellisenseClientBase
{
private readonly MonitorClient _monitorClient;
public MonitorIntellisenseClient(MonitorClient monitorClient)
{
_monitorClient = monitorClient;
schemaState = LoadSchemaState(monitorClient.LoadMetadata());
}
private GlobalState LoadSchemaState(WorkspaceResponse metadata)
{
var globalState = GlobalState.Default;
var members = new List<Symbol>();
foreach (var table in metadata.Tables)
{
var columnSymbols = table.Columns.Select(x => new ColumnSymbol(x.Name, x.Type.ToSymbolType()));
var tableSymbol = new TableSymbol(table.Name, columnSymbols);
members.Add(tableSymbol);
}
var databaseSymbol = new DatabaseSymbol(metadata.Workspaces.First().Id, members);
return globalState.WithDatabase(databaseSymbol);
}
public override void UpdateDatabase(string databaseName)
{
var workspace = _monitorClient.LoadMetadata();
schemaState = LoadSchemaState(workspace);
}
}
}

View File

@@ -0,0 +1,9 @@
namespace Microsoft.Kusto.ServiceLayer.DataSource.Monitor.Responses.Models
{
public class ColumnsModel
{
public string Name { get; set; }
public string Type { get; set; }
public string Description { get; set; }
}
}

View File

@@ -0,0 +1,10 @@
namespace Microsoft.Kusto.ServiceLayer.DataSource.Monitor.Responses.Models
{
public class TableGroupsModel
{
public string Id { get; set; }
public string Name { get; set; }
public string Source { get; set; }
public string[] Tables { get; set; }
}
}

View File

@@ -0,0 +1,13 @@
namespace Microsoft.Kusto.ServiceLayer.DataSource.Monitor.Responses.Models
{
public class TablesModel
{
public string Id { get; set; }
public string Name { get; set; }
public string TimeSpanColumn { get; set; }
public ColumnsModel[] Columns { get; set; }
}
}

View File

@@ -0,0 +1,12 @@
namespace Microsoft.Kusto.ServiceLayer.DataSource.Monitor.Responses.Models
{
public class WorkspacesModel
{
public string Id { get; set; }
public string Name { get; set; }
public string Region { get; set; }
public string ResourceId { get; set; }
public string[] TableGroups { get; set; }
public string[] Tables { get; set; }
}
}

View File

@@ -0,0 +1,11 @@
using Microsoft.Kusto.ServiceLayer.DataSource.Monitor.Responses.Models;
namespace Microsoft.Kusto.ServiceLayer.DataSource.Monitor.Responses
{
public class WorkspaceResponse
{
public TableGroupsModel[] TableGroups { get; set; }
public TablesModel[] Tables { get; set; }
public WorkspacesModel[] Workspaces { get; set; }
}
}

View File

@@ -62,7 +62,7 @@ namespace Microsoft.Kusto.ServiceLayer.Connection
_connectionDetails = connectionDetails;
_dataSourceFactory = dataSourceFactory;
_ownerUri = ownerUri;
_dataSource = dataSourceFactory.Create(DataSourceType.Kusto, connectionDetails, ownerUri);
_dataSource = dataSourceFactory.Create(connectionDetails, ownerUri);
_connectionRetryPolicy = connectionRetryPolicy ?? RetryPolicyFactory.CreateNoRetryPolicy();
_commandRetryPolicy = commandRetryPolicy ?? RetryPolicyFactory.CreateNoRetryPolicy();
@@ -191,7 +191,7 @@ namespace Microsoft.Kusto.ServiceLayer.Connection
{
_connectionRetryPolicy.ExecuteAction(() =>
{
_dataSource = _dataSourceFactory.Create(DataSourceType.Kusto, _connectionDetails, _ownerUri);
_dataSource = _dataSourceFactory.Create(_connectionDetails, _ownerUri);
});
}
}