3647 Kusto Refresh Logic (#1067)

* 3647 Renamed ParentMetadata to DatabaseMetadata in FolderMetadata.cs. Removed Refresh function from IDataSource and DataSourceBase since it's not used outside of KustoDataSource. Added unit tests for ServerNode.cs. Removed unused constructor from TreeNode and unused AddChild function.

* 3647 Refactored KustoDataSource. Moved Set logic for DatabaseMetadata and columns into separate functions

* 3647 Reverted Rename of ParentMetadata in FolderMetadata. Reverted removal of Refresh function from DataSourceBase and IDatasource

* 3647 Renamed SmoModel directories to DataSourceModel in Kusto projects.

* 3647 Removed reference to Kusto.ServiceLayer in SqlTools.ServiceLayer
This commit is contained in:
Justin M
2020-09-08 08:01:39 -07:00
committed by GitHub
parent 479e5242ff
commit cd1e4f5ec5
12 changed files with 266 additions and 103 deletions

View File

@@ -77,8 +77,9 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
/// <inheritdoc/>
public abstract IEnumerable<DataSourceObjectMetadata> GetChildObjects(DataSourceObjectMetadata parentMetadata, bool includeSizeDetails = false);
/// <param name="includeDatabase"></param>
/// <inheritdoc/>
public abstract void Refresh();
public abstract void Refresh(bool includeDatabase);
/// <inheritdoc/>
public abstract void Refresh(DataSourceObjectMetadata objectMetadata);

View File

@@ -73,7 +73,8 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
/// <summary>
/// Refresh object list for entire cluster.
/// </summary>
void Refresh();
/// <param name="includeDatabase"></param>
void Refresh(bool includeDatabase);
/// <summary>
/// Refresh object list for given object.

View File

@@ -347,40 +347,45 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
{
if (_databaseMetadata == null)
{
CancellationTokenSource source = new CancellationTokenSource();
CancellationToken token = source.Token;
// Getting database names when we are connected to a specific database should not happen.
ValidationUtils.IsNotNull(DatabaseName, nameof(DatabaseName));
var query = ".show databases" + (this.ClusterName.IndexOf(AriaProxyURL, StringComparison.CurrentCultureIgnoreCase) == -1 ? " | project DatabaseName, PrettyName" : "");
if (includeSizeDetails == true){
query = ".show cluster extents | summarize sum(OriginalSize) by tostring(DatabaseName)";
}
using (var reader = ExecuteQuery(query, token))
{
_databaseMetadata = reader.ToEnumerable()
.Where(row => !string.IsNullOrWhiteSpace(row["DatabaseName"].ToString()))
.Select(row => new DatabaseMetadata
{
ClusterName = this.ClusterName,
MetadataType = DataSourceMetadataType.Database,
MetadataTypeName = DataSourceMetadataType.Database.ToString(),
SizeInMB = includeSizeDetails == true ? row["sum_OriginalSize"].ToString() : null,
Name = row["DatabaseName"].ToString(),
PrettyName = includeSizeDetails == true ? row["DatabaseName"].ToString(): (String.IsNullOrEmpty(row["PrettyName"]?.ToString()) ? row["DatabaseName"].ToString() : row["PrettyName"].ToString()),
Urn = $"{this.ClusterName}.{row["DatabaseName"].ToString()}"
})
.Materialize()
.OrderBy(row => row.Name, StringComparer.Ordinal); // case-sensitive
}
SetDatabaseMetadata(includeSizeDetails);
}
return _databaseMetadata;
}
private void SetDatabaseMetadata(bool includeSizeDetails)
{
CancellationTokenSource source = new CancellationTokenSource();
CancellationToken token = source.Token;
// Getting database names when we are connected to a specific database should not happen.
ValidationUtils.IsNotNull(DatabaseName, nameof(DatabaseName));
var query = ".show databases" + (this.ClusterName.IndexOf(AriaProxyURL, StringComparison.CurrentCultureIgnoreCase) == -1 ? " | project DatabaseName, PrettyName" : "");
if (includeSizeDetails == true){
query = ".show cluster extents | summarize sum(OriginalSize) by tostring(DatabaseName)";
}
using (var reader = ExecuteQuery(query, token))
{
_databaseMetadata = reader.ToEnumerable()
.Where(row => !string.IsNullOrWhiteSpace(row["DatabaseName"].ToString()))
.Select(row => new DatabaseMetadata
{
ClusterName = this.ClusterName,
MetadataType = DataSourceMetadataType.Database,
MetadataTypeName = DataSourceMetadataType.Database.ToString(),
SizeInMB = includeSizeDetails == true ? row["sum_OriginalSize"].ToString() : null,
Name = row["DatabaseName"].ToString(),
PrettyName = includeSizeDetails == true ? row["DatabaseName"].ToString(): (String.IsNullOrEmpty(row["PrettyName"]?.ToString()) ? row["DatabaseName"].ToString() : row["PrettyName"].ToString()),
Urn = $"{this.ClusterName}.{row["DatabaseName"].ToString()}"
})
.Materialize()
.OrderBy(row => row.Name, StringComparer.Ordinal); // case-sensitive
}
}
/// <inheritdoc/>
public override bool Exists(DataSourceObjectMetadata objectMetadata)
{
@@ -509,11 +514,17 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
return markers.ToArray();
}
/// <inheritdoc/>
public override void Refresh()
/// <summary>
/// Clears everything
/// </summary>
/// <param name="includeDatabase"></param>
public override void Refresh(bool includeDatabase)
{
// This class caches objects. Throw them away so that the next call will re-query the data source for the objects.
_databaseMetadata = null;
if (includeDatabase)
{
_databaseMetadata = null;
}
_tableMetadata = new ConcurrentDictionary<string, IEnumerable<TableMetadata>>();
_columnMetadata = new ConcurrentDictionary<string, IEnumerable<DataSourceObjectMetadata>>();
_folderMetadata = new ConcurrentDictionary<string, IEnumerable<FolderMetadata>>();
@@ -528,49 +539,45 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
switch(objectMetadata.MetadataType)
{
case DataSourceMetadataType.Cluster:
Refresh();
Refresh(true);
SetDatabaseMetadata(false);
break;
case DataSourceMetadataType.Database:
_tableMetadata.TryRemove(objectMetadata.Name, out _);
Refresh(false);
LoadTableSchema(objectMetadata);
LoadFunctionSchema(objectMetadata);
break;
case DataSourceMetadataType.Table:
var tm = objectMetadata as TableMetadata;
_columnMetadata.TryRemove(GenerateMetadataKey(tm.DatabaseName, tm.Name), out _);
var table = objectMetadata as TableMetadata;
_columnMetadata.TryRemove(GenerateMetadataKey(table.DatabaseName, table.Name), out _);
SetTableSchema(table);
break;
case DataSourceMetadataType.Column:
// Remove column metadata for the whole table
var cm = objectMetadata as ColumnMetadata;
_columnMetadata.TryRemove(GenerateMetadataKey(cm.DatabaseName, cm.TableName), out _);
break;
case DataSourceMetadataType.Function:
var fm = objectMetadata as FunctionMetadata;
_functionMetadata.TryRemove(GenerateMetadataKey(fm.DatabaseName, fm.Name), out _);
break;
case DataSourceMetadataType.Folder:
Refresh(false);
var folder = objectMetadata as FolderMetadata;
_folderMetadata.TryRemove(GenerateMetadataKey(folder.ParentMetadata.Name, folder.Name), out _);
LoadTableSchema(folder.ParentMetadata);
LoadFunctionSchema(folder.ParentMetadata);
break;
default:
throw new ArgumentException($"Unexpected type {objectMetadata.MetadataType}.");
}
}
/// <inheritdoc/>
public override IEnumerable<DataSourceObjectMetadata> GetChildObjects(DataSourceObjectMetadata objectMetadata, bool includeSizeDetails)
public override IEnumerable<DataSourceObjectMetadata> GetChildObjects(DataSourceObjectMetadata objectMetadata,
bool includeSizeDetails = false)
{
ValidationUtils.IsNotNull(objectMetadata, nameof(objectMetadata));
switch(objectMetadata.MetadataType)
switch (objectMetadata.MetadataType)
{
case DataSourceMetadataType.Cluster: // show databases
return GetDatabaseMetadata(includeSizeDetails);
case DataSourceMetadataType.Database: // show folders, tables, and functions
return includeSizeDetails
? GetTablesForDashboard(objectMetadata)
@@ -589,7 +596,7 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
}
}
public override DiagnosticsInfo GetDiagnostics(DataSourceObjectMetadata objectMetadata)
public override DiagnosticsInfo GetDiagnostics(DataSourceObjectMetadata objectMetadata)
{
ValidationUtils.IsNotNull(objectMetadata, nameof(objectMetadata));
@@ -765,16 +772,23 @@ namespace Microsoft.Kusto.ServiceLayer.DataSource
return _columnMetadata[key];
}
SetTableSchema(tableMetadata);
return _columnMetadata.ContainsKey(key)
? _columnMetadata[key]
: Enumerable.Empty<DataSourceObjectMetadata>();
}
private void SetTableSchema(TableMetadata tableMetadata)
{
IEnumerable<ColumnInfo> columnInfos = GetColumnInfos(tableMetadata.DatabaseName, tableMetadata.Name);
if (!columnInfos.Any())
{
return Enumerable.Empty<DataSourceObjectMetadata>();
return;
}
SetColumnMetadata(tableMetadata.DatabaseName, tableMetadata.Name, columnInfos);
return _columnMetadata[key];
}
private void SetFolderMetadataForTables(DataSourceObjectMetadata objectMetadata, IEnumerable<TableInfo> tableInfos, string rootTableFolderKey)

View File

@@ -36,7 +36,7 @@
<ProjectReference Include="../Microsoft.SqlTools.ManagedBatchParser/Microsoft.SqlTools.ManagedBatchParser.csproj" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="ObjectExplorer\SmoModel\TreeNodeDefinition.xml" />
<EmbeddedResource Include="ObjectExplorer\DataSourceModel\TreeNodeDefinition.xml" />
<EmbeddedResource Include="Localization\sr.resx" />
<None Include="Localization\sr.strings" />
</ItemGroup>

View File

@@ -47,7 +47,7 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.DataSourceModel
/// <summary>
/// Returns the label to display to the user.
/// </summary>
internal string GetConnectionLabel()
private string GetConnectionLabel()
{
string userName = connectionSummary.UserName;

View File

@@ -30,7 +30,7 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes
private string nodePath;
private string label;
private string nodePathName;
public const char PathPartSeperator = '/';
private const char PathPartSeperator = '/';
/// <summary>
/// Object metadata
@@ -43,27 +43,17 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes
public IDataSource DataSource { get; set; }
/// <summary>
/// Constructor with no required inputs
/// Constructor with DataSource and DataSourceObjectMetadata
/// </summary>
public TreeNode(IDataSource dataSource, DataSourceObjectMetadata objectMetadata)
/// <param name="dataSource"></param>
/// <param name="objectMetadata"></param>
protected TreeNode(IDataSource dataSource, DataSourceObjectMetadata objectMetadata)
{
DataSource = dataSource;
ObjectMetadata = objectMetadata;
NodeValue = objectMetadata.Name;
}
/// <summary>
/// Constructor that accepts a label to identify the node
/// </summary>
/// <param name="value">Label identifying the node</param>
public TreeNode(string value, IDataSource dataSource, DataSourceObjectMetadata objectMetadata)
: this(dataSource, objectMetadata)
{
// We intentionally do not valid this being null or empty since
// some nodes may need to set it
NodeValue = value;
}
private object buildingMetadataLock = new object();
/// <summary>
@@ -133,7 +123,7 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes
/// for many nodes such as the server, the display label will be different
/// to the value.
/// </summary>
public string Label {
protected string Label {
get
{
if(label == null)
@@ -290,17 +280,6 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes
return new ReadOnlyCollection<TreeNode>(children);
}
/// <summary>
/// Adds a child to the list of children under this node
/// </summary>
/// <param name="newChild"><see cref="TreeNode"/></param>
public void AddChild(TreeNode newChild)
{
Validate.IsNotNull(nameof(newChild), newChild);
children.Add(newChild);
newChild.Parent = this;
}
/// <summary>
/// Optional context to help with lookup of children
/// </summary>
@@ -326,7 +305,7 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes
return Parent as T;
}
protected virtual void PopulateChildren(bool refresh, string name, CancellationToken cancellationToken)
protected void PopulateChildren(bool refresh, string name, CancellationToken cancellationToken)
{
Logger.Write(TraceEventType.Verbose, string.Format(CultureInfo.InvariantCulture, "Populating oe node :{0}", this.GetNodePath()));
Debug.Assert(IsAlwaysLeaf == false);
@@ -368,25 +347,22 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes
}
}
protected IEnumerable<TreeNode> ExpandChildren(TreeNode parent, bool refresh, string name, bool includeSystemObjects, CancellationToken cancellationToken)
protected IEnumerable<TreeNode> ExpandChildren(TreeNode parent, bool refresh, string name,
bool includeSystemObjects, CancellationToken cancellationToken)
{
List<TreeNode> allChildren = new List<TreeNode>();
try
{
OnExpandPopulateNonFolders(allChildren, parent, refresh, name, cancellationToken);
return OnExpandPopulateNonFolders(parent, refresh, name, cancellationToken);
}
catch(Exception ex)
catch (Exception ex)
{
string error = string.Format(CultureInfo.InvariantCulture, "Failed expanding oe children. parent:{0} error:{1} inner:{2} stacktrace:{3}",
parent != null ? parent.GetNodePath() : "", ex.Message, ex.InnerException != null ? ex.InnerException.Message : "", ex.StackTrace);
string error = string.Format(CultureInfo.InvariantCulture,
"Failed expanding oe children. parent:{0} error:{1} inner:{2} stacktrace:{3}",
parent != null ? parent.GetNodePath() : "", ex.Message,
ex.InnerException != null ? ex.InnerException.Message : "", ex.StackTrace);
Logger.Write(TraceEventType.Error, error);
throw ex;
}
finally
{
}
return allChildren;
}
/// <summary>
@@ -394,7 +370,7 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes
/// </summary>
/// <param name="allChildren">List to which nodes should be added</param>
/// <param name="parent">Parent the nodes are being added to</param>
private void OnExpandPopulateNonFolders(IList<TreeNode> allChildren, TreeNode parent, bool refresh, string name, CancellationToken cancellationToken)
private List<TreeNode> OnExpandPopulateNonFolders(TreeNode parent, bool refresh, string name, CancellationToken cancellationToken)
{
Logger.Write(TraceEventType.Verbose, string.Format(CultureInfo.InvariantCulture, "child factory parent :{0}", parent.GetNodePath()));
@@ -406,9 +382,15 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes
if (parent.DataSource != null)
{
if (refresh)
{
parent.DataSource.Refresh(parent.ObjectMetadata);
}
objectMetadataList = parent.DataSource.GetChildObjects(parent.ObjectMetadata);
}
List<TreeNode> allChildren = new List<TreeNode>();
foreach (var objectMetadata in objectMetadataList)
{
cancellationToken.ThrowIfCancellationRequested();
@@ -423,6 +405,7 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes
}
}
return allChildren;
}
catch (Exception ex)
{
@@ -436,7 +419,12 @@ namespace Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes
/// <summary>
/// The glue between the DataSource and the Object Explorer models. Creates the right tree node for each data source type
/// </summary>
protected TreeNode CreateChild(TreeNode parent, DataSourceObjectMetadata childMetadata)
/// <param name="parent"></param>
/// <param name="childMetadata"></param>
/// <returns></returns>
/// <exception cref="ArgumentException"></exception>
private TreeNode CreateChild(TreeNode parent, DataSourceObjectMetadata childMetadata)
{
ValidationUtils.IsNotNull(parent, nameof(parent));
ValidationUtils.IsNotNull(childMetadata, nameof(childMetadata));

View File

@@ -33,7 +33,6 @@
<ProjectReference Include="../Microsoft.SqlTools.Hosting/Microsoft.SqlTools.Hosting.csproj" />
<ProjectReference Include="../Microsoft.SqlTools.Credentials/Microsoft.SqlTools.Credentials.csproj" />
<ProjectReference Include="../Microsoft.SqlTools.ManagedBatchParser/Microsoft.SqlTools.ManagedBatchParser.csproj" />
<ProjectReference Include="../Microsoft.Kusto.ServiceLayer/Microsoft.Kusto.ServiceLayer.csproj" />
<ProjectReference Include="../Microsoft.InsightsGenerator/Microsoft.InsightsGenerator.csproj" />
</ItemGroup>
<ItemGroup>

View File

@@ -0,0 +1,160 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using Microsoft.Kusto.ServiceLayer.Connection.Contracts;
using Microsoft.Kusto.ServiceLayer.DataSource;
using Microsoft.Kusto.ServiceLayer.DataSource.Metadata;
using Microsoft.Kusto.ServiceLayer.ObjectExplorer.DataSourceModel;
using Microsoft.Kusto.ServiceLayer.ObjectExplorer.Nodes;
using Microsoft.Kusto.ServiceLayer.Utility;
using Microsoft.SqlTools.Extensibility;
using Moq;
using NUnit.Framework;
namespace Microsoft.Kusto.ServiceLayer.UnitTests.ObjectExplorer.DataSourceModel
{
public class ServerNodeTests
{
[Test]
public void ServerNode_ThrowsException_NullConnectionParam()
{
ConnectionCompleteParams param = null;
var serviceProviderMock = new Mock<IMultiServiceProvider>();
Assert.Throws<ArgumentNullException>(() => new ServerNode(param, serviceProviderMock.Object, null, new DatabaseMetadata()));
}
[Test]
public void ServerNode_ThrowsException_NullConnectionSummary()
{
var param = new ConnectionCompleteParams
{
ConnectionSummary = null
};
var serviceProviderMock = new Mock<IMultiServiceProvider>();
Assert.Throws<ArgumentNullException>(() => new ServerNode(param, serviceProviderMock.Object, null, new DatabaseMetadata()));
}
[Test]
public void ServerNode_ThrowsException_NullServiceProvider()
{
var param = new ConnectionCompleteParams
{
ConnectionSummary = new ConnectionSummary()
};
Assert.Throws<ArgumentNullException>(() => new ServerNode(param, null, null, new DatabaseMetadata()));
}
[TestCase(CommonConstants.MasterDatabaseName, "User1", "Server Name (SQL Server 2020 - User1)")]
[TestCase(CommonConstants.MasterDatabaseName, "", "Server Name (SQL Server 2020)")]
[TestCase(CommonConstants.MsdbDatabaseName, "User2", "Server Name (SQL Server 2020 - User2)")]
[TestCase(CommonConstants.MsdbDatabaseName, "", "Server Name (SQL Server 2020)")]
[TestCase(CommonConstants.ModelDatabaseName, "User3", "Server Name (SQL Server 2020 - User3)")]
[TestCase(CommonConstants.ModelDatabaseName, "", "Server Name (SQL Server 2020)")]
[TestCase(CommonConstants.TempDbDatabaseName, "User4", "Server Name (SQL Server 2020 - User4)")]
[TestCase(CommonConstants.TempDbDatabaseName, "", "Server Name (SQL Server 2020)")]
[TestCase("Database1", "User5", "Server Name (SQL Server 2020 - User5, Database1)")]
[TestCase("Database1", "", "Server Name (SQL Server 2020 - Database1)")]
public void GetConnectionLabel_Sets_Label_For_Params(string databaseName, string userName, string expected)
{
var connectionParams = new ConnectionCompleteParams
{
ConnectionSummary = new ConnectionSummary
{
DatabaseName = databaseName,
UserName = userName,
ServerName = "Server Name"
},
ServerInfo = new ServerInfo
{
ServerVersion = "2020"
}
};
var serviceProviderMock = new Mock<IMultiServiceProvider>();
var dataSourceMock = new Mock<IDataSource>();
var metadata = new DataSourceObjectMetadata();
var serverNode = new ServerNode(connectionParams, serviceProviderMock.Object, dataSourceMock.Object,
metadata);
var nodeInfo = serverNode.ToNodeInfo();
Assert.AreEqual(expected, nodeInfo.Label);
}
[Test]
public void Refresh_Calls_DataSourceRefresh()
{
var connectionParams = new ConnectionCompleteParams
{
ConnectionSummary = new ConnectionSummary
{
UserName = "UserName",
DatabaseName = "DatabaseName",
ServerName = "ServerName"
},
ServerInfo = new ServerInfo
{
ServerVersion = "Version"
}
};
var serviceProviderMock = new Mock<IMultiServiceProvider>();
var dataSourceMock = new Mock<IDataSource>();
var metadata = new DataSourceObjectMetadata();
var serverNode = new ServerNode(connectionParams, serviceProviderMock.Object, dataSourceMock.Object,
metadata);
serverNode.Refresh(CancellationToken.None);
dataSourceMock.Verify(x => x.Refresh(It.IsAny<DataSourceObjectMetadata>()), Times.Once());
}
[Test]
public void Refresh_Returns_Children()
{
var childMetadata = new DataSourceObjectMetadata
{
MetadataTypeName = DataSourceMetadataType.Database.ToString(),
MetadataType = DataSourceMetadataType.Database,
Name = "Database1"
};
var connectionParams = new ConnectionCompleteParams
{
ConnectionSummary = new ConnectionSummary
{
UserName = "UserName",
DatabaseName = "DatabaseName",
ServerName = "ServerName"
},
ServerInfo = new ServerInfo
{
ServerVersion = "Version"
}
};
var serviceProviderMock = new Mock<IMultiServiceProvider>();
var dataSourceMock = new Mock<IDataSource>();
dataSourceMock.Setup(x => x.GetChildObjects(It.IsAny<DataSourceObjectMetadata>(), It.IsAny<bool>()))
.Returns(new List<DataSourceObjectMetadata> {childMetadata});
var parentMetadata = new DataSourceObjectMetadata();
var serverNode = new ServerNode(connectionParams, serviceProviderMock.Object, dataSourceMock.Object,
parentMetadata);
var children = serverNode.Refresh(CancellationToken.None);
Assert.AreEqual(1, children.Count);
var child = children.First();
Assert.AreEqual(childMetadata.MetadataTypeName, child.NodeType);
Assert.AreEqual(NodeTypes.Database, child.NodeTypeId);
Assert.AreEqual(childMetadata.Name, child.NodeValue);
}
}
}