ckaczor/sqltoolsservice
1
0
Fork 0
mirror of https://github.com/ckaczor/sqltoolsservice.git synced 2026-02-16 18:47:57 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove InsightsGenerator project (#1764)

Browse Source
This commit is contained in:
Karl Burtram
2022-12-07 10:56:12 -08:00
committed by GitHub
parent eee37cb2ce
commit c9f310cb47
20 changed files with 0 additions and 1985 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/Microsoft.InsightsGenerator/DataArray.cs
View File

@@ -1,26 +0,0 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
namespace Microsoft.InsightsGenerator
{
public class DataArray
{
public enum DataType
{
Number,
String,
DateTime
}
public string[] ColumnNames { get; set; }
public string[] TransformedColumnNames { get; set; }
public DataType[] ColumnDataType { get; set; }
public object[][] Cells { get; set; }
}
}

164
src/Microsoft.InsightsGenerator/DataTransformation.cs
View File

@@ -1,164 +0,0 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Collections.Generic;
namespace Microsoft.InsightsGenerator
{
public class DataTransformer
{
private class ColumnInfo
{
public int ColumnIndex { get; set; }
public int DistinctValues { get; set; }
public DataArray.DataType DataType { get; set; }
}
public DataArray Transform(DataArray array)
{
if (array == null || array.Cells == null || array.Cells.Length == 0)
{
return array;
}
DataArray.DataType[] columnDataType;
array.TransformedColumnNames = GetColumnLabels(array , out columnDataType);
array.ColumnDataType = columnDataType;
return array;
}
private string[] GetColumnLabels(DataArray array, out DataArray.DataType[] columnDataType)
{
columnDataType = new DataArray.DataType[array.ColumnNames.Length];
int columnCount = array.Cells[0].Length;
Dictionary<DataArray.DataType, List<ColumnInfo>> columnInfo = new Dictionary<DataArray.DataType, List<ColumnInfo>>();
for (int column = 0; column < columnCount; ++column)
{
int distinctValues;
DataArray.DataType dataType = GetColumnType(array, column, out distinctValues);
columnDataType[column] = dataType;
if (!columnInfo.ContainsKey(dataType))
{
columnInfo.Add(dataType, new List<ColumnInfo>());
}
columnInfo[dataType].Add(new ColumnInfo()
{
ColumnIndex = column,
DistinctValues = distinctValues,
DataType = dataType
});
}
bool containsDateTime = columnInfo.ContainsKey(DataArray.DataType.DateTime);
string[] labels = new string[columnCount];
if (containsDateTime)
{
List<ColumnInfo> dateColumns = columnInfo[DataArray.DataType.DateTime];
for (int i = 0; i < dateColumns.Count; ++i)
{
labels[dateColumns[i].ColumnIndex] = "input_t_" + i;
}
if (columnInfo.ContainsKey(DataArray.DataType.String))
{
List<ColumnInfo> stringColumns = columnInfo[DataArray.DataType.String];
for (int i = 0; i < stringColumns.Count; ++i)
{
labels[stringColumns[i].ColumnIndex] = "slicer_" + i;
}
}
}
else
{
if (columnInfo.ContainsKey(DataArray.DataType.String))
{
int maxDistinctValue = Int32.MaxValue;
int maxColumnIndex = -1;
int maxColumnLabelIndex = 0;
List<ColumnInfo> stringColumns = columnInfo[DataArray.DataType.String];
for (int i = 0; i < stringColumns.Count; ++i)
{
if (maxDistinctValue == Int32.MaxValue || maxDistinctValue < stringColumns[i].DistinctValues)
{
maxDistinctValue = stringColumns[i].DistinctValues;
maxColumnIndex = i;
maxColumnLabelIndex = stringColumns[i].ColumnIndex;
}
}
labels[maxColumnLabelIndex] = "input_g_0";
int adjustIndex = 0;
for (int i = 0; i < stringColumns.Count; ++i)
{
if (i != maxColumnIndex)
{
labels[stringColumns[i].ColumnIndex] = "slicer_" + (i - adjustIndex);
}
else
{
++adjustIndex;
}
}
}
}
if (columnInfo.ContainsKey(DataArray.DataType.Number))
{
List<ColumnInfo> numberColumns = columnInfo[DataArray.DataType.Number];
for (int i = 0; i < numberColumns.Count; ++i)
{
labels[numberColumns[i].ColumnIndex] = "output_" + i;
}
}
return labels;
}
private DataArray.DataType GetColumnType(DataArray array, int column, out int distinctValues)
{
// count number of distinct values
HashSet<object> values = new HashSet<object>();
for (int row = 0; row < array.Cells.Length; ++row)
{
if (!values.Contains(array.Cells[row][column]))
{
values.Add(array.Cells[row][column]);
}
}
distinctValues = values.Count;
// return the provided type if available
if (array.ColumnDataType != null && array.ColumnDataType.Length > column)
{
return array.ColumnDataType[column];
}
else
{
// determine the type from the first value in array
object firstValue = array.Cells[0][column];
string firstValueString = firstValue.ToString();
long longValue;
double doubleValue;
if (long.TryParse(firstValueString, out longValue) || double.TryParse(firstValueString, out doubleValue))
{
return DataArray.DataType.Number;
}
DateTime dateValue;
if (DateTime.TryParse(firstValueString, out dateValue))
{
return DataArray.DataType.DateTime;
}
return DataArray.DataType.String;
}
}
}
}

21
src/Microsoft.InsightsGenerator/Microsoft.InsightsGenerator.csproj
View File

@@ -1,21 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Nullable>disable</Nullable>
<EnableDefaultItems>false</EnableDefaultItems>
<EnableDefaultCompileItems>false</EnableDefaultCompileItems>
<EnableDefaultEmbeddedResourceItems>false</EnableDefaultEmbeddedResourceItems>
<EnableDefaultNoneItems>false</EnableDefaultNoneItems>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<DefineConstants>$(DefineConstants);TRACE</DefineConstants>
<PreserveCompilationContext>true</PreserveCompilationContext>
<DebugType>portable</DebugType>
</PropertyGroup>
<ItemGroup>
<Compile Include="**\*.cs" Exclude="**/obj/**/*.cs" />
</ItemGroup>
<ItemGroup>
<Content Include="Templates\Templates.txt">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
</Project>

44
src/Microsoft.InsightsGenerator/Properties/AssemblyInfo.cs
View File

@@ -1,44 +0,0 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Microsoft InsightsGenerator")]
[assembly: AssemblyDescription("Provides Microsoft InsightsGenerator services.")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft")]
[assembly: AssemblyProduct("SMicrosoft InsightsGenerator")]
[assembly: AssemblyCopyright("<22> Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("5b6bd4c4-7352-4762-9ad2-578b3fbd1685")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
[assembly: AssemblyInformationalVersion("1.0.0.0")]
[assembly: InternalsVisibleTo("Microsoft.InsightsGenerator.UnitTests")]

209
src/Microsoft.InsightsGenerator/RulesEngine.cs
View File

@@ -1,209 +0,0 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace Microsoft.InsightsGenerator
{
public class RulesEngine
{
public static List<Template> Templates;
public static List<string> TopListHashHeaders = new List<string>{ "#top", "#averageSlice", "#topPerslice" , "#bottom"};
public RulesEngine()
{
Templates = GetTemplates();
}
public static ColumnHeaders TemplateParser(string templateContent)
{
ColumnHeaders ch = new ColumnHeaders();
var processedText = Regex.Replace(templateContent, @",|\\n", "");
ch.Template = templateContent;
List<string> keyvalue = processedText.Split(' ').Select(s => s.Trim()).ToList();
foreach (string s in keyvalue)
{
if (s.StartsWith("#"))
{
string headers = s.Substring(1, s.Length - 1);
if (headers.StartsWith("#"))
{
ch.DoubleHashValues.Add( "##" + headers.Substring(1, headers.Length - 1));
}
else
{
if (headers != "placeHolder")
{
ch.SingleHashValues.Add("#" + headers);
}
}
}
}
return ch;
}
/// <summary>
/// Find a matched template based on the single hash values
/// </summary>
/// <param name="singleHashHeaders"></param>
/// <returns></returns>
public static string FindMatchedTemplate(List<List<string>> singleHashHeaders, DataArray columnInfo)
{
var resultTemplate = new StringBuilder();
Templates ??= GetTemplates();
var headersWithSingleHash = GetTopHeadersWithHash(singleHashHeaders);
foreach (var template in Templates)
{
var columnHeaders = TemplateParser(template.Content);
var singleHashFromTemplate = columnHeaders.SingleHashValues.Distinct();
var isMatched = true;
// all the values from template needs to be found in the input from SigGen
foreach (var hashFromTemplate in singleHashFromTemplate)
{
if (!headersWithSingleHash.Contains(hashFromTemplate.ToLower()))
{
isMatched = false;
break;
}
}
if (isMatched)
{
// Replace # and ## values in template with actual values
resultTemplate.AppendLine(ReplaceHashesInTemplate(singleHashHeaders, columnInfo, template) + "\n");
}
}
// No matched Template found
return resultTemplate.ToString();
}
private static string ReplaceHashesInTemplate(List<List<string>>singleHashList, DataArray columnInfo, Template template)
{
StringBuilder modifiedTemp = new StringBuilder(template.Content);
// Replace single hash values
foreach (var line in singleHashList)
{
// Example of a single list (line):
// "top", "3", " input (value) %OfValue ", " input (value) %OfValue ", " input (value) %OfValue "
var headerInputs = line.ToArray();
string header = "#" + headerInputs[0];
if (TopListHashHeaders.Contains(header))
{
if(!modifiedTemp.ToString().Contains(header))
{
continue;
}
//First replace the header with the second value in the list
modifiedTemp.Replace(header, headerInputs[1]);
StringBuilder remainingStr = new StringBuilder();
for (int i = 2; i < headerInputs.Length; i++)
{
// Append all the rest of the elemet in the array separated by new line
remainingStr.AppendLine(headerInputs[i]);
}
modifiedTemp.Replace("#placeHolder", remainingStr.ToString());
}
else
{
modifiedTemp.Replace("#" + headerInputs[0], headerInputs[1]);
}
}
// Replace double hash values
var transformedColumnArray = columnInfo.TransformedColumnNames.ToArray();
var columnArray = columnInfo.ColumnNames.ToArray();
for (int p = 0; p < columnInfo.TransformedColumnNames.Length; p++)
{
modifiedTemp.Replace("##" + transformedColumnArray[p], columnArray[p]);
}
return modifiedTemp.ToString();
}
private static List<string> GetTopHeadersWithHash(List<List<string>> singleHashHeaders)
{
var topHeaderList = new List<string>();
foreach (var list in singleHashHeaders)
{
topHeaderList.Add("#" + list.First().ToLower());
}
return topHeaderList;
}
/// <summary>
/// Retrieve all the templates and template ids
/// </summary>
/// <returns>All the template & id comnbination </returns>
public static List<Template> GetTemplates()
{
var templateHolder = new List<Template>();
string assemblyPath = System.Reflection.Assembly.GetExecutingAssembly().Location;
string assemblyDirectoryPath = System.IO.Path.GetDirectoryName(assemblyPath);
string templateFilePath = Path.Combine(assemblyDirectoryPath, "Templates", "templates.txt");
using (StreamReader streamReader = new StreamReader(templateFilePath, Encoding.UTF8))
{
int temId = 0;
var wholeText = streamReader.ReadToEnd();
var templateArray = wholeText.Split(new[] { "Template " }, StringSplitOptions.None).ToList();
foreach (var line in templateArray.Where(r => r != string.Empty))
{
var parts = line.Split(new[] { Environment.NewLine }, StringSplitOptions.None).ToList();
temId = int.Parse(parts[0]);
templateHolder.Add(new Template(temId, parts[1]));
}
return templateHolder;
}
}
}
/// <summary>
/// Template container to hold the template id + template body combination
/// </summary>
public class Template
{
public Template(int id, string content)
{
Id = id;
Content = content;
}
public int Id { get; set; }
public string Content { get; set; }
}
public class ColumnHeaders
{
public ColumnHeaders()
{
SingleHashValues = new List<string>();
DoubleHashValues = new List<string>();
Template = null;
}
public List<string> SingleHashValues { get; set; }
public List<string> DoubleHashValues { get; set; }
public string Template { get; set; }
}
}

464
src/Microsoft.InsightsGenerator/SigGen.cs
View File

@@ -1,464 +0,0 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Collections.Generic;
using System.Linq;
namespace Microsoft.InsightsGenerator
{
class SignatureGenerator
{
private DataArray Table;
public SignatureGeneratorResult Result;
public SignatureGenerator(DataArray table)
{
this.Table = table;
Result = new SignatureGeneratorResult();
}
public SignatureGeneratorResult Learn()
{
var stringInputIndexes = new List<int>();
var timeInputIndexes = new List<int>();
var slicerIndexes = new List<int>();
var outputIndexes = new List<int>();
for (var i = 0; i < Table.TransformedColumnNames.Length; i++)
{
if (Table.TransformedColumnNames[i].Contains("input_g"))
{
stringInputIndexes.Add(i);
}
if (Table.TransformedColumnNames[i].Contains("input_t"))
{
timeInputIndexes.Add(i);
}
if (Table.TransformedColumnNames[i].Contains("slicer"))
{
slicerIndexes.Add(i);
}
if (Table.TransformedColumnNames[i].Contains("output"))
{
outputIndexes.Add(i);
}
}
foreach (int stringIndex in stringInputIndexes)
{
foreach (int outputIndex in outputIndexes)
{
ExecuteStringInputInsights(stringIndex, outputIndex);
foreach (int slicerIndex in slicerIndexes)
{
ExecuteStringInputSlicerInsights(stringIndex, outputIndex, slicerIndex);
}
}
}
return Result;
}
public void ExecuteStringInputInsights(int inputCol, int outputCol)
{
var n = Table.Cells.Length;
if (Table.Cells.Length >8) {
n = 3;
}
OverallAverageInsights(outputCol);
OverallBottomInsights(n, inputCol, outputCol);
OverallMaxInsights(outputCol);
OverallMinInsights(outputCol);
OverallSumInsights(outputCol);
OverallTopInsights(n, inputCol, outputCol);
UniqueInputsInsight(inputCol);
}
public void ExecuteStringInputSlicerInsights(int inputCol, int outputCol, int slicerCol)
{
var n = Table.Cells.Length;
if (Table.Cells.Length > 8)
{
n = 3;
}
if (Table.Cells.Length > 50)
{
n = 5;
}
SlicedMaxInsights(slicerCol, outputCol);
SlicedAverageInsights(slicerCol, outputCol);
SlicedBottomInsights(n, inputCol, slicerCol, outputCol);
SlicedSumInsights(slicerCol, outputCol);
SlicedPercentageInsights(slicerCol, outputCol);
SlicedSumInsights(slicerCol, outputCol);
SlicedMinInsights(slicerCol, outputCol);
SlicedTopInsights(n, inputCol, slicerCol, outputCol);
}
public void UniqueInputsInsight(int inputCol)
{
List<string> insight = new List<string>();
// Adding the insight identifier
insight.Add(SignatureGeneratorResult.uniqueInputsIdentifier);
var uniqueInputs = GetUniqueColumValues(inputCol);
insight.Add(uniqueInputs.Length.ToString());
insight.AddRange(uniqueInputs);
Result.Insights.Add(insight);
}
public void OverallTopInsights(long n, int inputColumn, int outputColumn)
{
List<string> insight = new List<string>();
// Adding the insight identifier
insight.Add(SignatureGeneratorResult.topInsightIdentifier);
insight.AddRange(GenericTop(Table.Cells, n, inputColumn, outputColumn));
Result.Insights.Add(insight);
}
public void SlicedTopInsights(long n, int inputColumn, int sliceColumn, int outputColumn)
{
List<string> insight = new List<string>();
// Adding the insight identifier
insight.Add(SignatureGeneratorResult.topSliceInsightIdentifier);
object[] slices = GetUniqueColumValues(sliceColumn);
insight.Add(slices.Length.ToString());
foreach (var slice in slices)
{
insight.Add(slice.ToString());
var sliceTable = CreateSliceBucket(sliceColumn, slice.ToString());
insight.AddRange(GenericTop(sliceTable, n, inputColumn, outputColumn));
}
Result.Insights.Add(insight);
}
public List<string> GenericTop(Object[][] table, long n, int inputColumn, int outputColumn)
{
List<string> insight = new List<string>();
Object[][] sortedTable = SortCellsByColumn(table, outputColumn);
double outputSum = CalculateColumnSum(sortedTable, outputColumn);
for (int i = sortedTable.Length - 1; i >= 0 && i >= sortedTable.Length - n; i--)
{
double percent = Percentage(Double.Parse(sortedTable[i][outputColumn].ToString()), outputSum);
string temp = String.Format("{0} ({1}) {2}%", sortedTable[i][inputColumn].ToString(), sortedTable[i][outputColumn].ToString(), percent);
insight.Add(temp);
}
// Adding the count of the result
insight.Insert(0, insight.Count.ToString());
return insight;
}
public void OverallBottomInsights(long n, int inputColumn, int outputColumn)
{
List<string> insight = new List<string>();
// Adding the insight identifier
insight.Add(SignatureGeneratorResult.bottomInsightIdentifier);
insight.AddRange(GenericBottom(Table.Cells, n, inputColumn, outputColumn));
Result.Insights.Add(insight);
}
public void SlicedBottomInsights(long n, int inputColumn, int sliceColumn, int outputColumn)
{
List<string> insight = new List<string>();
// Adding the insight identifier
insight.Add(SignatureGeneratorResult.bottomSliceInsightIdentifier);
object[] slices = GetUniqueColumValues(sliceColumn);
insight.Add(slices.Length.ToString());
foreach (var slice in slices)
{
insight.Add(slice.ToString());
var sliceTable = CreateSliceBucket(sliceColumn, slice.ToString());
insight.AddRange(GenericBottom(sliceTable, n, inputColumn, outputColumn));
}
Result.Insights.Add(insight);
}
public List<string> GenericBottom(Object[][] table, long n, int inputColumn, int outputColumn)
{
List<string> insight = new List<string>();
Object[][] sortedTable = SortCellsByColumn(table, outputColumn);
double outputSum = CalculateColumnSum(sortedTable, outputColumn);
for (int i = 0; i < n && i < sortedTable.Length; i++)
{
double percent = Percentage(Double.Parse(sortedTable[i][outputColumn].ToString()), outputSum);
string temp = String.Format("{0} ({1}) {2}%", sortedTable[i][inputColumn].ToString(), sortedTable[i][outputColumn].ToString(), percent);
insight.Add(temp);
}
// Adding the count of the result
insight.Insert(0, insight.Count.ToString());
return insight;
}
public void OverallAverageInsights(int colIndex)
{
var outputList = new List<string>();
outputList.Add(SignatureGeneratorResult.averageInsightIdentifier);
outputList.Add(CalculateColumnAverage(Table.Cells, colIndex).ToString());
Result.Insights.Add(outputList);
}
public void OverallSumInsights(int colIndex)
{
var outputList = new List<string>();
outputList.Add(SignatureGeneratorResult.sumInsightIdentifier);
outputList.Add(CalculateColumnSum(Table.Cells, colIndex).ToString());
Result.Insights.Add(outputList);
}
public void OverallMaxInsights(int colIndex)
{
var outputList = new List<string>();
outputList.Add(SignatureGeneratorResult.maxInsightIdentifier);
outputList.Add(CalculateColumnMax(Table.Cells, colIndex).ToString());
Result.Insights.Add(outputList);
}
public void OverallMinInsights(int colIndex)
{
var outputList = new List<string>();
outputList.Add(SignatureGeneratorResult.minInsightIdentifier);
outputList.Add(CalculateColumnMin(Table.Cells, colIndex).ToString());
Result.Insights.Add(outputList);
}
public void SlicedSumInsights(int sliceIndex, int colIndex)
{
var insight = new List<string>();
insight.Add(SignatureGeneratorResult.sumSliceInsightIdentifier);
object[] slices = GetUniqueColumValues(sliceIndex);
insight.Add(slices.Length.ToString());
foreach (var slice in slices)
{
insight.Add(slice.ToString());
var sliceTable = CreateSliceBucket(sliceIndex, slice.ToString());
insight.Add(CalculateColumnSum(sliceTable, colIndex).ToString());
}
Result.Insights.Add(insight);
}
public void SlicedMaxInsights(int sliceIndex, int colIndex)
{
var insight = new List<string>();
insight.Add(SignatureGeneratorResult.maxSliceInsightIdentifier);
object[] slices = GetUniqueColumValues(sliceIndex);
insight.Add(slices.Length.ToString());
foreach (var slice in slices)
{
insight.Add(slice.ToString());
var sliceTable = CreateSliceBucket(sliceIndex, slice.ToString());
insight.Add(CalculateColumnMax(sliceTable, colIndex).ToString());
}
Result.Insights.Add(insight);
}
public void SlicedMinInsights(int sliceIndex, int colIndex)
{
var insight = new List<string>();
insight.Add(SignatureGeneratorResult.minSliceInsightIdentifier);
object[] slices = GetUniqueColumValues(sliceIndex);
insight.Add(slices.Length.ToString());
foreach (var slice in slices)
{
insight.Add(slice.ToString());
var sliceTable = CreateSliceBucket(sliceIndex, slice.ToString());
insight.Add(CalculateColumnMin(sliceTable, colIndex).ToString());
}
Result.Insights.Add(insight);
}
public void SlicedAverageInsights(int sliceIndex, int colIndex)
{
var insight = new List<string>();
insight.Add(SignatureGeneratorResult.sumSliceInsightIdentifier);
object[] slices = GetUniqueColumValues(sliceIndex);
insight.Add(slices.Length.ToString());
foreach (var slice in slices)
{
insight.Add(slice.ToString());
var sliceTable = CreateSliceBucket(sliceIndex, slice.ToString());
insight.Add(CalculateColumnAverage(sliceTable, colIndex).ToString());
}
Result.Insights.Add(insight);
}
public void SlicedPercentageInsights(int sliceIndex, int colIndex)
{
var insight = new List<string>();
insight.Add(SignatureGeneratorResult.percentageSliceInsightIdentifier);
object[] slices = GetUniqueColumValues(sliceIndex);
insight.Add(slices.Length.ToString());
double totalSum = CalculateColumnSum(Table.Cells, colIndex);
foreach (var slice in slices)
{
insight.Add(slice.ToString());
var sliceTable = CreateSliceBucket(sliceIndex, slice.ToString());
double sliceSum = CalculateColumnSum(sliceTable, colIndex);
var percentagePerSlice = Percentage(sliceSum, totalSum);
insight.Add(percentagePerSlice.ToString());
}
Result.Insights.Add(insight);
}
private double CalculateColumnAverage(object[][] rows, int colIndex)
{
return Math.Round(CalculateColumnSum(rows, colIndex) / rows.Length, 2);
}
private double CalculateColumnSum(object[][] rows, int colIndex)
{
return Math.Round(rows.Sum(row => double.Parse(row[colIndex].ToString())), 2);
}
private double CalculateColumnPercentage(object[][] rows, int colIndex)
{
return rows.Sum(row => double.Parse(row[colIndex].ToString()));
}
private double CalculateColumnMin(object[][] rows, int colIndex)
{
return rows.Min(row => double.Parse(row[colIndex].ToString()));
}
private double CalculateColumnMax(object[][] rows, int colIndex)
{
return rows.Max(row => double.Parse(row[colIndex].ToString()));
}
private string[] GetUniqueColumValues(int colIndex)
{
return Table.Cells.Select(row => row[colIndex].ToString()).Distinct().ToArray();
}
public Object[][] CreateSliceBucket(int sliceColIndex, string sliceValue)
{
List<Object[]> slicedTable = new List<object[]>();
foreach (var row in Table.Cells)
{
if (row[sliceColIndex].Equals(sliceValue))
{
slicedTable.Add(DeepCloneRow(row));
}
}
return slicedTable.ToArray();
}
public object[][] SortCellsByColumn(Object[][] table, int colIndex)
{
var cellCopy = DeepCloneTable(table);
Comparer<Object> comparer = Comparer<Object>.Default;
switch (this.Table.ColumnDataType[colIndex])
{
case DataArray.DataType.Number:
Array.Sort<Object[]>(cellCopy, (x, y) => comparer.Compare(double.Parse(x[colIndex].ToString()), double.Parse(y[colIndex].ToString())));
break;
case DataArray.DataType.String:
Array.Sort<Object[]>(cellCopy, (x, y) => String.Compare(x[colIndex].ToString(), y[colIndex].ToString()));
break;
case DataArray.DataType.DateTime:
Array.Sort<Object[]>(cellCopy, (x, y) => DateTime.Compare(DateTime.Parse(x[colIndex].ToString()), DateTime.Parse(y[colIndex].ToString())));
break;
}
return cellCopy;
}
public Object[][] DeepCloneTable(object[][] table)
{
return table.Select(a => a.ToArray()).ToArray();
}
public Object[] DeepCloneRow(object[] row)
{
return row.Select(a => a).ToArray();
}
public double Percentage(double value, double sum)
{
return Math.Round((double)((value / sum) * 100), 2);
}
}
}
public class SignatureGeneratorResult
{
public SignatureGeneratorResult()
{
Insights = new List<List<string>>();
}
public List<List<string>> Insights { get; set; }
public static string topInsightIdentifier = "top";
public static string bottomInsightIdentifier = "bottom";
public static string topSliceInsightIdentifier = "topPerSlice";
public static string bottomSliceInsightIdentifier = "bottomPerSlice";
public static string averageInsightIdentifier = "average";
public static string sumInsightIdentifier = "sum";
public static string maxInsightIdentifier = "max";
public static string minInsightIdentifier = "min";
public static string averageSliceInsightIdentifier = "averagePerSlice";
public static string sumSliceInsightIdentifier = "sumPerSlice";
public static string percentageSliceInsightIdentifier = "percentagePerSlice";
public static string maxSliceInsightIdentifier = "maxPerSlice";
public static string minSliceInsightIdentifier = "minPerSlice";
public static string uniqueInputsIdentifier = "uniqueInputs";
}
/** Some general format about the output
* "time"/"string"
* "top", "3", " input (value) %OfValue ", " input (value) %OfValue ", " input (value) %OfValue "
* "top", "1", " input (value) %OfValue "
* "bottom", "3", " input (value) %OfValue ", " input (value) %OfValue ", " input (value) %OfValue "
* "average", "100"
* "mean", "100"
* "median", "100"
* "averageSlice", "#slice","nameofslice", "100", "nameofslice", "100", "nameofslice", "100"
* "topPerslice", "#slice", "nameofslice", "3", " input (value) %OfValue ", " input (value) %OfValue ", " input (value) %OfValue ",
* "nameofslice", "3", " input (value) %OfValue ", " input (value) %OfValue ", " input (value) %OfValue ",
* "nameofslice", "3", " input (value) %OfValue ", " input (value) %OfValue ", " input (value) %OfValue "
* ....
*
**/

52
src/Microsoft.InsightsGenerator/Templates/Templates.txt
View File

@@ -1,52 +0,0 @@
Template 9
There were #uniqueInputs ##input_g_0 (s), the top #top highest total ##output_0 were as follows:\n #placeHolder
Template 16
For the #slices ##SlicePar_GG_1(s), the percentage of ##OutPar_N_C_1 on #time were \n #stHData\n this was compared with #Etime where #ESlices ##SlicePar_GG_1\n #EstHData \n.
Template 17
For the #slices ##SlicePar_GG_1(s), the percentage of ##OutPar_N_C_1 for each group during the week of #time was \n #stHData\n, compared to the #Etime where #ESlices ##SlicePar_GG_1\n #EstHData \n.
Template 21
The data pattern indicates that #st have #volumeType ##OutPar_N_C_1 \n.
Template 22
The largest increase of ##OutPar_N_C_1 was on #time when it increased #increasesize% above the average value.
Template 23
The largest decrease in ##OutPar_N_C_1 was on #time when it fell below #reducesize% of the average value.
Template 24
The data trend indicates a consistent #trend over time. \n
Template 1
For the #averageSlice ##slicer_0, the volume of each is: #placeHolder
Template 2
The ##InPar_GG_1 that had largest of each ##SlicePar_GG_1 are (only top #n ##InPar_GG_1) \n
Template 3
#input had #TPer% for ##SlicePar_GG_1 #Slice \n"
Template 5
The ##InPar_GG_1 that had largest of each ##SlicePar_GG_1 are (only top #n ##InPar_GG_1) \n
Template 6
The largest and smallest ##SlicePar_GG_1 for the top #n ##InPar_GG_1 were are as follows \n
Template 7
#inp had #maxSlice the largest ##SlicePar_GG_1 #maxPer% and #minSlice the smallest ##SlicePar_GG_1 #minPer% \n
Template 8
#inp had a total of #OutPar_N_C_1 ##OutPar_N_C_1 that constitues #Per% \n \n
Template 10
There were #uniqueInputs ##InPar_GG_1(s), the total ##OutPar_N_C_1 were as follows:\n #st
Template 11
in the week #time the data contained #slices ##SlicePar_GG_1, the top #n ##SlicePar_GG_1 with highest ##OutPar_N_C_1 were as follows:\n #stHData this was compared with week #Etime with the following top #En of #Eslice ##SlicePar_GG_1\n #EstData\n
Template 12
in the week #time the data contained #slices ##SlicePar_GG_1, the top #n ##SlicePar_GG_1 with highest ##OutPar_N_C_1 were as follows:\n #stHData\n
Template 13
On day #time the data contained #slices ##SlicePar_GG_1, the top #n ##SlicePar_GG_1 with highest ##OutPar_N_C_1 were as follows:\n #stHData this was compared with day #Etime with the following top #n ##SlicePar_GG_1\n #EstHData\n
Template 14
in month #time the data contained #slices ##SlicePar_GG_1, the top #n ##SlicePar_GG_1 with highest ##OutPar_N_C_1 were as follows:\n #stData this was compared with month #Etime with the following top #n ##SlicePar_GG_1\n #EstData\n
Template 15
in month #time the data contained #slices ##SlicePar_GG_1, the top #n ##SlicePar_GG_1 with highest ##OutPar_N_C_1 were as follows:\n #stHData\n
Template 18
in the week #time the data contained #slices ##SlicePar_GG_1, the % of ##OutPar_N_C_1 for each ##SlicePar_GG_1 were as follows:\n #stHData this was compared with week #Etime with the following #En of #Eslice ##SlicePar_GG_1\n #EstHData \n
Template 19
in month #time the data contained #slices ##SlicePar_GG_1, the % of ##OutPar_N_C_1 for each ##SlicePar_GG_1 were as follows:\n #stData this was compared with month #Etime with the following #n ##SlicePar_GG_1\n #EstData\n
Template 20
in month #time the data contained #slices ##SlicePar_GG_1, the % of ##OutPar_N_C_1 for each ##SlicePar_GG_1 were as follows:\n #stData\n
Template 21
Looking at the pattern of the data it appears that #st has the #volumeType ##OutPar_N_C_1 \n
Template 22
The largest increase in ##OutPar_N_C_1 were observed on #time with an increase of #increasesize% of the avarage value of ##OutPar_N_C_1 \n
Template 23
The bottom #bottom total ##output_0 were as follows:\n #placeHolder

56
src/Microsoft.InsightsGenerator/Workflow.cs
View File

@@ -1,56 +0,0 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.InsightsGenerator
{
public class Workflow
{
public Task<string> ProcessInputData(DataArray rulesData,
CancellationToken cancellationToken = new CancellationToken())
{
// added cancellationToken just in case for future
cancellationToken.ThrowIfCancellationRequested();
//Get the signature result
SignatureGenerator siggen = new SignatureGenerator(rulesData);
return Task.Run(() =>
{
try
{
DataTransformer transformer = new DataTransformer();
transformer.Transform(rulesData);
SignatureGeneratorResult result = siggen.Learn();
// call the rules engine processor
string insights = null;
if (result?.Insights == null)
{
// Console.WriteLine("Failure in generating insights, Input not recognized!");
}
else
{
insights = RulesEngine.FindMatchedTemplate(result.Insights, rulesData);
// Console.WriteLine(
// $"Good News! Insights generator has provided you the chart text: \n{insights}\n");
}
return insights;
}
catch (Exception)
{
// Console.WriteLine(ex.ToString());
throw;
}
}, cancellationToken);
}
}
}

4
src/Microsoft.SqlTools.ServiceLayer/HostLoader.cs
View File

@@ -19,7 +19,6 @@ using Microsoft.SqlTools.ServiceLayer.DisasterRecovery;
using Microsoft.SqlTools.ServiceLayer.EditData;
using Microsoft.SqlTools.ServiceLayer.FileBrowser;
using Microsoft.SqlTools.ServiceLayer.Hosting;
using Microsoft.SqlTools.ServiceLayer.InsightsGenerator;
using Microsoft.SqlTools.ServiceLayer.LanguageExtensibility;
using Microsoft.SqlTools.ServiceLayer.LanguageServices;
using Microsoft.SqlTools.ServiceLayer.Metadata;
@@ -160,9 +159,6 @@ namespace Microsoft.SqlTools.ServiceLayer
NotebookConvertService.Instance.InitializeService(serviceHost);
serviceProvider.RegisterSingleService(NotebookConvertService.Instance);
InsightsGeneratorService.Instance.InitializeService(serviceHost);
serviceProvider.RegisterSingleService(InsightsGeneratorService.Instance);
#if INCLUDE_MIGRATION
MigrationService.Instance.InitializeService(serviceHost);
serviceProvider.RegisterSingleService(MigrationService.Instance);

46
src/Microsoft.SqlTools.ServiceLayer/InsightsGenerator/Contracts/InsightsGeneratorRequest.cs
View File

@@ -1,46 +0,0 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using Microsoft.SqlTools.Hosting.Protocol.Contracts;
using Microsoft.SqlTools.ServiceLayer.Utility;
using Microsoft.SqlTools.Utility;
namespace Microsoft.SqlTools.ServiceLayer.InsightsGenerator.Contracts
{
public class AccessibleChartData
{
public string[] Columns { get; set; }
public string[][] Rows { get; set; }
}
/// <summary>
/// Query insights generator parameters
/// </summary>
public class QueryInsightsGeneratorParams : GeneralRequestDetails
{
public AccessibleChartData Data { get; set; }
}
/// <summary>
/// Query insights generator result
/// </summary>
public class InsightsGeneratorResult : ResultStatus
{
public string InsightsText { get; set; }
}
/// <summary>
/// Query insights generato request type
/// </summary>
public class QueryInsightsGeneratorRequest
{
/// <summary>
/// Request definition
/// </summary>
public static readonly
RequestType<QueryInsightsGeneratorParams, InsightsGeneratorResult> Type =
RequestType<QueryInsightsGeneratorParams, InsightsGeneratorResult>.Create("insights/query");
}
}

78
src/Microsoft.SqlTools.ServiceLayer/InsightsGenerator/InsightsGeneratorService.cs
View File

@@ -1,78 +0,0 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Threading.Tasks;
using Microsoft.SqlTools.Hosting.Protocol;
using Microsoft.SqlTools.ServiceLayer.Hosting;
using Microsoft.SqlTools.ServiceLayer.InsightsGenerator.Contracts;
using Microsoft.InsightsGenerator;
namespace Microsoft.SqlTools.ServiceLayer.InsightsGenerator
{
/// <summary>
/// Service responsible for securing credentials in a platform-neutral manner. This provides
/// a generic API for read, save and delete credentials
/// </summary>
public class InsightsGeneratorService
{
/// <summary>
/// Singleton service instance
/// </summary>
private static Lazy<InsightsGeneratorService> instance
= new Lazy<InsightsGeneratorService>(() => new InsightsGeneratorService());
/// <summary>
/// Gets the singleton service instance
/// </summary>
public static InsightsGeneratorService Instance
{
get
{
return instance.Value;
}
}
/// <summary>
/// Initializes the service instance
/// </summary>
public void InitializeService(ServiceHost serviceHost)
{
// Insights Generatoe request handlers
serviceHost.SetRequestHandler(QueryInsightsGeneratorRequest.Type, HandleQueryInsightGeneratorRequest);
}
internal async Task HandleQueryInsightGeneratorRequest(QueryInsightsGeneratorParams parameters, RequestContext<InsightsGeneratorResult> requestContext)
{
Microsoft.InsightsGenerator.DataArray dataArray = new Microsoft.InsightsGenerator.DataArray(){
ColumnNames = parameters.Data.Columns,
Cells = parameters.Data.Rows
};
Workflow insightWorkFlow = new Workflow();
try
{
string insightText = await insightWorkFlow.ProcessInputData(dataArray);
insightText = insightText.Replace("\\n", "");
await requestContext.SendResult(new InsightsGeneratorResult()
{
InsightsText = insightText,
Success = true,
ErrorMessage = null
});
}
catch (Exception ex)
{
await requestContext.SendResult(new InsightsGeneratorResult()
{
Success = false,
ErrorMessage = ex.Message
});
}
}
}
}

1
src/Microsoft.SqlTools.ServiceLayer/Microsoft.SqlTools.ServiceLayer.csproj
View File

@@ -67,7 +67,6 @@
<ProjectReference Include="../Microsoft.SqlTools.Hosting/Microsoft.SqlTools.Hosting.csproj" />
<ProjectReference Include="../Microsoft.SqlTools.Credentials/Microsoft.SqlTools.Credentials.csproj" />
<ProjectReference Include="../Microsoft.SqlTools.ManagedBatchParser/Microsoft.SqlTools.ManagedBatchParser.csproj" />
<ProjectReference Include="../Microsoft.InsightsGenerator/Microsoft.InsightsGenerator.csproj" />
</ItemGroup>
<ItemGroup>
<Content Include="..\..\Notice.txt">