Files
sqltoolsservice/test/Microsoft.InsightsGenerator.UnitTests/DataTransformationTests.cs
Karl Burtram 5cf5b59a0d Insights generator project (#1066)
* InsightsGenerator project template files

* Add insights projects to SLN

* Setting up siggen class (#1003)

* Setting up siggen class

* fixed the learn method

* Refactoring code
Fixed compile errors

* renamed results to result

* Basic transformation logic (#1004)

* Fix a couple bugs and add a simple test (#1007)

* Fix a couple bugs and add a simple test

* More tests and bug fix

* Nara/workflow (#1006)

* added a queue processor

* ordered  using statements

* Armemon/analytics (#1008)

* Basic transformation logic

* changed some structure of siggen

* added sum and average method, as well as select rows by input name

* add insights to results

* min, max added

Co-authored-by: Karl Burtram <karlb@microsoft.com>
Co-authored-by: Aasim Khan <aasimkhan30@gmail.com>
Co-authored-by: Arslan Memon <armemon@microsoft.com>

* Added rules engine base implementation (#1005)

* Added rules engine base implementation

* update comments

* addressing comments

* adding template text to columnheaders object

* adding template text to columnheaders object

* fixing columnheaders class

* Added test

* Added Template Parser unit test in Test project

* Deleted unnecessary files and reverted the files that were modified by mistake

Co-authored-by: Jinjing Arima <jiarima@microsoft.com>

* Insights generator message handler placeholder (#1013)

* Aasim/insights/insight methods (#1014)

* Basic transformation logic

* changed some structure of siggen

* Added top and bottom insight functions

* Added top, bottom insights
Added tests for top, bottom insights

* Armemon/insights2 (#1011)

* max and min insightsperslice, and tests

* got rid of unneccesssary function

* get indexes

Co-authored-by: Arslan Memon <armemon@microsoft.com>

* Armemon/insights2 (#1012)

* max and min insightsperslice, and tests

* got rid of unneccesssary function

* get indexes

* learn for stringinputtyype

* add learn implentation

Co-authored-by: Arslan Memon <armemon@microsoft.com>

* Added Tests
Removed duplicate methods

Co-authored-by: Karl Burtram <karlb@microsoft.com>
Co-authored-by: arslan9955 <53170027+arslan9955@users.noreply.github.com>
Co-authored-by: Arslan Memon <armemon@microsoft.com>

* Armemon/insights2 (#1016)

* Basic transformation logic

* changed some structure of siggen

* Added top and bottom insight functions

* Added top, bottom insights
Added tests for top, bottom insights

* max and min insightsperslice, and tests

* got rid of unneccesssary function

* get indexes

* learn for stringinputtyype

* add learn implentation

* add unique inputs

* fix merge error

* add to result

Co-authored-by: Karl Burtram <karlb@microsoft.com>
Co-authored-by: Aasim Khan <aasimkhan30@gmail.com>
Co-authored-by: Arslan Memon <armemon@microsoft.com>

* Added all the templates (#1015)

* Added all the templates
Added a method to find matched template

* Added a function to replace # and ## values in a template

* Added ReplaceHashesInTemplate call

* Added comments

* Updated the template txt

* Updated GetTopHeadersWithHash function to add #toplist

* Updated the logic per offline discussion with Hermineh

* Update request handler contract to take array (#1020)

* added rulesengine findmatchingtemplate (#1019)

* Add support for getting DacFx deploy options from a publish profile (#995)

* add support for getting options from a publish profile

* update comments

* set values for default options if they aren't specified in the publish profile

* addressing comments

* Updating to latest DacFx for a bug fix (#1010)

* added rulesengine findmatchingtemplate

* Update DacFx deploy and generate script with options (#998)

* update deploy and generate script to accept deployment options

* add tests

* add test with option set to true

* merge

* merge

* incorporated FindMatchedTemplate

Co-authored-by: Kim Santiago <31145923+kisantia@users.noreply.github.com>
Co-authored-by: Udeesha Gautam <46980425+udeeshagautam@users.noreply.github.com>

* -Added logic for Insights Generator Service Handler (#1017)

* -Added logic for Insights Generator Service Handler

* Fixed some logic

* Adding workflow test

* Update transform and add tests (#1024)

* Jiarima/fix rules engine logic (#1021)

* Added all the templates
Added a method to find matched template

* Added a function to replace # and ## values in a template

* Added ReplaceHashesInTemplate call

* Added comments

* Updated the template txt

* Updated GetTopHeadersWithHash function to add #toplist

* Updated the logic per offline discussion with Hermineh

* Update with the fixes

* Updated template and foreach conditions

* Added distinct

* Updated tests according to the logic change (#1026)

* Nara/remove queing (#1023)

* loc update (#914)

* loc update

* loc updates

* Add support for getting DacFx deploy options from a publish profile (#995)

* add support for getting options from a publish profile

* update comments

* set values for default options if they aren't specified in the publish profile

* addressing comments

* Updating to latest DacFx for a bug fix (#1010)

* Update DacFx deploy and generate script with options (#998)

* update deploy and generate script to accept deployment options

* add tests

* add test with option set to true

* intermediate check in for merge, transformed not working

* intermediate check in for merge, transformed not working

* added test case

* merged

Co-authored-by: khoiph1 <khoiph@microsoft.com>
Co-authored-by: Kim Santiago <31145923+kisantia@users.noreply.github.com>
Co-authored-by: Udeesha Gautam <46980425+udeeshagautam@users.noreply.github.com>

* Output data types from transform (#1029)

* Fix bug process input_g (#1030)

* Fixed the insight generator service (#1028)

* Jiarima/added more testings (#1031)

* Added another test
Updated ReplaceHashesInTemplate function to return string instead of Template

* Added third test

* Merged

* Reverted the workflow file to match with the one in hack/insights

* Bugs fixes to hook insights up to ADS (#1033)

* Bug fixes for hack insights (#1032)

* Fixed the minColumn index bug in Data Transformation
Fixed the template matching logic.

* Adding changes from PR

* Try to fix Workflow tests

* Readd workflow tests

* Fix template load location

Co-authored-by: Aasim Khan <aasimkhan30@gmail.com>
Co-authored-by: Nara <NaraVen@users.noreply.github.com>
Co-authored-by: arslan9955 <53170027+arslan9955@users.noreply.github.com>
Co-authored-by: Arslan Memon <armemon@microsoft.com>
Co-authored-by: gadudhbh <68879970+gadudhbh@users.noreply.github.com>
Co-authored-by: Jinjing Arima <jiarima@microsoft.com>
Co-authored-by: jiarima <68882862+jiarima@users.noreply.github.com>
Co-authored-by: Kim Santiago <31145923+kisantia@users.noreply.github.com>
Co-authored-by: Udeesha Gautam <46980425+udeeshagautam@users.noreply.github.com>
Co-authored-by: khoiph1 <khoiph@microsoft.com>
2020-09-03 14:15:51 -07:00

199 lines
7.8 KiB
C#

//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using Xunit;
namespace Microsoft.InsightsGenerator.UnitTests
{
/// <summary>
/// DataTransformation tests
/// </summary>
public class DataTransformerTests
{
[Fact]
public void Tranform_NullInput()
{
DataTransformer transformer = new DataTransformer();
DataArray array = null;
array = transformer.Transform(array);
Assert.Null(array);
}
[Fact]
public void Tranform_TimeSlicerCount_DeduceTypes()
{
DataTransformer transformer = new DataTransformer();
object[][] cells = new object[2][];
cells[0] = new object[3] { "1/15/2020", "Redmond", 50 };
cells[1] = new object[3] { "1/25/2020", "Bellevue", 75 };
DataArray array = new DataArray()
{
ColumnNames = new string[] { "Date", "City", "Count" },
Cells = cells
};
array = transformer.Transform(array);
Assert.Equal(array.TransformedColumnNames[0], "input_t_0");
Assert.Equal(array.TransformedColumnNames[1], "slicer_0");
Assert.Equal(array.TransformedColumnNames[2], "output_0");
}
[Fact]
public void Tranform_TimeSlicerCount_ProvidedTypes()
{
DataTransformer transformer = new DataTransformer();
object[][] cells = new object[2][];
cells[0] = new object[3] { "1/15/2020", "Redmond", 50 };
cells[1] = new object[3] { "1/25/2020", "Bellevue", 75 };
DataArray array = new DataArray()
{
ColumnNames = new string[] { "Date", "City", "Count" },
ColumnDataType = new DataArray.DataType[] {
DataArray.DataType.String,
DataArray.DataType.DateTime,
DataArray.DataType.Number },
Cells = cells
};
array = transformer.Transform(array);
Assert.Equal(array.TransformedColumnNames[0], "slicer_0");
Assert.Equal(array.TransformedColumnNames[1], "input_t_0");
Assert.Equal(array.TransformedColumnNames[2], "output_0");
}
[Fact]
public void Tranform_TimeSlicerSlicerCount()
{
DataTransformer transformer = new DataTransformer();
object[][] cells = new object[5][];
cells[0] = new object[4] { "1/15/2020", "Redmond", "1st Street", 50 };
cells[1] = new object[4] { "1/25/2020", "Redmond", "2nd Street", 75 };
cells[2] = new object[4] { "1/10/2020", "Bellevue", "3rd Street", 125 };
cells[3] = new object[4] { "1/13/2020", "Bellevue", "4th Street", 55 };
cells[4] = new object[4] { "1/20/2020", "Bellevue", "5th Street", 95 };
DataArray array = new DataArray()
{
ColumnNames = new string[] { "Date", "City", "Address", "Count" },
Cells = cells
};
array = transformer.Transform(array);
Assert.Equal(array.TransformedColumnNames[0], "input_t_0");
Assert.Equal(array.TransformedColumnNames[1], "slicer_0");
Assert.Equal(array.TransformedColumnNames[2], "slicer_1");
Assert.Equal(array.TransformedColumnNames[3], "output_0");
}
[Fact]
public void Tranform_TimeSlicerCountSlicer()
{
DataTransformer transformer = new DataTransformer();
object[][] cells = new object[5][];
cells[0] = new object[4] { "1/15/2020", "1st Street", 50, "Redmond" };
cells[1] = new object[4] { "1/25/2020", "2nd Street", 75, "Redmond" };
cells[2] = new object[4] { "1/10/2020", "3rd Street", 125, "Bellevue" };
cells[3] = new object[4] { "1/13/2020", "4th Street", 55, "Bellevue" };
cells[4] = new object[4] { "1/20/2020", "5th Street", 95, "Bellevue" };
DataArray array = new DataArray()
{
ColumnNames = new string[] { "Date", "Address", "Count", "City" },
Cells = cells
};
array = transformer.Transform(array);
Assert.Equal(array.TransformedColumnNames[0], "input_t_0");
Assert.Equal(array.TransformedColumnNames[1], "slicer_0");
Assert.Equal(array.TransformedColumnNames[2], "output_0");
Assert.Equal(array.TransformedColumnNames[3], "slicer_1");
}
[Fact]
public void Tranform_TimeSlicerCountCount()
{
DataTransformer transformer = new DataTransformer();
object[][] cells = new object[2][];
cells[0] = new object[4] { "1/15/2020", "1st Street", 50, 110 };
cells[1] = new object[4] { "1/25/2020", "2nd Street", 75, 160 };
DataArray array = new DataArray()
{
ColumnNames = new string[] { "Date", "Adress", "Count1", "Count2" },
Cells = cells
};
array = transformer.Transform(array);
Assert.Equal(array.TransformedColumnNames[0], "input_t_0");
Assert.Equal(array.TransformedColumnNames[1], "slicer_0");
Assert.Equal(array.TransformedColumnNames[2], "output_0");
Assert.Equal(array.TransformedColumnNames[3], "output_1");
}
[Fact]
public void Tranform_GroupSlicerTime()
{
DataTransformer transformer = new DataTransformer();
object[][] cells = new object[2][];
cells[0] = new object[3] { "1st Street", "Redmond", 110 };
cells[1] = new object[3] { "2nd Street", "Bellevue", 160 };
DataArray array = new DataArray()
{
ColumnNames = new string[] { "Address", "City", "Count" },
Cells = cells
};
array = transformer.Transform(array);
Assert.Equal(array.TransformedColumnNames[0], "input_g_0");
Assert.Equal(array.TransformedColumnNames[1], "slicer_0");
Assert.Equal(array.TransformedColumnNames[2], "output_0");
}
[Fact]
public void Tranform_SlicewrGroupSlicerTime()
{
DataTransformer transformer = new DataTransformer();
object[][] cells = new object[2][];
cells[0] = new object[4] { "1st Street", "Redmond", "North", 110 };
cells[1] = new object[4] { "2nd Street", "Redmond", "East", 160 };
DataArray array = new DataArray()
{
ColumnNames = new string[] { "Address", "City", "Direction", "Count" },
Cells = cells
};
array = transformer.Transform(array);
Assert.Equal(array.TransformedColumnNames[0], "slicer_0");
Assert.Equal(array.TransformedColumnNames[1], "input_g_0");
Assert.Equal(array.TransformedColumnNames[2], "slicer_1");
Assert.Equal(array.TransformedColumnNames[3], "output_0");
}
[Fact]
public void Tranform_SlicerGroupTime()
{
DataTransformer transformer = new DataTransformer();
object[][] cells = new object[2][];
cells[0] = new object[3] { "1st Street", "Redmond", 110 };
cells[1] = new object[3] { "2nd Street", "Redmond", 160 };
DataArray array = new DataArray()
{
ColumnNames = new string[] { "Address", "City", "Count" },
Cells = cells
};
array = transformer.Transform(array);
Assert.Equal(array.TransformedColumnNames[0], "slicer_0");
Assert.Equal(array.TransformedColumnNames[1], "input_g_0");
Assert.Equal(array.TransformedColumnNames[2], "output_0");
}
}
}