diff --git a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/Contracts/SaveResultsRequest.cs b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/Contracts/SaveResultsRequest.cs index 467003fc..cb286ea8 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/Contracts/SaveResultsRequest.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/Contracts/SaveResultsRequest.cs @@ -84,7 +84,7 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts public string Delimiter { get; set; } /// - /// either CR, CRLF or LF to seperate rows in CSV + /// either CR, CRLF or LF to separate rows in CSV /// public string LineSeperator { get; set; } @@ -123,6 +123,28 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts //TODO: define config for save as JSON } + /// + /// Parameters for saving results as a Markdown table + /// + public class SaveResultsAsMarkdownRequestParams : SaveResultsRequestParams + { + /// + /// Encoding of the CSV file + /// + public string Encoding { get; set; } + + /// + /// Whether to include column names as header for the table. + /// + public bool IncludeHeaders { get; set; } + + /// + /// Character sequence to separate a each row in the table. Should be either CR, CRLF, or + /// LF. If not provided, defaults to the system default line ending sequence. + /// + public string? LineSeparator { get; set; } + } + /// /// Parameters to save results as XML /// @@ -179,6 +201,16 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts RequestType Type = RequestType.Create("query/saveJson"); } + + /// + /// Request type to save results as a Markdown table + /// + public class SaveResultsAsMarkdownRequest + { + public static readonly + RequestType Type = + RequestType.Create("query/saveMarkdown"); + } /// /// Request type to save results as XML diff --git a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsCsvFileStreamWriter.cs b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsCsvFileStreamWriter.cs index 7e302392..cd3b60f9 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsCsvFileStreamWriter.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsCsvFileStreamWriter.cs @@ -61,18 +61,7 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.DataStorage } textIdentifierString = textIdentifier.ToString(); - Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); - try - { - encoding = int.TryParse(requestParams.Encoding, out int codePage) - ? Encoding.GetEncoding(codePage) - : Encoding.GetEncoding(requestParams.Encoding); - } - catch - { - // Fallback encoding when specified codepage is invalid - encoding = Encoding.GetEncoding("utf-8"); - } + encoding = ParseEncoding(requestParams.Encoding, Encoding.UTF8); // Output the header if the user requested it if (requestParams.IncludeHeaders) diff --git a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsMarkdownFileStreamFactory.cs b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsMarkdownFileStreamFactory.cs new file mode 100644 index 00000000..961dcf1a --- /dev/null +++ b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsMarkdownFileStreamFactory.cs @@ -0,0 +1,68 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts; +using Microsoft.SqlTools.ServiceLayer.SqlContext; +using Microsoft.SqlTools.ServiceLayer.Utility; + +namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.DataStorage +{ + public class SaveAsMarkdownFileStreamFactory : IFileStreamFactory + { + private readonly SaveResultsAsMarkdownRequestParams _saveRequestParams; + + /// + /// Constructs and initializes a new instance of . + /// + /// Parameters for the save as request + public SaveAsMarkdownFileStreamFactory(SaveResultsAsMarkdownRequestParams requestParams) + { + this._saveRequestParams = requestParams; + } + + /// + public QueryExecutionSettings QueryExecutionSettings { get; set; } + + /// + /// Throw at all times. + [Obsolete("Not implemented for export factories.")] + public string CreateFile() + { + throw new InvalidOperationException("CreateFile not implemented for export factories"); + } + + /// + /// + /// Returns an instance of the . + /// + public IFileStreamReader GetReader(string fileName) + { + return new ServiceBufferFileStreamReader( + new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), + this.QueryExecutionSettings); + } + + /// + /// + /// Returns an instance of the . + /// + public IFileStreamWriter GetWriter(string fileName, IReadOnlyList columns) + { + return new SaveAsMarkdownFileStreamWriter( + new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite), + this._saveRequestParams, + columns); + } + + /// + public void DisposeFile(string fileName) + { + FileUtilities.SafeFileDelete(fileName); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsMarkdownFileStreamWriter.cs b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsMarkdownFileStreamWriter.cs new file mode 100644 index 00000000..3581d47c --- /dev/null +++ b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsMarkdownFileStreamWriter.cs @@ -0,0 +1,101 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Web; +using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts; + +namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.DataStorage +{ + /// + /// Writer for exporting results to a Markdown table. + /// + public class SaveAsMarkdownFileStreamWriter : SaveAsStreamWriter + { + private const string Delimiter = "|"; + private static readonly Regex NewlineRegex = new Regex(@"(\r\n|\n|\r)", RegexOptions.Compiled); + + private readonly Encoding _encoding; + private readonly string _lineSeparator; + + public SaveAsMarkdownFileStreamWriter( + Stream stream, + SaveResultsAsMarkdownRequestParams requestParams, + IReadOnlyList columns) + : base(stream, requestParams, columns) + { + // Parse the request params + this._lineSeparator = string.IsNullOrEmpty(requestParams.LineSeparator) + ? Environment.NewLine + : requestParams.LineSeparator; + this._encoding = ParseEncoding(requestParams.Encoding, Encoding.UTF8); + + // Output the header if requested + if (requestParams.IncludeHeaders) + { + // Write the column header + IEnumerable selectedColumnNames = columns.Skip(this.ColumnStartIndex) + .Take(this.ColumnCount) + .Select(c => EncodeMarkdownField(c.ColumnName)); + string headerLine = string.Join(Delimiter, selectedColumnNames); + + this.WriteLine($"{Delimiter}{headerLine}{Delimiter}"); + + // Write the separator row + var separatorBuilder = new StringBuilder(Delimiter); + for (int i = 0; i < this.ColumnCount; i++) + { + separatorBuilder.Append($"---{Delimiter}"); + } + + this.WriteLine(separatorBuilder.ToString()); + } + } + + /// + public override void WriteRow(IList row, IReadOnlyList columns) + { + IEnumerable selectedCells = row.Skip(this.ColumnStartIndex) + .Take(this.ColumnCount) + .Select(c => EncodeMarkdownField(c.DisplayValue)); + string rowLine = string.Join(Delimiter, selectedCells); + + this.WriteLine($"{Delimiter}{rowLine}{Delimiter}"); + } + + internal static string EncodeMarkdownField(string? field) + { + // Special case for nulls + if (field == null) + { + return "NULL"; + } + + // Escape HTML entities, since Markdown supports inline HTML + field = HttpUtility.HtmlEncode(field); + + // Escape pipe delimiters + field = field.Replace(@"|", @"\|"); + + // @TODO: Allow option to encode multiple whitespace characters as   + + // Replace newlines with br tags, since cell values must be single line + field = NewlineRegex.Replace(field, @"
"); + + return field; + } + + private void WriteLine(string line) + { + byte[] bytes = this._encoding.GetBytes(line + this._lineSeparator); + this.FileStream.Write(bytes); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsWriterBase.cs b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsWriterBase.cs index 9fa71d7d..a93f1577 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsWriterBase.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/SaveAsWriterBase.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Text; using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts; using Microsoft.SqlTools.Utility; @@ -106,6 +107,37 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.DataStorage FileStream.Flush(); } + /// + /// Attempts to parse the provided and return an encoding that + /// matches the encoding name or codepage number. + /// + /// Encoding name or codepage number to parse. + /// + /// Encoding to return if no encoding of provided name/codepage number exists. + /// + /// + /// Desired encoding object or the if the desired + /// encoding could not be found. + /// + protected static Encoding ParseEncoding(string encoding, Encoding fallbackEncoding) + { + // If the encoding is a number, we try to look up a codepage encoding using the + // parsed number as a codepage. If it is not a number, attempt to look up an + // encoding with the provided encoding name. If getting the encoding fails in + // either case, we will return the fallback encoding. + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + try + { + return int.TryParse(encoding, out int codePage) + ? Encoding.GetEncoding(codePage) + : Encoding.GetEncoding(encoding); + } + catch + { + return fallbackEncoding; + } + } + #region IDisposable Implementation private bool disposed; diff --git a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/QueryExecutionService.cs b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/QueryExecutionService.cs index ac0d4d91..84c495a9 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/QueryExecutionService.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/QueryExecutionService.cs @@ -98,6 +98,12 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution /// internal IFileStreamFactory JsonFileFactory { get; set; } + /// + /// File factory to be used to create Markdown files from result sets. + /// + /// Internal to allow overriding in unit testing. + internal IFileStreamFactory? MarkdownFileFactory { get; set; } + /// /// File factory to be used to create XML files from result sets. Set to internal in order /// to allow overriding in unit testing @@ -174,6 +180,7 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution serviceHost.SetRequestHandler(SaveResultsAsCsvRequest.Type, HandleSaveResultsAsCsvRequest); serviceHost.SetRequestHandler(SaveResultsAsExcelRequest.Type, HandleSaveResultsAsExcelRequest); serviceHost.SetRequestHandler(SaveResultsAsJsonRequest.Type, HandleSaveResultsAsJsonRequest); + serviceHost.SetRequestHandler(SaveResultsAsMarkdownRequest.Type, this.HandleSaveResultsAsMarkdownRequest); serviceHost.SetRequestHandler(SaveResultsAsXmlRequest.Type, HandleSaveResultsAsXmlRequest); serviceHost.SetRequestHandler(QueryExecutionPlanRequest.Type, HandleExecutionPlanRequest); serviceHost.SetRequestHandler(SimpleExecuteRequest.Type, HandleSimpleExecuteRequest); @@ -518,6 +525,25 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution await SaveResultsHelper(saveParams, requestContext, jsonFactory); } + /// + /// Processes a request to save a result set to a file in Markdown format. + /// + /// Parameters for the request + /// Context of the request + internal async Task HandleSaveResultsAsMarkdownRequest( + SaveResultsAsMarkdownRequestParams saveParams, + RequestContext requestContext) + { + // Use the default markdown file factory if we haven't overridden it + IFileStreamFactory markdownFactory = this.MarkdownFileFactory ?? + new SaveAsMarkdownFileStreamFactory(saveParams) + { + QueryExecutionSettings = this.Settings.QueryExecutionSettings, + }; + + await this.SaveResultsHelper(saveParams, requestContext, markdownFactory); + } + /// /// Process request to save a resultSet to a file in XML format /// diff --git a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/SerializationService.cs b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/SerializationService.cs index c3b7a30a..56390cee 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/SerializationService.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/QueryExecution/SerializationService.cs @@ -243,6 +243,9 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution SaveRequestParams = CreateCsvRequestParams() }; break; + case "markdown": + factory = new SaveAsMarkdownFileStreamFactory(CreateMarkdownRequestParams()); + break; case "xml": factory = new SaveAsXmlFileStreamFactory() { @@ -304,6 +307,18 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution MaxCharsToStore = this.requestParams.MaxCharsToStore }; } + + private SaveResultsAsMarkdownRequestParams CreateMarkdownRequestParams() => + new SaveResultsAsMarkdownRequestParams + { + FilePath = this.requestParams.FilePath, + BatchIndex = 0, + ResultSetIndex = 0, + IncludeHeaders = this.requestParams.IncludeHeaders, + LineSeparator = this.requestParams.LineSeparator, + Encoding = this.requestParams.Encoding, + }; + private SaveResultsAsXmlRequestParams CreateXmlRequestParams() { return new SaveResultsAsXmlRequestParams diff --git a/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/DataStorage/SaveAsMarkdownFileStreamWriterTests.cs b/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/DataStorage/SaveAsMarkdownFileStreamWriterTests.cs new file mode 100644 index 00000000..2f8a3882 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/DataStorage/SaveAsMarkdownFileStreamWriterTests.cs @@ -0,0 +1,376 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts; +using Microsoft.SqlTools.ServiceLayer.QueryExecution.DataStorage; +using Microsoft.SqlTools.ServiceLayer.UnitTests.Utility; +using NUnit.Framework; + +namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.DataStorage +{ + public class SaveAsMarkdownFileStreamWriterTests + { + // Regex: Matches '|' not preceded by a '\' + private static readonly Regex UnescapedPipe = new Regex(@"(? _ = new SaveAsMarkdownFileStreamWriter( + null, + new SaveResultsAsMarkdownRequestParams(), + Array.Empty() + ); + + // Assert + Assert.That(action, Throws.ArgumentNullException); + } + + [Test] + public void Constructor_NullColumns() + { + // Act + TestDelegate action = () => _ = new SaveAsMarkdownFileStreamWriter( + Stream.Null, + new SaveResultsAsMarkdownRequestParams(), + null + ); + + // Assert + Assert.That(action, Throws.ArgumentNullException); + } + + [Test] + public void Constructor_WithoutSelectionWithHeader_WritesHeaderWithAllColumns() + { + // Setup: + // ... Create a request params that has no selection made, headers should be printed + // ... Create a set of columns + // --- Create a memory location to store the output + var requestParams = new SaveResultsAsMarkdownRequestParams { IncludeHeaders = true }; + var (columns, _) = GetTestValues(2); + byte[] output = new byte[8192]; + using var outputStream = new MemoryStream(output); + + // If: I construct a Markdown file writer + using var writer = new SaveAsMarkdownFileStreamWriter(outputStream, requestParams, columns); + + // Then: + // ... It should have written a line + string[] lines = ParseWriterOutput(output, Environment.NewLine); + Assert.That(lines.Length, Is.EqualTo(2), "Expected two lines of output"); + + // ... It should have written a header line like |col1|col2| + // ... It should have written a separator line like |---|---| + ValidateLine(lines[0], columns.Select(c => c.ColumnName)); + ValidateLine(lines[1], Enumerable.Repeat("---", columns.Length)); + } + + [Test] + public void Constructor_WithSelectionWithHeader_WritesHeaderWithSelectedColumns() + { + // Setup: + // ... Create a request params that has no selection made, headers should be printed + // ... Create a set of columns + // --- Create a memory location to store the output + var requestParams = new SaveResultsAsMarkdownRequestParams + { + IncludeHeaders = true, + ColumnStartIndex = 1, + ColumnEndIndex = 2, + RowStartIndex = 0, // Including b/c it is required to be a "save selection" + RowEndIndex = 10, + }; + var (columns, _) = GetTestValues(4); + byte[] output = new byte[8192]; + using var outputStream = new MemoryStream(output); + + // If: I construct a Markdown file writer + using var writer = new SaveAsMarkdownFileStreamWriter(outputStream, requestParams, columns); + + // Then: + // ... It should have written a line + string[] lines = ParseWriterOutput(output, Environment.NewLine); + Assert.That(lines.Length, Is.EqualTo(2), "Expected two lines of output"); + + // ... It should have written a header line like |col1|col2| + // ... It should have written a separator line like |---|---| + ValidateLine(lines[0], columns.Skip(1).Take(2).Select(c => c.ColumnName)); + ValidateLine(lines[1], Enumerable.Repeat("---", 2)); + } + + [Test] + public void Constructor_WithoutSelectionWithoutHeader_DoesNotWriteHeader() + { + // Setup: + // ... Create a request params that has no selection made, headers should not be printed + // ... Create a set of columns + // --- Create a memory location to store the output + var requestParams = new SaveResultsAsMarkdownRequestParams { IncludeHeaders = false }; + var (columns, _) = GetTestValues(2); + byte[] output = new byte[8192]; + using var outputStream = new MemoryStream(output); + + // If: I construct a Markdown file writer + using var writer = new SaveAsMarkdownFileStreamWriter(outputStream, requestParams, columns); + + // Then: + // ... It not have written anything + string[] lines = ParseWriterOutput(output, Environment.NewLine); + Assert.That(lines, Is.Empty); + } + + [TestCase("Something\rElse")] // Contains carriage return + [TestCase("Something\nElse")] // Contains line feed + [TestCase("Something\r\nElse")] // Contains carriage return + public void EncodeMarkdownField_ContainsNewlineCharacters_ShouldConvertToBr(string field) + { + // If: I Markdown encode a field that has a newline + string output = SaveAsMarkdownFileStreamWriter.EncodeMarkdownField(field); + + // Then: It should replace it the newline character(s) with a
+ const string expected = "Something
Else"; + Assert.That(output, Is.EqualTo(expected)); + } + + [Test] + public void EncodeMarkdownField_ContainsDelimiter_ShouldBeEscaped() + { + // If: I Markdown encode a field that has a pipe in it + const string input = "|Something|Else|"; + string output = SaveAsMarkdownFileStreamWriter.EncodeMarkdownField(input); + + // Then: It should escape the pipe character + const string expected = @"\|Something\|Else\|"; + Assert.AreEqual(expected, output); + } + + // @TODO: Convert excess whitespace to   on user choice + // [TestCase("\tSomething")] // Starts with tab + // [TestCase("Something\t")] // Ends with tab + // [TestCase(" Something")] // Starts with space + // [TestCase("Something ")] // Ends with space + // [TestCase(" Something ")] // Starts and ends with space + // [TestCase("Something else")] // Contains multiple consecutive spaces + // public void EncodeMarkdownField_WhitespaceAtFrontOrBack_ShouldBeWrapped(string field) + // { + // // Setup: Create MarkdownFileStreamWriter that specifies the text identifier and field separator + // var writer = GetWriterForEncodingTests(null, null, null); + // + // // If: I Markdown encode a field that has forbidden characters in it + // string output = writer.EncodeMarkdownField(field); + // + // // Then: It should wrap it in quotes + // Assert.True(Regex.IsMatch(output, "^\".*\"$", RegexOptions.Singleline)); + // } + + [Test] + public void EncodeMarkdownField_ContainsHtmlEntityCharacters_ShouldConvertToHtmlEntities() + { + // If: I Markdown encode a field that has html entity characters in it + const string input = "<<>>&®±ßüÁ"; + string output = SaveAsMarkdownFileStreamWriter.EncodeMarkdownField(input); + + // Then: The entity characters should be HTML encoded + const string expected = "<<>>&®±ßüÁ"; + Assert.That(output, Is.EqualTo(expected)); + } + + [Test] + public void EncodeMarkdownField_Null() + { + // If: I Markdown encode a null + string output = SaveAsMarkdownFileStreamWriter.EncodeMarkdownField(null); + + // Then: there should be a string version of null returned + Assert.That(output, Is.EqualTo("NULL")); + } + + [Test] + public void WriteRow_WithoutColumnSelection() + { + // Setup: + // ... Create a request params that has no selection made or header enabled + // ... Create a set of data to write + // ... Create a memory location to store the data + var requestParams = new SaveResultsAsMarkdownRequestParams(); + var (columns, data) = GetTestValues(2); + byte[] output = new byte[8192]; + using var outputStream = new MemoryStream(output); + + // If: I write a row + using (var writer = new SaveAsMarkdownFileStreamWriter(outputStream, requestParams, columns)) + { + writer.WriteRow(data, columns); + } + + // Then: It should write one line with the two cells + string[] lines = ParseWriterOutput(output, Environment.NewLine); + Assert.That(lines.Length, Is.EqualTo(1), "Expected one line of output"); + + ValidateLine(lines[0], data.Select(c => c.DisplayValue)); + } + + [Test] + public void WriteRow_WithColumnSelection() + { + // Setup: + // ... Create a request params that selects n-1 columns from the front and back + // ... Create a set of data to write + // ... Create a memory location to store the data + var requestParams = new SaveResultsAsMarkdownRequestParams + { + ColumnStartIndex = 1, + ColumnEndIndex = 2, + RowStartIndex = 0, // Including b/c it is required to be a "save selection" + RowEndIndex = 10 + }; + var (columns, data) = GetTestValues(4); + byte[] output = new byte[8192]; + using var outputStream = new MemoryStream(output); + + // If: I write a row + using (var writer = new SaveAsMarkdownFileStreamWriter(outputStream, requestParams, columns)) + { + writer.WriteRow(data, columns); + } + + // Then: + // ... It should have written one line with the two cells written + var lines = ParseWriterOutput(output, Environment.NewLine); + Assert.That(lines.Length, Is.EqualTo(1), "Expected one line of output"); + + ValidateLine(lines[0], data.Skip(1).Take(2).Select(c => c.DisplayValue)); + } + + [Test] + public void WriteRow_EncodingTest() + { + // Setup: + // ... Create default request params + // ... Create a set of data to write that contains characters that should be encoded + // ... Create a memory location to store the data + // @TODO: Add case to test string for non-breaking spaces + var requestParams = new SaveResultsAsMarkdownRequestParams(); + var columns = new[] { new DbColumnWrapper(new TestDbColumn("column")) }; + var data = new[] { new DbCellValue { DisplayValue = "|Something|\n|<<>>&|" } }; + byte[] output = new byte[8192]; + using var outputStream = new MemoryStream(output); + + // If: I write a row + using (var writer = new SaveAsMarkdownFileStreamWriter(outputStream, requestParams, columns)) + { + writer.WriteRow(data, columns); + } + + // Then: + // ... It should have written one line with the data properly encoded + string[] lines = ParseWriterOutput(output, Environment.NewLine); + Assert.That(lines.Length, Is.EqualTo(1), "Expected one line of output"); + + ValidateLine(lines[0], new[] { "\\|Something\\|
\\|<<>>&\\|" }); + } + + [Test] + public void WriteRow_CustomLineSeparator() + { + // Setup: + // ... Create a request params that has custom line separator + // ... Create a set of data to write + // ... Create a memory location to store the data + var requestParams = new SaveResultsAsMarkdownRequestParams + { + LineSeparator = "$$", + IncludeHeaders = true, + }; + var (columns, data) = GetTestValues(2); + byte[] output = new byte[8192]; + using var outputStream = new MemoryStream(output); + + // If: I set write a row + using (var writer = new SaveAsMarkdownFileStreamWriter(outputStream, requestParams, columns)) + { + writer.WriteRow(data, columns); + } + + // Then: + // ... The lines should be split by the custom line separator + var lines = ParseWriterOutput(output, "$$"); + Assert.That(lines.Length, Is.EqualTo(3), "Expected three lines of output"); + + // Note: Header output has been tested in constructor tests + } + + [Test] + public void WriteRow_CustomEncoding() + { + // Setup: + // ... Create a request params that uses a custom encoding + // ... Create a set of data to write + // ... Create a memory location to store the data + var requestParams = new SaveResultsAsMarkdownRequestParams { Encoding = "utf-16", }; + var data = new[] { new DbCellValue { DisplayValue = "ü" } }; + var columns = new[] { new DbColumnWrapper(new TestDbColumn("column1")) }; + byte[] output = new byte[8192]; + using var outputStream = new MemoryStream(output); + + // If: I write a row + using (var writer = new SaveAsMarkdownFileStreamWriter(outputStream, requestParams, columns)) + { + writer.WriteRow(data, columns); + } + + // Then: + // ... It should have written the umlaut as an HTML entity in utf-16le + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + string outputString = Encoding.Unicode.GetString(output).TrimEnd('\0', '\r', '\n'); + Assert.That(outputString, Is.EqualTo("|ü|")); + } + + private static (DbColumnWrapper[] columns, DbCellValue[] cells) GetTestValues(int columnCount) + { + var data = new DbCellValue[columnCount]; + var columns = new DbColumnWrapper[columnCount]; + for (int i = 0; i < columnCount; i++) + { + data[i] = new DbCellValue { DisplayValue = $"item{i}" }; + columns[i] = new DbColumnWrapper(new TestDbColumn($"column{i}")); + } + return (columns, data); + } + + private static string[] ParseWriterOutput(byte[] output, string lineSeparator) + { + string outputString = Encoding.UTF8.GetString(output).Trim('\0'); + string[] lines = outputString.Split(lineSeparator); + + // Make sure the file ends with a new line and return all but the meaningful lines + Assert.That(lines[^1], Is.Empty, "Output did not end with a newline"); + return lines.Take(lines.Length - 1).ToArray(); + } + + private static void ValidateLine(string line, IEnumerable expectedCells) + { + string[] cells = UnescapedPipe.Split(line); + string[] expectedCellsArray = expectedCells as string[] ?? expectedCells.ToArray(); + Assert.That(cells.Length - 2, Is.EqualTo(expectedCellsArray.Length), "Wrong number of cells in output"); + + Assert.That(cells[0], Is.Empty, "Row did not start with |"); + Assert.That(cells[^1], Is.Empty, "Row did not end with |"); + + for (int i = 0; i < expectedCellsArray.Length; i++) + { + Assert.That(cells[i + 1], Is.EqualTo(expectedCellsArray[i]), "Wrong cell value"); + } + } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/SaveResults/SerializationServiceTests.cs b/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/SaveResults/SerializationServiceTests.cs index 980137c5..bf650f4f 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/SaveResults/SerializationServiceTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/SaveResults/SerializationServiceTests.cs @@ -60,19 +60,9 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults protected Mock HostMock { get; private set; } protected SerializationService SerializationService { get; private set; } - [Test] - public async Task SaveResultsAsCsvNoHeaderSuccess() - { - await TestSaveAsCsvSuccess(false); - } - - [Test] - public async Task SaveResultsAsCsvWithHeaderSuccess() - { - await TestSaveAsCsvSuccess(true); - } - - private async Task TestSaveAsCsvSuccess(bool includeHeaders) + [TestCase(true)] + [TestCase(false)] + public async Task TestSaveAsCsvSuccess(bool includeHeaders) { await this.RunFileSaveTest(async (filePath) => { @@ -102,18 +92,47 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults }); } - [Test] - public async Task SaveResultsAsCsvNoHeaderMultiRequestSuccess() + [TestCase(true)] + [TestCase(false)] + public Task TestSaveAsMarkdownSuccess(bool includeHeaders) { - await TestSaveAsCsvMultiRequestSuccess(false); + return this.RunFileSaveTest(async filePath => + { + // Give: + // ... A simple data set that requires 1 message + var saveParams = new SerializeDataStartRequestParams + { + FilePath = filePath, + Columns = DefaultColumns, + Rows = DefaultData, + IsLastBatch = true, + SaveFormat = "markdown", + IncludeHeaders = includeHeaders, + }; + + // When: I attempt to save this to a file + var efv = new EventFlowValidator() + .AddStandardResultValidator() + .Complete(); + + await SerializationService.RunSerializeStartRequest(saveParams, efv.Object); + + // Then: + // ... There should not have been any errors + efv.Validate(); + + // ... And the file should look as expected + VerifyContents.VerifyMarkdownMatchesData( + saveParams.Rows, + saveParams.Columns, + saveParams.IncludeHeaders, + saveParams.FilePath); + }); } - [Test] - public async Task SaveResultsAsCsvWithHeaderMultiRequestSuccess() - { - await TestSaveAsCsvMultiRequestSuccess(true); - } - private async Task TestSaveAsCsvMultiRequestSuccess(bool includeHeaders) + [TestCase(true)] + [TestCase(false)] + public async Task TestSaveAsCsvMultiRequestSuccess(bool includeHeaders) { Action setParams = (serializeParams) => { serializeParams.SaveFormat = "csv"; @@ -149,6 +168,22 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults await this.TestSerializeDataMultiRequestSuccess(setParams, validation); } + [TestCase(true)] + [TestCase(false)] + public async Task SaveAsMarkdownMultiRequestSuccess(bool includeHeaders) + { + Action setParams = serializeParams => + { + serializeParams.SaveFormat = "markdown"; + serializeParams.IncludeHeaders = includeHeaders; + }; + Action validation = filePath => + { + VerifyContents.VerifyMarkdownMatchesData(DefaultData, DefaultColumns, includeHeaders, filePath); + }; + await this.TestSerializeDataMultiRequestSuccess(setParams, validation); + } + private async Task TestSerializeDataMultiRequestSuccess(Action setStandardParams, Action verify) { await this.RunFileSaveTest(async (filePath) => @@ -225,9 +260,9 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults { return efv.AddResultValidation(r => { - Assert.NotNull(r); - Assert.Null(r.Messages); - Assert.True(r.Succeeded); + Assert.That(r, Is.Not.Null, "Result should not be null"); + Assert.That(r.Messages, Is.Null, "No messages should be attached to the result"); + Assert.That(r.Succeeded, Is.True, "Result should indicate request succeeded"); }); } } @@ -236,10 +271,10 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults { public static void VerifyCsvMatchesData(DbCellValue[][] data, ColumnInfo[] columns, bool includeHeaders, string filePath) { - Assert.True(File.Exists(filePath), "Expected file to have been written"); + Assert.That(filePath, Does.Exist, "Expected file to have been written"); string[] lines = File.ReadAllLines(filePath); int expectedLength = includeHeaders ? data.Length + 1 : data.Length; - Assert.AreEqual(expectedLength, lines.Length); + Assert.That(lines.Length, Is.EqualTo(expectedLength), "Incorrect number of lines in result"); int lineIndex = 0; if (includeHeaders) { @@ -248,7 +283,7 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults } for (int dataIndex =0; dataIndex < data.Length && lineIndex < lines.Length; dataIndex++, lineIndex++) { - AssertLineEquals(lines[lineIndex], data[dataIndex].Select((d) => GetCsvPrintValue(d)).ToArray()); + AssertLineEquals(lines[lineIndex], data[dataIndex].Select(GetCsvPrintValue).ToArray()); } } @@ -260,17 +295,54 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults private static void AssertLineEquals(string line, string[] expected) { var actual = line.Split(','); - Assert.True(actual.Length == expected.Length, $"Line '{line}' does not match values {string.Join(",", expected)}"); + Assert.That(actual.Length, Is.EqualTo(expected.Length), + $"Line '{line}' does not match values {string.Join(",", expected)}"); for (int i = 0; i < actual.Length; i++) { - Assert.True(expected[i] == actual[i], $"Line '{line}' does not match values '{string.Join(",", expected)}' as '{expected[i]}' does not equal '{actual[i]}'"); + Assert.That(actual[i], Is.EqualTo(expected[i]), + $"Line '{line}' does not match values '{string.Join(",", expected)}' as '{expected[i]}' does not equal '{actual[i]}'"); } } + public static void VerifyMarkdownMatchesData( + DbCellValue[][] data, + ColumnInfo[] columns, + bool includeHeaders, + string filePath) + { + Assert.That(filePath, Does.Exist, "Expected file to be written"); + string[] lines = File.ReadAllLines(filePath); + + int expectedLength = includeHeaders ? data.Length + 2 : data.Length; + Assert.That(lines.Length, Is.EqualTo(expectedLength), "Incorrect number of lines in output"); + + int lineOffset = 0; + if (includeHeaders) + { + // First line is |col1|col2|... + var firstLineExpected = $"|{string.Join("|", columns.Select(c => c.Name))}|"; + Assert.That(lines[0], Is.EqualTo(firstLineExpected), "Header row does not match expected"); + // Second line is |---|---|... + var secondLineExpected = $"|{string.Join("", Enumerable.Repeat("---|", columns.Length))}"; + Assert.That(lines[1], Is.EqualTo(secondLineExpected), "Separator row does not match expected"); + + lineOffset = 2; + } + + for (int i = 0; i < data.Length; i++) + { + var expectedLine = $"|{string.Join("|", data[i].Select(GetMarkdownPrintValue).ToArray())}|"; + Assert.That(lines[i + lineOffset], Is.EqualTo(expectedLine), "Data row does not match expected"); + } + } + + private static string GetMarkdownPrintValue(DbCellValue d) => + d.IsNull ? "NULL" : d.DisplayValue; + public static void VerifyJsonMatchesData(DbCellValue[][] data, ColumnInfo[] columns, string filePath) { - // ... Upon deserialization to an array of dictionaries - Assert.True(File.Exists(filePath), "Expected file to have been written"); + // ... Upon deserialization to an array of dictionaries + Assert.That(filePath, Does.Exist, "Expected file to have been written"); string output = File.ReadAllText(filePath); Dictionary[] outputObject = JsonConvert.DeserializeObject[]>(output); @@ -278,18 +350,18 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults // ... There should be 2 items in the array, // ... The item should have three fields, and three values, assigned appropriately // ... The deserialized values should match the display value - Assert.AreEqual(data.Length, outputObject.Length); + Assert.That(outputObject.Length, Is.EqualTo(data.Length), "Incorrect number of records in output"); for (int rowIndex = 0; rowIndex < outputObject.Length; rowIndex++) { Dictionary item = outputObject[rowIndex]; - Assert.AreEqual(columns.Length, item.Count); + Assert.That(item.Count, Is.EqualTo(columns.Length), $"Incorrect number of cells for record {rowIndex}"); for (int columnIndex = 0; columnIndex < columns.Length; columnIndex++) { var key = columns[columnIndex].Name; - Assert.True(item.ContainsKey(key)); + Assert.That(item, Contains.Key(key), $"Record {rowIndex} does not contain column {key}"); DbCellValue value = data[rowIndex][columnIndex]; object expectedValue = GetJsonExpectedValue(value, columns[columnIndex]); - Assert.AreEqual(expectedValue, item[key]); + Assert.That(item[key], Is.EqualTo(expectedValue), $"Record {rowIndex}, column {key} contains incorrect value"); } } } @@ -313,8 +385,8 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults } public static void VerifyXmlMatchesData(DbCellValue[][] data, ColumnInfo[] columns, string filePath) { - // ... Upon deserialization to an array of dictionaries - Assert.True(File.Exists(filePath), "Expected file to have been written"); + // ... Upon deserialization to an array of dictionaries + Assert.That(filePath, Does.Exist, "Expected file to have been written"); string output = File.ReadAllText(filePath); XmlDocument xmlDoc = new XmlDocument(); xmlDoc.LoadXml(output); @@ -325,7 +397,7 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults string xpath = "data/row"; var rows = xmlDoc.SelectNodes(xpath); - Assert.AreEqual(data.Length, rows.Count); + Assert.That(rows.Count, Is.EqualTo(data.Length), "Incorrect number of records in output"); for (int rowIndex = 0; rowIndex < rows.Count; rowIndex++) { var rowValue = rows.Item(rowIndex); @@ -335,10 +407,10 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults { var columnName = columns[columnIndex].Name; var xmlColumn = xmlCols.FirstOrDefault(x => x.Name == columnName); - Assert.NotNull(xmlColumn); + Assert.That(xmlColumn, Is.Not.Null, $"Record {rowIndex} does not contain column {columnName}"); DbCellValue value = data[rowIndex][columnIndex]; object expectedValue = GetXmlExpectedValue(value); - Assert.AreEqual(expectedValue, xmlColumn.InnerText); + Assert.That(xmlColumn.InnerText, Is.EqualTo(expectedValue), $"Invalid value for record {rowIndex}, column {columnName}"); } } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/SaveResults/ServiceIntegrationTests.cs b/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/SaveResults/ServiceIntegrationTests.cs index e84c4664..c51e126f 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/SaveResults/ServiceIntegrationTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.UnitTests/QueryExecution/SaveResults/ServiceIntegrationTests.cs @@ -247,6 +247,129 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults #endregion + #region Markdown Tests + + [Test] + public async Task SaveResultsMarkdown_NonExistentQuery() + { + // Given: A working query and workspace service + WorkspaceService ws = Common.GetPrimedWorkspaceService(null); + QueryExecutionService qes = Common.GetPrimedExecutionService(null, false, false, false, ws); + + // If: I attempt to save a result set from a query that doesn't exist + var saveParams = new SaveResultsAsMarkdownRequestParams + { + OwnerUri = Constants.OwnerUri, // Won't exist because nothing has executed + }; + var evf = new EventFlowValidator() + .AddStandardErrorValidation() + .Complete(); + await qes.HandleSaveResultsAsMarkdownRequest(saveParams, evf.Object); + + // Then: + // ... An error event should have been fired + // ... No success event should have been fired + evf.Validate(); + } + + [Test] + public async Task SaveResultAsMarkdown_Failure() + { + // Given: + // ... A working query and workspace service + WorkspaceService ws = Common.GetPrimedWorkspaceService(Constants.StandardQuery); + QueryExecutionService qes = Common.GetPrimedExecutionService( + Common.ExecutionPlanTestDataSet, + true, + false, + false, + ws, + out ConcurrentDictionary storage); + + // ... The query execution service has executed a query with results + var executeParams = new ExecuteDocumentSelectionParams { QuerySelection = null, OwnerUri = Constants.OwnerUri }; + var executeRequest = RequestContextMocks.Create(null); + await qes.HandleExecuteRequest(executeParams, executeRequest.Object); + await qes.WorkTask; + await qes.ActiveQueries[Constants.OwnerUri].ExecutionTask; + + // If: I attempt to save a result set and get it to throw because of invalid column selection + var saveParams = new SaveResultsAsMarkdownRequestParams + { + BatchIndex = 0, + FilePath = "qqq", + OwnerUri = Constants.OwnerUri, + ResultSetIndex = 0, + ColumnStartIndex = -1, + ColumnEndIndex = 100, + RowStartIndex = 0, + RowEndIndex = 5 + }; + qes.MarkdownFileFactory = GetMarkdownStreamFactory(storage, saveParams); + var efv = new EventFlowValidator() + .AddStandardErrorValidation() + .Complete(); + + await qes.HandleSaveResultsAsMarkdownRequest(saveParams, efv.Object); + await qes.ActiveQueries[saveParams.OwnerUri] + .Batches[saveParams.BatchIndex] + .ResultSets[saveParams.ResultSetIndex] + .SaveTasks[saveParams.FilePath]; + + // Then: + // ... An error event should have been fired + // ... No success event should have been fired + efv.Validate(); + } + + [Test] + public async Task SaveResultsAsMarkdown_Success() + { + // Given: + // ... A working query and workspace service + WorkspaceService ws = Common.GetPrimedWorkspaceService(Constants.StandardQuery); + QueryExecutionService qes = Common.GetPrimedExecutionService( + Common.ExecutionPlanTestDataSet, + true, + false, + false, + ws, + out ConcurrentDictionary storage); + + // ... The query execution service has executed a query with results + var executeParams = new ExecuteDocumentSelectionParams { QuerySelection = null, OwnerUri = Constants.OwnerUri }; + var executeRequest = RequestContextMocks.Create(null); + await qes.HandleExecuteRequest(executeParams, executeRequest.Object); + await qes.WorkTask; + await qes.ActiveQueries[Constants.OwnerUri].ExecutionTask; + + // If: I attempt to save a result set from a query + var saveParams = new SaveResultsAsMarkdownRequestParams + { + OwnerUri = Constants.OwnerUri, + FilePath = "qqq", + BatchIndex = 0, + ResultSetIndex = 0 + }; + qes.MarkdownFileFactory = GetMarkdownStreamFactory(storage, saveParams); + var efv = new EventFlowValidator() + .AddStandardResultValidator() + .Complete(); + + await qes.HandleSaveResultsAsMarkdownRequest(saveParams, efv.Object); + await qes.ActiveQueries[saveParams.OwnerUri] + .Batches[saveParams.BatchIndex] + .ResultSets[saveParams.ResultSetIndex] + .SaveTasks[saveParams.FilePath]; + + // Then: + // ... I should have a successful result + // ... There should not have been an error + efv.Validate(); + } + + #endregion + #region XML tests [Test] @@ -507,6 +630,23 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.QueryExecution.SaveResults return mock.Object; } + private static IFileStreamFactory GetMarkdownStreamFactory( + IDictionary storage, + SaveResultsAsMarkdownRequestParams saveParams) + { + var mock = new Mock(); + mock.Setup(fsf => fsf.GetReader(It.IsAny())) + .Returns(output => new ServiceBufferFileStreamReader(new MemoryStream(storage[output]), new QueryExecutionSettings())); + mock.Setup(fsf => fsf.GetWriter(It.IsAny(), It.IsAny>())) + .Returns>((output, columns) => + { + storage.Add(output, new byte[8192]); + return new SaveAsMarkdownFileStreamWriter(new MemoryStream(storage[output]), saveParams, columns); + }); + + return mock.Object; + } + private static IFileStreamFactory GetXmlStreamFactory( IDictionary storage, SaveResultsAsXmlRequestParams saveParams)