ckaczor/sqltoolsservice
1
0
Fork 0
mirror of https://github.com/ckaczor/sqltoolsservice.git synced 2026-01-19 01:25:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Feat/result streaming (#721)

Browse Source 
This changes adds the following two notifications from the results processing within a batch. These new notifications allows a consumer to stream results from a resultset instead of getting them all at once after the entire resultset has been fetched.

ResultsAvailable
This is issued after at least 1 row has been fetched for this resultset.

ResultsUpdated
This is issued periodically as more rows are available on this resultset. The final send of this notification when all rows have been fetched has the property 'Complete' set to true in the ResultSummary object.

Detailed Change Log:
* Initial completed implementation of QueryResults stream feature. 3 unittests still need fixing

* Fix for the 3 failing test. I will look into making MockBehavior strict again for the three tests later

* Making GetReader/GetWriter use filestream objects in FileShare.ReadWrite mode so the file can be concurrently read and written

* Changing resultsAvailable also to fire off on a timer instead of after 1st row

* adding a project for clr TableValuedFunction to produce result set with delays after each row. This is helpful in end to end testing.

* Fixing up some tests and simplifying implementation of result update timer

* Address review comments

* Some test fixes

* Disabled flaky test verification
This commit is contained in:
Arvind Ranasaria
2018-11-26 10:24:54 -08:00
committed by GitHub
parent 688e128c4c
commit 6dd9a4b5f1
46 changed files with 18070 additions and 7316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
src/Microsoft.SqlTools.ServiceLayer/QueryExecution/ResultSet.cs
View File

@@ -1,20 +1,22 @@
//
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Data.Common;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts;
using Microsoft.SqlTools.ServiceLayer.QueryExecution.DataStorage;
using Microsoft.SqlTools.ServiceLayer.Utility;
using Microsoft.SqlTools.Utility;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Data.Common;
using System.Diagnostics;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
{
@@ -30,7 +32,8 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
private const string NameOfForXmlColumn = "XML_F52E2B61-18A1-11d1-B105-00805F49916B";
private const string NameOfForJsonColumn = "JSON_F52E2B61-18A1-11d1-B105-00805F49916B";
private const string YukonXmlShowPlanColumn = "Microsoft SQL Server 2005 XML Showplan";
private const uint MaxResultsTimerPulseMilliseconds = 1000;
private const uint MinResultTimerPulseMilliseconds = 10;
#endregion
#region Member Variables
@@ -52,9 +55,15 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
/// <summary>
/// Whether or not the result set has been read in from the database,
/// set as internal in order to fake value in unit tests
/// set as internal in order to fake value in unit tests.
/// This gets set as soon as we start reading.
/// </summary>
internal bool hasBeenRead;
internal bool hasStartedRead = false;
/// <summary>
/// Set when all results have been read for this resultSet from the server
/// </summary>
private bool hasCompletedRead = false;
/// <summary>
/// Whether resultSet is a 'for xml' or 'for json' result
@@ -69,7 +78,7 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
/// <summary>
/// Row count to use in special scenarios where we want to override the number of rows.
/// </summary>
private long? rowCountOverride;
private long? rowCountOverride=null;
/// <summary>
/// The special action which applied to this result set
@@ -82,6 +91,8 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
/// </summary>
internal long totalBytesWritten;
internal readonly Timer resultsTimer;
#endregion
/// <summary>
@@ -103,8 +114,10 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
// Store the factory
fileStreamFactory = factory;
hasBeenRead = false;
hasStartedRead = false;
hasCompletedRead = false;
SaveTasks = new ConcurrentDictionary<string, Task>();
resultsTimer = new Timer(SendResultAvailableOrUpdated);
}
#region Eventing
@@ -123,7 +136,7 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
public delegate Task SaveAsFailureAsyncEventHandler(SaveResultsRequestParams parameters, string message);
/// <summary>
/// Asynchronous handler for when a resultset has completed
/// Asynchronous handler for when a resultset is available/updated/completed
/// </summary>
/// <param name="resultSet">The result set that completed</param>
public delegate Task ResultSetAsyncEventHandler(ResultSet resultSet);
@@ -133,6 +146,17 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
/// </summary>
public event ResultSetAsyncEventHandler ResultCompletion;
/// <summary>
/// Event that will be called when the resultSet first becomes available. This is as soon as we start reading the results.
/// </summary>
public event ResultSetAsyncEventHandler ResultAvailable;
/// <summary>
/// Event that will be called when additional rows in the result set are available (rowCount available has increased)
/// </summary>
public event ResultSetAsyncEventHandler ResultUpdated;
#endregion
#region Properties
@@ -155,7 +179,7 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
/// <summary>
/// The number of rows for this result set
/// </summary>
public long RowCount => rowCountOverride ?? fileOffsets.Count;
public long RowCount => rowCountOverride != null ? Math.Min(rowCountOverride.Value, fileOffsets.Count) : fileOffsets.Count;
/// <summary>
/// All save tasks currently saving this ResultSet
@@ -175,11 +199,11 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
Id = Id,
BatchId = BatchId,
RowCount = RowCount,
SpecialAction = hasBeenRead ? ProcessSpecialAction() : null
Complete = hasCompletedRead,
SpecialAction = hasCompletedRead ? ProcessSpecialAction() : null
};
}
}
#endregion
#region Public Methods
@@ -195,8 +219,8 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
/// <returns>The requested row</returns>
public IList<DbCellValue> GetRow(long rowId)
{
// Sanity check to make sure that results have been read beforehand
if (!hasBeenRead)
// Sanity check to make sure that results read has started
if (!hasStartedRead)
{
throw new InvalidOperationException(SR.QueryServiceResultSetNotRead);
}
@@ -221,8 +245,8 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
/// <returns>A subset of results</returns>
public Task<ResultSetSubset> GetSubset(long startRow, int rowCount)
{
// Sanity check to make sure that the results have been read beforehand
if (!hasBeenRead)
// Sanity check to make sure that results read has started
if (!hasStartedRead)
{
throw new InvalidOperationException(SR.QueryServiceResultSetNotRead);
}
@@ -286,11 +310,11 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
/// <returns>An execution plan object</returns>
public Task<ExecutionPlan> GetExecutionPlan()
{
// Process the action just incase is hasn't been yet
// Process the action just in case it hasn't been yet
ProcessSpecialAction();
// Sanity check to make sure that the results have been read beforehand
if (!hasBeenRead)
// Sanity check to make sure that results read has started
if (!hasStartedRead)
{
throw new InvalidOperationException(SR.QueryServiceResultSetNotRead);
}
@@ -333,6 +357,7 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
public async Task ReadResultToEnd(DbDataReader dbDataReader, CancellationToken cancellationToken)
{
// Sanity check to make sure we got a reader
//
Validate.IsNotNull(nameof(dbDataReader), dbDataReader);
try
@@ -340,38 +365,57 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
// Verify the request hasn't been cancelled
cancellationToken.ThrowIfCancellationRequested();
// Mark that result has been read
hasBeenRead = true;
StorageDataReader dataReader = new StorageDataReader(dbDataReader);
// Open a writer for the file
//
var fileWriter = fileStreamFactory.GetWriter(outputFileName);
using (fileWriter)
{
// If we can initialize the columns using the column schema, use that
//
if (!dataReader.DbDataReader.CanGetColumnSchema())
{
throw new InvalidOperationException(SR.QueryServiceResultSetNoColumnSchema);
}
Columns = dataReader.Columns;
// Check if result set is 'for xml/json'. If it is, set isJson/isXml value in column metadata
//
SingleColumnXmlJsonResultSet();
// Mark that read of result has started
//
hasStartedRead = true;
while (await dataReader.ReadAsync(cancellationToken))
{
fileOffsets.Add(totalBytesWritten);
totalBytesWritten += fileWriter.WriteRow(dataReader);
// If we have never triggered the timer to start sending the results available/updated notification
// then: Trigger the timer to start sending results update notification
//
if (LastUpdatedSummary == null)
{
// Invoke the timer to send available/update result set notification immediately
//
resultsTimer.Change(0, Timeout.Infinite);
}
}
CheckForIsJson();
}
// Check if resultset is 'for xml/json'. If it is, set isJson/isXml value in column metadata
SingleColumnXmlJsonResultSet();
CheckForIsJson();
}
finally
{
// Fire off a result set completion event if we have one
if (ResultCompletion != null)
{
await ResultCompletion(this);
}
hasCompletedRead = true; // set the flag to indicate that we are done reading
// Make a final call to ResultUpdated by invoking the timer to send update result set notification immediately
//
resultsTimer.Change(0, Timeout.Infinite);
// and finally:
// Make a call to send ResultCompletion and await for it to Complete
//
await (ResultCompletion?.Invoke(this) ?? Task.CompletedTask);
}
}
@@ -381,8 +425,8 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
/// <param name="internalId">Internal ID of the row</param>
public void RemoveRow(long internalId)
{
// Make sure that the results have been read
if (!hasBeenRead)
// Sanity check to make sure that results read has started
if (!hasStartedRead)
{
throw new InvalidOperationException(SR.QueryServiceResultSetNotRead);
}
@@ -436,7 +480,7 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
Validate.IsNotNull(nameof(fileFactory), fileFactory);
// Make sure the resultset has finished being read
if (!hasBeenRead)
if (!hasCompletedRead)
{
throw new InvalidOperationException(SR.QueryServiceSaveAsResultSetNotComplete);
}
@@ -526,6 +570,7 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
public void Dispose()
{
resultsTimer.Dispose();
Dispose(true);
GC.SuppressFinalize(this);
}
@@ -564,7 +609,59 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
#endregion
#region Private Helper Methods
/// <summary>
/// Sends the ResultsUpdated message if the number of rows has changed since last send.
/// </summary>
/// <param name="stateInfo"></param>
private async void SendResultAvailableOrUpdated (object stateInfo = null)
{
ResultSet currentResultSetSnapshot = (ResultSet) MemberwiseClone();
if (LastUpdatedSummary == null) // We need to send results available message.
{
// Fire off results Available task and await for it complete
//
await (ResultAvailable?.Invoke(currentResultSetSnapshot) ?? Task.CompletedTask);
ResultAvailable = null; // set this to null as we need to call ResultAvailable only once
}
else // We need to send results updated message.
{
// If there has been no change in rowCount since last update and we are not done yet then log and increase the timer duration
//
if (!currentResultSetSnapshot.hasCompletedRead && LastUpdatedSummary.RowCount == currentResultSetSnapshot.RowCount)
{
Logger.Write(TraceEventType.Warning, $"The result set:{Summary} has not made any progress in last {ResultTimerInterval} milliseconds and the read of resultset is not completed yet!");
ResultsIntervalMultiplier++;
}
// Fire off results updated task and await for it complete
//
await (ResultUpdated?.Invoke(currentResultSetSnapshot) ?? Task.CompletedTask);
}
// Update the LastUpdatedSummary to be the value captured in current snapshot
//
LastUpdatedSummary = currentResultSetSnapshot.Summary;
// Setup timer for the next callback
if (currentResultSetSnapshot.hasCompletedRead)
{
//If we have already completed reading then we are done and we do not need to send any more updates. Switch off timer.
//
resultsTimer.Change(Timeout.Infinite, Timeout.Infinite);
}
else
{
resultsTimer.Change(ResultTimerInterval, Timeout.Infinite);
}
}
private uint ResultsIntervalMultiplier { get; set; } = 1;
internal uint ResultTimerInterval => Math.Max(Math.Min(MaxResultsTimerPulseMilliseconds, (uint)RowCount / 500 /* 1 millisec per 500 rows*/), MinResultTimerPulseMilliseconds * ResultsIntervalMultiplier);
internal ResultSetSummary LastUpdatedSummary { get; set; } = null;
/// <summary>
/// If the result set represented by this class corresponds to a single XML
/// column that contains results of "for xml" query, set isXml = true
@@ -574,7 +671,7 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
private void SingleColumnXmlJsonResultSet()
{
if (Columns?.Length == 1 && RowCount != 0)
if (Columns?.Length == 1)
{
if (Columns[0].ColumnName.Equals(NameOfForXmlColumn, StringComparison.Ordinal))
{
@@ -636,7 +733,8 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution
private async Task<long> AppendRowToBuffer(DbDataReader dbDataReader)
{
Validate.IsNotNull(nameof(dbDataReader), dbDataReader);
if (!hasBeenRead)
// Sanity check to make sure that results read has started
if (!hasStartedRead)
{
throw new InvalidOperationException(SR.QueryServiceResultSetNotRead);
}