ckaczor/sqltoolsservice
1
0
Fork 0
mirror of https://github.com/ckaczor/sqltoolsservice.git synced 2026-01-13 17:23:02 -05:00
Code Issues Packages Projects Releases Wiki Activity

fixed perf tests and added more scenarios (#683)

Browse Source 
* fixed perf tests and added more scenarios
This commit is contained in:
Leila Lali
2018-08-27 10:57:41 -07:00
committed by GitHub
parent 69961992bb
commit aa2b30f486
15 changed files with 870 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
sqltoolsservice.sln
View File

@@ -220,4 +220,4 @@ Global
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {B31CDF4B-2851-45E5-8C5F-BE97125D9DD8}
EndGlobalSection
EndGlobal
EndGlobal

3
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs
View File

@@ -25,8 +25,7 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
}
Logger.Initialize("testdriver", LogLevel.Verbose);
return TestRunner.RunTests(args, "Microsoft.SqlTools.ServiceLayer.PerfTests.").Result;
return TestRunner.Instance.RunTests(args, "Microsoft.SqlTools.ServiceLayer.PerfTests.").Result;
}
}
}

119
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs
View File

@@ -18,85 +18,94 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
[CreateTestDb(TestServerType.Azure)]
public async Task ConnectAzureTest()
{
TestServerType serverType = TestServerType.Azure;
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
const string query = Scripts.TestDbSimpleSelectQuery;
testService.WriteToFile(queryTempFile.FilePath, query);
DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification
TestServerType serverType = TestServerType.Azure;
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
TextDocument = new TextDocumentItem
const string query = Scripts.TestDbSimpleSelectQuery;
testService.WriteToFile(queryTempFile.FilePath, query);
DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification
{
Uri = queryTempFile.FilePath,
LanguageId = "enu",
Version = 1,
Text = query
}
};
TextDocument = new TextDocumentItem
{
Uri = queryTempFile.FilePath,
LanguageId = "enu",
Version = 1,
Text = query
}
};
await testService.RequestOpenDocumentNotification(openParams);
await testService.RequestOpenDocumentNotification(openParams);
Thread.Sleep(500);
var connected = await testService.CalculateRunTime(async () =>
{
var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName);
return await testService.Connect(queryTempFile.FilePath, connectParams);
}, true);
Assert.True(connected, "Connection was not successful");
}
Thread.Sleep(500);
var connected = await testService.CalculateRunTime(async () =>
{
var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName);
return await testService.Connect(queryTempFile.FilePath, connectParams);
}, timer);
Assert.True(connected, "Connection was not successful");
}
});
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task ConnectOnPremTest()
{
TestServerType serverType = TestServerType.OnPrem;
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
const string query = Scripts.TestDbSimpleSelectQuery;
testService.WriteToFile(queryTempFile.FilePath, query);
TestServerType serverType = TestServerType.OnPrem;
DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
TextDocument = new TextDocumentItem
const string query = Scripts.TestDbSimpleSelectQuery;
testService.WriteToFile(queryTempFile.FilePath, query);
DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification
{
Uri = queryTempFile.FilePath,
LanguageId = "enu",
Version = 1,
Text = query
}
};
TextDocument = new TextDocumentItem
{
Uri = queryTempFile.FilePath,
LanguageId = "enu",
Version = 1,
Text = query
}
};
await testService.RequestOpenDocumentNotification(openParams);
await testService.RequestOpenDocumentNotification(openParams);
Thread.Sleep(500);
var connected = await testService.CalculateRunTime(async () =>
{
var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName);
return await testService.Connect(queryTempFile.FilePath, connectParams);
}, true);
Assert.True(connected, "Connection was not successful");
}
Thread.Sleep(500);
var connected = await testService.CalculateRunTime(async () =>
{
var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName);
return await testService.Connect(queryTempFile.FilePath, connectParams);
}, timer);
Assert.True(connected, "Connection was not successful");
}
});
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task DisconnectTest()
{
TestServerType serverType = TestServerType.OnPrem;
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName);
Thread.Sleep(1000);
var connected = await testService.CalculateRunTime(() => testService.Disconnect(queryTempFile.FilePath), true);
Assert.True(connected);
}
TestServerType serverType = TestServerType.OnPrem;
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName);
Thread.Sleep(1000);
var connected = await testService.CalculateRunTime(() => testService.Disconnect(queryTempFile.FilePath), timer);
Assert.True(connected);
}
});
}
}

264
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs
View File

@@ -21,107 +21,122 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
[CreateTestDb(TestServerType.OnPrem)]
public async Task HoverTestOnPrem()
{
TestServerType serverType = TestServerType.OnPrem;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
const string query = Scripts.TestDbSimpleSelectQuery;
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName);
Hover hover = await testService.CalculateRunTime(() => testService.RequestHover(queryTempFile.FilePath, query, 0, Scripts.TestDbComplexSelectQueries.Length + 1), true);
Assert.NotNull(hover);
await testService.Disconnect(queryTempFile.FilePath);
}
TestServerType serverType = TestServerType.OnPrem;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
const string query = Scripts.TestDbSimpleSelectQuery;
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName);
await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, waitForIntelliSense: true);
Hover hover = await testService.CalculateRunTime(() => testService.RequestHover(queryTempFile.FilePath, query, 0, Scripts.TestDbComplexSelectQueries.Length + 1), timer);
Assert.NotNull(hover);
await testService.Disconnect(queryTempFile.FilePath);
}
});
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task SuggestionsTest()
{
TestServerType serverType = TestServerType.OnPrem;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
const string query = Scripts.TestDbSimpleSelectQuery;
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName);
await ValidateCompletionResponse(testService, queryTempFile.FilePath, false, Common.PerfTestDatabaseName, true);
await ValidateCompletionResponse(testService, queryTempFile.FilePath, true, Common.PerfTestDatabaseName, false);
await testService.Disconnect(queryTempFile.FilePath);
}
TestServerType serverType = TestServerType.OnPrem;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
const string query = Scripts.TestDbSimpleSelectQuery;
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName);
await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, timer: null, waitForIntelliSense: true);
await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, timer: timer, waitForIntelliSense: false);
await testService.Disconnect(queryTempFile.FilePath);
}
});
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task DiagnosticsTests()
{
TestServerType serverType = TestServerType.OnPrem;
SqlTestDb.CreateNew(serverType, doNotCleanupDb: true, databaseName: Common.PerfTestDatabaseName, query: Scripts.CreateDatabaseObjectsQuery);
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName);
TestServerType serverType = TestServerType.OnPrem;
SqlTestDb.CreateNew(serverType, doNotCleanupDb: true, databaseName: Common.PerfTestDatabaseName, query: Scripts.CreateDatabaseObjectsQuery);
Thread.Sleep(500);
var contentChanges = new TextDocumentChangeEvent[1];
contentChanges[0] = new TextDocumentChangeEvent()
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
Range = new Range
await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName);
Thread.Sleep(500);
var contentChanges = new TextDocumentChangeEvent[1];
contentChanges[0] = new TextDocumentChangeEvent()
{
Start = new Position
Range = new Range
{
Line = 0,
Character = 5
Start = new Position
{
Line = 0,
Character = 5
},
End = new Position
{
Line = 0,
Character = 6
}
},
End = new Position
{
Line = 0,
Character = 6
}
},
RangeLength = 1,
Text = "z"
};
DidChangeTextDocumentParams changeParams = new DidChangeTextDocumentParams
{
ContentChanges = contentChanges,
TextDocument = new VersionedTextDocumentIdentifier
RangeLength = 1,
Text = "z"
};
DidChangeTextDocumentParams changeParams = new DidChangeTextDocumentParams
{
Version = 2,
Uri = queryTempFile.FilePath
}
};
ContentChanges = contentChanges,
TextDocument = new VersionedTextDocumentIdentifier
{
Version = 2,
Uri = queryTempFile.FilePath
}
};
TestTimer timer = new TestTimer() { PrintResult = true };
await testService.RequestChangeTextDocumentNotification(changeParams);
await testService.ExecuteWithTimeout(timer, 60000, async () =>
{
var completeEvent = await testService.Driver.WaitForEvent(PublishDiagnosticsNotification.Type, 15000);
return completeEvent?.Diagnostics != null && completeEvent.Diagnostics.Length > 0;
});
await testService.Disconnect(queryTempFile.FilePath);
}
await testService.RequestChangeTextDocumentNotification(changeParams);
await testService.ExecuteWithTimeout(timer, 60000, async () =>
{
var completeEvent = await testService.Driver.WaitForEvent(PublishDiagnosticsNotification.Type, 15000);
return completeEvent?.Diagnostics != null && completeEvent.Diagnostics.Length > 0;
});
await testService.Disconnect(queryTempFile.FilePath);
}
});
}
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task BindingCacheColdAzureSimpleQuery()
{
TestServerType serverType = TestServerType.Azure;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false);
}
TestServerType serverType = TestServerType.Azure;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false, timer);
}
});
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task BindingCacheColdOnPremSimpleQuery()
{
TestServerType serverType = TestServerType.OnPrem;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false);
}
TestServerType serverType = TestServerType.OnPrem;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false, timer);
}
});
}
@@ -129,76 +144,94 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
[CreateTestDb(TestServerType.Azure)]
public async Task BindingCacheWarmAzureSimpleQuery()
{
TestServerType serverType = TestServerType.Azure;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
const string query = Scripts.TestDbSimpleSelectQuery;
await VerifyBindingLoadScenario(testService, serverType, query, true);
}
TestServerType serverType = TestServerType.Azure;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
const string query = Scripts.TestDbSimpleSelectQuery;
await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
}
});
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task BindingCacheWarmOnPremSimpleQuery()
{
TestServerType serverType = TestServerType.OnPrem;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
const string query = Scripts.TestDbSimpleSelectQuery;
await VerifyBindingLoadScenario(testService, serverType, query, true);
}
TestServerType serverType = TestServerType.OnPrem;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
const string query = Scripts.TestDbSimpleSelectQuery;
await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
}
});
}
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task BindingCacheColdAzureComplexQuery()
{
TestServerType serverType = TestServerType.Azure;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries,false);
}
TestServerType serverType = TestServerType.Azure;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false, timer);
}
});
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task BindingCacheColdOnPremComplexQuery()
{
TestServerType serverType = TestServerType.OnPrem;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false);
}
TestServerType serverType = TestServerType.OnPrem;
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false, timer);
}
});
}
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task BindingCacheWarmAzureComplexQuery()
{
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
string query = Scripts.TestDbComplexSelectQueries;
const TestServerType serverType = TestServerType.Azure;
await VerifyBindingLoadScenario(testService, serverType, query, true);
}
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
string query = Scripts.TestDbComplexSelectQueries;
const TestServerType serverType = TestServerType.Azure;
await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
}
});
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task BindingCacheWarmOnPremComplexQuery()
{
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
string query = Scripts.TestDbComplexSelectQueries;
const TestServerType serverType = TestServerType.OnPrem;
await VerifyBindingLoadScenario(testService, serverType, query, true);
}
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
string query = Scripts.TestDbComplexSelectQueries;
const TestServerType serverType = TestServerType.OnPrem;
await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
}
});
}
#region Private Helper Methods
@@ -207,34 +240,36 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
TestServiceDriverProvider testService,
TestServerType serverType,
string query,
bool preLoad,
bool preLoad,
TestTimer timer,
[CallerMemberName] string testName = "")
{
string databaseName = Common.PerfTestDatabaseName;
if (preLoad)
{
await VerifyCompletationLoaded(testService, serverType, Scripts.TestDbSimpleSelectQuery,
databaseName, printResult: false, testName: testName);
databaseName, null, testName: testName);
Console.WriteLine("Intellisense cache loaded.");
}
await VerifyCompletationLoaded(testService, serverType, query, databaseName,
printResult: true, testName: testName);
timer, testName: testName);
}
private async Task VerifyCompletationLoaded(
TestServiceDriverProvider testService,
TestServerType serverType,
string query,
private async Task VerifyCompletationLoaded(
TestServiceDriverProvider testService,
TestServerType serverType,
string query,
string databaseName,
bool printResult,
TestTimer timer,
string testName)
{
using (SelfCleaningTempFile testTempFile = new SelfCleaningTempFile())
{
testService.WriteToFile(testTempFile.FilePath, query);
await testService.ConnectForQuery(serverType, query, testTempFile.FilePath, databaseName);
await ValidateCompletionResponse(testService, testTempFile.FilePath, printResult, databaseName,
waitForIntelliSense: true, testName: testName);
await ValidateCompletionResponse(testService, testTempFile.FilePath, databaseName,
waitForIntelliSense: true, timer: timer, testName: testName);
await testService.Disconnect(testTempFile.FilePath);
}
}
@@ -242,14 +277,17 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
private static async Task ValidateCompletionResponse(
TestServiceDriverProvider testService,
string ownerUri,
bool printResult,
string databaseName,
bool waitForIntelliSense,
bool waitForIntelliSense,
TestTimer timer = null,
[CallerMemberName] string testName = "")
{
TestTimer timer = new TestTimer() { PrintResult = printResult };
if (timer == null)
{
timer = new TestTimer { PrintResult = false };
}
bool isReady = !waitForIntelliSense;
await testService.ExecuteWithTimeout(timer, 150000, async () =>
await testService.ExecuteWithTimeout(timer, 550000, async () =>
{
if (isReady)
{

146
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ObjectExplorerTests.cs Normal file
View File

@@ -0,0 +1,146 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts;
using Microsoft.SqlTools.ServiceLayer.Test.Common;
using Xunit;
namespace Microsoft.SqlTools.ServiceLayer.PerfTests
{
public class ObjectExplorerTests
{
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task CreateSessionAzure()
{
TestServerType serverType = TestServerType.Azure;
await VerifyCreateSession(serverType);
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task CreateSessionOnPrem()
{
TestServerType serverType = TestServerType.OnPrem;
await VerifyCreateSession(serverType);
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task ExpandDatabasesOnPrem()
{
TestServerType serverType = TestServerType.OnPrem;
await VerifyExpand(serverType, SqlTestDb.MasterDatabaseName);
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task ExpandOneDatabaseOnPrem()
{
TestServerType serverType = TestServerType.OnPrem;
await VerifyExpand(serverType, Common.PerfTestDatabaseName);
}
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task ExpandDatabasesAzure()
{
TestServerType serverType = TestServerType.Azure;
await VerifyExpand(serverType, SqlTestDb.MasterDatabaseName);
}
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task ExpandOneDatabaseAzure()
{
TestServerType serverType = TestServerType.Azure;
await VerifyExpand(serverType, Common.PerfTestDatabaseName);
}
private async Task VerifyCreateSession(TestServerType serverType, [CallerMemberName] string testName = "")
{
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
ConnectParams connectParams = TestServiceProvider.Instance.ConnectionProfileService.GetConnectionParameters(serverType, SqlTestDb.MasterDatabaseName);
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
var result = await testService.CalculateRunTime(() => testService.RequestObjectExplorerCreateSession(connectParams.Connection), timer);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.False(string.IsNullOrEmpty(result.SessionId), "Session id cannot be empty");
await testService.RequestObjectExplorerCloseSession(new ObjectExplorer.Contracts.CloseSessionParams
{
SessionId = result.SessionId
});
await testService.Disconnect(queryTempFile.FilePath);
}
}, testName);
}
private async Task VerifyExpand(TestServerType serverType, string databaseName, [CallerMemberName] string testName = "")
{
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
ConnectParams connectParams = TestServiceProvider.Instance.ConnectionProfileService.GetConnectionParameters(serverType, databaseName);
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
var result = await testService.CalculateRunTime(() => testService.RequestObjectExplorerCreateSession(connectParams.Connection), null);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.False(string.IsNullOrEmpty(result.SessionId), "Session id cannot be empty");
await ExpandDatabase(testService, result.SessionId, result.RootNode, timer);
await testService.RequestObjectExplorerCloseSession(new ObjectExplorer.Contracts.CloseSessionParams
{
SessionId = result.SessionId
});
await testService.Disconnect(queryTempFile.FilePath);
}
}, testName);
}
private async Task<bool> ExpandDatabase(TestServiceDriverProvider testService, string sessionId, NodeInfo nodeInfo, TestTimer timer)
{
if (nodeInfo == null) return false;
bool foundNode = nodeInfo.NodePath.Contains("Database") || nodeInfo.NodeType == "Database";
var expandResult = await testService.CalculateRunTime(() => testService.RequestObjectExplorerExpand(new ObjectExplorer.Contracts.ExpandParams
{
SessionId = sessionId,
NodePath = nodeInfo.NodePath
}, 50000), foundNode ? timer : null);
Assert.NotNull(expandResult);
Assert.NotNull(expandResult.Nodes);
Assert.False(expandResult.Nodes == null, "Nodes are not valid");
if (!foundNode)
{
foreach (var node in expandResult.Nodes)
{
if (await ExpandDatabase(testService, sessionId, node, timer))
{
return true;
}
}
return false;
}
else
{
Console.WriteLine("Node Expanded " + nodeInfo.NodePath);
return true;
}
}
}
}

166
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs
View File

@@ -5,6 +5,7 @@
using System;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts.ExecuteRequests;
using Microsoft.SqlTools.ServiceLayer.Test.Common;
@@ -17,82 +18,137 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
[Fact]
public async Task QueryResultSummaryOnPremTest()
{
TestServerType serverType = TestServerType.OnPrem;
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
const string query = Scripts.MasterBasicQuery;
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
var queryResult = await testService.CalculateRunTime(() => testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query), true);
Assert.NotNull(queryResult);
Assert.True(queryResult.BatchSummaries.Any(x => x.ResultSetSummaries.Any(r => r.RowCount > 0)));
await testService.Disconnect(queryTempFile.FilePath);
}
await QueryResultSummaryOnPremTest(TestServerType.OnPrem, Scripts.MasterBasicQuery);
}
[Fact]
public async Task QueryResultFirstOnPremTest()
{
TestServerType serverType = TestServerType.OnPrem;
await QueryResultFirstOnPremTest(TestServerType.OnPrem, Scripts.MasterBasicQuery);
}
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
const string query = Scripts.MasterBasicQuery;
[Fact]
public async Task LongQueryResultSummaryOnPremTest()
{
await QueryResultSummaryOnPremTest(TestServerType.OnPrem, Scripts.MasterLongQuery);
}
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
[Fact]
public async Task LongQueryResultFirstOnPremTest()
{
await QueryResultFirstOnPremTest(TestServerType.OnPrem, Scripts.MasterLongQuery);
}
var queryResult = await testService.CalculateRunTime(async () =>
{
await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query);
return await testService.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 100);
}, true);
[Fact]
public async Task QueryResultSummaryOnAzureTest()
{
await QueryResultSummaryOnPremTest(TestServerType.Azure, Scripts.MasterBasicQuery);
}
Assert.NotNull(queryResult);
Assert.NotNull(queryResult.ResultSubset);
Assert.True(queryResult.ResultSubset.Rows.Any());
[Fact]
public async Task QueryResultFirstOnAzureTest()
{
await QueryResultFirstOnPremTest(TestServerType.Azure, Scripts.MasterBasicQuery);
}
await testService.Disconnect(queryTempFile.FilePath);
}
[Fact]
public async Task LongQueryResultSummaryOnAzureTest()
{
await QueryResultSummaryOnPremTest(TestServerType.Azure, Scripts.MasterLongQuery);
}
[Fact]
public async Task LongQueryResultFirstOnAzureTest()
{
await QueryResultFirstOnPremTest(TestServerType.Azure, Scripts.MasterLongQuery);
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task CancelQueryOnPremTest()
{
TestServerType serverType = TestServerType.OnPrem;
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
await testService.ConnectForQuery(serverType, Scripts.DelayQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName);
var queryParams = new ExecuteDocumentSelectionParams
{
OwnerUri = queryTempFile.FilePath,
QuerySelection = null
};
TestServerType serverType = TestServerType.OnPrem;
var result = await testService.Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams);
if (result != null)
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
TestTimer timer = new TestTimer() { PrintResult = true };
await testService.ExecuteWithTimeout(timer, 100000, async () =>
await testService.ConnectForQuery(serverType, Scripts.DelayQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName);
var queryParams = new ExecuteDocumentSelectionParams
{
var cancelQueryResult = await testService.CancelQuery(queryTempFile.FilePath);
return true;
}, TimeSpan.FromMilliseconds(10));
}
else
{
Assert.True(false, "Failed to run the query");
}
OwnerUri = queryTempFile.FilePath,
QuerySelection = null
};
await testService.Disconnect(queryTempFile.FilePath);
}
testService.WriteToFile(queryTempFile.FilePath, Scripts.MasterLongQuery);
var result = await testService.Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams);
if (result != null)
{
await testService.ExecuteWithTimeout(timer, 100000, async () =>
{
var cancelQueryResult = await testService.CancelQuery(queryTempFile.FilePath);
return true;
}, TimeSpan.FromMilliseconds(10));
}
else
{
Assert.True(false, "Failed to run the query");
await testService.Disconnect(queryTempFile.FilePath);
}
}
});
}
private async Task QueryResultSummaryOnPremTest(TestServerType serverType, string query, [CallerMemberName] string testName = "")
{
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
testService.WriteToFile(queryTempFile.FilePath, query);
var queryResult = await testService.CalculateRunTime(
() => testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, 50000),
timer);
Assert.NotNull(queryResult);
Assert.True(queryResult.BatchSummaries.Any(x => x.ResultSetSummaries.Any(r => r.RowCount > 0)));
await testService.Disconnect(queryTempFile.FilePath);
}
}, testName);
}
private async Task QueryResultFirstOnPremTest(TestServerType serverType, string query, [CallerMemberName] string testName = "")
{
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
testService.WriteToFile(queryTempFile.FilePath, query);
await testService.RunQueryAndWaitToStart(queryTempFile.FilePath, 50000);
await testService.ExecuteWithTimeout(timer, 500000, async () =>
{
var queryResult = await testService.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 100);
if (queryResult != null)
{
Assert.NotNull(queryResult);
Assert.NotNull(queryResult.ResultSubset);
Assert.True(queryResult.ResultSubset.Rows.Any());
}
return queryResult != null;
}, TimeSpan.FromMilliseconds(10));
await testService.Disconnect(queryTempFile.FilePath);
}
}, testName);
}
}
}

54
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs
View File

@@ -14,39 +14,45 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
[Fact]
public async Task TestSaveResultsToCsvTest()
{
TestServerType serverType = TestServerType.OnPrem;
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
const string query = Scripts.MasterBasicQuery;
TestServerType serverType = TestServerType.OnPrem;
// Execute a query
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query);
await testService.CalculateRunTime(() => testService.SaveAsCsv(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), true);
await testService.Disconnect(queryTempFile.FilePath);
}
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
const string query = Scripts.MasterBasicQuery;
// Execute a query
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query);
await testService.CalculateRunTime(() => testService.SaveAsCsv(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), timer);
await testService.Disconnect(queryTempFile.FilePath);
}
});
}
[Fact]
public async Task TestSaveResultsToJsonTest()
{
TestServerType serverType = TestServerType.OnPrem;
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
const string query = Scripts.MasterBasicQuery;
TestServerType serverType = TestServerType.OnPrem;
// Execute a query
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query);
await testService.CalculateRunTime(() => testService.SaveAsJson(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), true);
await testService.Disconnect(queryTempFile.FilePath);
}
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
const string query = Scripts.MasterBasicQuery;
// Execute a query
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query);
await testService.CalculateRunTime(() => testService.SaveAsJson(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), timer);
await testService.Disconnect(queryTempFile.FilePath);
}
});
}
}

79
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ScriptinTests.cs Normal file
View File

@@ -0,0 +1,79 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts;
using Microsoft.SqlTools.ServiceLayer.Scripting.Contracts;
using Microsoft.SqlTools.ServiceLayer.Test.Common;
using Xunit;
namespace Microsoft.SqlTools.ServiceLayer.PerfTests
{
public class ScriptingTests
{
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task ScripTableAzure()
{
TestServerType serverType = TestServerType.Azure;
await VerifyScriptTable(serverType);
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task ScripTableOnPrem()
{
TestServerType serverType = TestServerType.OnPrem;
await VerifyScriptTable(serverType);
}
private async Task VerifyScriptTable(TestServerType serverType, [CallerMemberName] string testName = "")
{
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
await testService.ConnectForQuery(serverType, string.Empty, queryTempFile.FilePath, Common.PerfTestDatabaseName);
List<ScriptingObject> scriptingObjects = new List<ScriptingObject>()
{
new ScriptingObject
{
Schema = "Person",
Name = "Address",
Type = "Table"
}
};
ScriptingParams scriptingParams = new ScriptingParams
{
OwnerUri = queryTempFile.FilePath,
Operation = ScriptingOperationType.Create,
FilePath = queryTempFile.FilePath,
ScriptOptions = new ScriptOptions
{
ScriptCreateDrop = "ScriptCreate",
},
ScriptDestination = "ToEditor",
ScriptingObjects = scriptingObjects
};
var result = await testService.CalculateRunTime(() => testService.RequestScript(scriptingParams), timer);
Assert.NotNull(result);
Assert.NotNull(result.Script);
Assert.False(string.IsNullOrEmpty(result.Script), "Script result is invalid");
await testService.Disconnect(queryTempFile.FilePath);
}
}, testName);
}
}
}

2
test/Microsoft.SqlTools.ServiceLayer.Test.Common/Scripts/Scripts.cs
View File

@@ -14,6 +14,8 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
private const string ResourceNameRefix = "Microsoft.SqlTools.ServiceLayer.Test.Common.Scripts.";
public const string MasterBasicQuery = "SELECT * FROM sys.all_columns"; //basic queries should return at least 10000 rows
public const string MasterLongQuery = @"SELECT * FROM sys.all_columns a1
JOIN sys.all_columns a2 on a1.object_id = a2.object_id";
public const string DelayQuery = "WAITFOR DELAY '00:01:00'";

21
test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestResult.cs
View File

@@ -3,10 +3,31 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using Newtonsoft.Json;
namespace Microsoft.SqlTools.ServiceLayer.Test.Common
{
public class TestResult
{
[JsonProperty("elapsedTime")]
public double ElapsedTime { get; set; }
[JsonProperty("metricValue")]
public double MetricValue { get; set; }
[JsonProperty("iterations")]
public double[] Iterations { get; set; }
[JsonProperty("ninetiethPercentile")]
public double NinetiethPercentile { get; set; }
[JsonProperty("fiftiethPercentile")]
public double FiftiethPercentile { get; set; }
[JsonProperty("average")]
public double Average { get; set; }
[JsonProperty("primaryMetric")]
public string PrimaryMetric { get; set; }
}
}

127
test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestServiceDriverProvider.cs
View File

@@ -9,11 +9,13 @@ using System.Threading.Tasks;
using Microsoft.SqlTools.Hosting.Protocol.Contracts;
using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts;
using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts;
using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts;
using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts.ExecuteRequests;
using Microsoft.SqlTools.ServiceLayer.Scripting.Contracts;
using Microsoft.SqlTools.ServiceLayer.SqlContext;
using Microsoft.SqlTools.ServiceLayer.TestDriver.Driver;
using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility;
using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts;
using Xunit;
@@ -29,7 +31,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
public TestServiceDriverProvider()
{
Driver = new ServiceTestDriver();
Driver = new ServiceTestDriver(TestRunner.Instance.ExecutableFilePath);
Driver.Start().Wait();
this.isRunning = true;
}
@@ -228,6 +230,18 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
return result;
}
/// <summary>
/// Request a list of completion items for a position in a block of text
/// </summary>
public async Task RequestRebuildIntelliSense(string ownerUri)
{
var rebuildIntelliSenseParams = new RebuildIntelliSenseParams();
rebuildIntelliSenseParams.OwnerUri = ownerUri;
await Driver.SendEvent(RebuildIntelliSenseNotification.Type, rebuildIntelliSenseParams);
}
/// <summary>
/// Request a a hover tooltop
/// </summary>
@@ -284,6 +298,15 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
// Write the query text to a backing file
WriteToFile(ownerUri, query);
return await RunQueryAndWaitToComplete(ownerUri, timeoutMilliseconds);
}
/// <summary>
/// Run a query using a given connection bound to a URI
/// </summary>
public async Task<QueryCompleteParams> RunQueryAndWaitToComplete(string ownerUri, int timeoutMilliseconds = 5000)
{
var queryParams = new ExecuteDocumentSelectionParams
{
OwnerUri = ownerUri,
@@ -302,6 +325,79 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
}
}
/// <summary>
/// Run a query using a given connection bound to a URI
/// </summary>
public async Task<BatchEventParams> RunQueryAndWaitToStart(string ownerUri, string query, int timeoutMilliseconds = 5000)
{
// Write the query text to a backing file
WriteToFile(ownerUri, query);
return await RunQueryAndWaitToStart(ownerUri, timeoutMilliseconds);
}
/// <summary>
/// Run a query using a given connection bound to a URI
/// </summary>
public async Task<BatchEventParams> RunQueryAndWaitToStart(string ownerUri, int timeoutMilliseconds = 5000)
{
var queryParams = new ExecuteDocumentSelectionParams
{
OwnerUri = ownerUri,
QuerySelection = null
};
var result = await Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams);
if (result != null)
{
var eventResult = await Driver.WaitForEvent(BatchStartEvent.Type, timeoutMilliseconds);
return eventResult;
}
else
{
return null;
}
}
public async Task<SessionCreatedParameters> RequestObjectExplorerCreateSession(ConnectionDetails connectionDetails, int timeoutMilliseconds = 5000)
{
var result = await Driver.SendRequest(CreateSessionRequest.Type, connectionDetails);
if (result != null)
{
var eventResult = await Driver.WaitForEvent(CreateSessionCompleteNotification.Type, timeoutMilliseconds);
return eventResult;
}
else
{
return null;
}
}
public async Task<ExpandResponse> RequestObjectExplorerExpand(ExpandParams expandParams, int timeoutMilliseconds = 5000)
{
var result = await Driver.SendRequest(ExpandRequest.Type, expandParams);
if (result)
{
var eventResult = await Driver.WaitForEvent(ExpandCompleteNotification.Type, timeoutMilliseconds);
return eventResult;
}
else
{
return null;
}
}
public async Task<ScriptingResult> RequestScript(ScriptingParams scriptingParams, int timeoutMilliseconds = 5000)
{
var result = await Driver.SendRequest(ScriptingRequest.Type, scriptingParams);
return result;
}
public async Task<CloseSessionResponse> RequestObjectExplorerCloseSession(CloseSessionParams closeSessionParams, int timeoutMilliseconds = 5000)
{
return await Driver.SendRequest(CloseSessionRequest.Type, closeSessionParams);
}
/// <summary>
/// Run a query using a given connection bound to a URI. This method only waits for the initial response from query
/// execution (QueryExecuteResult). It is up to the caller to wait for the QueryCompleteEvent if they are interested.
@@ -324,7 +420,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
await ConnectForQuery(serverType, query, queryTempFile.FilePath, databaseName);
var queryResult = await CalculateRunTime(() => RunQueryAndWaitToComplete(queryTempFile.FilePath, query, 50000), false);
var queryResult = await CalculateRunTime(() => RunQueryAndWaitToComplete(queryTempFile.FilePath, query, 50000));
Assert.NotNull(queryResult);
Assert.NotNull(queryResult.BatchSummaries);
@@ -332,11 +428,29 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
}
}
public async Task<T> CalculateRunTime<T>(Func<Task<T>> testToRun, bool printResult, [CallerMemberName] string testName = "")
public static async Task RunTestIterations(Func<TestTimer, Task> testToRun, [CallerMemberName] string testName = "")
{
TestTimer timer = new TestTimer() { PrintResult = printResult };
TestTimer timer = new TestTimer() { PrintResult = true };
for (int i = 0; i < TestRunner.Instance.NumberOfRuns; i++)
{
Console.WriteLine("Iteration Number: " + i);
await testToRun(timer);
}
timer.Print(testName);
}
public async Task<T> CalculateRunTime<T>(Func<Task<T>> testToRun, TestTimer timer = null)
{
if (timer != null)
{
timer.Start();
}
T result = await testToRun();
timer.EndAndPrint(testName);
if (timer != null)
{
timer.End();
}
return result;
}
@@ -344,11 +458,12 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
public async Task ExecuteWithTimeout(TestTimer timer, int timeout, Func<Task<bool>> repeatedCode,
TimeSpan? delay = null, [CallerMemberName] string testName = "")
{
timer.Start();
while (true)
{
if (await repeatedCode())
{
timer.EndAndPrint(testName);
timer.End();
break;
}
if (timer.TotalMilliSecondsUntilNow >= timeout)

48
test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestTimer.cs
View File

@@ -3,9 +3,12 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
namespace Microsoft.SqlTools.ServiceLayer.Test.Common
@@ -17,10 +20,16 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
{
private static string resultFolder = InitResultFolder();
private List<double> iterations = new List<double>();
private static string InitResultFolder()
{
string resultFodler = Environment.GetEnvironmentVariable("ResultFolder");
if (string.IsNullOrEmpty(resultFodler))
{
resultFodler = TestRunner.Instance.ResultFolder;
}
if (string.IsNullOrEmpty(resultFodler))
{
string assemblyLocation = System.Reflection.Assembly.GetEntryAssembly().Location;
resultFodler = Path.GetDirectoryName(assemblyLocation);
@@ -43,18 +52,39 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
public void End()
{
EndDateTime = DateTime.UtcNow;
iterations.Add(TotalMilliSeconds);
if (PrintResult)
{
Console.WriteLine("Result: " + TotalMilliSeconds);
}
}
public void EndAndPrint([CallerMemberName] string testName = "")
{
End();
Print(testName);
}
public void Print([CallerMemberName] string testName = "")
{
if (PrintResult)
{
var iterationArray = iterations.ToArray();
double elapsed = Percentile(iterationArray, 0.5);
var currentColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "Test Name: {0} Run time in milliSeconds: {1}", testName, TotalMilliSeconds));
Console.ForegroundColor = currentColor;
string resultContent = Newtonsoft.Json.JsonConvert.SerializeObject(new TestResult { ElapsedTime = TotalMilliSeconds });
string resultContent = Newtonsoft.Json.JsonConvert.SerializeObject(new TestResult
{
ElapsedTime = TotalMilliSeconds,
MetricValue = elapsed,
PrimaryMetric = "ElapsedTimeMetric",
Iterations = iterationArray,
FiftiethPercentile = Percentile(iterationArray, 0.5),
NinetiethPercentile = Percentile(iterationArray, 0.9),
Average = iterations.Where(x => x > 0).Average()
});
string fileName = testName + ".json";
string resultFilePath = string.IsNullOrEmpty(resultFolder) ? fileName : Path.Combine(resultFolder, fileName);
File.WriteAllText(resultFilePath, resultContent);
@@ -62,6 +92,22 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
}
}
private static double Percentile(double[] sequence, double excelPercentile)
{
Array.Sort(sequence);
int N = sequence.Length;
double n = (N - 1) * excelPercentile + 1;
// Another method: double n = (N + 1) * excelPercentile;
if (n == 1d) return sequence[0];
else if (n == N) return sequence[N - 1];
else
{
int k = (int)n;
double d = n - k;
return sequence[k - 1] + d * (sequence[k] - sequence[k - 1]);
}
}
public double TotalMilliSeconds
{
get

4
test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs
View File

@@ -43,9 +43,9 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver
private DateTime startTime;
public ServiceTestDriver()
public ServiceTestDriver(string executableFilePath = null)
{
string serviceHostExecutable = Environment.GetEnvironmentVariable(ServiceHostEnvironmentVariable);
string serviceHostExecutable = executableFilePath != null ? executableFilePath : Environment.GetEnvironmentVariable(ServiceHostEnvironmentVariable);
string serviceHostArguments = "--enable-logging";
if (string.IsNullOrWhiteSpace(serviceHostExecutable))
{

14
test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/Constants.cs Normal file
View File

@@ -0,0 +1,14 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Utility
{
public class Constants
{
public const string NumberOfRunsEnvironmentVariable = "NumberOfRuns";
public const string ExecutableFileEnvironmentVariable = "SQLTOOLSSERVICE_EXE";
public const string ResultFolderEnvironmentVariable = "ResultFolder";
}
}

82
test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs
View File

@@ -5,6 +5,7 @@
using System;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
@@ -15,9 +16,86 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Utility
{
public class TestRunner
{
public static async Task<int> RunTests(string[] tests, string testNamespace)
private int numberOfRuns = 20;
protected int DefaultNumberOfRuns = 2;
public static TestRunner Instance { get; } = new TestRunner();
public string[] Tests { get; set; }
public int NumberOfRuns { get; set; }
public string ExecutableFilePath { get; set; }
private TestRunner()
{
foreach (var test in tests)
InitParameters();
}
private void ParseArguments(string[] args)
{
int index = 0;
while (index < args.Length - 1)
{
string arg = args[index++];
string argValue = args[index++];
switch (arg)
{
case "/t":
case "/T":
case "/tests":
Tests = argValue.Split(" ");
break;
case "/n":
case "/N":
case "/numberOfRuns":
int value;
if (Int32.TryParse(argValue, out value))
{
NumberOfRuns = value;
}
break;
case "/r":
case "/R":
case "/Result":
ResultFolder = argValue;
break;
case "/s":
case "/S":
case "/Service":
ExecutableFilePath = argValue;
break;
}
}
if ((Tests == null || Tests.Length == 0) && args.Length >= 1)
{
Tests = args;
}
}
public string ResultFolder = InitResultFolder();
private static string InitResultFolder()
{
return Environment.GetEnvironmentVariable("ResultFolder");
}
private void InitParameters()
{
string numberOfRunsEnv = Environment.GetEnvironmentVariable(Constants.NumberOfRunsEnvironmentVariable);
if (!Int32.TryParse(numberOfRunsEnv, out numberOfRuns))
{
numberOfRuns = DefaultNumberOfRuns;
}
NumberOfRuns = numberOfRuns;
ExecutableFilePath = Environment.GetEnvironmentVariable(Constants.ExecutableFileEnvironmentVariable);
}
public async Task<int> RunTests(string[] args, string testNamespace)
{
ParseArguments(args);
foreach (var test in Tests)
{
try
{