ckaczor/sqltoolsservice
1
0
Fork 0
mirror of https://github.com/ckaczor/sqltoolsservice.git synced 2026-02-16 10:58:30 -05:00
Code Issues Packages Projects Releases Wiki Activity

fixed perf tests and added more scenarios (#683)

Browse Source 
* fixed perf tests and added more scenarios
This commit is contained in:
Leila Lali
2018-08-27 10:57:41 -07:00
committed by GitHub
parent 69961992bb
commit aa2b30f486
15 changed files with 870 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
sqltoolsservice.sln
View File

@@ -220,4 +220,4 @@ Global
GlobalSection(ExtensibilityGlobals) = postSolution GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {B31CDF4B-2851-45E5-8C5F-BE97125D9DD8} SolutionGuid = {B31CDF4B-2851-45E5-8C5F-BE97125D9DD8}
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal

3
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs
View File

@@ -25,8 +25,7 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
} }
Logger.Initialize("testdriver", LogLevel.Verbose); Logger.Initialize("testdriver", LogLevel.Verbose);
return TestRunner.Instance.RunTests(args, "Microsoft.SqlTools.ServiceLayer.PerfTests.").Result;
return TestRunner.RunTests(args, "Microsoft.SqlTools.ServiceLayer.PerfTests.").Result;
} }
} }
} }

119
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs
View File

@@ -18,85 +18,94 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
[CreateTestDb(TestServerType.Azure)] [CreateTestDb(TestServerType.Azure)]
public async Task ConnectAzureTest() public async Task ConnectAzureTest()
{ {
TestServerType serverType = TestServerType.Azure; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
const string query = Scripts.TestDbSimpleSelectQuery; TestServerType serverType = TestServerType.Azure;
testService.WriteToFile(queryTempFile.FilePath, query); using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification
{ {
TextDocument = new TextDocumentItem const string query = Scripts.TestDbSimpleSelectQuery;
testService.WriteToFile(queryTempFile.FilePath, query);
DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification
{ {
Uri = queryTempFile.FilePath, TextDocument = new TextDocumentItem
LanguageId = "enu", {
Version = 1, Uri = queryTempFile.FilePath,
Text = query LanguageId = "enu",
} Version = 1,
}; Text = query
}
};
await testService.RequestOpenDocumentNotification(openParams); await testService.RequestOpenDocumentNotification(openParams);
Thread.Sleep(500); Thread.Sleep(500);
var connected = await testService.CalculateRunTime(async () => var connected = await testService.CalculateRunTime(async () =>
{ {
var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName); var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName);
return await testService.Connect(queryTempFile.FilePath, connectParams); return await testService.Connect(queryTempFile.FilePath, connectParams);
}, true); }, timer);
Assert.True(connected, "Connection was not successful"); Assert.True(connected, "Connection was not successful");
} }
});
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.OnPrem)] [CreateTestDb(TestServerType.OnPrem)]
public async Task ConnectOnPremTest() public async Task ConnectOnPremTest()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
const string query = Scripts.TestDbSimpleSelectQuery; TestServerType serverType = TestServerType.OnPrem;
testService.WriteToFile(queryTempFile.FilePath, query);
DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
TextDocument = new TextDocumentItem const string query = Scripts.TestDbSimpleSelectQuery;
testService.WriteToFile(queryTempFile.FilePath, query);
DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification
{ {
Uri = queryTempFile.FilePath, TextDocument = new TextDocumentItem
LanguageId = "enu", {
Version = 1, Uri = queryTempFile.FilePath,
Text = query LanguageId = "enu",
} Version = 1,
}; Text = query
}
};
await testService.RequestOpenDocumentNotification(openParams); await testService.RequestOpenDocumentNotification(openParams);
Thread.Sleep(500); Thread.Sleep(500);
var connected = await testService.CalculateRunTime(async () => var connected = await testService.CalculateRunTime(async () =>
{ {
var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName); var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName);
return await testService.Connect(queryTempFile.FilePath, connectParams); return await testService.Connect(queryTempFile.FilePath, connectParams);
}, true); }, timer);
Assert.True(connected, "Connection was not successful"); Assert.True(connected, "Connection was not successful");
} }
});
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.OnPrem)] [CreateTestDb(TestServerType.OnPrem)]
public async Task DisconnectTest() public async Task DisconnectTest()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName); TestServerType serverType = TestServerType.OnPrem;
Thread.Sleep(1000);
var connected = await testService.CalculateRunTime(() => testService.Disconnect(queryTempFile.FilePath), true); using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
Assert.True(connected); using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
} {
await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName);
Thread.Sleep(1000);
var connected = await testService.CalculateRunTime(() => testService.Disconnect(queryTempFile.FilePath), timer);
Assert.True(connected);
}
});
} }
} }

264
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs
View File

@@ -21,107 +21,122 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
[CreateTestDb(TestServerType.OnPrem)] [CreateTestDb(TestServerType.OnPrem)]
public async Task HoverTestOnPrem() public async Task HoverTestOnPrem()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{ {
const string query = Scripts.TestDbSimpleSelectQuery; TestServerType serverType = TestServerType.OnPrem;
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName); using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
Hover hover = await testService.CalculateRunTime(() => testService.RequestHover(queryTempFile.FilePath, query, 0, Scripts.TestDbComplexSelectQueries.Length + 1), true); using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
Assert.NotNull(hover); {
await testService.Disconnect(queryTempFile.FilePath); const string query = Scripts.TestDbSimpleSelectQuery;
} await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName);
await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, waitForIntelliSense: true);
Hover hover = await testService.CalculateRunTime(() => testService.RequestHover(queryTempFile.FilePath, query, 0, Scripts.TestDbComplexSelectQueries.Length + 1), timer);
Assert.NotNull(hover);
await testService.Disconnect(queryTempFile.FilePath);
}
});
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.OnPrem)] [CreateTestDb(TestServerType.OnPrem)]
public async Task SuggestionsTest() public async Task SuggestionsTest()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{ {
const string query = Scripts.TestDbSimpleSelectQuery; TestServerType serverType = TestServerType.OnPrem;
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName); using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await ValidateCompletionResponse(testService, queryTempFile.FilePath, false, Common.PerfTestDatabaseName, true); using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
await ValidateCompletionResponse(testService, queryTempFile.FilePath, true, Common.PerfTestDatabaseName, false); {
await testService.Disconnect(queryTempFile.FilePath); const string query = Scripts.TestDbSimpleSelectQuery;
} await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName);
await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, timer: null, waitForIntelliSense: true);
await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, timer: timer, waitForIntelliSense: false);
await testService.Disconnect(queryTempFile.FilePath);
}
});
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.OnPrem)] [CreateTestDb(TestServerType.OnPrem)]
public async Task DiagnosticsTests() public async Task DiagnosticsTests()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
SqlTestDb.CreateNew(serverType, doNotCleanupDb: true, databaseName: Common.PerfTestDatabaseName, query: Scripts.CreateDatabaseObjectsQuery);
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{ {
await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName); TestServerType serverType = TestServerType.OnPrem;
SqlTestDb.CreateNew(serverType, doNotCleanupDb: true, databaseName: Common.PerfTestDatabaseName, query: Scripts.CreateDatabaseObjectsQuery);
Thread.Sleep(500); using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
var contentChanges = new TextDocumentChangeEvent[1]; using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
contentChanges[0] = new TextDocumentChangeEvent()
{ {
Range = new Range await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName);
Thread.Sleep(500);
var contentChanges = new TextDocumentChangeEvent[1];
contentChanges[0] = new TextDocumentChangeEvent()
{ {
Start = new Position Range = new Range
{ {
Line = 0, Start = new Position
Character = 5 {
Line = 0,
Character = 5
},
End = new Position
{
Line = 0,
Character = 6
}
}, },
End = new Position RangeLength = 1,
{ Text = "z"
Line = 0, };
Character = 6 DidChangeTextDocumentParams changeParams = new DidChangeTextDocumentParams
}
},
RangeLength = 1,
Text = "z"
};
DidChangeTextDocumentParams changeParams = new DidChangeTextDocumentParams
{
ContentChanges = contentChanges,
TextDocument = new VersionedTextDocumentIdentifier
{ {
Version = 2, ContentChanges = contentChanges,
Uri = queryTempFile.FilePath TextDocument = new VersionedTextDocumentIdentifier
} {
}; Version = 2,
Uri = queryTempFile.FilePath
}
};
TestTimer timer = new TestTimer() { PrintResult = true }; await testService.RequestChangeTextDocumentNotification(changeParams);
await testService.RequestChangeTextDocumentNotification(changeParams); await testService.ExecuteWithTimeout(timer, 60000, async () =>
await testService.ExecuteWithTimeout(timer, 60000, async () => {
{ var completeEvent = await testService.Driver.WaitForEvent(PublishDiagnosticsNotification.Type, 15000);
var completeEvent = await testService.Driver.WaitForEvent(PublishDiagnosticsNotification.Type, 15000); return completeEvent?.Diagnostics != null && completeEvent.Diagnostics.Length > 0;
return completeEvent?.Diagnostics != null && completeEvent.Diagnostics.Length > 0; });
}); await testService.Disconnect(queryTempFile.FilePath);
await testService.Disconnect(queryTempFile.FilePath); }
} });
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.Azure)] [CreateTestDb(TestServerType.Azure)]
public async Task BindingCacheColdAzureSimpleQuery() public async Task BindingCacheColdAzureSimpleQuery()
{ {
TestServerType serverType = TestServerType.Azure; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false); TestServerType serverType = TestServerType.Azure;
} using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false, timer);
}
});
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.OnPrem)] [CreateTestDb(TestServerType.OnPrem)]
public async Task BindingCacheColdOnPremSimpleQuery() public async Task BindingCacheColdOnPremSimpleQuery()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false); TestServerType serverType = TestServerType.OnPrem;
} using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false, timer);
}
});
} }
@@ -129,76 +144,94 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
[CreateTestDb(TestServerType.Azure)] [CreateTestDb(TestServerType.Azure)]
public async Task BindingCacheWarmAzureSimpleQuery() public async Task BindingCacheWarmAzureSimpleQuery()
{ {
TestServerType serverType = TestServerType.Azure; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{ {
const string query = Scripts.TestDbSimpleSelectQuery; TestServerType serverType = TestServerType.Azure;
await VerifyBindingLoadScenario(testService, serverType, query, true); using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
} using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
const string query = Scripts.TestDbSimpleSelectQuery;
await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
}
});
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.OnPrem)] [CreateTestDb(TestServerType.OnPrem)]
public async Task BindingCacheWarmOnPremSimpleQuery() public async Task BindingCacheWarmOnPremSimpleQuery()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{ {
const string query = Scripts.TestDbSimpleSelectQuery; TestServerType serverType = TestServerType.OnPrem;
await VerifyBindingLoadScenario(testService, serverType, query, true);
} using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
const string query = Scripts.TestDbSimpleSelectQuery;
await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
}
});
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.Azure)] [CreateTestDb(TestServerType.Azure)]
public async Task BindingCacheColdAzureComplexQuery() public async Task BindingCacheColdAzureComplexQuery()
{ {
TestServerType serverType = TestServerType.Azure; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries,false); TestServerType serverType = TestServerType.Azure;
}
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false, timer);
}
});
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.OnPrem)] [CreateTestDb(TestServerType.OnPrem)]
public async Task BindingCacheColdOnPremComplexQuery() public async Task BindingCacheColdOnPremComplexQuery()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false); TestServerType serverType = TestServerType.OnPrem;
} using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false, timer);
}
});
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.Azure)] [CreateTestDb(TestServerType.Azure)]
public async Task BindingCacheWarmAzureComplexQuery() public async Task BindingCacheWarmAzureComplexQuery()
{ {
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
string query = Scripts.TestDbComplexSelectQueries; using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
const TestServerType serverType = TestServerType.Azure; using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await VerifyBindingLoadScenario(testService, serverType, query, true); {
} string query = Scripts.TestDbComplexSelectQueries;
const TestServerType serverType = TestServerType.Azure;
await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
}
});
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.OnPrem)] [CreateTestDb(TestServerType.OnPrem)]
public async Task BindingCacheWarmOnPremComplexQuery() public async Task BindingCacheWarmOnPremComplexQuery()
{ {
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
string query = Scripts.TestDbComplexSelectQueries; using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
const TestServerType serverType = TestServerType.OnPrem; using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await VerifyBindingLoadScenario(testService, serverType, query, true); {
} string query = Scripts.TestDbComplexSelectQueries;
const TestServerType serverType = TestServerType.OnPrem;
await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
}
});
} }
#region Private Helper Methods #region Private Helper Methods
@@ -207,34 +240,36 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
TestServiceDriverProvider testService, TestServiceDriverProvider testService,
TestServerType serverType, TestServerType serverType,
string query, string query,
bool preLoad, bool preLoad,
TestTimer timer,
[CallerMemberName] string testName = "") [CallerMemberName] string testName = "")
{ {
string databaseName = Common.PerfTestDatabaseName; string databaseName = Common.PerfTestDatabaseName;
if (preLoad) if (preLoad)
{ {
await VerifyCompletationLoaded(testService, serverType, Scripts.TestDbSimpleSelectQuery, await VerifyCompletationLoaded(testService, serverType, Scripts.TestDbSimpleSelectQuery,
databaseName, printResult: false, testName: testName); databaseName, null, testName: testName);
Console.WriteLine("Intellisense cache loaded."); Console.WriteLine("Intellisense cache loaded.");
} }
await VerifyCompletationLoaded(testService, serverType, query, databaseName, await VerifyCompletationLoaded(testService, serverType, query, databaseName,
printResult: true, testName: testName); timer, testName: testName);
} }
private async Task VerifyCompletationLoaded( private async Task VerifyCompletationLoaded(
TestServiceDriverProvider testService, TestServiceDriverProvider testService,
TestServerType serverType, TestServerType serverType,
string query, string query,
string databaseName, string databaseName,
bool printResult, TestTimer timer,
string testName) string testName)
{ {
using (SelfCleaningTempFile testTempFile = new SelfCleaningTempFile()) using (SelfCleaningTempFile testTempFile = new SelfCleaningTempFile())
{ {
testService.WriteToFile(testTempFile.FilePath, query); testService.WriteToFile(testTempFile.FilePath, query);
await testService.ConnectForQuery(serverType, query, testTempFile.FilePath, databaseName); await testService.ConnectForQuery(serverType, query, testTempFile.FilePath, databaseName);
await ValidateCompletionResponse(testService, testTempFile.FilePath, printResult, databaseName, await ValidateCompletionResponse(testService, testTempFile.FilePath, databaseName,
waitForIntelliSense: true, testName: testName); waitForIntelliSense: true, timer: timer, testName: testName);
await testService.Disconnect(testTempFile.FilePath); await testService.Disconnect(testTempFile.FilePath);
} }
} }
@@ -242,14 +277,17 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
private static async Task ValidateCompletionResponse( private static async Task ValidateCompletionResponse(
TestServiceDriverProvider testService, TestServiceDriverProvider testService,
string ownerUri, string ownerUri,
bool printResult,
string databaseName, string databaseName,
bool waitForIntelliSense, bool waitForIntelliSense,
TestTimer timer = null,
[CallerMemberName] string testName = "") [CallerMemberName] string testName = "")
{ {
TestTimer timer = new TestTimer() { PrintResult = printResult }; if (timer == null)
{
timer = new TestTimer { PrintResult = false };
}
bool isReady = !waitForIntelliSense; bool isReady = !waitForIntelliSense;
await testService.ExecuteWithTimeout(timer, 150000, async () => await testService.ExecuteWithTimeout(timer, 550000, async () =>
{ {
if (isReady) if (isReady)
{ {

146
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ObjectExplorerTests.cs Normal file
View File

@@ -0,0 +1,146 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts;
using Microsoft.SqlTools.ServiceLayer.Test.Common;
using Xunit;
namespace Microsoft.SqlTools.ServiceLayer.PerfTests
{
public class ObjectExplorerTests
{
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task CreateSessionAzure()
{
TestServerType serverType = TestServerType.Azure;
await VerifyCreateSession(serverType);
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task CreateSessionOnPrem()
{
TestServerType serverType = TestServerType.OnPrem;
await VerifyCreateSession(serverType);
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task ExpandDatabasesOnPrem()
{
TestServerType serverType = TestServerType.OnPrem;
await VerifyExpand(serverType, SqlTestDb.MasterDatabaseName);
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task ExpandOneDatabaseOnPrem()
{
TestServerType serverType = TestServerType.OnPrem;
await VerifyExpand(serverType, Common.PerfTestDatabaseName);
}
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task ExpandDatabasesAzure()
{
TestServerType serverType = TestServerType.Azure;
await VerifyExpand(serverType, SqlTestDb.MasterDatabaseName);
}
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task ExpandOneDatabaseAzure()
{
TestServerType serverType = TestServerType.Azure;
await VerifyExpand(serverType, Common.PerfTestDatabaseName);
}
private async Task VerifyCreateSession(TestServerType serverType, [CallerMemberName] string testName = "")
{
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
ConnectParams connectParams = TestServiceProvider.Instance.ConnectionProfileService.GetConnectionParameters(serverType, SqlTestDb.MasterDatabaseName);
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
var result = await testService.CalculateRunTime(() => testService.RequestObjectExplorerCreateSession(connectParams.Connection), timer);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.False(string.IsNullOrEmpty(result.SessionId), "Session id cannot be empty");
await testService.RequestObjectExplorerCloseSession(new ObjectExplorer.Contracts.CloseSessionParams
{
SessionId = result.SessionId
});
await testService.Disconnect(queryTempFile.FilePath);
}
}, testName);
}
private async Task VerifyExpand(TestServerType serverType, string databaseName, [CallerMemberName] string testName = "")
{
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
ConnectParams connectParams = TestServiceProvider.Instance.ConnectionProfileService.GetConnectionParameters(serverType, databaseName);
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
var result = await testService.CalculateRunTime(() => testService.RequestObjectExplorerCreateSession(connectParams.Connection), null);
Assert.NotNull(result);
Assert.True(result.Success);
Assert.False(string.IsNullOrEmpty(result.SessionId), "Session id cannot be empty");
await ExpandDatabase(testService, result.SessionId, result.RootNode, timer);
await testService.RequestObjectExplorerCloseSession(new ObjectExplorer.Contracts.CloseSessionParams
{
SessionId = result.SessionId
});
await testService.Disconnect(queryTempFile.FilePath);
}
}, testName);
}
private async Task<bool> ExpandDatabase(TestServiceDriverProvider testService, string sessionId, NodeInfo nodeInfo, TestTimer timer)
{
if (nodeInfo == null) return false;
bool foundNode = nodeInfo.NodePath.Contains("Database") || nodeInfo.NodeType == "Database";
var expandResult = await testService.CalculateRunTime(() => testService.RequestObjectExplorerExpand(new ObjectExplorer.Contracts.ExpandParams
{
SessionId = sessionId,
NodePath = nodeInfo.NodePath
}, 50000), foundNode ? timer : null);
Assert.NotNull(expandResult);
Assert.NotNull(expandResult.Nodes);
Assert.False(expandResult.Nodes == null, "Nodes are not valid");
if (!foundNode)
{
foreach (var node in expandResult.Nodes)
{
if (await ExpandDatabase(testService, sessionId, node, timer))
{
return true;
}
}
return false;
}
else
{
Console.WriteLine("Node Expanded " + nodeInfo.NodePath);
return true;
}
}
}
}

166
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs
View File

@@ -5,6 +5,7 @@
using System; using System;
using System.Linq; using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts.ExecuteRequests; using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts.ExecuteRequests;
using Microsoft.SqlTools.ServiceLayer.Test.Common; using Microsoft.SqlTools.ServiceLayer.Test.Common;
@@ -17,82 +18,137 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
[Fact] [Fact]
public async Task QueryResultSummaryOnPremTest() public async Task QueryResultSummaryOnPremTest()
{ {
TestServerType serverType = TestServerType.OnPrem; await QueryResultSummaryOnPremTest(TestServerType.OnPrem, Scripts.MasterBasicQuery);
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
const string query = Scripts.MasterBasicQuery;
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
var queryResult = await testService.CalculateRunTime(() => testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query), true);
Assert.NotNull(queryResult);
Assert.True(queryResult.BatchSummaries.Any(x => x.ResultSetSummaries.Any(r => r.RowCount > 0)));
await testService.Disconnect(queryTempFile.FilePath);
}
} }
[Fact] [Fact]
public async Task QueryResultFirstOnPremTest() public async Task QueryResultFirstOnPremTest()
{ {
TestServerType serverType = TestServerType.OnPrem; await QueryResultFirstOnPremTest(TestServerType.OnPrem, Scripts.MasterBasicQuery);
}
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) [Fact]
using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) public async Task LongQueryResultSummaryOnPremTest()
{ {
const string query = Scripts.MasterBasicQuery; await QueryResultSummaryOnPremTest(TestServerType.OnPrem, Scripts.MasterLongQuery);
}
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); [Fact]
public async Task LongQueryResultFirstOnPremTest()
{
await QueryResultFirstOnPremTest(TestServerType.OnPrem, Scripts.MasterLongQuery);
}
var queryResult = await testService.CalculateRunTime(async () => [Fact]
{ public async Task QueryResultSummaryOnAzureTest()
await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query); {
return await testService.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 100); await QueryResultSummaryOnPremTest(TestServerType.Azure, Scripts.MasterBasicQuery);
}, true); }
Assert.NotNull(queryResult); [Fact]
Assert.NotNull(queryResult.ResultSubset); public async Task QueryResultFirstOnAzureTest()
Assert.True(queryResult.ResultSubset.Rows.Any()); {
await QueryResultFirstOnPremTest(TestServerType.Azure, Scripts.MasterBasicQuery);
}
await testService.Disconnect(queryTempFile.FilePath); [Fact]
} public async Task LongQueryResultSummaryOnAzureTest()
{
await QueryResultSummaryOnPremTest(TestServerType.Azure, Scripts.MasterLongQuery);
}
[Fact]
public async Task LongQueryResultFirstOnAzureTest()
{
await QueryResultFirstOnPremTest(TestServerType.Azure, Scripts.MasterLongQuery);
} }
[Fact] [Fact]
[CreateTestDb(TestServerType.OnPrem)] [CreateTestDb(TestServerType.OnPrem)]
public async Task CancelQueryOnPremTest() public async Task CancelQueryOnPremTest()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
await testService.ConnectForQuery(serverType, Scripts.DelayQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName); TestServerType serverType = TestServerType.OnPrem;
var queryParams = new ExecuteDocumentSelectionParams
{
OwnerUri = queryTempFile.FilePath,
QuerySelection = null
};
var result = await testService.Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams); using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
if (result != null) using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
TestTimer timer = new TestTimer() { PrintResult = true }; await testService.ConnectForQuery(serverType, Scripts.DelayQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName);
await testService.ExecuteWithTimeout(timer, 100000, async () => var queryParams = new ExecuteDocumentSelectionParams
{ {
var cancelQueryResult = await testService.CancelQuery(queryTempFile.FilePath); OwnerUri = queryTempFile.FilePath,
return true; QuerySelection = null
}, TimeSpan.FromMilliseconds(10)); };
}
else
{
Assert.True(false, "Failed to run the query");
}
await testService.Disconnect(queryTempFile.FilePath); testService.WriteToFile(queryTempFile.FilePath, Scripts.MasterLongQuery);
}
var result = await testService.Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams);
if (result != null)
{
await testService.ExecuteWithTimeout(timer, 100000, async () =>
{
var cancelQueryResult = await testService.CancelQuery(queryTempFile.FilePath);
return true;
}, TimeSpan.FromMilliseconds(10));
}
else
{
Assert.True(false, "Failed to run the query");
await testService.Disconnect(queryTempFile.FilePath);
}
}
});
}
private async Task QueryResultSummaryOnPremTest(TestServerType serverType, string query, [CallerMemberName] string testName = "")
{
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
testService.WriteToFile(queryTempFile.FilePath, query);
var queryResult = await testService.CalculateRunTime(
() => testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, 50000),
timer);
Assert.NotNull(queryResult);
Assert.True(queryResult.BatchSummaries.Any(x => x.ResultSetSummaries.Any(r => r.RowCount > 0)));
await testService.Disconnect(queryTempFile.FilePath);
}
}, testName);
}
private async Task QueryResultFirstOnPremTest(TestServerType serverType, string query, [CallerMemberName] string testName = "")
{
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
testService.WriteToFile(queryTempFile.FilePath, query);
await testService.RunQueryAndWaitToStart(queryTempFile.FilePath, 50000);
await testService.ExecuteWithTimeout(timer, 500000, async () =>
{
var queryResult = await testService.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 100);
if (queryResult != null)
{
Assert.NotNull(queryResult);
Assert.NotNull(queryResult.ResultSubset);
Assert.True(queryResult.ResultSubset.Rows.Any());
}
return queryResult != null;
}, TimeSpan.FromMilliseconds(10));
await testService.Disconnect(queryTempFile.FilePath);
}
}, testName);
} }
} }
} }

54
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs
View File

@@ -14,39 +14,45 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
[Fact] [Fact]
public async Task TestSaveResultsToCsvTest() public async Task TestSaveResultsToCsvTest()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
const string query = Scripts.MasterBasicQuery; TestServerType serverType = TestServerType.OnPrem;
// Execute a query using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile())
await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query); using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await testService.CalculateRunTime(() => testService.SaveAsCsv(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), true); {
await testService.Disconnect(queryTempFile.FilePath); const string query = Scripts.MasterBasicQuery;
}
// Execute a query
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query);
await testService.CalculateRunTime(() => testService.SaveAsCsv(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), timer);
await testService.Disconnect(queryTempFile.FilePath);
}
});
} }
[Fact] [Fact]
public async Task TestSaveResultsToJsonTest() public async Task TestSaveResultsToJsonTest()
{ {
TestServerType serverType = TestServerType.OnPrem; await TestServiceDriverProvider.RunTestIterations(async (timer) =>
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile())
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
{ {
const string query = Scripts.MasterBasicQuery; TestServerType serverType = TestServerType.OnPrem;
// Execute a query using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile())
await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query); using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
await testService.CalculateRunTime(() => testService.SaveAsJson(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), true); {
await testService.Disconnect(queryTempFile.FilePath); const string query = Scripts.MasterBasicQuery;
}
// Execute a query
await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query);
await testService.CalculateRunTime(() => testService.SaveAsJson(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), timer);
await testService.Disconnect(queryTempFile.FilePath);
}
});
} }
} }

79
test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ScriptinTests.cs Normal file
View File

@@ -0,0 +1,79 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts;
using Microsoft.SqlTools.ServiceLayer.Scripting.Contracts;
using Microsoft.SqlTools.ServiceLayer.Test.Common;
using Xunit;
namespace Microsoft.SqlTools.ServiceLayer.PerfTests
{
public class ScriptingTests
{
[Fact]
[CreateTestDb(TestServerType.Azure)]
public async Task ScripTableAzure()
{
TestServerType serverType = TestServerType.Azure;
await VerifyScriptTable(serverType);
}
[Fact]
[CreateTestDb(TestServerType.OnPrem)]
public async Task ScripTableOnPrem()
{
TestServerType serverType = TestServerType.OnPrem;
await VerifyScriptTable(serverType);
}
private async Task VerifyScriptTable(TestServerType serverType, [CallerMemberName] string testName = "")
{
await TestServiceDriverProvider.RunTestIterations(async (timer) =>
{
using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{
await testService.ConnectForQuery(serverType, string.Empty, queryTempFile.FilePath, Common.PerfTestDatabaseName);
List<ScriptingObject> scriptingObjects = new List<ScriptingObject>()
{
new ScriptingObject
{
Schema = "Person",
Name = "Address",
Type = "Table"
}
};
ScriptingParams scriptingParams = new ScriptingParams
{
OwnerUri = queryTempFile.FilePath,
Operation = ScriptingOperationType.Create,
FilePath = queryTempFile.FilePath,
ScriptOptions = new ScriptOptions
{
ScriptCreateDrop = "ScriptCreate",
},
ScriptDestination = "ToEditor",
ScriptingObjects = scriptingObjects
};
var result = await testService.CalculateRunTime(() => testService.RequestScript(scriptingParams), timer);
Assert.NotNull(result);
Assert.NotNull(result.Script);
Assert.False(string.IsNullOrEmpty(result.Script), "Script result is invalid");
await testService.Disconnect(queryTempFile.FilePath);
}
}, testName);
}
}
}

2
test/Microsoft.SqlTools.ServiceLayer.Test.Common/Scripts/Scripts.cs
View File

@@ -14,6 +14,8 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
private const string ResourceNameRefix = "Microsoft.SqlTools.ServiceLayer.Test.Common.Scripts."; private const string ResourceNameRefix = "Microsoft.SqlTools.ServiceLayer.Test.Common.Scripts.";
public const string MasterBasicQuery = "SELECT * FROM sys.all_columns"; //basic queries should return at least 10000 rows public const string MasterBasicQuery = "SELECT * FROM sys.all_columns"; //basic queries should return at least 10000 rows
public const string MasterLongQuery = @"SELECT * FROM sys.all_columns a1
JOIN sys.all_columns a2 on a1.object_id = a2.object_id";
public const string DelayQuery = "WAITFOR DELAY '00:01:00'"; public const string DelayQuery = "WAITFOR DELAY '00:01:00'";

21
test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestResult.cs
View File

@@ -3,10 +3,31 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information. // Licensed under the MIT license. See LICENSE file in the project root for full license information.
// //
using Newtonsoft.Json;
namespace Microsoft.SqlTools.ServiceLayer.Test.Common namespace Microsoft.SqlTools.ServiceLayer.Test.Common
{ {
public class TestResult public class TestResult
{ {
[JsonProperty("elapsedTime")]
public double ElapsedTime { get; set; } public double ElapsedTime { get; set; }
[JsonProperty("metricValue")]
public double MetricValue { get; set; }
[JsonProperty("iterations")]
public double[] Iterations { get; set; }
[JsonProperty("ninetiethPercentile")]
public double NinetiethPercentile { get; set; }
[JsonProperty("fiftiethPercentile")]
public double FiftiethPercentile { get; set; }
[JsonProperty("average")]
public double Average { get; set; }
[JsonProperty("primaryMetric")]
public string PrimaryMetric { get; set; }
} }
} }

127
test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestServiceDriverProvider.cs
View File

@@ -9,11 +9,13 @@ using System.Threading.Tasks;
using Microsoft.SqlTools.Hosting.Protocol.Contracts; using Microsoft.SqlTools.Hosting.Protocol.Contracts;
using Microsoft.SqlTools.ServiceLayer.Connection.Contracts; using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts; using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts;
using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts;
using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts; using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts;
using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts.ExecuteRequests; using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts.ExecuteRequests;
using Microsoft.SqlTools.ServiceLayer.Scripting.Contracts; using Microsoft.SqlTools.ServiceLayer.Scripting.Contracts;
using Microsoft.SqlTools.ServiceLayer.SqlContext; using Microsoft.SqlTools.ServiceLayer.SqlContext;
using Microsoft.SqlTools.ServiceLayer.TestDriver.Driver; using Microsoft.SqlTools.ServiceLayer.TestDriver.Driver;
using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility;
using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts; using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts;
using Xunit; using Xunit;
@@ -29,7 +31,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
public TestServiceDriverProvider() public TestServiceDriverProvider()
{ {
Driver = new ServiceTestDriver(); Driver = new ServiceTestDriver(TestRunner.Instance.ExecutableFilePath);
Driver.Start().Wait(); Driver.Start().Wait();
this.isRunning = true; this.isRunning = true;
} }
@@ -228,6 +230,18 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
return result; return result;
} }
/// <summary>
/// Request a list of completion items for a position in a block of text
/// </summary>
public async Task RequestRebuildIntelliSense(string ownerUri)
{
var rebuildIntelliSenseParams = new RebuildIntelliSenseParams();
rebuildIntelliSenseParams.OwnerUri = ownerUri;
await Driver.SendEvent(RebuildIntelliSenseNotification.Type, rebuildIntelliSenseParams);
}
/// <summary> /// <summary>
/// Request a a hover tooltop /// Request a a hover tooltop
/// </summary> /// </summary>
@@ -284,6 +298,15 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
// Write the query text to a backing file // Write the query text to a backing file
WriteToFile(ownerUri, query); WriteToFile(ownerUri, query);
return await RunQueryAndWaitToComplete(ownerUri, timeoutMilliseconds);
}
/// <summary>
/// Run a query using a given connection bound to a URI
/// </summary>
public async Task<QueryCompleteParams> RunQueryAndWaitToComplete(string ownerUri, int timeoutMilliseconds = 5000)
{
var queryParams = new ExecuteDocumentSelectionParams var queryParams = new ExecuteDocumentSelectionParams
{ {
OwnerUri = ownerUri, OwnerUri = ownerUri,
@@ -302,6 +325,79 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
} }
} }
/// <summary>
/// Run a query using a given connection bound to a URI
/// </summary>
public async Task<BatchEventParams> RunQueryAndWaitToStart(string ownerUri, string query, int timeoutMilliseconds = 5000)
{
// Write the query text to a backing file
WriteToFile(ownerUri, query);
return await RunQueryAndWaitToStart(ownerUri, timeoutMilliseconds);
}
/// <summary>
/// Run a query using a given connection bound to a URI
/// </summary>
public async Task<BatchEventParams> RunQueryAndWaitToStart(string ownerUri, int timeoutMilliseconds = 5000)
{
var queryParams = new ExecuteDocumentSelectionParams
{
OwnerUri = ownerUri,
QuerySelection = null
};
var result = await Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams);
if (result != null)
{
var eventResult = await Driver.WaitForEvent(BatchStartEvent.Type, timeoutMilliseconds);
return eventResult;
}
else
{
return null;
}
}
public async Task<SessionCreatedParameters> RequestObjectExplorerCreateSession(ConnectionDetails connectionDetails, int timeoutMilliseconds = 5000)
{
var result = await Driver.SendRequest(CreateSessionRequest.Type, connectionDetails);
if (result != null)
{
var eventResult = await Driver.WaitForEvent(CreateSessionCompleteNotification.Type, timeoutMilliseconds);
return eventResult;
}
else
{
return null;
}
}
public async Task<ExpandResponse> RequestObjectExplorerExpand(ExpandParams expandParams, int timeoutMilliseconds = 5000)
{
var result = await Driver.SendRequest(ExpandRequest.Type, expandParams);
if (result)
{
var eventResult = await Driver.WaitForEvent(ExpandCompleteNotification.Type, timeoutMilliseconds);
return eventResult;
}
else
{
return null;
}
}
public async Task<ScriptingResult> RequestScript(ScriptingParams scriptingParams, int timeoutMilliseconds = 5000)
{
var result = await Driver.SendRequest(ScriptingRequest.Type, scriptingParams);
return result;
}
public async Task<CloseSessionResponse> RequestObjectExplorerCloseSession(CloseSessionParams closeSessionParams, int timeoutMilliseconds = 5000)
{
return await Driver.SendRequest(CloseSessionRequest.Type, closeSessionParams);
}
/// <summary> /// <summary>
/// Run a query using a given connection bound to a URI. This method only waits for the initial response from query /// Run a query using a given connection bound to a URI. This method only waits for the initial response from query
/// execution (QueryExecuteResult). It is up to the caller to wait for the QueryCompleteEvent if they are interested. /// execution (QueryExecuteResult). It is up to the caller to wait for the QueryCompleteEvent if they are interested.
@@ -324,7 +420,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
{ {
await ConnectForQuery(serverType, query, queryTempFile.FilePath, databaseName); await ConnectForQuery(serverType, query, queryTempFile.FilePath, databaseName);
var queryResult = await CalculateRunTime(() => RunQueryAndWaitToComplete(queryTempFile.FilePath, query, 50000), false); var queryResult = await CalculateRunTime(() => RunQueryAndWaitToComplete(queryTempFile.FilePath, query, 50000));
Assert.NotNull(queryResult); Assert.NotNull(queryResult);
Assert.NotNull(queryResult.BatchSummaries); Assert.NotNull(queryResult.BatchSummaries);
@@ -332,11 +428,29 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
} }
} }
public async Task<T> CalculateRunTime<T>(Func<Task<T>> testToRun, bool printResult, [CallerMemberName] string testName = "") public static async Task RunTestIterations(Func<TestTimer, Task> testToRun, [CallerMemberName] string testName = "")
{ {
TestTimer timer = new TestTimer() { PrintResult = printResult }; TestTimer timer = new TestTimer() { PrintResult = true };
for (int i = 0; i < TestRunner.Instance.NumberOfRuns; i++)
{
Console.WriteLine("Iteration Number: " + i);
await testToRun(timer);
}
timer.Print(testName);
}
public async Task<T> CalculateRunTime<T>(Func<Task<T>> testToRun, TestTimer timer = null)
{
if (timer != null)
{
timer.Start();
}
T result = await testToRun(); T result = await testToRun();
timer.EndAndPrint(testName); if (timer != null)
{
timer.End();
}
return result; return result;
} }
@@ -344,11 +458,12 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
public async Task ExecuteWithTimeout(TestTimer timer, int timeout, Func<Task<bool>> repeatedCode, public async Task ExecuteWithTimeout(TestTimer timer, int timeout, Func<Task<bool>> repeatedCode,
TimeSpan? delay = null, [CallerMemberName] string testName = "") TimeSpan? delay = null, [CallerMemberName] string testName = "")
{ {
timer.Start();
while (true) while (true)
{ {
if (await repeatedCode()) if (await repeatedCode())
{ {
timer.EndAndPrint(testName); timer.End();
break; break;
} }
if (timer.TotalMilliSecondsUntilNow >= timeout) if (timer.TotalMilliSecondsUntilNow >= timeout)

48
test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestTimer.cs
View File

@@ -3,9 +3,12 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information. // Licensed under the MIT license. See LICENSE file in the project root for full license information.
// //
using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility;
using System; using System;
using System.Collections.Generic;
using System.Globalization; using System.Globalization;
using System.IO; using System.IO;
using System.Linq;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
namespace Microsoft.SqlTools.ServiceLayer.Test.Common namespace Microsoft.SqlTools.ServiceLayer.Test.Common
@@ -17,10 +20,16 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
{ {
private static string resultFolder = InitResultFolder(); private static string resultFolder = InitResultFolder();
private List<double> iterations = new List<double>();
private static string InitResultFolder() private static string InitResultFolder()
{ {
string resultFodler = Environment.GetEnvironmentVariable("ResultFolder"); string resultFodler = Environment.GetEnvironmentVariable("ResultFolder");
if (string.IsNullOrEmpty(resultFodler)) if (string.IsNullOrEmpty(resultFodler))
{
resultFodler = TestRunner.Instance.ResultFolder;
}
if (string.IsNullOrEmpty(resultFodler))
{ {
string assemblyLocation = System.Reflection.Assembly.GetEntryAssembly().Location; string assemblyLocation = System.Reflection.Assembly.GetEntryAssembly().Location;
resultFodler = Path.GetDirectoryName(assemblyLocation); resultFodler = Path.GetDirectoryName(assemblyLocation);
@@ -43,18 +52,39 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
public void End() public void End()
{ {
EndDateTime = DateTime.UtcNow; EndDateTime = DateTime.UtcNow;
iterations.Add(TotalMilliSeconds);
if (PrintResult)
{
Console.WriteLine("Result: " + TotalMilliSeconds);
}
} }
public void EndAndPrint([CallerMemberName] string testName = "") public void EndAndPrint([CallerMemberName] string testName = "")
{ {
End(); End();
Print(testName);
}
public void Print([CallerMemberName] string testName = "")
{
if (PrintResult) if (PrintResult)
{ {
var iterationArray = iterations.ToArray();
double elapsed = Percentile(iterationArray, 0.5);
var currentColor = Console.ForegroundColor; var currentColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Green; Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "Test Name: {0} Run time in milliSeconds: {1}", testName, TotalMilliSeconds)); Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "Test Name: {0} Run time in milliSeconds: {1}", testName, TotalMilliSeconds));
Console.ForegroundColor = currentColor; Console.ForegroundColor = currentColor;
string resultContent = Newtonsoft.Json.JsonConvert.SerializeObject(new TestResult { ElapsedTime = TotalMilliSeconds }); string resultContent = Newtonsoft.Json.JsonConvert.SerializeObject(new TestResult
{
ElapsedTime = TotalMilliSeconds,
MetricValue = elapsed,
PrimaryMetric = "ElapsedTimeMetric",
Iterations = iterationArray,
FiftiethPercentile = Percentile(iterationArray, 0.5),
NinetiethPercentile = Percentile(iterationArray, 0.9),
Average = iterations.Where(x => x > 0).Average()
});
string fileName = testName + ".json"; string fileName = testName + ".json";
string resultFilePath = string.IsNullOrEmpty(resultFolder) ? fileName : Path.Combine(resultFolder, fileName); string resultFilePath = string.IsNullOrEmpty(resultFolder) ? fileName : Path.Combine(resultFolder, fileName);
File.WriteAllText(resultFilePath, resultContent); File.WriteAllText(resultFilePath, resultContent);
@@ -62,6 +92,22 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
} }
} }
private static double Percentile(double[] sequence, double excelPercentile)
{
Array.Sort(sequence);
int N = sequence.Length;
double n = (N - 1) * excelPercentile + 1;
// Another method: double n = (N + 1) * excelPercentile;
if (n == 1d) return sequence[0];
else if (n == N) return sequence[N - 1];
else
{
int k = (int)n;
double d = n - k;
return sequence[k - 1] + d * (sequence[k] - sequence[k - 1]);
}
}
public double TotalMilliSeconds public double TotalMilliSeconds
{ {
get get

4
test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs
View File

@@ -43,9 +43,9 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver
private DateTime startTime; private DateTime startTime;
public ServiceTestDriver() public ServiceTestDriver(string executableFilePath = null)
{ {
string serviceHostExecutable = Environment.GetEnvironmentVariable(ServiceHostEnvironmentVariable); string serviceHostExecutable = executableFilePath != null ? executableFilePath : Environment.GetEnvironmentVariable(ServiceHostEnvironmentVariable);
string serviceHostArguments = "--enable-logging"; string serviceHostArguments = "--enable-logging";
if (string.IsNullOrWhiteSpace(serviceHostExecutable)) if (string.IsNullOrWhiteSpace(serviceHostExecutable))
{ {

14
test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/Constants.cs Normal file
View File

@@ -0,0 +1,14 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Utility
{
public class Constants
{
public const string NumberOfRunsEnvironmentVariable = "NumberOfRuns";
public const string ExecutableFileEnvironmentVariable = "SQLTOOLSSERVICE_EXE";
public const string ResultFolderEnvironmentVariable = "ResultFolder";
}
}

82
test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs
View File

@@ -5,6 +5,7 @@
using System; using System;
using System.Globalization; using System.Globalization;
using System.IO;
using System.Linq; using System.Linq;
using System.Reflection; using System.Reflection;
using System.Threading.Tasks; using System.Threading.Tasks;
@@ -15,9 +16,86 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Utility
{ {
public class TestRunner public class TestRunner
{ {
public static async Task<int> RunTests(string[] tests, string testNamespace) private int numberOfRuns = 20;
protected int DefaultNumberOfRuns = 2;
public static TestRunner Instance { get; } = new TestRunner();
public string[] Tests { get; set; }
public int NumberOfRuns { get; set; }
public string ExecutableFilePath { get; set; }
private TestRunner()
{ {
foreach (var test in tests) InitParameters();
}
private void ParseArguments(string[] args)
{
int index = 0;
while (index < args.Length - 1)
{
string arg = args[index++];
string argValue = args[index++];
switch (arg)
{
case "/t":
case "/T":
case "/tests":
Tests = argValue.Split(" ");
break;
case "/n":
case "/N":
case "/numberOfRuns":
int value;
if (Int32.TryParse(argValue, out value))
{
NumberOfRuns = value;
}
break;
case "/r":
case "/R":
case "/Result":
ResultFolder = argValue;
break;
case "/s":
case "/S":
case "/Service":
ExecutableFilePath = argValue;
break;
}
}
if ((Tests == null || Tests.Length == 0) && args.Length >= 1)
{
Tests = args;
}
}
public string ResultFolder = InitResultFolder();
private static string InitResultFolder()
{
return Environment.GetEnvironmentVariable("ResultFolder");
}
private void InitParameters()
{
string numberOfRunsEnv = Environment.GetEnvironmentVariable(Constants.NumberOfRunsEnvironmentVariable);
if (!Int32.TryParse(numberOfRunsEnv, out numberOfRuns))
{
numberOfRuns = DefaultNumberOfRuns;
}
NumberOfRuns = numberOfRuns;
ExecutableFilePath = Environment.GetEnvironmentVariable(Constants.ExecutableFileEnvironmentVariable);
}
public async Task<int> RunTests(string[] args, string testNamespace)
{
ParseArguments(args);
foreach (var test in Tests)
{ {
try try
{ {