diff --git a/sqltoolsservice.sln b/sqltoolsservice.sln index bcef0a92..c1b33806 100644 --- a/sqltoolsservice.sln +++ b/sqltoolsservice.sln @@ -220,4 +220,4 @@ Global GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {B31CDF4B-2851-45E5-8C5F-BE97125D9DD8} EndGlobalSection -EndGlobal +EndGlobal \ No newline at end of file diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs index fd8423d8..1462df66 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs @@ -25,8 +25,7 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests } Logger.Initialize("testdriver", LogLevel.Verbose); - - return TestRunner.RunTests(args, "Microsoft.SqlTools.ServiceLayer.PerfTests.").Result; + return TestRunner.Instance.RunTests(args, "Microsoft.SqlTools.ServiceLayer.PerfTests.").Result; } } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs index 81462d69..687ca44a 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs @@ -18,85 +18,94 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests [CreateTestDb(TestServerType.Azure)] public async Task ConnectAzureTest() { - TestServerType serverType = TestServerType.Azure; - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - const string query = Scripts.TestDbSimpleSelectQuery; - testService.WriteToFile(queryTempFile.FilePath, query); - - DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification + TestServerType serverType = TestServerType.Azure; + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) { - TextDocument = new TextDocumentItem + const string query = Scripts.TestDbSimpleSelectQuery; + testService.WriteToFile(queryTempFile.FilePath, query); + + DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification { - Uri = queryTempFile.FilePath, - LanguageId = "enu", - Version = 1, - Text = query - } - }; + TextDocument = new TextDocumentItem + { + Uri = queryTempFile.FilePath, + LanguageId = "enu", + Version = 1, + Text = query + } + }; - await testService.RequestOpenDocumentNotification(openParams); + await testService.RequestOpenDocumentNotification(openParams); - Thread.Sleep(500); - var connected = await testService.CalculateRunTime(async () => - { - var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName); - return await testService.Connect(queryTempFile.FilePath, connectParams); - }, true); - Assert.True(connected, "Connection was not successful"); - } + Thread.Sleep(500); + var connected = await testService.CalculateRunTime(async () => + { + var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName); + return await testService.Connect(queryTempFile.FilePath, connectParams); + }, timer); + Assert.True(connected, "Connection was not successful"); + } + }); } [Fact] [CreateTestDb(TestServerType.OnPrem)] public async Task ConnectOnPremTest() { - TestServerType serverType = TestServerType.OnPrem; - - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - const string query = Scripts.TestDbSimpleSelectQuery; - testService.WriteToFile(queryTempFile.FilePath, query); + TestServerType serverType = TestServerType.OnPrem; - DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) { - TextDocument = new TextDocumentItem + const string query = Scripts.TestDbSimpleSelectQuery; + testService.WriteToFile(queryTempFile.FilePath, query); + + DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification { - Uri = queryTempFile.FilePath, - LanguageId = "enu", - Version = 1, - Text = query - } - }; + TextDocument = new TextDocumentItem + { + Uri = queryTempFile.FilePath, + LanguageId = "enu", + Version = 1, + Text = query + } + }; - await testService.RequestOpenDocumentNotification(openParams); + await testService.RequestOpenDocumentNotification(openParams); - Thread.Sleep(500); - var connected = await testService.CalculateRunTime(async () => - { - var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName); - return await testService.Connect(queryTempFile.FilePath, connectParams); - }, true); - Assert.True(connected, "Connection was not successful"); - } + Thread.Sleep(500); + var connected = await testService.CalculateRunTime(async () => + { + var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName); + return await testService.Connect(queryTempFile.FilePath, connectParams); + }, timer); + Assert.True(connected, "Connection was not successful"); + } + }); } [Fact] [CreateTestDb(TestServerType.OnPrem)] public async Task DisconnectTest() { - TestServerType serverType = TestServerType.OnPrem; - - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName); - Thread.Sleep(1000); - var connected = await testService.CalculateRunTime(() => testService.Disconnect(queryTempFile.FilePath), true); - Assert.True(connected); - } + TestServerType serverType = TestServerType.OnPrem; + + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName); + Thread.Sleep(1000); + var connected = await testService.CalculateRunTime(() => testService.Disconnect(queryTempFile.FilePath), timer); + Assert.True(connected); + } + }); } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs index 8b4a7617..9765aae4 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs @@ -21,107 +21,122 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests [CreateTestDb(TestServerType.OnPrem)] public async Task HoverTestOnPrem() { - TestServerType serverType = TestServerType.OnPrem; - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - const string query = Scripts.TestDbSimpleSelectQuery; - await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName); - Hover hover = await testService.CalculateRunTime(() => testService.RequestHover(queryTempFile.FilePath, query, 0, Scripts.TestDbComplexSelectQueries.Length + 1), true); - Assert.NotNull(hover); - await testService.Disconnect(queryTempFile.FilePath); - } + TestServerType serverType = TestServerType.OnPrem; + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + { + const string query = Scripts.TestDbSimpleSelectQuery; + await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName); + await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, waitForIntelliSense: true); + Hover hover = await testService.CalculateRunTime(() => testService.RequestHover(queryTempFile.FilePath, query, 0, Scripts.TestDbComplexSelectQueries.Length + 1), timer); + Assert.NotNull(hover); + await testService.Disconnect(queryTempFile.FilePath); + } + }); } [Fact] [CreateTestDb(TestServerType.OnPrem)] public async Task SuggestionsTest() { - TestServerType serverType = TestServerType.OnPrem; - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - const string query = Scripts.TestDbSimpleSelectQuery; - await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName); - await ValidateCompletionResponse(testService, queryTempFile.FilePath, false, Common.PerfTestDatabaseName, true); - await ValidateCompletionResponse(testService, queryTempFile.FilePath, true, Common.PerfTestDatabaseName, false); - await testService.Disconnect(queryTempFile.FilePath); - } + TestServerType serverType = TestServerType.OnPrem; + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + { + const string query = Scripts.TestDbSimpleSelectQuery; + await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName); + await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, timer: null, waitForIntelliSense: true); + await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, timer: timer, waitForIntelliSense: false); + await testService.Disconnect(queryTempFile.FilePath); + } + }); } [Fact] [CreateTestDb(TestServerType.OnPrem)] public async Task DiagnosticsTests() { - TestServerType serverType = TestServerType.OnPrem; - SqlTestDb.CreateNew(serverType, doNotCleanupDb: true, databaseName: Common.PerfTestDatabaseName, query: Scripts.CreateDatabaseObjectsQuery); - - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName); + TestServerType serverType = TestServerType.OnPrem; + SqlTestDb.CreateNew(serverType, doNotCleanupDb: true, databaseName: Common.PerfTestDatabaseName, query: Scripts.CreateDatabaseObjectsQuery); - Thread.Sleep(500); - var contentChanges = new TextDocumentChangeEvent[1]; - contentChanges[0] = new TextDocumentChangeEvent() + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) { - Range = new Range + await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName); + + Thread.Sleep(500); + var contentChanges = new TextDocumentChangeEvent[1]; + contentChanges[0] = new TextDocumentChangeEvent() { - Start = new Position + Range = new Range { - Line = 0, - Character = 5 + Start = new Position + { + Line = 0, + Character = 5 + }, + End = new Position + { + Line = 0, + Character = 6 + } }, - End = new Position - { - Line = 0, - Character = 6 - } - }, - RangeLength = 1, - Text = "z" - }; - DidChangeTextDocumentParams changeParams = new DidChangeTextDocumentParams - { - ContentChanges = contentChanges, - TextDocument = new VersionedTextDocumentIdentifier + RangeLength = 1, + Text = "z" + }; + DidChangeTextDocumentParams changeParams = new DidChangeTextDocumentParams { - Version = 2, - Uri = queryTempFile.FilePath - } - }; + ContentChanges = contentChanges, + TextDocument = new VersionedTextDocumentIdentifier + { + Version = 2, + Uri = queryTempFile.FilePath + } + }; - TestTimer timer = new TestTimer() { PrintResult = true }; - await testService.RequestChangeTextDocumentNotification(changeParams); - await testService.ExecuteWithTimeout(timer, 60000, async () => - { - var completeEvent = await testService.Driver.WaitForEvent(PublishDiagnosticsNotification.Type, 15000); - return completeEvent?.Diagnostics != null && completeEvent.Diagnostics.Length > 0; - }); - await testService.Disconnect(queryTempFile.FilePath); - } + await testService.RequestChangeTextDocumentNotification(changeParams); + await testService.ExecuteWithTimeout(timer, 60000, async () => + { + var completeEvent = await testService.Driver.WaitForEvent(PublishDiagnosticsNotification.Type, 15000); + return completeEvent?.Diagnostics != null && completeEvent.Diagnostics.Length > 0; + }); + await testService.Disconnect(queryTempFile.FilePath); + } + }); } [Fact] [CreateTestDb(TestServerType.Azure)] public async Task BindingCacheColdAzureSimpleQuery() { - TestServerType serverType = TestServerType.Azure; - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false); - } + TestServerType serverType = TestServerType.Azure; + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false, timer); + } + }); } [Fact] [CreateTestDb(TestServerType.OnPrem)] public async Task BindingCacheColdOnPremSimpleQuery() { - TestServerType serverType = TestServerType.OnPrem; - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false); - } + TestServerType serverType = TestServerType.OnPrem; + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false, timer); + } + }); } @@ -129,76 +144,94 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests [CreateTestDb(TestServerType.Azure)] public async Task BindingCacheWarmAzureSimpleQuery() { - TestServerType serverType = TestServerType.Azure; - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - const string query = Scripts.TestDbSimpleSelectQuery; - await VerifyBindingLoadScenario(testService, serverType, query, true); - } + TestServerType serverType = TestServerType.Azure; + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + { + const string query = Scripts.TestDbSimpleSelectQuery; + await VerifyBindingLoadScenario(testService, serverType, query, true, timer); + } + }); } [Fact] [CreateTestDb(TestServerType.OnPrem)] public async Task BindingCacheWarmOnPremSimpleQuery() { - TestServerType serverType = TestServerType.OnPrem; - - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - const string query = Scripts.TestDbSimpleSelectQuery; - await VerifyBindingLoadScenario(testService, serverType, query, true); - } + TestServerType serverType = TestServerType.OnPrem; + + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + { + const string query = Scripts.TestDbSimpleSelectQuery; + await VerifyBindingLoadScenario(testService, serverType, query, true, timer); + } + }); } [Fact] [CreateTestDb(TestServerType.Azure)] public async Task BindingCacheColdAzureComplexQuery() { - TestServerType serverType = TestServerType.Azure; - - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries,false); - } + TestServerType serverType = TestServerType.Azure; + + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false, timer); + } + }); } [Fact] [CreateTestDb(TestServerType.OnPrem)] public async Task BindingCacheColdOnPremComplexQuery() { - TestServerType serverType = TestServerType.OnPrem; - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false); - } + TestServerType serverType = TestServerType.OnPrem; + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false, timer); + } + }); } [Fact] [CreateTestDb(TestServerType.Azure)] public async Task BindingCacheWarmAzureComplexQuery() { - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - string query = Scripts.TestDbComplexSelectQueries; - const TestServerType serverType = TestServerType.Azure; - await VerifyBindingLoadScenario(testService, serverType, query, true); - } + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + string query = Scripts.TestDbComplexSelectQueries; + const TestServerType serverType = TestServerType.Azure; + await VerifyBindingLoadScenario(testService, serverType, query, true, timer); + } + }); } [Fact] [CreateTestDb(TestServerType.OnPrem)] public async Task BindingCacheWarmOnPremComplexQuery() { - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - string query = Scripts.TestDbComplexSelectQueries; - const TestServerType serverType = TestServerType.OnPrem; - await VerifyBindingLoadScenario(testService, serverType, query, true); - } + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + string query = Scripts.TestDbComplexSelectQueries; + const TestServerType serverType = TestServerType.OnPrem; + await VerifyBindingLoadScenario(testService, serverType, query, true, timer); + } + }); } #region Private Helper Methods @@ -207,34 +240,36 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests TestServiceDriverProvider testService, TestServerType serverType, string query, - bool preLoad, + bool preLoad, + TestTimer timer, [CallerMemberName] string testName = "") { string databaseName = Common.PerfTestDatabaseName; if (preLoad) { await VerifyCompletationLoaded(testService, serverType, Scripts.TestDbSimpleSelectQuery, - databaseName, printResult: false, testName: testName); + databaseName, null, testName: testName); Console.WriteLine("Intellisense cache loaded."); } await VerifyCompletationLoaded(testService, serverType, query, databaseName, - printResult: true, testName: testName); + timer, testName: testName); } - private async Task VerifyCompletationLoaded( - TestServiceDriverProvider testService, - TestServerType serverType, - string query, + private async Task VerifyCompletationLoaded( + TestServiceDriverProvider testService, + TestServerType serverType, + string query, string databaseName, - bool printResult, + TestTimer timer, string testName) { + using (SelfCleaningTempFile testTempFile = new SelfCleaningTempFile()) { testService.WriteToFile(testTempFile.FilePath, query); await testService.ConnectForQuery(serverType, query, testTempFile.FilePath, databaseName); - await ValidateCompletionResponse(testService, testTempFile.FilePath, printResult, databaseName, - waitForIntelliSense: true, testName: testName); + await ValidateCompletionResponse(testService, testTempFile.FilePath, databaseName, + waitForIntelliSense: true, timer: timer, testName: testName); await testService.Disconnect(testTempFile.FilePath); } } @@ -242,14 +277,17 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests private static async Task ValidateCompletionResponse( TestServiceDriverProvider testService, string ownerUri, - bool printResult, string databaseName, - bool waitForIntelliSense, + bool waitForIntelliSense, + TestTimer timer = null, [CallerMemberName] string testName = "") { - TestTimer timer = new TestTimer() { PrintResult = printResult }; + if (timer == null) + { + timer = new TestTimer { PrintResult = false }; + } bool isReady = !waitForIntelliSense; - await testService.ExecuteWithTimeout(timer, 150000, async () => + await testService.ExecuteWithTimeout(timer, 550000, async () => { if (isReady) { diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ObjectExplorerTests.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ObjectExplorerTests.cs new file mode 100644 index 00000000..973de41e --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ObjectExplorerTests.cs @@ -0,0 +1,146 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using Microsoft.SqlTools.ServiceLayer.Connection.Contracts; +using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts; +using Microsoft.SqlTools.ServiceLayer.Test.Common; +using Xunit; + +namespace Microsoft.SqlTools.ServiceLayer.PerfTests +{ + public class ObjectExplorerTests + { + [Fact] + [CreateTestDb(TestServerType.Azure)] + public async Task CreateSessionAzure() + { + TestServerType serverType = TestServerType.Azure; + await VerifyCreateSession(serverType); + } + + [Fact] + [CreateTestDb(TestServerType.OnPrem)] + public async Task CreateSessionOnPrem() + { + TestServerType serverType = TestServerType.OnPrem; + await VerifyCreateSession(serverType); + } + + [Fact] + [CreateTestDb(TestServerType.OnPrem)] + public async Task ExpandDatabasesOnPrem() + { + TestServerType serverType = TestServerType.OnPrem; + await VerifyExpand(serverType, SqlTestDb.MasterDatabaseName); + } + + [Fact] + [CreateTestDb(TestServerType.OnPrem)] + public async Task ExpandOneDatabaseOnPrem() + { + TestServerType serverType = TestServerType.OnPrem; + await VerifyExpand(serverType, Common.PerfTestDatabaseName); + } + + [Fact] + [CreateTestDb(TestServerType.Azure)] + public async Task ExpandDatabasesAzure() + { + TestServerType serverType = TestServerType.Azure; + await VerifyExpand(serverType, SqlTestDb.MasterDatabaseName); + } + + [Fact] + [CreateTestDb(TestServerType.Azure)] + public async Task ExpandOneDatabaseAzure() + { + TestServerType serverType = TestServerType.Azure; + await VerifyExpand(serverType, Common.PerfTestDatabaseName); + } + + private async Task VerifyCreateSession(TestServerType serverType, [CallerMemberName] string testName = "") + { + await TestServiceDriverProvider.RunTestIterations(async (timer) => + { + ConnectParams connectParams = TestServiceProvider.Instance.ConnectionProfileService.GetConnectionParameters(serverType, SqlTestDb.MasterDatabaseName); + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + { + var result = await testService.CalculateRunTime(() => testService.RequestObjectExplorerCreateSession(connectParams.Connection), timer); + + Assert.NotNull(result); + Assert.True(result.Success); + Assert.False(string.IsNullOrEmpty(result.SessionId), "Session id cannot be empty"); + + await testService.RequestObjectExplorerCloseSession(new ObjectExplorer.Contracts.CloseSessionParams + { + SessionId = result.SessionId + }); + await testService.Disconnect(queryTempFile.FilePath); + } + }, testName); + } + + + private async Task VerifyExpand(TestServerType serverType, string databaseName, [CallerMemberName] string testName = "") + { + await TestServiceDriverProvider.RunTestIterations(async (timer) => + { + ConnectParams connectParams = TestServiceProvider.Instance.ConnectionProfileService.GetConnectionParameters(serverType, databaseName); + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + { + var result = await testService.CalculateRunTime(() => testService.RequestObjectExplorerCreateSession(connectParams.Connection), null); + + Assert.NotNull(result); + Assert.True(result.Success); + Assert.False(string.IsNullOrEmpty(result.SessionId), "Session id cannot be empty"); + + await ExpandDatabase(testService, result.SessionId, result.RootNode, timer); + + await testService.RequestObjectExplorerCloseSession(new ObjectExplorer.Contracts.CloseSessionParams + { + SessionId = result.SessionId + }); + await testService.Disconnect(queryTempFile.FilePath); + } + }, testName); + } + + private async Task ExpandDatabase(TestServiceDriverProvider testService, string sessionId, NodeInfo nodeInfo, TestTimer timer) + { + if (nodeInfo == null) return false; + bool foundNode = nodeInfo.NodePath.Contains("Database") || nodeInfo.NodeType == "Database"; + var expandResult = await testService.CalculateRunTime(() => testService.RequestObjectExplorerExpand(new ObjectExplorer.Contracts.ExpandParams + { + SessionId = sessionId, + NodePath = nodeInfo.NodePath + }, 50000), foundNode ? timer : null); + + Assert.NotNull(expandResult); + Assert.NotNull(expandResult.Nodes); + Assert.False(expandResult.Nodes == null, "Nodes are not valid"); + if (!foundNode) + { + foreach (var node in expandResult.Nodes) + { + if (await ExpandDatabase(testService, sessionId, node, timer)) + { + return true; + } + } + return false; + } + else + { + Console.WriteLine("Node Expanded " + nodeInfo.NodePath); + return true; + } + } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs index 7254ec75..1a735b9d 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs @@ -5,6 +5,7 @@ using System; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading.Tasks; using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts.ExecuteRequests; using Microsoft.SqlTools.ServiceLayer.Test.Common; @@ -17,82 +18,137 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests [Fact] public async Task QueryResultSummaryOnPremTest() { - TestServerType serverType = TestServerType.OnPrem; - - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) - { - const string query = Scripts.MasterBasicQuery; - - await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); - var queryResult = await testService.CalculateRunTime(() => testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query), true); - - Assert.NotNull(queryResult); - Assert.True(queryResult.BatchSummaries.Any(x => x.ResultSetSummaries.Any(r => r.RowCount > 0))); - - await testService.Disconnect(queryTempFile.FilePath); - } + await QueryResultSummaryOnPremTest(TestServerType.OnPrem, Scripts.MasterBasicQuery); } [Fact] public async Task QueryResultFirstOnPremTest() { - TestServerType serverType = TestServerType.OnPrem; + await QueryResultFirstOnPremTest(TestServerType.OnPrem, Scripts.MasterBasicQuery); + } - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) - { - const string query = Scripts.MasterBasicQuery; + [Fact] + public async Task LongQueryResultSummaryOnPremTest() + { + await QueryResultSummaryOnPremTest(TestServerType.OnPrem, Scripts.MasterLongQuery); + } - await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); + [Fact] + public async Task LongQueryResultFirstOnPremTest() + { + await QueryResultFirstOnPremTest(TestServerType.OnPrem, Scripts.MasterLongQuery); + } - var queryResult = await testService.CalculateRunTime(async () => - { - await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query); - return await testService.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 100); - }, true); + [Fact] + public async Task QueryResultSummaryOnAzureTest() + { + await QueryResultSummaryOnPremTest(TestServerType.Azure, Scripts.MasterBasicQuery); + } - Assert.NotNull(queryResult); - Assert.NotNull(queryResult.ResultSubset); - Assert.True(queryResult.ResultSubset.Rows.Any()); + [Fact] + public async Task QueryResultFirstOnAzureTest() + { + await QueryResultFirstOnPremTest(TestServerType.Azure, Scripts.MasterBasicQuery); + } - await testService.Disconnect(queryTempFile.FilePath); - } + [Fact] + public async Task LongQueryResultSummaryOnAzureTest() + { + await QueryResultSummaryOnPremTest(TestServerType.Azure, Scripts.MasterLongQuery); + } + + [Fact] + public async Task LongQueryResultFirstOnAzureTest() + { + await QueryResultFirstOnPremTest(TestServerType.Azure, Scripts.MasterLongQuery); } [Fact] [CreateTestDb(TestServerType.OnPrem)] public async Task CancelQueryOnPremTest() { - TestServerType serverType = TestServerType.OnPrem; - - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - await testService.ConnectForQuery(serverType, Scripts.DelayQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName); - var queryParams = new ExecuteDocumentSelectionParams - { - OwnerUri = queryTempFile.FilePath, - QuerySelection = null - }; + TestServerType serverType = TestServerType.OnPrem; - var result = await testService.Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams); - if (result != null) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) { - TestTimer timer = new TestTimer() { PrintResult = true }; - await testService.ExecuteWithTimeout(timer, 100000, async () => + await testService.ConnectForQuery(serverType, Scripts.DelayQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName); + var queryParams = new ExecuteDocumentSelectionParams { - var cancelQueryResult = await testService.CancelQuery(queryTempFile.FilePath); - return true; - }, TimeSpan.FromMilliseconds(10)); - } - else - { - Assert.True(false, "Failed to run the query"); - } + OwnerUri = queryTempFile.FilePath, + QuerySelection = null + }; - await testService.Disconnect(queryTempFile.FilePath); - } + testService.WriteToFile(queryTempFile.FilePath, Scripts.MasterLongQuery); + + var result = await testService.Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams); + if (result != null) + { + await testService.ExecuteWithTimeout(timer, 100000, async () => + { + var cancelQueryResult = await testService.CancelQuery(queryTempFile.FilePath); + return true; + }, TimeSpan.FromMilliseconds(10)); + } + else + { + Assert.True(false, "Failed to run the query"); + + await testService.Disconnect(queryTempFile.FilePath); + } + } + }); + + } + + private async Task QueryResultSummaryOnPremTest(TestServerType serverType, string query, [CallerMemberName] string testName = "") + { + await TestServiceDriverProvider.RunTestIterations(async (timer) => + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); + testService.WriteToFile(queryTempFile.FilePath, query); + var queryResult = await testService.CalculateRunTime( + () => testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, 50000), + timer); + + Assert.NotNull(queryResult); + Assert.True(queryResult.BatchSummaries.Any(x => x.ResultSetSummaries.Any(r => r.RowCount > 0))); + + await testService.Disconnect(queryTempFile.FilePath); + } + }, testName); + } + + private async Task QueryResultFirstOnPremTest(TestServerType serverType, string query, [CallerMemberName] string testName = "") + { + await TestServiceDriverProvider.RunTestIterations(async (timer) => + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); + testService.WriteToFile(queryTempFile.FilePath, query); + await testService.RunQueryAndWaitToStart(queryTempFile.FilePath, 50000); + await testService.ExecuteWithTimeout(timer, 500000, async () => + { + var queryResult = await testService.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 100); + if (queryResult != null) + { + Assert.NotNull(queryResult); + Assert.NotNull(queryResult.ResultSubset); + Assert.True(queryResult.ResultSubset.Rows.Any()); + } + return queryResult != null; + }, TimeSpan.FromMilliseconds(10)); + + await testService.Disconnect(queryTempFile.FilePath); + } + }, testName); } } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs index 6caf02d3..301de872 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs @@ -14,39 +14,45 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests [Fact] public async Task TestSaveResultsToCsvTest() { - TestServerType serverType = TestServerType.OnPrem; - - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) - using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile()) - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - const string query = Scripts.MasterBasicQuery; + TestServerType serverType = TestServerType.OnPrem; - // Execute a query - await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); - await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query); - await testService.CalculateRunTime(() => testService.SaveAsCsv(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), true); - await testService.Disconnect(queryTempFile.FilePath); - } + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile()) + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + const string query = Scripts.MasterBasicQuery; + + // Execute a query + await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); + await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query); + await testService.CalculateRunTime(() => testService.SaveAsCsv(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), timer); + await testService.Disconnect(queryTempFile.FilePath); + } + }); } [Fact] public async Task TestSaveResultsToJsonTest() { - TestServerType serverType = TestServerType.OnPrem; - - using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) - using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile()) - using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + await TestServiceDriverProvider.RunTestIterations(async (timer) => { - const string query = Scripts.MasterBasicQuery; + TestServerType serverType = TestServerType.OnPrem; - // Execute a query - await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); - await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query); - await testService.CalculateRunTime(() => testService.SaveAsJson(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), true); - await testService.Disconnect(queryTempFile.FilePath); - } + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile()) + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + { + const string query = Scripts.MasterBasicQuery; + + // Execute a query + await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName); + await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query); + await testService.CalculateRunTime(() => testService.SaveAsJson(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), timer); + await testService.Disconnect(queryTempFile.FilePath); + } + }); } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ScriptinTests.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ScriptinTests.cs new file mode 100644 index 00000000..b90ff9c3 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ScriptinTests.cs @@ -0,0 +1,79 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using Microsoft.SqlTools.ServiceLayer.Connection.Contracts; +using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts; +using Microsoft.SqlTools.ServiceLayer.Scripting.Contracts; +using Microsoft.SqlTools.ServiceLayer.Test.Common; +using Xunit; + +namespace Microsoft.SqlTools.ServiceLayer.PerfTests +{ + public class ScriptingTests + { + [Fact] + [CreateTestDb(TestServerType.Azure)] + public async Task ScripTableAzure() + { + TestServerType serverType = TestServerType.Azure; + await VerifyScriptTable(serverType); + } + + [Fact] + [CreateTestDb(TestServerType.OnPrem)] + public async Task ScripTableOnPrem() + { + TestServerType serverType = TestServerType.OnPrem; + await VerifyScriptTable(serverType); + } + + + private async Task VerifyScriptTable(TestServerType serverType, [CallerMemberName] string testName = "") + { + await TestServiceDriverProvider.RunTestIterations(async (timer) => + { + using (TestServiceDriverProvider testService = new TestServiceDriverProvider()) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + { + await testService.ConnectForQuery(serverType, string.Empty, queryTempFile.FilePath, Common.PerfTestDatabaseName); + List scriptingObjects = new List() + { + new ScriptingObject + { + Schema = "Person", + Name = "Address", + Type = "Table" + } + }; + ScriptingParams scriptingParams = new ScriptingParams + { + OwnerUri = queryTempFile.FilePath, + Operation = ScriptingOperationType.Create, + FilePath = queryTempFile.FilePath, + ScriptOptions = new ScriptOptions + { + ScriptCreateDrop = "ScriptCreate", + }, + ScriptDestination = "ToEditor", + ScriptingObjects = scriptingObjects + + }; + var result = await testService.CalculateRunTime(() => testService.RequestScript(scriptingParams), timer); + + Assert.NotNull(result); + Assert.NotNull(result.Script); + + Assert.False(string.IsNullOrEmpty(result.Script), "Script result is invalid"); + + await testService.Disconnect(queryTempFile.FilePath); + } + }, testName); + } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/Scripts/Scripts.cs b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/Scripts/Scripts.cs index 64737382..86b047b8 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/Scripts/Scripts.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/Scripts/Scripts.cs @@ -14,6 +14,8 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common private const string ResourceNameRefix = "Microsoft.SqlTools.ServiceLayer.Test.Common.Scripts."; public const string MasterBasicQuery = "SELECT * FROM sys.all_columns"; //basic queries should return at least 10000 rows + public const string MasterLongQuery = @"SELECT * FROM sys.all_columns a1 + JOIN sys.all_columns a2 on a1.object_id = a2.object_id"; public const string DelayQuery = "WAITFOR DELAY '00:01:00'"; diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestResult.cs b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestResult.cs index 7c4b0c4b..ebeae64a 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestResult.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestResult.cs @@ -3,10 +3,31 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using Newtonsoft.Json; + namespace Microsoft.SqlTools.ServiceLayer.Test.Common { public class TestResult { + [JsonProperty("elapsedTime")] public double ElapsedTime { get; set; } + + [JsonProperty("metricValue")] + public double MetricValue { get; set; } + + [JsonProperty("iterations")] + public double[] Iterations { get; set; } + + [JsonProperty("ninetiethPercentile")] + public double NinetiethPercentile { get; set; } + + [JsonProperty("fiftiethPercentile")] + public double FiftiethPercentile { get; set; } + + [JsonProperty("average")] + public double Average { get; set; } + + [JsonProperty("primaryMetric")] + public string PrimaryMetric { get; set; } } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestServiceDriverProvider.cs b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestServiceDriverProvider.cs index 37c67741..a2a4720f 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestServiceDriverProvider.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestServiceDriverProvider.cs @@ -9,11 +9,13 @@ using System.Threading.Tasks; using Microsoft.SqlTools.Hosting.Protocol.Contracts; using Microsoft.SqlTools.ServiceLayer.Connection.Contracts; using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts; +using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts; using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts; using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts.ExecuteRequests; using Microsoft.SqlTools.ServiceLayer.Scripting.Contracts; using Microsoft.SqlTools.ServiceLayer.SqlContext; using Microsoft.SqlTools.ServiceLayer.TestDriver.Driver; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility; using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts; using Xunit; @@ -29,7 +31,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common public TestServiceDriverProvider() { - Driver = new ServiceTestDriver(); + Driver = new ServiceTestDriver(TestRunner.Instance.ExecutableFilePath); Driver.Start().Wait(); this.isRunning = true; } @@ -228,6 +230,18 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common return result; } + /// + /// Request a list of completion items for a position in a block of text + /// + public async Task RequestRebuildIntelliSense(string ownerUri) + { + var rebuildIntelliSenseParams = new RebuildIntelliSenseParams(); + rebuildIntelliSenseParams.OwnerUri = ownerUri; + + await Driver.SendEvent(RebuildIntelliSenseNotification.Type, rebuildIntelliSenseParams); + } + + /// /// Request a a hover tooltop /// @@ -284,6 +298,15 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common // Write the query text to a backing file WriteToFile(ownerUri, query); + return await RunQueryAndWaitToComplete(ownerUri, timeoutMilliseconds); + } + + /// + /// Run a query using a given connection bound to a URI + /// + public async Task RunQueryAndWaitToComplete(string ownerUri, int timeoutMilliseconds = 5000) + { + var queryParams = new ExecuteDocumentSelectionParams { OwnerUri = ownerUri, @@ -302,6 +325,79 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common } } + /// + /// Run a query using a given connection bound to a URI + /// + public async Task RunQueryAndWaitToStart(string ownerUri, string query, int timeoutMilliseconds = 5000) + { + // Write the query text to a backing file + WriteToFile(ownerUri, query); + + return await RunQueryAndWaitToStart(ownerUri, timeoutMilliseconds); + } + + /// + /// Run a query using a given connection bound to a URI + /// + public async Task RunQueryAndWaitToStart(string ownerUri, int timeoutMilliseconds = 5000) + { + var queryParams = new ExecuteDocumentSelectionParams + { + OwnerUri = ownerUri, + QuerySelection = null + }; + + var result = await Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams); + if (result != null) + { + var eventResult = await Driver.WaitForEvent(BatchStartEvent.Type, timeoutMilliseconds); + return eventResult; + } + else + { + return null; + } + } + + public async Task RequestObjectExplorerCreateSession(ConnectionDetails connectionDetails, int timeoutMilliseconds = 5000) + { + var result = await Driver.SendRequest(CreateSessionRequest.Type, connectionDetails); + if (result != null) + { + var eventResult = await Driver.WaitForEvent(CreateSessionCompleteNotification.Type, timeoutMilliseconds); + return eventResult; + } + else + { + return null; + } + } + + public async Task RequestObjectExplorerExpand(ExpandParams expandParams, int timeoutMilliseconds = 5000) + { + var result = await Driver.SendRequest(ExpandRequest.Type, expandParams); + if (result) + { + var eventResult = await Driver.WaitForEvent(ExpandCompleteNotification.Type, timeoutMilliseconds); + return eventResult; + } + else + { + return null; + } + } + + public async Task RequestScript(ScriptingParams scriptingParams, int timeoutMilliseconds = 5000) + { + var result = await Driver.SendRequest(ScriptingRequest.Type, scriptingParams); + return result; + } + + public async Task RequestObjectExplorerCloseSession(CloseSessionParams closeSessionParams, int timeoutMilliseconds = 5000) + { + return await Driver.SendRequest(CloseSessionRequest.Type, closeSessionParams); + } + /// /// Run a query using a given connection bound to a URI. This method only waits for the initial response from query /// execution (QueryExecuteResult). It is up to the caller to wait for the QueryCompleteEvent if they are interested. @@ -324,7 +420,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) { await ConnectForQuery(serverType, query, queryTempFile.FilePath, databaseName); - var queryResult = await CalculateRunTime(() => RunQueryAndWaitToComplete(queryTempFile.FilePath, query, 50000), false); + var queryResult = await CalculateRunTime(() => RunQueryAndWaitToComplete(queryTempFile.FilePath, query, 50000)); Assert.NotNull(queryResult); Assert.NotNull(queryResult.BatchSummaries); @@ -332,11 +428,29 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common } } - public async Task CalculateRunTime(Func> testToRun, bool printResult, [CallerMemberName] string testName = "") + public static async Task RunTestIterations(Func testToRun, [CallerMemberName] string testName = "") { - TestTimer timer = new TestTimer() { PrintResult = printResult }; + TestTimer timer = new TestTimer() { PrintResult = true }; + for (int i = 0; i < TestRunner.Instance.NumberOfRuns; i++) + { + Console.WriteLine("Iteration Number: " + i); + + await testToRun(timer); + } + timer.Print(testName); + } + + public async Task CalculateRunTime(Func> testToRun, TestTimer timer = null) + { + if (timer != null) + { + timer.Start(); + } T result = await testToRun(); - timer.EndAndPrint(testName); + if (timer != null) + { + timer.End(); + } return result; } @@ -344,11 +458,12 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common public async Task ExecuteWithTimeout(TestTimer timer, int timeout, Func> repeatedCode, TimeSpan? delay = null, [CallerMemberName] string testName = "") { + timer.Start(); while (true) { if (await repeatedCode()) { - timer.EndAndPrint(testName); + timer.End(); break; } if (timer.TotalMilliSecondsUntilNow >= timeout) diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestTimer.cs b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestTimer.cs index 783f483a..f81c3762 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestTimer.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestTimer.cs @@ -3,9 +3,12 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility; using System; +using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Runtime.CompilerServices; namespace Microsoft.SqlTools.ServiceLayer.Test.Common @@ -17,10 +20,16 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common { private static string resultFolder = InitResultFolder(); + private List iterations = new List(); + private static string InitResultFolder() { string resultFodler = Environment.GetEnvironmentVariable("ResultFolder"); if (string.IsNullOrEmpty(resultFodler)) + { + resultFodler = TestRunner.Instance.ResultFolder; + } + if (string.IsNullOrEmpty(resultFodler)) { string assemblyLocation = System.Reflection.Assembly.GetEntryAssembly().Location; resultFodler = Path.GetDirectoryName(assemblyLocation); @@ -43,18 +52,39 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common public void End() { EndDateTime = DateTime.UtcNow; + iterations.Add(TotalMilliSeconds); + if (PrintResult) + { + Console.WriteLine("Result: " + TotalMilliSeconds); + } } public void EndAndPrint([CallerMemberName] string testName = "") { End(); + Print(testName); + } + + public void Print([CallerMemberName] string testName = "") + { if (PrintResult) { + var iterationArray = iterations.ToArray(); + double elapsed = Percentile(iterationArray, 0.5); var currentColor = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "Test Name: {0} Run time in milliSeconds: {1}", testName, TotalMilliSeconds)); Console.ForegroundColor = currentColor; - string resultContent = Newtonsoft.Json.JsonConvert.SerializeObject(new TestResult { ElapsedTime = TotalMilliSeconds }); + string resultContent = Newtonsoft.Json.JsonConvert.SerializeObject(new TestResult + { + ElapsedTime = TotalMilliSeconds, + MetricValue = elapsed, + PrimaryMetric = "ElapsedTimeMetric", + Iterations = iterationArray, + FiftiethPercentile = Percentile(iterationArray, 0.5), + NinetiethPercentile = Percentile(iterationArray, 0.9), + Average = iterations.Where(x => x > 0).Average() + }); string fileName = testName + ".json"; string resultFilePath = string.IsNullOrEmpty(resultFolder) ? fileName : Path.Combine(resultFolder, fileName); File.WriteAllText(resultFilePath, resultContent); @@ -62,6 +92,22 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common } } + private static double Percentile(double[] sequence, double excelPercentile) + { + Array.Sort(sequence); + int N = sequence.Length; + double n = (N - 1) * excelPercentile + 1; + // Another method: double n = (N + 1) * excelPercentile; + if (n == 1d) return sequence[0]; + else if (n == N) return sequence[N - 1]; + else + { + int k = (int)n; + double d = n - k; + return sequence[k - 1] + d * (sequence[k] - sequence[k - 1]); + } + } + public double TotalMilliSeconds { get diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs index cda0b063..578707e4 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs @@ -43,9 +43,9 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver private DateTime startTime; - public ServiceTestDriver() + public ServiceTestDriver(string executableFilePath = null) { - string serviceHostExecutable = Environment.GetEnvironmentVariable(ServiceHostEnvironmentVariable); + string serviceHostExecutable = executableFilePath != null ? executableFilePath : Environment.GetEnvironmentVariable(ServiceHostEnvironmentVariable); string serviceHostArguments = "--enable-logging"; if (string.IsNullOrWhiteSpace(serviceHostExecutable)) { diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/Constants.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/Constants.cs new file mode 100644 index 00000000..47ca8f94 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/Constants.cs @@ -0,0 +1,14 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Utility +{ + public class Constants + { + public const string NumberOfRunsEnvironmentVariable = "NumberOfRuns"; + public const string ExecutableFileEnvironmentVariable = "SQLTOOLSSERVICE_EXE"; + public const string ResultFolderEnvironmentVariable = "ResultFolder"; + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs index e0f443b1..9625b531 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs @@ -5,6 +5,7 @@ using System; using System.Globalization; +using System.IO; using System.Linq; using System.Reflection; using System.Threading.Tasks; @@ -15,9 +16,86 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Utility { public class TestRunner { - public static async Task RunTests(string[] tests, string testNamespace) + private int numberOfRuns = 20; + protected int DefaultNumberOfRuns = 2; + + public static TestRunner Instance { get; } = new TestRunner(); + public string[] Tests { get; set; } + public int NumberOfRuns { get; set; } + public string ExecutableFilePath { get; set; } + + private TestRunner() { - foreach (var test in tests) + InitParameters(); + } + + private void ParseArguments(string[] args) + { + int index = 0; + while (index < args.Length - 1) + { + string arg = args[index++]; + string argValue = args[index++]; + switch (arg) + { + case "/t": + case "/T": + case "/tests": + Tests = argValue.Split(" "); + break; + case "/n": + case "/N": + case "/numberOfRuns": + int value; + if (Int32.TryParse(argValue, out value)) + { + NumberOfRuns = value; + } + break; + case "/r": + case "/R": + case "/Result": + ResultFolder = argValue; + break; + case "/s": + case "/S": + case "/Service": + ExecutableFilePath = argValue; + break; + } + } + + if ((Tests == null || Tests.Length == 0) && args.Length >= 1) + { + Tests = args; + } + } + + public string ResultFolder = InitResultFolder(); + + private static string InitResultFolder() + { + return Environment.GetEnvironmentVariable("ResultFolder"); + } + + private void InitParameters() + { + string numberOfRunsEnv = Environment.GetEnvironmentVariable(Constants.NumberOfRunsEnvironmentVariable); + + if (!Int32.TryParse(numberOfRunsEnv, out numberOfRuns)) + { + numberOfRuns = DefaultNumberOfRuns; + } + + NumberOfRuns = numberOfRuns; + + ExecutableFilePath = Environment.GetEnvironmentVariable(Constants.ExecutableFileEnvironmentVariable); + } + + public async Task RunTests(string[] args, string testNamespace) + { + ParseArguments(args); + foreach (var test in Tests) { try {