From db1e4ae3513e42341ef869e0c40fae3ad0d34044 Mon Sep 17 00:00:00 2001 From: Benjamin Russell Date: Fri, 18 Nov 2016 17:46:56 -0800 Subject: [PATCH] Fix Code Coverage (#151) This is another large code review. I want to make a few more changes, but since these changes will stand on their own, I'll hold back on making this change set any larger than it already is. Changes in this request: To address Microsoft/vscode-mssql#326, instead of doing taskkill on the service layer when WaitForExit is executed, we now make an educated guess at which service layer was spawned when the test starts and do a Process.Kill on it when we shut down the test. All the perf tests have been moved into a new project. This was done to keep them easily separated from code coverage test runs. At the same time the perf tests were separated into separate classes for logical categorization. This process will likely be repeated on the stress tests. The tests can still easily be ran from Visual Studio Test Explorer To address Microsoft/vscode-mssql#349, a new SelfCleaningFile class was created to allow for easy cleanup of temporary files generated for integration tests via using blocks. Due to some of the refactoring done while moving the perf tests to a new project, the TestBase class had to be switched to more of a helper class style. As such, all tests that use inherit from TestBase now create a TestBase object on start via a using block. This also simplifies the cleanup at the end of the test. * Solution for hanging code coverage runs Code coverage runs would hang in certain scenarios if a test failed before the service process could be spawned. The taskkill command would fail to find the service process. The test would then wait for opencover to exit, but it would not since the service process it had spawned would still be running, causing the test run to hang indefinitely. Solution was to capture the service process after it launched and explicitly kill it when shutting down the test driver. * Setting the test name in the propery in the class and removign the parameter from each method * New project for perf tests * Reworking integration tests to cleanup temp files * Changes as per @llali review comments * Adding copyright notices * Renaming TestBase => TestHelper * Renaming SelfCleaningFile => SelfCleaningTempFile * Removing code that sets TestName property * Fixing compilation error due to removed code --- sqltoolsservice.sln | 7 + ...soft.SqlTools.ServiceLayer.PerfTests.xproj | 22 + .../Program.cs | 31 + .../Properties/AssemblyInfo.cs | 19 + .../Tests/Common.cs | 76 +++ .../Tests/ConnectionTests.cs | 97 +++ .../Tests/IntellisenseTests.cs | 214 +++++++ .../Tests/QueryExecutionTests.cs | 90 +++ .../Tests/SaveResultsTests.cs | 52 ++ .../project.json | 29 + .../LanguageServer/LanguageServiceTests.cs | 90 +-- .../Driver/ServiceTestDriver.cs | 66 +- ...oft.SqlTools.ServiceLayer.TestDriver.xproj | 7 +- .../Scripts/Scripts.cs | 37 ++ .../Tests/ConnectionTests.cs | 37 +- .../Tests/LanguageServiceTests.cs | 152 ++--- .../Tests/PerformanceTests.cs | 582 ------------------ .../Tests/QueryExecutionTests.cs | 333 +++++----- .../Tests/StressTests.cs | 103 ++-- .../Tests/{TestBase.cs => TestHelper.cs} | 117 ++-- .../Tests/WorkspaceTests.cs | 10 +- .../Utility/SelfCleaningTempFile.cs | 53 ++ .../Utility/TestRunner.cs | 76 +++ 23 files changed, 1249 insertions(+), 1051 deletions(-) create mode 100644 test/Microsoft.SqlTools.ServiceLayer.PerfTests/Microsoft.SqlTools.ServiceLayer.PerfTests.xproj create mode 100644 test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs create mode 100644 test/Microsoft.SqlTools.ServiceLayer.PerfTests/Properties/AssemblyInfo.cs create mode 100644 test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/Common.cs create mode 100644 test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs create mode 100644 test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs create mode 100644 test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs create mode 100644 test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs create mode 100644 test/Microsoft.SqlTools.ServiceLayer.PerfTests/project.json create mode 100644 test/Microsoft.SqlTools.ServiceLayer.TestDriver/Scripts/Scripts.cs delete mode 100644 test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/PerformanceTests.cs rename test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/{TestBase.cs => TestHelper.cs} (71%) create mode 100644 test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/SelfCleaningTempFile.cs create mode 100644 test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs diff --git a/sqltoolsservice.sln b/sqltoolsservice.sln index 26e27176..630d13a2 100644 --- a/sqltoolsservice.sln +++ b/sqltoolsservice.sln @@ -51,6 +51,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CodeCoverage", "CodeCoverag test\CodeCoverage\runintegration.bat = test\CodeCoverage\runintegration.bat EndProjectSection EndProject +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.SqlTools.ServiceLayer.PerfTests", "test\Microsoft.SqlTools.ServiceLayer.PerfTests\Microsoft.SqlTools.ServiceLayer.PerfTests.xproj", "{7E5968AB-83D7-4738-85A2-416A50F13D2F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -69,6 +71,10 @@ Global {CC785604-6277-4878-8DA9-360C47158E96}.Debug|Any CPU.Build.0 = Debug|Any CPU {CC785604-6277-4878-8DA9-360C47158E96}.Release|Any CPU.ActiveCfg = Release|Any CPU {CC785604-6277-4878-8DA9-360C47158E96}.Release|Any CPU.Build.0 = Release|Any CPU + {7E5968AB-83D7-4738-85A2-416A50F13D2F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7E5968AB-83D7-4738-85A2-416A50F13D2F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7E5968AB-83D7-4738-85A2-416A50F13D2F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7E5968AB-83D7-4738-85A2-416A50F13D2F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -79,5 +85,6 @@ Global {CC785604-6277-4878-8DA9-360C47158E96} = {AB9CA2B8-6F70-431C-8A1D-67479D8A7BE4} {B7D21727-2926-452B-9610-3ADB0BB6D789} = {F9978D78-78FE-4E92-A7D6-D436B7683EF6} {87D9C7D9-18F4-4AB9-B20D-66C02B6075E2} = {AB9CA2B8-6F70-431C-8A1D-67479D8A7BE4} + {7E5968AB-83D7-4738-85A2-416A50F13D2F} = {AB9CA2B8-6F70-431C-8A1D-67479D8A7BE4} EndGlobalSection EndGlobal diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Microsoft.SqlTools.ServiceLayer.PerfTests.xproj b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Microsoft.SqlTools.ServiceLayer.PerfTests.xproj new file mode 100644 index 00000000..bfdf13be --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Microsoft.SqlTools.ServiceLayer.PerfTests.xproj @@ -0,0 +1,22 @@ + + + + 14.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + 7e5968ab-83d7-4738-85a2-416a50f13d2f + Microsoft.SqlTools.ServiceLayer.PerfTests + .\obj + .\bin\ + v4.5.2 + + + 2.0 + + + + + + \ No newline at end of file diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs new file mode 100644 index 00000000..aa2dc482 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs @@ -0,0 +1,31 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Driver; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility; +using Microsoft.SqlTools.ServiceLayer.Utility; + +namespace Microsoft.SqlTools.ServiceLayer.PerfTests +{ + public class Program + { + internal static int Main(string[] args) + { + if (args.Length < 1) + { + Console.WriteLine("Microsoft.SqlTools.ServiceLayer.PerfTests.exe [tests]" + Environment.NewLine + + " [tests] is a space-separated list of tests to run." + Environment.NewLine + + " They are qualified within the Microsoft.SqlTools.ServiceLayer.TestDriver.PerfTests namespace" + Environment.NewLine + + $"Be sure to set the environment variable {ServiceTestDriver.ServiceHostEnvironmentVariable} to the full path of the sqltoolsservice executable."); + return 0; + } + + Logger.Initialize("testdriver", LogLevel.Verbose); + + return TestRunner.RunTests(args, "Microsoft.SqlTools.ServiceLayer.PerfTests.Tests.").Result; + } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Properties/AssemblyInfo.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..825d07ad --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Properties/AssemblyInfo.cs @@ -0,0 +1,19 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Microsoft.SqlTools.ServiceLayer.PerfTests")] +[assembly: AssemblyTrademark("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("7e5968ab-83d7-4738-85a2-416a50f13d2f")] diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/Common.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/Common.cs new file mode 100644 index 00000000..0011d226 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/Common.cs @@ -0,0 +1,76 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Tests; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility; +using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts; +using Xunit; + +namespace Microsoft.SqlTools.ServiceLayer.PerfTests.Tests +{ + public class Common + { + internal static async Task ExecuteWithTimeout(TestTimer timer, int timeout, Func> repeatedCode, + TimeSpan? delay = null, [CallerMemberName] string testName = "") + { + while (true) + { + if (await repeatedCode()) + { + timer.EndAndPrint(testName); + break; + } + if (timer.TotalMilliSecondsUntilNow >= timeout) + { + Assert.True(false, $"{testName} timed out after {timeout} milliseconds"); + break; + } + if (delay.HasValue) + { + await Task.Delay(delay.Value); + } + } + } + + internal static async Task ConnectAsync(TestHelper testHelper, TestServerType serverType, string query, string ownerUri) + { + testHelper.WriteToFile(ownerUri, query); + + DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification + { + TextDocument = new TextDocumentItem + { + Uri = ownerUri, + LanguageId = "enu", + Version = 1, + Text = query + } + }; + + await testHelper.RequestOpenDocumentNotification(openParams); + + Thread.Sleep(500); + var connectParams = await testHelper.GetDatabaseConnectionAsync(serverType); + bool connected = await testHelper.Connect(ownerUri, connectParams); + Assert.True(connected, "Connection is successful"); + Console.WriteLine($"Connection to {connectParams.Connection.ServerName} is successful"); + + return connected; + } + + internal static async Task CalculateRunTime(Func> testToRun, [CallerMemberName] string testName = "") + { + TestTimer timer = new TestTimer(); + T result = await testToRun(); + timer.EndAndPrint(testName); + + return result; + } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs new file mode 100644 index 00000000..a5edd3d5 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs @@ -0,0 +1,97 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Scripts; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Tests; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility; +using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts; +using Xunit; + +namespace Microsoft.SqlTools.ServiceLayer.PerfTests.Tests +{ + public class ConnectionTests + { + + [Fact] + public async Task ConnectAzureTest() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + const string query = Scripts.SimpleQuery; + testHelper.WriteToFile(queryTempFile.FilePath, query); + + DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification + { + TextDocument = new TextDocumentItem + { + Uri = queryTempFile.FilePath, + LanguageId = "enu", + Version = 1, + Text = query + } + }; + + await testHelper.RequestOpenDocumentNotification(openParams); + + Thread.Sleep(500); + var connected = await Common.CalculateRunTime(async () => + { + var connectParams = await testHelper.GetDatabaseConnectionAsync(TestServerType.Azure); + return await testHelper.Connect(queryTempFile.FilePath, connectParams); + }); + Assert.True(connected, "Connection was not successful"); + } + } + + [Fact] + public async Task ConnectOnPremTest() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + const string query = Scripts.SimpleQuery; + testHelper.WriteToFile(queryTempFile.FilePath, query); + + DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification + { + TextDocument = new TextDocumentItem + { + Uri = queryTempFile.FilePath, + LanguageId = "enu", + Version = 1, + Text = query + } + }; + + await testHelper.RequestOpenDocumentNotification(openParams); + + Thread.Sleep(500); + var connected = await Common.CalculateRunTime(async () => + { + var connectParams = await testHelper.GetDatabaseConnectionAsync(TestServerType.OnPrem); + return await testHelper.Connect(queryTempFile.FilePath, connectParams); + }); + Assert.True(connected, "Connection was not successful"); + } + } + + [Fact] + public async Task DisconnectTest() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + await Common.ConnectAsync(testHelper, TestServerType.OnPrem, Scripts.SimpleQuery, queryTempFile.FilePath); + Thread.Sleep(1000); + var connected = await Common.CalculateRunTime(() => testHelper.Disconnect(queryTempFile.FilePath)); + Assert.True(connected); + } + } + + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs new file mode 100644 index 00000000..642007c5 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs @@ -0,0 +1,214 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Scripts; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Tests; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility; +using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts; +using Xunit; + +namespace Microsoft.SqlTools.ServiceLayer.PerfTests.Tests +{ + public class IntellisenseTests + { + [Fact] + public async Task HoverTestOnPrem() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + const string query = Scripts.SimpleQuery; + await Common.ConnectAsync(testHelper, TestServerType.OnPrem, query, queryTempFile.FilePath); + Hover hover = await Common.CalculateRunTime(() => testHelper.RequestHover(queryTempFile.FilePath, query, 0, 15)); + Assert.NotNull(hover); + await testHelper.Disconnect(queryTempFile.FilePath); + } + } + + [Fact] + public async Task SuggestionsTest() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + const string query = Scripts.SimpleQuery; + await Common.ConnectAsync(testHelper, TestServerType.OnPrem, query, queryTempFile.FilePath); + await ValidateCompletionResponse(testHelper, queryTempFile.FilePath, query, null); + await ValidateCompletionResponse(testHelper, queryTempFile.FilePath, query); + await testHelper.Disconnect(queryTempFile.FilePath); + } + } + + [Fact] + public async Task DiagnosticsTests() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + await Common.ConnectAsync(testHelper, TestServerType.OnPrem, Scripts.SimpleQuery, queryTempFile.FilePath); + + Thread.Sleep(500); + var contentChanges = new TextDocumentChangeEvent[1]; + contentChanges[0] = new TextDocumentChangeEvent() + { + Range = new Range + { + Start = new Position + { + Line = 0, + Character = 5 + }, + End = new Position + { + Line = 0, + Character = 6 + } + }, + RangeLength = 1, + Text = "z" + }; + DidChangeTextDocumentParams changeParams = new DidChangeTextDocumentParams + { + ContentChanges = contentChanges, + TextDocument = new VersionedTextDocumentIdentifier + { + Version = 2, + Uri = queryTempFile.FilePath + } + }; + + TestTimer timer = new TestTimer(); + await testHelper.RequestChangeTextDocumentNotification(changeParams); + await Common.ExecuteWithTimeout(timer, 60000, async () => + { + var completeEvent = await testHelper.Driver.WaitForEvent(PublishDiagnosticsNotification.Type, 15000); + return completeEvent?.Diagnostics != null && completeEvent.Diagnostics.Length > 0; + }); + await testHelper.Disconnect(queryTempFile.FilePath); + } + } + + [Fact] + public async Task BindingCacheColdAzureSimpleQuery() + { + using (TestHelper testHelper = new TestHelper()) + { + await VerifyBindingLoadScenario(testHelper, TestServerType.Azure, Scripts.SimpleQuery); + } + } + + [Fact] + public async Task BindingCacheColdOnPremSimpleQuery() + { + using (TestHelper testHelper = new TestHelper()) + { + await VerifyBindingLoadScenario(testHelper, TestServerType.OnPrem, Scripts.SimpleQuery); + } + } + + [Fact] + public async Task BindingCacheWarmAzureSimpleQuery() + { + using (TestHelper testHelper = new TestHelper()) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + { + const string query = Scripts.SimpleQuery; + const TestServerType serverType = TestServerType.Azure; + await Common.ConnectAsync(testHelper, serverType, query, queryTempFile.FilePath); + Thread.Sleep(10000); + await VerifyBindingLoadScenario(testHelper, serverType, query); + } + } + + [Fact] + public async Task BindingCacheWarmOnPremSimpleQuery() + { + using (TestHelper testHelper = new TestHelper()) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + { + const string query = Scripts.SimpleQuery; + const TestServerType serverType = TestServerType.OnPrem; + await Common.ConnectAsync(testHelper, serverType, query, queryTempFile.FilePath); + Thread.Sleep(10000); + await VerifyBindingLoadScenario(testHelper, serverType, query); + } + } + + [Fact] + public async Task BindingCacheColdAzureComplexQuery() + { + using (TestHelper testHelper = new TestHelper()) + { + await VerifyBindingLoadScenario(testHelper, TestServerType.Azure, Scripts.ComplexQuery); + } + } + + [Fact] + public async Task BindingCacheColdOnPremComplexQuery() + { + using (TestHelper testHelper = new TestHelper()) + { + await VerifyBindingLoadScenario(testHelper, TestServerType.OnPrem, Scripts.ComplexQuery); + } + } + + [Fact] + public async Task BindingCacheWarmAzureComplexQuery() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + string query = Scripts.ComplexQuery; + const TestServerType serverType = TestServerType.Azure; + await Common.ConnectAsync(testHelper, serverType, query, queryTempFile.FilePath); + Thread.Sleep(10000); + await VerifyBindingLoadScenario(testHelper, serverType, query); + } + } + + [Fact] + public async Task BindingCacheWarmOnPremComplexQuery() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + string query = Scripts.ComplexQuery; + const TestServerType serverType = TestServerType.OnPrem; + await Common.ConnectAsync(testHelper, serverType, query, queryTempFile.FilePath); + Thread.Sleep(10000); + await VerifyBindingLoadScenario(testHelper, serverType, query); + } + } + + #region Private Helper Methods + + private static async Task VerifyBindingLoadScenario(TestHelper testHelper, TestServerType serverType, string query, [CallerMemberName] string testName = "") + { + using(SelfCleaningTempFile testTempFile = new SelfCleaningTempFile()) { + testHelper.WriteToFile(testTempFile.FilePath, query); + await Common.ConnectAsync(testHelper, serverType, query, testTempFile.FilePath); + await ValidateCompletionResponse(testHelper, testTempFile.FilePath, query, testName); + await testHelper.Disconnect(testTempFile.FilePath); + } + } + + private static async Task ValidateCompletionResponse(TestHelper testHelper, string ownerUri, string query, [CallerMemberName] string testName="") + { + TestTimer timer = new TestTimer(); + await Common.ExecuteWithTimeout(timer, 60000, async () => + { + CompletionItem[] completions = await testHelper.RequestCompletion(ownerUri, query, 0, 15); + return completions != null && completions.Any(x => x.Label == "master"); + }, testName:testName); + } + + #endregion + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs new file mode 100644 index 00000000..1b8ff999 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs @@ -0,0 +1,90 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Scripts; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Tests; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility; +using Xunit; + +namespace Microsoft.SqlTools.ServiceLayer.PerfTests.Tests +{ + public class QueryExecutionTests + { + [Fact] + public async Task QueryResultSummaryOnPremTest() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + const string query = Scripts.SimpleQuery; + + await Common.ConnectAsync(testHelper, TestServerType.OnPrem, query, queryTempFile.FilePath); + var queryResult = await Common.CalculateRunTime(() => testHelper.RunQuery(queryTempFile.FilePath, query)); + + Assert.NotNull(queryResult); + Assert.True(queryResult.BatchSummaries.Any(x => x.ResultSetSummaries.Any(r => r.RowCount > 0))); + + await testHelper.Disconnect(queryTempFile.FilePath); + } + } + + [Fact] + public async Task QueryResultFirstOnPremTest() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + const string query = Scripts.SimpleQuery; + + await Common.ConnectAsync(testHelper, TestServerType.OnPrem, query, queryTempFile.FilePath); + + var queryResult = await Common.CalculateRunTime(async () => + { + await testHelper.RunQuery(queryTempFile.FilePath, query); + return await testHelper.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 100); + }); + + Assert.NotNull(queryResult); + Assert.NotNull(queryResult.ResultSubset); + Assert.True(queryResult.ResultSubset.Rows.Any()); + + await testHelper.Disconnect(queryTempFile.FilePath); + } + } + + [Fact] + public async Task CancelQueryOnPremTest() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + await Common.ConnectAsync(testHelper, TestServerType.OnPrem, Scripts.DelayQuery, queryTempFile.FilePath); + var queryParams = new QueryExecuteParams + { + OwnerUri = queryTempFile.FilePath, + QuerySelection = null + }; + + var result = await testHelper.Driver.SendRequest(QueryExecuteRequest.Type, queryParams); + if (result != null && string.IsNullOrEmpty(result.Messages)) + { + TestTimer timer = new TestTimer(); + await Common.ExecuteWithTimeout(timer, 100000, + async () => await testHelper.CancelConnect(queryTempFile.FilePath), TimeSpan.FromMilliseconds(10)); + } + else + { + Assert.True(false, "Failed to run the query"); + } + + await testHelper.Disconnect(queryTempFile.FilePath); + } + } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs new file mode 100644 index 00000000..adc0e7a9 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs @@ -0,0 +1,52 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System.Threading.Tasks; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Scripts; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Tests; +using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility; +using Xunit; + +namespace Microsoft.SqlTools.ServiceLayer.PerfTests.Tests +{ + public class SaveResultsTests + { + [Fact] + public async Task TestSaveResultsToCsvTest() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + const string query = Scripts.SimpleQuery; + + // Execute a query + await Common.ConnectAsync(testHelper, TestServerType.OnPrem, query, queryTempFile.FilePath); + await testHelper.RunQuery(queryTempFile.FilePath, query); + await Common.CalculateRunTime(() => testHelper.SaveAsCsv(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0)); + await testHelper.Disconnect(queryTempFile.FilePath); + } + } + + [Fact] + public async Task TestSaveResultsToJsonTest() + { + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (SelfCleaningTempFile outputTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + const string query = Scripts.SimpleQuery; + const TestServerType serverType = TestServerType.OnPrem; + + // Execute a query + await Common.ConnectAsync(testHelper, serverType, query, queryTempFile.FilePath); + await testHelper.RunQuery(queryTempFile.FilePath, query); + await Common.CalculateRunTime(() => testHelper.SaveAsJson(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0)); + await testHelper.Disconnect(queryTempFile.FilePath); + } + } + + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.PerfTests/project.json b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/project.json new file mode 100644 index 00000000..abb50518 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.PerfTests/project.json @@ -0,0 +1,29 @@ +{ + "version": "1.0.0-*", + + "dependencies": { + "xunit": "2.1.0", + "dotnet-test-xunit": "1.0.0-rc2-192208-24", + "Microsoft.SqlTools.ServiceLayer": { + "target": "project" + }, + "Microsoft.SqlTools.ServiceLayer.TestDriver": "1.0.0-*" + }, + + "testRunner": "xunit", + + "frameworks": { + "netcoreapp1.0": { + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version" : "1.0.0" + } + }, + "imports": [ + "dotnet5.4", + "portable-net451+win8" + ] + } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test/LanguageServer/LanguageServiceTests.cs b/test/Microsoft.SqlTools.ServiceLayer.Test/LanguageServer/LanguageServiceTests.cs index bc976498..c8c22986 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.Test/LanguageServer/LanguageServiceTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.Test/LanguageServer/LanguageServiceTests.cs @@ -3,28 +3,13 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. // -using System; -using System.Collections.Generic; -using System.Data; -using System.Data.Common; -using System.IO; -using System.Reflection; using Microsoft.SqlTools.ServiceLayer.Connection; -using Microsoft.SqlTools.ServiceLayer.Connection.Contracts; -using Microsoft.SqlTools.ServiceLayer.Credentials; using Microsoft.SqlTools.ServiceLayer.LanguageServices; -using Microsoft.SqlTools.ServiceLayer.QueryExecution; -using Microsoft.SqlTools.ServiceLayer.SqlContext; -using Microsoft.SqlTools.ServiceLayer.Test.QueryExecution; -using Microsoft.SqlTools.ServiceLayer.Test.Utility; -using Microsoft.SqlTools.ServiceLayer.Workspace; using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts; using Microsoft.SqlTools.Test.Utility; -using Moq; -using Moq.Protected; using Xunit; -namespace Microsoft.SqlTools.ServiceLayer.Test.LanguageServices +namespace Microsoft.SqlTools.ServiceLayer.Test.LanguageServer { /// /// Tests for the ServiceHost Language Service tests @@ -145,14 +130,32 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.LanguageServices #region "General Language Service tests" - #if LIVE_CONNECTION_TESTS + + private static void GetLiveAutoCompleteTestObjects( + out TextDocumentPosition textDocument, + out ScriptFile scriptFile, + out ConnectionInfo connInfo) + { + textDocument = new TextDocumentPosition + { + TextDocument = new TextDocumentIdentifier {Uri = TestObjects.ScriptUri}, + Position = new Position + { + Line = 0, + Character = 0 + } + }; + + connInfo = TestObjects.InitLiveConnectionInfo(out scriptFile); + } + /// /// Test the service initialization code path and verify nothing throws /// // Test is causing failures in build lab..investigating to reenable [Fact] - public void ServiceInitiailzation() + public void ServiceInitialization() { try { @@ -178,8 +181,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.LanguageServices ScriptFile scriptFile; ConnectionInfo connInfo = TestObjects.InitLiveConnectionInfo(out scriptFile); - ScriptParseInfo scriptInfo = new ScriptParseInfo(); - scriptInfo.IsConnected = true; + ScriptParseInfo scriptInfo = new ScriptParseInfo {IsConnected = true}; AutoCompleteHelper.PrepopulateCommonMetadata(connInfo, scriptInfo, null); } @@ -193,7 +195,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.LanguageServices TextDocumentPosition textDocument; ConnectionInfo connInfo; ScriptFile scriptFile; - Common.GetAutoCompleteTestObjects(out textDocument, out scriptFile, out connInfo); + GetLiveAutoCompleteTestObjects(out textDocument, out scriptFile, out connInfo); textDocument.Position.Character = 7; scriptFile.Contents = "select "; @@ -209,52 +211,6 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.LanguageServices #endif - private Hosting.ServiceHost GetTestServiceHost() - { - // set up the host details and profile paths - var hostDetails = new HostDetails("Test Service Host", "SQLToolsService", new Version(1,0)); - SqlToolsContext context = new SqlToolsContext(hostDetails); - - // Grab the instance of the service host - Hosting.ServiceHost host = Hosting.ServiceHost.Instance; - - // Start the service - host.Start().Wait(); - - return host; - } - - #endregion - - #region "Autocomplete Tests" - - /// - /// Creates a mock db command that returns a predefined result set - /// - public static DbCommand CreateTestCommand(Dictionary[][] data) - { - var commandMock = new Mock { CallBase = true }; - var commandMockSetup = commandMock.Protected() - .Setup("ExecuteDbDataReader", It.IsAny()); - - commandMockSetup.Returns(new TestDbDataReader(data)); - - return commandMock.Object; - } - - /// - /// Creates a mock db connection that returns predefined data when queried for a result set - /// - public DbConnection CreateMockDbConnection(Dictionary[][] data) - { - var connectionMock = new Mock { CallBase = true }; - connectionMock.Protected() - .Setup("CreateDbCommand") - .Returns(CreateTestCommand(data)); - - return connectionMock.Object; - } - #endregion } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs index 8eb49dc6..be5d2e39 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs @@ -9,7 +9,10 @@ // using System; +using System.Diagnostics; using System.IO; +using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.SqlTools.ServiceLayer.Connection.Contracts; using Microsoft.SqlTools.ServiceLayer.Hosting.Protocol; @@ -33,7 +36,11 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver public const string ServiceHostEnvironmentVariable = "SQLTOOLSSERVICE_EXE"; - public bool IsCoverageRun { get; set; } + public bool IsCoverageRun { get; set; } + + private Process[] serviceProcesses; + + private DateTime startTime; public ServiceTestDriver() { @@ -71,8 +78,9 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver coverageOutput = "coverage.xml"; } - serviceHostArguments = "-mergeoutput -target:" + serviceHostExecutable + " -targetargs:" + serviceHostArguments - + " -register:user -oldstyle -filter:\"+[Microsoft.SqlTools.*]* -[xunit*]*\" -output:" + coverageOutput + " -searchdirs:" + serviceHostDirectory; + serviceHostArguments = $"-mergeoutput -target:{serviceHostExecutable} -targetargs:{serviceHostArguments} " + + $"-register:user -oldstyle -filter:\"+[Microsoft.SqlTools.*]* -[xunit*]*\" -output:{coverageOutput} " + + $"-searchdirs:{serviceHostDirectory};"; serviceHostExecutable = coverageToolPath; this.IsCoverageRun = true; @@ -88,9 +96,28 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver /// public async Task Start() { + // Store the time we started + startTime = DateTime.Now; + + // Launch the process await this.protocolClient.Start(); await Task.Delay(1000); // Wait for the service host to start + // If this is a code coverage run, we need access to the service layer separate from open cover + if (IsCoverageRun) + { + CancellationTokenSource cancelSource = new CancellationTokenSource(); + Task getServiceProcess = GetServiceProcess(cancelSource.Token); + Task timeoutTask = Task.Delay(TimeSpan.FromSeconds(15), cancelSource.Token); + if (await Task.WhenAny(getServiceProcess, timeoutTask) == timeoutTask) + { + cancelSource.Cancel(); + throw new Exception("Failed to capture service process"); + } + } + + Console.WriteLine("Successfully launched service"); + // Setup events to queue for testing this.QueueEventsForType(ConnectionCompleteNotification.Type); this.QueueEventsForType(IntelliSenseReadyNotification.Type); @@ -103,7 +130,38 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver /// public async Task Stop() { - await this.protocolClient.Stop(); + if (IsCoverageRun) + { + // Kill all the processes in the list + foreach (Process p in serviceProcesses.Where(p => !p.HasExited)) + { + p.Kill(); + } + ServiceProcess?.WaitForExit(); + } + else + { + await this.protocolClient.Stop(); + } + } + + private async Task GetServiceProcess(CancellationToken token) + { + while (serviceProcesses == null && !token.IsCancellationRequested) + { + var processes = Process.GetProcessesByName("Microsoft.SqlTools.ServiceLayer") + .Where(p => p.StartTime >= startTime).ToArray(); + + // Wait a second if we can't find the process + if (processes.Any()) + { + serviceProcesses = processes; + } + else + { + await Task.Delay(TimeSpan.FromSeconds(1), token); + } + } } } } \ No newline at end of file diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Microsoft.SqlTools.ServiceLayer.TestDriver.xproj b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Microsoft.SqlTools.ServiceLayer.TestDriver.xproj index 1319ccb7..6fc6a045 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Microsoft.SqlTools.ServiceLayer.TestDriver.xproj +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Microsoft.SqlTools.ServiceLayer.TestDriver.xproj @@ -1,4 +1,4 @@ - + 14.0 @@ -15,5 +15,8 @@ 2.0 + + + - + \ No newline at end of file diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Scripts/Scripts.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Scripts/Scripts.cs new file mode 100644 index 00000000..f2c28877 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Scripts/Scripts.cs @@ -0,0 +1,37 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.IO; +using System.Reflection; + +namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Scripts +{ + public class Scripts + { + + public const string SimpleQuery = "SELECT * FROM sys.all_columns"; + + public const string DelayQuery = "WAITFOR DELAY '00:01:00'"; + + private static readonly Lazy ComplexQueryInstance = new Lazy(() => + { + try + { + string assemblyLocation = typeof(Scripts).GetTypeInfo().Assembly.Location; + string folderName = Path.GetDirectoryName(assemblyLocation); + string filePath = Path.Combine(folderName, "Scripts/AdventureWorks.sql"); + return File.ReadAllText(filePath); + } + catch (Exception ex) + { + Console.WriteLine($"Failed to load the sql script. error: {ex.Message}"); + return string.Empty; + } + }); + + public static string ComplexQuery { get { return ComplexQueryInstance.Value; } } + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/ConnectionTests.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/ConnectionTests.cs index 8a6f3f95..8aefb01b 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/ConnectionTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/ConnectionTests.cs @@ -13,7 +13,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Language Service end-to-end integration tests /// - public class ConnectionTests : TestBase + public class ConnectionTest { /// /// Try to connect with invalid credentials @@ -21,23 +21,19 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests [Fact] public async Task InvalidConnection() { - try - { - string ownerUri = System.IO.Path.GetTempFileName(); - bool connected = await Connect(ownerUri, ConnectionTestUtils.InvalidConnection, 300000); + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + bool connected = await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.InvalidConnection, 300000); Assert.False(connected, "Invalid connection is failed to connect"); - await Connect(ownerUri, ConnectionTestUtils.InvalidConnection, 300000); + await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.InvalidConnection, 300000); Thread.Sleep(1000); - await CancelConnect(ownerUri); + await testHelper.CancelConnect(queryTempFile.FilePath); - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } @@ -47,22 +43,17 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests [Fact] public async Task ListDatabasesTest() { - try - { - string ownerUri = System.IO.Path.GetTempFileName(); - bool connected = await Connect(ownerUri, ConnectionTestUtils.LocalhostConnection); + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + bool connected = await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.LocalhostConnection); Assert.True(connected, "Connection successful"); - var listDatabaseResult = await ListDatabases(ownerUri); + var listDatabaseResult = await testHelper.ListDatabases(queryTempFile.FilePath); Assert.True(listDatabaseResult.DatabaseNames.Length > 0); - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } - } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/LanguageServiceTests.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/LanguageServiceTests.cs index 74183c02..039b0b6f 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/LanguageServiceTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/LanguageServiceTests.cs @@ -3,10 +3,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. // -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts; @@ -20,7 +16,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Language Service end-to-end integration tests /// - public class LanguageServiceTests : TestBase + public class LanguageServiceTests { /// @@ -29,42 +25,38 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests [Fact] public async Task HoverTest() { - try + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { - string ownerUri = System.IO.Path.GetTempFileName(); string query = "SELECT * FROM sys.objects"; - WriteToFile(ownerUri, query); + testHelper.WriteToFile(queryTempFile.FilePath, query); - DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification() + DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification { - TextDocument = new TextDocumentItem() + TextDocument = new TextDocumentItem { - Uri = ownerUri, + Uri = queryTempFile.FilePath, LanguageId = "enu", Version = 1, Text = query } }; - await RequestOpenDocumentNotification(openParams); + await testHelper.RequestOpenDocumentNotification(openParams); Thread.Sleep(500); - bool connected = await Connect(ownerUri, ConnectionTestUtils.LocalhostConnection); - Assert.True(connected, "Connection is successful"); + bool connected = await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.LocalhostConnection); + Assert.True(connected, "Connection was not successful"); Thread.Sleep(10000); - Hover hover = await RequestHover(ownerUri, query, 0, 15); + Hover hover = await testHelper.RequestHover(queryTempFile.FilePath, query, 0, 15); - Assert.True(hover != null, "Hover tooltop is not null"); + Assert.True(hover != null, "Hover tooltop is null"); - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } @@ -74,48 +66,44 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests [Fact] public async Task CompletionTest() { - try - { - string ownerUri = System.IO.Path.GetTempFileName(); + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { string query = "SELECT * FROM sys.objects"; - WriteToFile(ownerUri, query); + testHelper.WriteToFile(queryTempFile.FilePath, query); - DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification() + DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification { - TextDocument = new TextDocumentItem() + TextDocument = new TextDocumentItem { - Uri = ownerUri, + Uri = queryTempFile.FilePath, LanguageId = "enu", Version = 1, Text = query } }; - await RequestOpenDocumentNotification(openParams); + await testHelper.RequestOpenDocumentNotification(openParams); Thread.Sleep(500); - bool connected = await Connect(ownerUri, ConnectionTestUtils.LocalhostConnection); + bool connected = await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.LocalhostConnection); Assert.True(connected, "Connection is successful"); Thread.Sleep(10000); - CompletionItem[] completions = await RequestCompletion(ownerUri, query, 0, 15); + CompletionItem[] completions = await testHelper.RequestCompletion(queryTempFile.FilePath, query, 0, 15); - Assert.True(completions != null && completions.Length > 0, "Completion items list is not null and not empty"); + Assert.True(completions != null && completions.Length > 0, "Completion items list is null or empty"); Thread.Sleep(50); - CompletionItem item = await RequestResolveCompletion(completions[0]); + await testHelper.RequestResolveCompletion(completions[0]); - Assert.True(completions != null && completions.Length > 0, "Completion items list is not null and not empty"); + Assert.True(completions != null && completions.Length > 0, "Completion items list is null or empty"); - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } @@ -125,42 +113,42 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests [Fact] public async Task DiagnosticsTests() { - try - { - string ownerUri = System.IO.Path.GetTempFileName(); - bool connected = await Connect(ownerUri, ConnectionTestUtils.LocalhostConnection); - Assert.True(connected, "Connection is successful"); + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + bool connected = await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.LocalhostConnection); + Assert.True(connected, "Connection was not successful"); Thread.Sleep(500); string query = "SELECT *** FROM sys.objects"; - DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification() + DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification { - TextDocument = new TextDocumentItem() + TextDocument = new TextDocumentItem { - Uri = ownerUri, + Uri = queryTempFile.FilePath, LanguageId = "enu", Version = 1, Text = query } }; - await RequestOpenDocumentNotification(openParams); + await testHelper.RequestOpenDocumentNotification(openParams); Thread.Sleep(100); var contentChanges = new TextDocumentChangeEvent[1]; - contentChanges[0] = new TextDocumentChangeEvent() + contentChanges[0] = new TextDocumentChangeEvent { - Range = new Range() + Range = new Range { - Start = new Position() + Start = new Position { Line = 0, Character = 5 }, - End = new Position() + End = new Position { Line = 0, Character = 6 @@ -176,24 +164,24 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests TextDocument = new VersionedTextDocumentIdentifier() { Version = 2, - Uri = ownerUri + Uri = queryTempFile.FilePath } }; - await RequestChangeTextDocumentNotification(changeParams); + await testHelper.RequestChangeTextDocumentNotification(changeParams); Thread.Sleep(100); - contentChanges[0] = new TextDocumentChangeEvent() + contentChanges[0] = new TextDocumentChangeEvent { - Range = new Range() + Range = new Range { - Start = new Position() + Start = new Position { Line = 0, Character = 5 }, - End = new Position() + End = new Position { Line = 0, Character = 6 @@ -203,25 +191,21 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests Text = "t" }; - changeParams = new DidChangeTextDocumentParams() + changeParams = new DidChangeTextDocumentParams { ContentChanges = contentChanges, - TextDocument = new VersionedTextDocumentIdentifier() + TextDocument = new VersionedTextDocumentIdentifier { Version = 3, - Uri = ownerUri + Uri = queryTempFile.FilePath } }; - await RequestChangeTextDocumentNotification(changeParams); + await testHelper.RequestChangeTextDocumentNotification(changeParams); Thread.Sleep(2500); - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } @@ -231,11 +215,11 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests [Fact] public async Task ChangeConfigurationTest() { - try - { - string ownerUri = System.IO.Path.GetTempFileName(); - bool connected = await Connect(ownerUri, ConnectionTestUtils.LocalhostConnection); - Assert.True(connected, "Connection is successful"); + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) + { + bool connected = await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.LocalhostConnection); + Assert.True(connected, "Connection was not successful"); Thread.Sleep(500); @@ -246,37 +230,29 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests Settings = settings }; - await RequestChangeConfigurationNotification(configParams); + await testHelper.RequestChangeConfigurationNotification(configParams); Thread.Sleep(2000); - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } [Fact] public async Task NotificationIsSentAfterOnConnectionAutoCompleteUpdate() { - try + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { // Connect - string ownerUri = System.IO.Path.GetTempFileName(); - await Connect(ownerUri, ConnectionTestUtils.LocalhostConnection); + await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.LocalhostConnection); // An event signalling that IntelliSense is ready should be sent shortly thereafter - var readyParams = await Driver.WaitForEvent(IntelliSenseReadyNotification.Type, 30000); + var readyParams = await testHelper.Driver.WaitForEvent(IntelliSenseReadyNotification.Type, 30000); Assert.NotNull(readyParams); - Assert.Equal(ownerUri, readyParams.OwnerUri); + Assert.Equal(queryTempFile.FilePath, readyParams.OwnerUri); - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/PerformanceTests.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/PerformanceTests.cs deleted file mode 100644 index 46372680..00000000 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/PerformanceTests.cs +++ /dev/null @@ -1,582 +0,0 @@ -// -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. -// - -using System; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts; -using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts; -using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility; -using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts; -using Xunit; - -namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests -{ - public class PerformanceTests : TestBase - { - - private static string ComplexQuery = LoadComplexScript(); - private static string SimpleQuery = "SELECT * FROM sys.all_columns"; - - private static string LoadComplexScript() - { - try - { - string assemblyLocation = Assembly.GetEntryAssembly().Location; - string folderName = Path.GetDirectoryName(assemblyLocation); - string filePath = Path.Combine(folderName, "Scripts/AdventureWorks.sql"); - return File.ReadAllText(filePath); - } - catch(Exception ex) - { - Console.WriteLine("Failed to load the sql script. error: " + ex.Message); - return ""; - } - } - - [Fact] - public async Task HoverTestOnPrem() - { - try - { - string ownerUri = Path.GetTempFileName(); - string query = SimpleQuery; - - await ConnectAsync(TestServerType.OnPrem, query, ownerUri); - Hover hover = await CalculateRunTime(async () => - { - return await RequestHover(ownerUri, query, 0, 15); - }); - Assert.True(hover != null, "Hover tool-tip is not null"); - - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task SuggestionsTest() - { - try - { - string query = SimpleQuery; - TestServerType serverType = TestServerType.OnPrem; - string ownerUri = Path.GetTempFileName(); - - WriteToFile(ownerUri, query); - - await ConnectAsync(serverType, query, ownerUri); - await ValidateCompletionResponse(ownerUri, query, null); - - await ValidateCompletionResponse(ownerUri, query); - - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task DiagnosticsTests() - { - try - { - string ownerUri = Path.GetTempFileName(); - string query = "SELECT * FROM sys.objects"; - - await ConnectAsync(TestServerType.OnPrem, query, ownerUri); - Thread.Sleep(500); - - var contentChanges = new TextDocumentChangeEvent[1]; - contentChanges[0] = new TextDocumentChangeEvent() - { - Range = new Range() - { - Start = new Position() - { - Line = 0, - Character = 5 - }, - End = new Position() - { - Line = 0, - Character = 6 - } - }, - RangeLength = 1, - Text = "z" - }; - - DidChangeTextDocumentParams changeParams = new DidChangeTextDocumentParams() - { - ContentChanges = contentChanges, - TextDocument = new VersionedTextDocumentIdentifier() - { - Version = 2, - Uri = ownerUri - } - }; - TestTimer timer = new TestTimer(); - await RequestChangeTextDocumentNotification(changeParams); - - while (true) - { - var completeEvent = await Driver.WaitForEvent(PublishDiagnosticsNotification.Type, 15000); - if (completeEvent != null && completeEvent.Diagnostics != null && completeEvent.Diagnostics.Length > 0) - { - timer.EndAndPrint(); - break; - } - if (timer.TotalMilliSecondsUntilNow >= 500000) - { - Assert.True(false, "Failed to get Diagnostics"); - break; - } - } - - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); - } - } - - private async Task ValidateCompletionResponse(string ownerUri, string query, [CallerMemberName] string testName = "") - { - TestTimer timer = new TestTimer(); - CompletionItem completion = null; - while (true) - { - CompletionItem[] completions = await RequestCompletion(ownerUri, query, 0, 15); - - completion = completions != null ? completions.FirstOrDefault(x => x.Label == "master") : null; - if (completion != null) - { - if (testName != null) - { - timer.EndAndPrint(testName); - } - break; - } - if (timer.TotalMilliSecondsUntilNow >= 500000) - { - Assert.True(false, "Failed to get a valid auto-complete list"); - break; - } - - Thread.Sleep(50); - } - } - - private async Task VerifyBindingLoadScenario(TestServerType serverType, string query, [CallerMemberName] string testName = "") - { - string ownerUri = Path.GetTempFileName(); - - WriteToFile(ownerUri, query); - - await ConnectAsync(serverType, query, ownerUri); - await ValidateCompletionResponse(ownerUri, query, testName); - - await Disconnect(ownerUri); - } - - [Fact] - public async Task BindingCacheColdAzureSimpleQuery() - { - try - { - string query = SimpleQuery; - Thread.Sleep(5000); - await VerifyBindingLoadScenario(TestServerType.Azure, query); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task BindingCacheColdOnPremSimpleQuery() - { - try - { - string query = SimpleQuery; - await VerifyBindingLoadScenario(TestServerType.OnPrem, query); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task BindingCacheWarmAzureSimpleQuery() - { - try - { - string query = SimpleQuery; - string ownerUri = Path.GetTempFileName(); - TestServerType serverType = TestServerType.Azure; - await ConnectAsync(serverType, query, ownerUri); - Thread.Sleep(10000); - await VerifyBindingLoadScenario(serverType, query); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task BindingCacheWarmOnPremSimpleQuery() - { - try - { - string query = SimpleQuery; - string ownerUri = Path.GetTempFileName(); - TestServerType serverType = TestServerType.OnPrem; - await ConnectAsync(serverType, query, ownerUri); - Thread.Sleep(10000); - await VerifyBindingLoadScenario(serverType, query); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task BindingCacheColdAzureComplexQuery() - { - try - { - string query = ComplexQuery; - await VerifyBindingLoadScenario(TestServerType.Azure, query); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task BindingCacheColdOnPremComplexQuery() - { - try - { - string query = ComplexQuery; - await VerifyBindingLoadScenario(TestServerType.OnPrem, query); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task BindingCacheWarmAzureComplexQuery() - { - try - { - string query = ComplexQuery; - string ownerUri = Path.GetTempFileName(); - TestServerType serverType = TestServerType.Azure; - await ConnectAsync(serverType, query, ownerUri); - Thread.Sleep(100000); - await VerifyBindingLoadScenario(serverType, query); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task BindingCacheWarmOnPremComplexQuery() - { - try - { - string query = ComplexQuery; - string ownerUri = Path.GetTempFileName(); - TestServerType serverType = TestServerType.OnPrem; - await ConnectAsync(serverType, query, ownerUri); - Thread.Sleep(10000); - await VerifyBindingLoadScenario(serverType, query); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task ConnectAzureTest() - { - try - { - string query = SimpleQuery; - string ownerUri = Path.GetTempFileName(); - TestServerType serverType = TestServerType.Azure; - WriteToFile(ownerUri, query); - - DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification() - { - TextDocument = new TextDocumentItem() - { - Uri = ownerUri, - LanguageId = "enu", - Version = 1, - Text = query - } - }; - - await RequestOpenDocumentNotification(openParams); - - Thread.Sleep(500); - var connected = await CalculateRunTime(async () => - { - var connectParams = await GetDatabaseConnectionAsync(serverType); - return await Connect(ownerUri, connectParams); - }); - Assert.True(connected, "Connection is successful"); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task ConnectOnPremTest() - { - try - { - string query = SimpleQuery; - string ownerUri = Path.GetTempFileName(); - TestServerType serverType = TestServerType.OnPrem; - WriteToFile(ownerUri, query); - - DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification() - { - TextDocument = new TextDocumentItem() - { - Uri = ownerUri, - LanguageId = "enu", - Version = 1, - Text = query - } - }; - - await RequestOpenDocumentNotification(openParams); - - Thread.Sleep(500); - var connected = await CalculateRunTime(async () => - { - var connectParams = await GetDatabaseConnectionAsync(serverType); - return await Connect(ownerUri, connectParams); - }); - Assert.True(connected, "Connection is successful"); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task DisconnectTest() - { - try - { - string query = SimpleQuery; - string ownerUri = Path.GetTempFileName(); - TestServerType serverType = TestServerType.OnPrem; - await ConnectAsync(serverType, query, ownerUri); - Thread.Sleep(1000); - var connected = await CalculateRunTime(async () => - { - return await base.Disconnect(ownerUri); - }); - Assert.True(connected); - } - finally - { - WaitForExit(); - } - } - - [Fact] - public async Task QueryResultSummaryOnPremTest() - { - string ownerUri = Path.GetTempFileName(); - TestServerType serverType = TestServerType.OnPrem; - string query = SimpleQuery; - - await ConnectAsync(serverType, query, ownerUri); - - var queryTask = await CalculateRunTime(async () => - { - return await RunQuery(ownerUri, query); - }); - - Assert.NotNull(queryTask); - Assert.True(queryTask.BatchSummaries.Any(x => x.ResultSetSummaries.Any( r => r.RowCount > 0))); - - await Disconnect(ownerUri); - } - - [Fact] - public async Task QueryResultFirstOnPremTest() - { - string ownerUri = Path.GetTempFileName(); - TestServerType serverType = TestServerType.OnPrem; - string query = SimpleQuery; - - await ConnectAsync(serverType, query, ownerUri); - - var queryResult = await CalculateRunTime(async () => - { - var queryTask = await RunQuery(ownerUri, query); - return await ExecuteSubset(ownerUri, 0, 0, 0, 100); - }); - - Assert.NotNull(queryResult); - Assert.NotNull(queryResult.ResultSubset); - Assert.True(queryResult.ResultSubset.Rows.Count() > 0); - - await Disconnect(ownerUri); - } - - - [Fact] - public async Task CancelQueryOnPremTest() - { - string ownerUri = Path.GetTempFileName(); - TestServerType serverType = TestServerType.OnPrem; - string query = "WAITFOR DELAY '00:01:00';"; - - await ConnectAsync(serverType, query, ownerUri); - var queryParams = new QueryExecuteParams(); - queryParams.OwnerUri = ownerUri; - queryParams.QuerySelection = null; - - var result = await Driver.SendRequest(QueryExecuteRequest.Type, queryParams); - if (result != null && string.IsNullOrEmpty(result.Messages)) - { - TestTimer timer = new TestTimer(); - - while (true) - { - var queryTask = await CancelQuery(ownerUri); - if (queryTask != null) - { - timer.EndAndPrint(); - break; - } - if (timer.TotalMilliSecondsUntilNow >= 100000) - { - Assert.True(false, "Failed to cancel query"); - break; - } - - Thread.Sleep(10); - } - } - else - { - Assert.True(false, "Failed to run the query"); - } - - await Disconnect(ownerUri); - } - - [Fact] - public async Task TestSaveResultsToCsvTest() - { - string ownerUri = Path.GetTempFileName(); - string query = SimpleQuery; - TestServerType serverType = TestServerType.OnPrem; - string output = Path.GetTempFileName(); - await ConnectAsync(serverType, query, ownerUri); - - // Execute a query - await RunQuery(ownerUri, query); - - var saveTask = await CalculateRunTime(async () => - { - return await SaveAsCsv(ownerUri, output, 0, 0); - }); - - await Disconnect(ownerUri); - } - - [Fact] - public async Task TestSaveResultsToJsonTest() - { - string ownerUri = Path.GetTempFileName(); - string query = SimpleQuery; - TestServerType serverType = TestServerType.OnPrem; - await ConnectAsync(serverType, query, ownerUri); - string output = Path.GetTempFileName(); - // Execute a query - await RunQuery(ownerUri, query); - - var saveTask = await CalculateRunTime(async () => - { - return await SaveAsJson(ownerUri, output, 0, 0); - }); - - await Disconnect(ownerUri); - } - - private async Task ConnectAsync(TestServerType serverType, string query, string ownerUri) - { - WriteToFile(ownerUri, query); - - DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification() - { - TextDocument = new TextDocumentItem() - { - Uri = ownerUri, - LanguageId = "enu", - Version = 1, - Text = query - } - }; - - await RequestOpenDocumentNotification(openParams); - - Thread.Sleep(500); - var connectParams = await GetDatabaseConnectionAsync(serverType); - bool connected = await Connect(ownerUri, connectParams); - Assert.True(connected, "Connection is successful"); - if (connected) - { - Console.WriteLine("Connection is successful"); - } - - return connected; - } - - private async Task CalculateRunTime(Func> testToRun, [CallerMemberName] string testName = "") - { - TestTimer timer = new TestTimer(); - T result = await testToRun(); - timer.EndAndPrint(testName); - - return result; - } - } -} diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/QueryExecutionTests.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/QueryExecutionTests.cs index a06e1e90..d57f91d9 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/QueryExecutionTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/QueryExecutionTests.cs @@ -14,219 +14,243 @@ using Xunit; namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests { - public class QueryExecutionTests : TestBase + public class QueryExecutionTests { [Fact] public async Task TestQueryCancelReliability() { - string ownerUri = System.IO.Path.GetTempFileName(); - string query = "SELECT * FROM sys.objects a CROSS JOIN sys.objects b CROSS JOIN sys.objects c"; + const string query = "SELECT * FROM sys.objects a CROSS JOIN sys.objects b CROSS JOIN sys.objects c"; - await Connect(ownerUri, ConnectionTestUtils.AzureTestServerConnection); - - // Run and cancel 100 queries - for (int i = 0; i < 100; i++) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { - var queryTask = RunQuery(ownerUri, query); + await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.AzureTestServerConnection); - var cancelResult = await CancelQuery(ownerUri); - Assert.NotNull(cancelResult); - Assert.True(string.IsNullOrEmpty(cancelResult.Messages)); + // Run and cancel 100 queries + for (int i = 0; i < 100; i++) + { + var queryTask = testHelper.RunQuery(queryTempFile.FilePath, query); - await queryTask; + var cancelResult = await testHelper.CancelQuery(queryTempFile.FilePath); + Assert.NotNull(cancelResult); + Assert.True(string.IsNullOrEmpty(cancelResult.Messages)); + + await queryTask; + } + + await testHelper.Disconnect(queryTempFile.FilePath); } - - await Disconnect(ownerUri); } [Fact] public async Task TestQueryDoesNotBlockOtherRequests() { - string ownerUri = System.IO.Path.GetTempFileName(); - string query = "SELECT * FROM sys.objects a CROSS JOIN sys.objects b CROSS JOIN sys.objects c"; + const string query = "SELECT * FROM sys.objects a CROSS JOIN sys.objects b CROSS JOIN sys.objects c"; - await Connect(ownerUri, ConnectionTestUtils.AzureTestServerConnection); - - // Start a long-running query - var queryTask = RunQuery(ownerUri, query, 60000); - - // Interact with the service. None of these requests should time out while waiting for the query to finish - for (int i = 0; i < 10; i++) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { - string ownerUri2 = System.IO.Path.GetTempFileName(); + await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.AzureTestServerConnection); - await Connect(ownerUri2, ConnectionTestUtils.AzureTestServerConnection); - Assert.NotNull(await RequestCompletion(ownerUri2, "SELECT * FROM sys.objects", 0, 10)); - await Disconnect(ownerUri2); + // Start a long-running query + var queryTask = testHelper.RunQuery(queryTempFile.FilePath, query, 60000); + + // Interact with the service. None of these requests should time out while waiting for the query to finish + for (int i = 0; i < 10; i++) + { + using (SelfCleaningTempFile queryFile2 = new SelfCleaningTempFile()) + { + await testHelper.Connect(queryFile2.FilePath, ConnectionTestUtils.AzureTestServerConnection); + Assert.NotNull(await testHelper.RequestCompletion(queryFile2.FilePath, "SELECT * FROM sys.objects", 0, 10)); + await testHelper.Disconnect(queryFile2.FilePath); + } + } + + await testHelper.CancelQuery(queryTempFile.FilePath); + await testHelper.Disconnect(queryTempFile.FilePath); } - - await CancelQuery(ownerUri); - await Disconnect(ownerUri); } [Fact] public async Task TestParallelQueryExecution() { - int queryCount = 10; + const int queryCount = 10; + const string query = "SELECT * FROM sys.objects"; - // Create n connections - string[] ownerUris = new string[queryCount]; - for (int i = 0; i < queryCount; i++) + using (TestHelper testHelper = new TestHelper()) { - ownerUris[i] = System.IO.Path.GetTempFileName(); - Assert.NotNull(await Connect(ownerUris[i], ConnectionTestUtils.AzureTestServerConnection)); - } + // Create n connections + SelfCleaningTempFile[] ownerUris = new SelfCleaningTempFile[queryCount]; + for (int i = 0; i < queryCount; i++) + { + ownerUris[i] = new SelfCleaningTempFile(); + Assert.NotNull(await testHelper.Connect(ownerUris[i].FilePath, ConnectionTestUtils.AzureTestServerConnection)); + } - // Run n queries at once - string query = "SELECT * FROM sys.objects"; - var queryTasks = new Task[queryCount]; - for (int i = 0; i < queryCount; i++) - { - queryTasks[i] = RunQuery(ownerUris[i], query); - } - await Task.WhenAll(queryTasks); + // Run n queries at once + var queryTasks = new Task[queryCount]; + for (int i = 0; i < queryCount; i++) + { + queryTasks[i] = testHelper.RunQuery(ownerUris[i].FilePath, query); + } + await Task.WhenAll(queryTasks); - // Verify that they all completed with results and Disconnect - for (int i = 0; i < queryCount; i++) - { - Assert.NotNull(queryTasks[i].Result); - Assert.NotNull(queryTasks[i].Result.BatchSummaries); - await Disconnect(ownerUris[i]); + // Verify that they all completed with results and Disconnect + for (int i = 0; i < queryCount; i++) + { + Assert.NotNull(queryTasks[i].Result); + Assert.NotNull(queryTasks[i].Result.BatchSummaries); + await testHelper.Disconnect(ownerUris[i].FilePath); + ownerUris[i].Dispose(); + } } } [Fact] public async Task TestSaveResultsDoesNotBlockOtherRequests() { - string ownerUri = System.IO.Path.GetTempFileName(); - string query = "SELECT * FROM sys.objects"; + const string query = "SELECT * FROM sys.objects"; - await Connect(ownerUri, ConnectionTestUtils.AzureTestServerConnection); - - // Execute a query - await RunQuery(ownerUri, query); - - // Spawn several tasks to save results - var saveTasks = new Task[100]; - for (int i = 0; i < 100; i++) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { - if (i % 2 == 0) + await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.AzureTestServerConnection); + + // Execute a query + await testHelper.RunQuery(queryTempFile.FilePath, query); + + // Spawn several tasks to save results + var saveTasks = new Task[100]; + for (int i = 0; i < 100; i++) { - saveTasks[i] = SaveAsCsv(ownerUri, System.IO.Path.GetTempFileName(), 0, 0); + if (i % 2 == 0) + { + saveTasks[i] = testHelper.SaveAsCsv(queryTempFile.FilePath, System.IO.Path.GetTempFileName(), 0, 0); + } + else + { + saveTasks[i] = testHelper.SaveAsJson(queryTempFile.FilePath, System.IO.Path.GetTempFileName(), 0, 0); + } } - else + + // Interact with the service. None of these requests should time out while waiting for the save results tasks to finish + for (int i = 0; i < 10; i++) { - saveTasks[i] = SaveAsJson(ownerUri, System.IO.Path.GetTempFileName(), 0, 0); + using(SelfCleaningTempFile queryFile2 = new SelfCleaningTempFile()) + { + await testHelper.Connect(queryFile2.FilePath, ConnectionTestUtils.AzureTestServerConnection); + Assert.NotNull(await testHelper.RequestCompletion(queryFile2.FilePath, "SELECT * FROM sys.objects", 0, 10)); + await testHelper.Disconnect(queryFile2.FilePath); + } } + + await Task.WhenAll(saveTasks); + + await testHelper.Disconnect(queryTempFile.FilePath); } - - // Interact with the service. None of these requests should time out while waiting for the save results tasks to finish - for (int i = 0; i < 10; i++) - { - string ownerUri2 = System.IO.Path.GetTempFileName(); - - await Connect(ownerUri2, ConnectionTestUtils.AzureTestServerConnection); - Assert.NotNull(await RequestCompletion(ownerUri2, "SELECT * FROM sys.objects", 0, 10)); - await Disconnect(ownerUri2); - } - - await Task.WhenAll(saveTasks); - - await Disconnect(ownerUri); } [Fact] public async Task TestQueryingSubsetDoesNotBlockOtherRequests() { - string ownerUri = System.IO.Path.GetTempFileName(); - string query = "SELECT * FROM sys.objects"; + const string query = "SELECT * FROM sys.objects"; - await Connect(ownerUri, ConnectionTestUtils.AzureTestServerConnection); - - // Execute a query - await RunQuery(ownerUri, query); - - // Spawn several tasks for subset requests - var subsetTasks = new Task[100]; - for (int i = 0; i < 100; i++) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { - subsetTasks[i] = ExecuteSubset(ownerUri, 0, 0, 0, 100); + await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.AzureTestServerConnection); + + // Execute a query + await testHelper.RunQuery(queryTempFile.FilePath, query); + + // Spawn several tasks for subset requests + var subsetTasks = new Task[100]; + for (int i = 0; i < 100; i++) + { + subsetTasks[i] = testHelper.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 100); + } + + // Interact with the service. None of these requests should time out while waiting for the subset tasks to finish + for (int i = 0; i < 10; i++) + { + using (SelfCleaningTempFile queryFile2 = new SelfCleaningTempFile()) + { + await testHelper.Connect(queryFile2.FilePath, ConnectionTestUtils.AzureTestServerConnection); + Assert.NotNull(await testHelper.RequestCompletion(queryFile2.FilePath, "SELECT * FROM sys.objects", 0, 10)); + await testHelper.Disconnect(queryFile2.FilePath); + } + } + + await Task.WhenAll(subsetTasks); + + await testHelper.Disconnect(queryTempFile.FilePath); } - - // Interact with the service. None of these requests should time out while waiting for the subset tasks to finish - for (int i = 0; i < 10; i++) - { - string ownerUri2 = System.IO.Path.GetTempFileName(); - - await Connect(ownerUri2, ConnectionTestUtils.AzureTestServerConnection); - Assert.NotNull(await RequestCompletion(ownerUri2, "SELECT * FROM sys.objects", 0, 10)); - await Disconnect(ownerUri2); - } - - await Task.WhenAll(subsetTasks); - - await Disconnect(ownerUri); } [Fact] public async Task TestCancelQueryWhileOtherOperationsAreInProgress() { - string ownerUri = System.IO.Path.GetTempFileName(); - string query = "SELECT * FROM sys.objects a CROSS JOIN sys.objects b"; - List tasks = new List(); + const string query = "SELECT * FROM sys.objects a CROSS JOIN sys.objects b"; - await Connect(ownerUri, ConnectionTestUtils.AzureTestServerConnection); - - // Execute a long-running query - var queryTask = RunQuery(ownerUri, query, 60000); - - // Queue up some tasks that interact with the service - for (int i = 0; i < 10; i++) + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { - string ownerUri2 = System.IO.Path.GetTempFileName(); + List tasks = new List(); - tasks.Add(Task.Run(async () => + await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.AzureTestServerConnection); + + // Execute a long-running query + var queryTask = testHelper.RunQuery(queryTempFile.FilePath, query, 60000); + + // Queue up some tasks that interact with the service + for (int i = 0; i < 10; i++) { - await Connect(ownerUri2, ConnectionTestUtils.AzureTestServerConnection); - await RequestCompletion(ownerUri2, "SELECT * FROM sys.objects", 0, 10); - await RunQuery(ownerUri2, "SELECT * FROM sys.objects"); - await Disconnect(ownerUri2); - })); + using (SelfCleaningTempFile queryFile2 = new SelfCleaningTempFile()) + { + tasks.Add(Task.Run(async () => + { + await testHelper.Connect(queryFile2.FilePath, ConnectionTestUtils.AzureTestServerConnection); + await testHelper.RequestCompletion(queryFile2.FilePath, "SELECT * FROM sys.objects", 0, 10); + await testHelper.RunQuery(queryFile2.FilePath, "SELECT * FROM sys.objects"); + await testHelper.Disconnect(queryFile2.FilePath); + })); + } + } + + // Cancel the long-running query + await testHelper.CancelQuery(queryTempFile.FilePath); + + await testHelper.Disconnect(queryTempFile.FilePath); } - - // Cancel the long-running query - await CancelQuery(ownerUri); - - await Disconnect(ownerUri); } [Fact] public async Task ExecuteBasicQueryTest() { - try + const string query = "SELECT * FROM sys.all_columns c"; + + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { - string ownerUri = System.IO.Path.GetTempFileName(); - bool connected = await Connect(ownerUri, ConnectionTestUtils.LocalhostConnection); + bool connected = await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.LocalhostConnection); Assert.True(connected, "Connection is successful"); Thread.Sleep(500); - string query = "SELECT * FROM sys.all_columns c"; - DidOpenTextDocumentNotification openParams = new DidOpenTextDocumentNotification() { TextDocument = new TextDocumentItem() { - Uri = ownerUri, + Uri = queryTempFile.FilePath, LanguageId = "enu", Version = 1, Text = query } }; - await RequestOpenDocumentNotification(openParams); + await testHelper.RequestOpenDocumentNotification(openParams); - var queryResult = await RunQuery(ownerUri, query); + var queryResult = await testHelper.RunQuery(queryTempFile.FilePath, query, 10000); Assert.NotNull(queryResult); Assert.NotNull(queryResult.BatchSummaries); @@ -241,69 +265,62 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests var subsetRequest = new QueryExecuteSubsetParams() { - OwnerUri = ownerUri, + OwnerUri = queryTempFile.FilePath, BatchIndex = 0, ResultSetIndex = 0, RowsStartIndex = 0, RowsCount = 100, }; - var querySubset = await RequestQueryExecuteSubset(subsetRequest); + var querySubset = await testHelper.RequestQueryExecuteSubset(subsetRequest); Assert.NotNull(querySubset); Assert.True(querySubset.ResultSubset.RowCount == 100); - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } [Fact] public async Task TestQueryingAfterCompletionRequests() { - try + const string query = "SELECT * FROM sys.objects"; + + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { - string ownerUri = System.IO.Path.GetTempFileName(); - string query = "SELECT * FROM sys.objects"; List tasks = new List(); - await Connect(ownerUri, ConnectionTestUtils.AzureTestServerConnection); + await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.AzureTestServerConnection); - Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10))); - var queryTask = RunQuery(ownerUri, query); + Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(testHelper.RequestCompletion(queryTempFile.FilePath, query, 0, 10))); + var queryTask = testHelper.RunQuery(queryTempFile.FilePath, query); tasks.Add(queryTask); await Task.WhenAll(tasks); Assert.NotNull(queryTask.Result); Assert.NotNull(queryTask.Result.BatchSummaries); - await Connect(ownerUri, ConnectionTestUtils.DataToolsTelemetryAzureConnection); + await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.DataToolsTelemetryAzureConnection); tasks.Clear(); - Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10))); - queryTask = RunQuery(ownerUri, query); + Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(testHelper.RequestCompletion(queryTempFile.FilePath, query, 0, 10))); + queryTask = testHelper.RunQuery(queryTempFile.FilePath, query); tasks.Add(queryTask); await Task.WhenAll(tasks); Assert.NotNull(queryTask.Result); Assert.NotNull(queryTask.Result.BatchSummaries); - await Connect(ownerUri, ConnectionTestUtils.SqlDataToolsAzureConnection); + await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.SqlDataToolsAzureConnection); tasks.Clear(); - Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10))); - queryTask = RunQuery(ownerUri, query); + Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(testHelper.RequestCompletion(queryTempFile.FilePath, query, 0, 10))); + queryTask = testHelper.RunQuery(queryTempFile.FilePath, query); tasks.Add(queryTask); await Task.WhenAll(tasks); Assert.NotNull(queryTask.Result); Assert.NotNull(queryTask.Result.BatchSummaries); - await Disconnect(ownerUri); - } - finally - { - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/StressTests.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/StressTests.cs index cbb603ca..bd39ee4a 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/StressTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/StressTests.cs @@ -13,7 +13,7 @@ using Xunit; namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests { - public class StressTests : TestBase + public class StressTests { /// /// Simulate typing by a user to stress test the language service @@ -21,24 +21,24 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests //[Fact] public async Task TestLanguageService() { - string textToType = - "SELECT * FROM sys.objects GO " + - "CREATE TABLE MyTable(" + - "FirstName CHAR," + - "LastName CHAR," + - "DateOfBirth DATETIME," + - "CONSTRAINT MyTableConstraint UNIQUE (FirstName, LastName, DateOfBirth)) GO " + - "INSERT INTO MyTable (FirstName, LastName, DateOfBirth) VALUES ('John', 'Doe', '19800101') GO " + - "SELECT * FROM MyTable GO " + - "ALTER TABLE MyTable DROP CONSTRAINT MyTableConstraint GO " + - "DROP TABLE MyTable GO "; - var ownerUri = System.IO.Path.GetTempFileName(); + const string textToType = "SELECT * FROM sys.objects GO " + + "CREATE TABLE MyTable(" + + "FirstName CHAR," + + "LastName CHAR," + + "DateOfBirth DATETIME," + + "CONSTRAINT MyTableConstraint UNIQUE (FirstName, LastName, DateOfBirth)) GO " + + "INSERT INTO MyTable (FirstName, LastName, DateOfBirth) VALUES ('John', 'Doe', '19800101') GO " + + "SELECT * FROM MyTable GO " + + "ALTER TABLE MyTable DROP CONSTRAINT MyTableConstraint GO " + + "DROP TABLE MyTable GO "; - try + + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { // Connect - bool connected = await Connect(ownerUri, ConnectionTestUtils.LocalhostConnection); - Assert.True(connected, "Connection is successful"); + bool connected = await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.LocalhostConnection); + Assert.True(connected, "Connection was not successful"); Thread.Sleep(10000); // Wait for intellisense to warm up @@ -50,7 +50,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests { for (int i = 0; i < textToType.Length; i++) { - System.IO.File.WriteAllText(ownerUri, textToType.Substring(0, i + 1)); + System.IO.File.WriteAllText(queryTempFile.FilePath, textToType.Substring(0, i + 1)); var contentChanges = new TextDocumentChangeEvent[1]; contentChanges[0] = new TextDocumentChangeEvent() @@ -78,30 +78,30 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests TextDocument = new VersionedTextDocumentIdentifier() { Version = ++version, - Uri = ownerUri + Uri = queryTempFile.FilePath } }; - await RequestChangeTextDocumentNotification(changeParams); + await testHelper.RequestChangeTextDocumentNotification(changeParams); Thread.Sleep(50); // If we just typed a space, request/resolve completion if (textToType[i] == ' ') { - var completions = await RequestCompletion(ownerUri, textToType.Substring(0, i + 1), 0, i + 1); - Assert.True(completions != null && completions.Length > 0, "Completion items list is not null and not empty"); + var completions = await testHelper.RequestCompletion(queryTempFile.FilePath, textToType.Substring(0, i + 1), 0, i + 1); + Assert.True(completions != null && completions.Length > 0, "Completion items list was null or empty"); Thread.Sleep(50); - var item = await RequestResolveCompletion(completions[0]); + var item = await testHelper.RequestResolveCompletion(completions[0]); Assert.NotNull(item); } } // Clear the text document - System.IO.File.WriteAllText(ownerUri, ""); + System.IO.File.WriteAllText(queryTempFile.FilePath, ""); var contentChanges2 = new TextDocumentChangeEvent[1]; contentChanges2[0] = new TextDocumentChangeEvent() @@ -129,23 +129,14 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests TextDocument = new VersionedTextDocumentIdentifier() { Version = ++version, - Uri = ownerUri + Uri = queryTempFile.FilePath } }; - await RequestChangeTextDocumentNotification(changeParams2); + await testHelper.RequestChangeTextDocumentNotification(changeParams2); } - await Disconnect(ownerUri); - } - finally - { - try - { - System.IO.File.Delete(ownerUri); - } - catch {} - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } @@ -155,13 +146,16 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests //[Fact] public async Task TestQueryExecutionService() { - string queryToRun = "SELECT * FROM sys.all_objects GO SELECT * FROM sys.objects GO SELECT * FROM sys.tables GO SELECT COUNT(*) FROM sys.objects"; - var ownerUri = System.IO.Path.GetTempFileName(); + const string queryToRun = "SELECT * FROM sys.all_objects GO " + + "SELECT * FROM sys.objects GO " + + "SELECT * FROM sys.tables GO " + + "SELECT COUNT(*) FROM sys.objects"; - try + using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile()) + using (TestHelper testHelper = new TestHelper()) { // Connect - bool connected = await Connect(ownerUri, ConnectionTestUtils.LocalhostConnection); + bool connected = await testHelper.Connect(queryTempFile.FilePath, ConnectionTestUtils.LocalhostConnection); Assert.True(connected, "Connection is successful"); // Run queries repeatedly @@ -169,7 +163,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests stopwatch.Start(); while (stopwatch.Elapsed < TimeSpan.FromMinutes(60)) { - var queryResult = await RunQuery(ownerUri, queryToRun, 10000); + var queryResult = await testHelper.RunQuery(queryTempFile.FilePath, queryToRun, 10000); Assert.NotNull(queryResult); Assert.NotNull(queryResult.BatchSummaries); @@ -179,24 +173,15 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests Assert.NotNull(queryResult.BatchSummaries[2].ResultSetSummaries); Assert.NotNull(queryResult.BatchSummaries[3].ResultSetSummaries); - Assert.NotNull(await ExecuteSubset(ownerUri, 0, 0, 0, 7)); - Assert.NotNull(await ExecuteSubset(ownerUri, 1, 0, 0, 7)); - Assert.NotNull(await ExecuteSubset(ownerUri, 2, 0, 0, 7)); - Assert.NotNull(await ExecuteSubset(ownerUri, 3, 0, 0, 1)); + Assert.NotNull(await testHelper.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 7)); + Assert.NotNull(await testHelper.ExecuteSubset(queryTempFile.FilePath, 1, 0, 0, 7)); + Assert.NotNull(await testHelper.ExecuteSubset(queryTempFile.FilePath, 2, 0, 0, 7)); + Assert.NotNull(await testHelper.ExecuteSubset(queryTempFile.FilePath, 3, 0, 0, 1)); Thread.Sleep(500); } - await Disconnect(ownerUri); - } - finally - { - try - { - System.IO.File.Delete(ownerUri); - } - catch {} - WaitForExit(); + await testHelper.Disconnect(queryTempFile.FilePath); } } @@ -211,7 +196,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests var connection = ConnectionTestUtils.LocalhostConnection; connection.Connection.Pooling = false; - try + using (TestHelper testHelper = new TestHelper()) { // Connect/disconnect repeatedly Stopwatch stopwatch = new Stopwatch(); @@ -219,18 +204,14 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests while (stopwatch.Elapsed < TimeSpan.FromMinutes(60)) { // Connect - bool connected = await Connect(ownerUri, connection); + bool connected = await testHelper.Connect(ownerUri, connection); Assert.True(connected, "Connection is successful"); // Disconnect - bool disconnected = await Disconnect(ownerUri); + bool disconnected = await testHelper.Disconnect(ownerUri); Assert.True(disconnected, "Disconnect is successful"); } } - finally - { - WaitForExit(); - } } } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/TestBase.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/TestHelper.cs similarity index 71% rename from test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/TestBase.cs rename to test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/TestHelper.cs index 5803d150..7f675941 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/TestBase.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/TestHelper.cs @@ -4,7 +4,6 @@ // using System; -using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using Microsoft.SqlTools.ServiceLayer.Connection.Contracts; @@ -21,11 +20,11 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Base class for all test suites run by the test driver /// - public class TestBase : IDisposable + public sealed class TestHelper : IDisposable { private bool isRunning = false; - public TestBase() + public TestHelper() { Driver = new ServiceTestDriver(); Driver.Start().Wait(); @@ -44,21 +43,13 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests { try { - this.isRunning = false; - - if (!Driver.IsCoverageRun) - { - Driver.Stop().Wait(); - } - else - { - var p = Process.Start("taskkill", "/IM Microsoft.SqlTools.ServiceLayer.exe /F"); - p.WaitForExit(); - Driver.ServiceProcess?.WaitForExit(); - } + this.isRunning = false; + Driver.Stop().Wait(); + Console.WriteLine("Successfully killed process."); } - catch - { + catch(Exception e) + { + Console.WriteLine($"Exception while waiting for service exit: {e.Message}"); } } @@ -77,7 +68,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// Request a new connection to be created /// /// True if the connection completed successfully - protected async Task Connect(string ownerUri, ConnectParams connectParams, int timeout = 15000) + public async Task Connect(string ownerUri, ConnectParams connectParams, int timeout = 15000) { connectParams.OwnerUri = ownerUri; var connectResult = await Driver.SendRequest(ConnectionRequest.Type, connectParams); @@ -95,7 +86,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request a disconnect /// - protected async Task Disconnect(string ownerUri) + public async Task Disconnect(string ownerUri) { var disconnectParams = new DisconnectParams(); disconnectParams.OwnerUri = ownerUri; @@ -107,7 +98,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request a cancel connect /// - protected async Task CancelConnect(string ownerUri) + public async Task CancelConnect(string ownerUri) { var cancelParams = new CancelConnectParams(); cancelParams.OwnerUri = ownerUri; @@ -118,7 +109,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request a cancel connect /// - protected async Task ListDatabases(string ownerUri) + public async Task ListDatabases(string ownerUri) { var listParams = new ListDatabasesParams(); listParams.OwnerUri = ownerUri; @@ -129,7 +120,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request the active SQL script is parsed for errors /// - protected async Task RequestQueryExecuteSubset(QueryExecuteSubsetParams subsetParams) + public async Task RequestQueryExecuteSubset(QueryExecuteSubsetParams subsetParams) { return await Driver.SendRequest(QueryExecuteSubsetRequest.Type, subsetParams); } @@ -137,7 +128,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request the active SQL script is parsed for errors /// - protected async Task RequestOpenDocumentNotification(DidOpenTextDocumentNotification openParams) + public async Task RequestOpenDocumentNotification(DidOpenTextDocumentNotification openParams) { await Driver.SendEvent(DidOpenTextDocumentNotification.Type, openParams); } @@ -145,7 +136,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request a configuration change notification /// - protected async Task RequestChangeConfigurationNotification(DidChangeConfigurationParams configParams) + public async Task RequestChangeConfigurationNotification(DidChangeConfigurationParams configParams) { await Driver.SendEvent(DidChangeConfigurationNotification.Type, configParams); } @@ -153,7 +144,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// /// Request the active SQL script is parsed for errors /// - protected async Task RequestChangeTextDocumentNotification(DidChangeTextDocumentParams changeParams) + public async Task RequestChangeTextDocumentNotification(DidChangeTextDocumentParams changeParams) { await Driver.SendEvent(DidChangeTextDocumentNotification.Type, changeParams); } @@ -161,7 +152,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request completion item resolve to look-up additional info /// - protected async Task RequestResolveCompletion(CompletionItem item) + public async Task RequestResolveCompletion(CompletionItem item) { var result = await Driver.SendRequest(CompletionResolveRequest.Type, item); return result; @@ -170,7 +161,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request a Read Credential for given credential id /// - protected async Task ReadCredential(string credentialId) + public async Task ReadCredential(string credentialId) { var credentialParams = new Credential(); credentialParams.CredentialId = credentialId; @@ -181,7 +172,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Returns database connection parameters for given server type /// - protected async Task GetDatabaseConnectionAsync(TestServerType serverType) + public async Task GetDatabaseConnectionAsync(TestServerType serverType) { ConnectionProfile connectionProfile = null; TestServerIdentity serverIdentiry = ConnectionTestUtils.TestServers.FirstOrDefault(x => x.ServerType == serverType); @@ -212,7 +203,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request a list of completion items for a position in a block of text /// - protected async Task RequestCompletion(string ownerUri, string text, int line, int character) + public async Task RequestCompletion(string ownerUri, string text, int line, int character) { // Write the text to a backing file lock (fileLock) @@ -234,7 +225,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request a a hover tooltop /// - protected async Task RequestHover(string ownerUri, string text, int line, int character) + public async Task RequestHover(string ownerUri, string text, int line, int character) { // Write the text to a backing file lock (fileLock) @@ -242,12 +233,15 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests System.IO.File.WriteAllText(ownerUri, text); } - var completionParams = new TextDocumentPosition(); - completionParams.TextDocument = new TextDocumentIdentifier(); - completionParams.TextDocument.Uri = ownerUri; - completionParams.Position = new Position(); - completionParams.Position.Line = line; - completionParams.Position.Character = character; + var completionParams = new TextDocumentPosition + { + TextDocument = new TextDocumentIdentifier {Uri = ownerUri}, + Position = new Position + { + Line = line, + Character = character + } + }; var result = await Driver.SendRequest(HoverRequest.Type, completionParams); return result; @@ -256,14 +250,16 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Run a query using a given connection bound to a URI /// - protected async Task RunQuery(string ownerUri, string query, int timeoutMilliseconds = 5000) + public async Task RunQuery(string ownerUri, string query, int timeoutMilliseconds = 5000) { // Write the query text to a backing file WriteToFile(ownerUri, query); - var queryParams = new QueryExecuteParams(); - queryParams.OwnerUri = ownerUri; - queryParams.QuerySelection = null; + var queryParams = new QueryExecuteParams + { + OwnerUri = ownerUri, + QuerySelection = null + }; var result = await Driver.SendRequest(QueryExecuteRequest.Type, queryParams); if (result != null && string.IsNullOrEmpty(result.Messages)) @@ -280,10 +276,9 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request to cancel an executing query /// - protected async Task CancelQuery(string ownerUri) + public async Task CancelQuery(string ownerUri) { - var cancelParams = new QueryCancelParams(); - cancelParams.OwnerUri = ownerUri; + var cancelParams = new QueryCancelParams {OwnerUri = ownerUri}; var result = await Driver.SendRequest(QueryCancelRequest.Type, cancelParams); return result; @@ -292,14 +287,16 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request to save query results as CSV /// - protected async Task SaveAsCsv(string ownerUri, string filename, int batchIndex, int resultSetIndex) + public async Task SaveAsCsv(string ownerUri, string filename, int batchIndex, int resultSetIndex) { - var saveParams = new SaveResultsAsCsvRequestParams(); - saveParams.OwnerUri = ownerUri; - saveParams.BatchIndex = batchIndex; - saveParams.ResultSetIndex = resultSetIndex; - saveParams.FilePath = filename; - + var saveParams = new SaveResultsAsCsvRequestParams + { + OwnerUri = ownerUri, + BatchIndex = batchIndex, + ResultSetIndex = resultSetIndex, + FilePath = filename + }; + var result = await Driver.SendRequest(SaveResultsAsCsvRequest.Type, saveParams); return result; } @@ -307,14 +304,16 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request to save query results as JSON /// - protected async Task SaveAsJson(string ownerUri, string filename, int batchIndex, int resultSetIndex) + public async Task SaveAsJson(string ownerUri, string filename, int batchIndex, int resultSetIndex) { - var saveParams = new SaveResultsAsJsonRequestParams(); - saveParams.OwnerUri = ownerUri; - saveParams.BatchIndex = batchIndex; - saveParams.ResultSetIndex = resultSetIndex; - saveParams.FilePath = filename; - + var saveParams = new SaveResultsAsJsonRequestParams + { + OwnerUri = ownerUri, + BatchIndex = batchIndex, + ResultSetIndex = resultSetIndex, + FilePath = filename + }; + var result = await Driver.SendRequest(SaveResultsAsJsonRequest.Type, saveParams); return result; } @@ -322,7 +321,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Request a subset of results from a query /// - protected async Task ExecuteSubset(string ownerUri, int batchIndex, int resultSetIndex, int rowStartIndex, int rowCount) + public async Task ExecuteSubset(string ownerUri, int batchIndex, int resultSetIndex, int rowStartIndex, int rowCount) { var subsetParams = new QueryExecuteSubsetParams(); subsetParams.OwnerUri = ownerUri; @@ -335,7 +334,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests return result; } - protected void WriteToFile(string ownerUri, string query) + public void WriteToFile(string ownerUri, string query) { lock (fileLock) { diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/WorkspaceTests.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/WorkspaceTests.cs index 0796dc44..59814534 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/WorkspaceTests.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/WorkspaceTests.cs @@ -13,7 +13,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests /// /// Language Service end-to-end integration tests /// - public class WorkspaceTests : TestBase + public class WorkspaceTests { /// /// Validate workspace lifecycle events @@ -21,7 +21,7 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests [Fact] public async Task InitializeRequestTest() { - try + using (TestHelper testHelper = new TestHelper()) { InitializeRequest initializeRequest = new InitializeRequest() { @@ -29,13 +29,9 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests Capabilities = new ClientCapabilities() }; - InitializeResult result = await Driver.SendRequest(InitializeRequest.Type, initializeRequest); + InitializeResult result = await testHelper.Driver.SendRequest(InitializeRequest.Type, initializeRequest); Assert.NotNull(result); } - finally - { - WaitForExit(); - } } } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/SelfCleaningTempFile.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/SelfCleaningTempFile.cs new file mode 100644 index 00000000..c991e3ff --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/SelfCleaningTempFile.cs @@ -0,0 +1,53 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.IO; + +namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Utility +{ + public class SelfCleaningTempFile : IDisposable + { + private bool disposed; + + public SelfCleaningTempFile() + { + FilePath = Path.GetTempFileName(); + } + + public string FilePath { get; private set; } + + #region IDisposable Implementation + + public void Dispose() + { + if (!disposed) + { + Dispose(true); + GC.SuppressFinalize(this); + } + } + + public void Dispose(bool disposing) + { + if (!disposed && disposing) + { + try + { + File.Delete(FilePath); + } + catch + { + Console.WriteLine($"Failed to cleanup {FilePath}"); + } + } + + disposed = true; + } + + #endregion + + } +} diff --git a/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs new file mode 100644 index 00000000..a3b6fc61 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs @@ -0,0 +1,76 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Utility +{ + public class TestRunner + { + public static async Task RunTests(string[] tests, string testNamespace) + { + foreach (var test in tests) + { + try + { + var testName = test.Contains(testNamespace) ? test.Replace(testNamespace, "") : test; + bool containsTestName = testName.Contains("."); + var className = containsTestName ? testName.Substring(0, testName.LastIndexOf('.')) : testName; + var methodName = containsTestName ? testName.Substring(testName.LastIndexOf('.') + 1) : null; + + + var type = Type.GetType(testNamespace + className); + if (type == null) + { + Console.WriteLine("Invalid class name"); + } + else + { + if (string.IsNullOrEmpty(methodName)) + { + var methods = type.GetMethods().Where(x => x.CustomAttributes.Any(a => a.AttributeType == typeof(FactAttribute))); + foreach (var method in methods) + { + await RunTest(type, method, method.Name); + } + } + else + { + MethodInfo methodInfo = type.GetMethod(methodName); + await RunTest(type, methodInfo, test); + } + } + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + return -1; + } + } + return 0; + } + + private static async Task RunTest(Type type, MethodBase methodInfo, string testName) + { + if (methodInfo == null) + { + Console.WriteLine("Invalid method name"); + } + else + { + using (var typeInstance = (IDisposable)Activator.CreateInstance(type)) + { + Console.WriteLine("Running test " + testName); + await (Task)methodInfo.Invoke(typeInstance, null); + Console.WriteLine("Test ran successfully: " + testName); + } + } + } + } +}