Test driver improvements (#117)

* Refactored the test driver to work with xunit in addition to the command line

* Fix behavior of property

src/Microsoft.SqlTools.ServiceLayer/Hosting/Protocol/MessageDispatcher.cs

@@ -163,8 +163,14 @@ namespace Microsoft.SqlTools.ServiceLayer.Hosting.Protocol
                     TParams typedParams = default(TParams);
                     if (eventMessage.Contents != null)
                     {
-                        // TODO: Catch parse errors!
+                        try
-                        typedParams = eventMessage.Contents.ToObject<TParams>();
+                        {
+                            typedParams = eventMessage.Contents.ToObject<TParams>();
+                        }
+                        catch (Exception ex)
+                        {
+                            Logger.Write(LogLevel.Verbose, ex.ToString());
+                        }
                     }
 
                     return eventHandler(typedParams, eventContext);

src/Microsoft.SqlTools.ServiceLayer/QueryExecution/Contracts/DbColumnWrapper.cs

@@ -152,6 +152,13 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts
             }
         }
 
+        /// <summary>
+        /// Default constructor, used for deserializing JSON RPC only
+        /// </summary>
+        public DbColumnWrapper()
+        {
+        }
+
         #region Properties
 
         /// <summary>

src/Microsoft.SqlTools.ServiceLayer/QueryExecution/Contracts/ResultMessage.cs

@@ -40,5 +40,12 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts
             Time = DateTime.Now.ToString("o");
             Message = message;
         }
+
+        /// <summary>
+        /// Default constructor, used for deserializing JSON RPC only
+        /// </summary>
+        public ResultMessage()
+        {
+        }
     }
 }

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs

@@ -8,9 +8,12 @@
 // License: https://github.com/PowerShell/PowerShellEditorServices/blob/develop/LICENSE
 //
 
+using System;
 using System.Threading.Tasks;
+using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
 using Microsoft.SqlTools.ServiceLayer.Hosting.Protocol;
 using Microsoft.SqlTools.ServiceLayer.Hosting.Protocol.Channel;
+using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts;
 
 namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver
 {
@@ -19,18 +22,38 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver
     /// </summary>
     public class ServiceTestDriver : TestDriverBase
     {
-        public ServiceTestDriver(string serviceHostExecutable)
+        /// <summary>
+        /// Environment variable that stores the path to the service host executable.
+        /// </summary>
+        public static string ServiceHostEnvironmentVariable
         {
-            var clientChannel = new StdioClientChannel(serviceHostExecutable);
+            get { return "SQLTOOLSSERVICE_EXE"; }
+        }
+
+        public ServiceTestDriver()
+        {
+            string serviceHostExecutable = Environment.GetEnvironmentVariable(ServiceHostEnvironmentVariable);
+
+            var clientChannel = new StdioClientChannel(serviceHostExecutable, "--enable-logging");
             this.protocolClient = new ProtocolEndpoint(clientChannel, MessageProtocolType.LanguageServer);
         }
 
+        /// <summary>
+        /// Start the test driver, and launch the sqltoolsservice executable
+        /// </summary>
         public async Task Start()
         {
             await this.protocolClient.Start();
             await Task.Delay(1000); // Wait for the service host to start
+
+            // Setup events to queue for testing
+            this.QueueEventsForType(ConnectionCompleteNotification.Type);
+            this.QueueEventsForType(QueryExecuteCompleteEvent.Type);
         }
 
+        /// <summary>
+        /// Stop the test driver, and shutdown the sqltoolsservice executable
+        /// </summary>
         public async Task Stop()
         {
             await this.protocolClient.Stop();

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Program.cs

@@ -4,10 +4,10 @@
 //
 
 using System;
-using System.Linq;
 using System.Reflection;
 using System.Threading.Tasks;
 using Microsoft.SqlTools.ServiceLayer.TestDriver.Driver;
+using Microsoft.SqlTools.ServiceLayer.Utility;
 
 namespace Microsoft.SqlTools.ServiceLayer.TestDriver
 {
@@ -17,48 +17,37 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver
         {
             if (args.Length < 1)
             {
-                Console.WriteLine(  "Microsoft.SqlTools.ServiceLayer.TestDriver.exe <service host executable> [tests]" + Environment.NewLine +
+                Console.WriteLine(  "Microsoft.SqlTools.ServiceLayer.TestDriver.exe [tests]" + Environment.NewLine +
-                                    "    <service host executable> is the path to the Microsoft.SqlTools.ServiceLayer.exe executable" + Environment.NewLine +
                                     "    [tests] is a space-separated list of tests to run." + Environment.NewLine + 
-                                    "            They are qualified within the Microsoft.SqlTools.ServiceLayer.TestDriver.Tests namespace");
+                                    "            They are qualified within the Microsoft.SqlTools.ServiceLayer.TestDriver.Tests namespace" + Environment.NewLine +
+                                    "Be sure to set the environment variable " + ServiceTestDriver.ServiceHostEnvironmentVariable + " to the full path of the sqltoolsservice executable.");
                 Environment.Exit(0);
             }
 
+            Logger.Initialize("testdriver", LogLevel.Verbose);
+
             Task.Run(async () => 
             {
-                var serviceHostExecutable = args[0];
+                foreach (var test in args)
-                var tests = args.Skip(1);
-
-                foreach (var test in tests)
                 {
-                    ServiceTestDriver driver = null;
-
                     try
                     {
-                        driver = new ServiceTestDriver(serviceHostExecutable);
-
                         var className = test.Substring(0, test.LastIndexOf('.'));
                         var methodName = test.Substring(test.LastIndexOf('.') + 1);
                         
                         var type = Type.GetType("Microsoft.SqlTools.ServiceLayer.TestDriver.Tests." + className);
-                        var typeInstance = Activator.CreateInstance(type);
+                        using (var typeInstance = (IDisposable)Activator.CreateInstance(type))
-                        MethodInfo methodInfo = type.GetMethod(methodName);
+                        {
+                            MethodInfo methodInfo = type.GetMethod(methodName);
 
-                        await driver.Start();
+                            Console.WriteLine("Running test " + test);
-                        Console.WriteLine("Running test " + test);
+                            await (Task)methodInfo.Invoke(typeInstance, null);
-                        await (Task)methodInfo.Invoke(typeInstance, new object[] {driver});
+                        }
                     }
                     catch (Exception ex)
                     {
                         Console.WriteLine(ex.ToString());
                     }
-                    finally
-                    {
-                        if (driver != null)
-                        {
-                            await driver.Stop();
-                        }
-                    }
                 }
             }).Wait();
         }

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/ExampleTests.cs

@@ -1,38 +0,0 @@
-using System;
-using System.Threading.Tasks;
-using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
-using Microsoft.SqlTools.ServiceLayer.TestDriver.Driver;
-
-namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests
-{
-    public class ExampleTests
-    {
-        /// <summary>
-        /// Example test that performs a connect, then disconnect.
-        /// All tests must have the same signature of returning an async Task
-        /// and taking in a ServiceTestDriver as a parameter.
-        /// </summary>
-        public async Task ConnectDisconnectTest(ServiceTestDriver driver)
-        {
-            var connectParams = new ConnectParams();
-            connectParams.OwnerUri = "file";
-            connectParams.Connection = new ConnectionDetails();
-            connectParams.Connection.ServerName = "localhost";
-            connectParams.Connection.AuthenticationType = "Integrated";
-            
-            var result = await driver.SendRequest(ConnectionRequest.Type, connectParams);
-            if (result)
-            {
-                await driver.WaitForEvent(ConnectionCompleteNotification.Type);
-
-                var disconnectParams = new DisconnectParams();
-                disconnectParams.OwnerUri = "file";
-                var result2 = await driver.SendRequest(DisconnectRequest.Type, disconnectParams);
-                if (result2)
-                {
-                    Console.WriteLine("success");
-                }
-            }
-        }
-    }
-}

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/QueryExecutionTests.cs

@@ -0,0 +1,55 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility;
+using Xunit;
+
+namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests
+{
+    public class QueryExecutionTests : TestBase
+    {
+        [Fact]
+        public async Task TestQueryingAfterCompletionRequests()
+        {
+            string ownerUri = System.IO.Path.GetTempFileName();
+            string query = "SELECT * FROM sys.objects";
+            List<Task> tasks = new List<Task>();
+
+            await Connect(ownerUri, ConnectionTestUtils.AzureTestServerConnection);
+            Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10)));
+            var queryTask = RunQuery(ownerUri, query);
+            tasks.Add(queryTask);
+            await Task.WhenAll(tasks);
+
+            Assert.NotNull(queryTask.Result);
+            Assert.NotNull(queryTask.Result.BatchSummaries);
+            
+            await Connect(ownerUri, ConnectionTestUtils.DataToolsTelemetryAzureConnection);
+            tasks.Clear();
+            Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10)));
+            queryTask = RunQuery(ownerUri, query);
+            tasks.Add(queryTask);
+            await Task.WhenAll(tasks);
+            
+            Assert.NotNull(queryTask.Result);
+            Assert.NotNull(queryTask.Result.BatchSummaries);
+
+            await Connect(ownerUri, ConnectionTestUtils.SqlDataToolsAzureConnection);
+            tasks.Clear();
+            Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10)));
+            queryTask = RunQuery(ownerUri, query);
+            tasks.Add(queryTask);
+            await Task.WhenAll(tasks);
+            
+            Assert.NotNull(queryTask.Result);
+            Assert.NotNull(queryTask.Result.BatchSummaries);
+
+            await Disconnect(ownerUri);
+        }
+    }
+}

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/TestBase.cs

@@ -0,0 +1,123 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
+using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts;
+using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts;
+using Microsoft.SqlTools.ServiceLayer.TestDriver.Driver;
+using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts;
+
+namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests
+{
+    /// <summary>
+    /// Base class for all test suites run by the test driver
+    /// </summary>
+    public class TestBase : IDisposable
+    {
+        public TestBase()
+        {
+            Driver = new ServiceTestDriver();
+            Driver.Start().Wait();
+        }
+
+        public void Dispose()
+        {
+            Driver.Stop().Wait();
+        }
+
+        /// <summary>
+        /// The driver object used to read/write data to the service
+        /// </summary>
+        public ServiceTestDriver Driver
+        {
+            get;
+            private set;
+        }
+
+        private object fileLock = new Object();
+
+        /// <summary>
+        /// Request a new connection to be created
+        /// </summary>
+        /// <returns>True if the connection completed successfully</returns>
+        protected async Task<bool> Connect(string ownerUri, ConnectParams connectParams)
+        {
+            connectParams.OwnerUri = ownerUri;
+            var connectResult = await Driver.SendRequest(ConnectionRequest.Type, connectParams);
+            if (connectResult)
+            {
+                var completeEvent = await Driver.WaitForEvent(ConnectionCompleteNotification.Type);
+                return !string.IsNullOrEmpty(completeEvent.ConnectionId);
+            }
+            else
+            {
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Request a disconnect
+        /// </summary>
+        protected async Task<bool> Disconnect(string ownerUri)
+        {
+            var disconnectParams = new DisconnectParams();
+            disconnectParams.OwnerUri = ownerUri;
+
+            var disconnectResult = await Driver.SendRequest(DisconnectRequest.Type, disconnectParams);
+            return disconnectResult;
+        }
+
+        /// <summary>
+        /// Request a list of completion items for a position in a block of text
+        /// </summary>
+        protected async Task<CompletionItem[]> RequestCompletion(string ownerUri, string text, int line, int character)
+        {
+            // Write the text to a backing file
+            lock (fileLock)
+            {
+                System.IO.File.WriteAllText(ownerUri, text);
+            }
+
+            var completionParams = new TextDocumentPosition();
+            completionParams.TextDocument = new TextDocumentIdentifier();
+            completionParams.TextDocument.Uri = ownerUri;
+            completionParams.Position = new Position();
+            completionParams.Position.Line = line;
+            completionParams.Position.Character = character;
+
+            var result = await Driver.SendRequest(CompletionRequest.Type, completionParams);
+            return result;
+        }
+
+        /// <summary>
+        /// Run a query using a given connection bound to a URI
+        /// </summary>
+        protected async Task<QueryExecuteCompleteParams> RunQuery(string ownerUri, string query)
+        {
+            // Write the query text to a backing file
+            lock (fileLock)
+            {
+                System.IO.File.WriteAllText(ownerUri, query);
+            }
+
+            var queryParams = new QueryExecuteParams();
+            queryParams.OwnerUri = ownerUri;
+            queryParams.QuerySelection = null;
+
+            var result = await Driver.SendRequest(QueryExecuteRequest.Type, queryParams);
+            if (result != null && string.IsNullOrEmpty(result.Messages))
+            {
+                var eventResult = await Driver.WaitForEvent(QueryExecuteCompleteEvent.Type);
+                return eventResult;
+            }
+            else
+            {
+                return null;
+            }
+        }
+    }
+}

test/Microsoft.SqlTools.ServiceLayer.TestDriver/project.json

@@ -6,10 +6,13 @@
     "emitEntryPoint": true
   },
   "dependencies": {
+    "xunit": "2.1.0",
+    "dotnet-test-xunit": "1.0.0-rc2-192208-24",
     "Microsoft.SqlTools.ServiceLayer": {
       "target": "project"
     }
   },
+  "testRunner": "xunit",
   "frameworks": {
     "netcoreapp1.0": {
       "dependencies": {