Code coverage improvements (#121)

These are test-only changes to improve code coverage so I'll merge directly.  Please review the commit and I'll pickup those changes in the next iteration.

* Add integration test batch file

* Exclude Linux and MacOS from Windows code coverage builds

* Enable code coverage for test driver e2e tests

* Use the windows only build for code coverage runs

src/Microsoft.SqlTools.ServiceLayer/Credentials/CredentialService.cs

@@ -48,7 +48,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Credentials
         /// Default constructor is private since it's a singleton class
         /// </summary>
         private CredentialService()
-            : this(null, new LinuxCredentialStore.StoreConfig() 
+            : this(null, new StoreConfig() 
                 { CredentialFolder = DefaultSecretsFolder, CredentialFile = DefaultSecretsFile, IsRelativeToUserHomeDir = true})
         {
         }
@@ -56,7 +56,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Credentials
         /// <summary>
         /// Internal for testing purposes only
         /// </summary>
-        internal CredentialService(ICredentialStore store, LinuxCredentialStore.StoreConfig config)
+        internal CredentialService(ICredentialStore store, StoreConfig config)
         {
             this.credStore = store != null ? store : GetStoreForOS(config);
         }
@@ -64,12 +64,13 @@ namespace Microsoft.SqlTools.ServiceLayer.Credentials
         /// <summary>
         /// Internal for testing purposes only
         /// </summary>
-        internal static ICredentialStore GetStoreForOS(LinuxCredentialStore.StoreConfig config)
+        internal static ICredentialStore GetStoreForOS(StoreConfig config)
         {
             if(RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
             {
                 return new Win32CredentialStore();
             }
+#if !WINDOWS_ONLY_BUILD
             else if(RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
             {
                 return new OSXCredentialStore();
@@ -78,6 +79,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Credentials
             {
                 return new LinuxCredentialStore(config);
             }
+#endif
             throw new InvalidOperationException("Platform not currently supported");
         }
 

src/Microsoft.SqlTools.ServiceLayer/Credentials/Linux/FileTokenStorage.cs

@@ -13,6 +13,9 @@ using Newtonsoft.Json;
 
 namespace Microsoft.SqlTools.ServiceLayer.Credentials.Linux
 {
+
+#if !WINDOWS_ONLY_BUILD
+
     public class FileTokenStorage
     {
         private const int OwnerAccessMode = 384; // Permission 0600 - owner read/write, nobody else has access
@@ -84,4 +87,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Credentials.Linux
             Interop.Sys.ChMod(filePath, OwnerAccessMode);            
         }
     }
+
+#endif
+
 }

src/Microsoft.SqlTools.ServiceLayer/Credentials/Linux/Interop.Errors.cs

@@ -8,6 +8,9 @@ using System.Runtime.InteropServices;
 
 namespace Microsoft.SqlTools.ServiceLayer.Credentials
 {
+
+#if !WINDOWS_ONLY_BUILD
+
     internal static partial class Interop
     {
         /// <summary>Common Unix errno error codes.</summary>
@@ -218,4 +221,6 @@ namespace Microsoft.SqlTools.ServiceLayer.Credentials
         }
     }
 
+#endif
+
 }

src/Microsoft.SqlTools.ServiceLayer/Credentials/Linux/Interop.Sys.cs

@@ -8,6 +8,9 @@ using System.Runtime.InteropServices;
 
 namespace Microsoft.SqlTools.ServiceLayer.Credentials
 {
+
+#if !WINDOWS_ONLY_BUILD
+
     internal static partial class Interop
     {
         internal static partial class Sys
@@ -37,6 +40,8 @@ namespace Microsoft.SqlTools.ServiceLayer.Credentials
                 internal const string SystemNative = "System.Native";
             }
         }
-        
     }
+
+#endif
+
 }

src/Microsoft.SqlTools.ServiceLayer/Credentials/Linux/LinuxCredentialStore.cs

@@ -3,7 +3,6 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 //
 
-
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -15,6 +14,18 @@ using Microsoft.SqlTools.ServiceLayer.Utility;
 
 namespace Microsoft.SqlTools.ServiceLayer.Credentials.Linux
 {
+    /// <summary>
+    /// Store configuration struct
+    /// </summary>
+    internal struct StoreConfig
+    {
+        public string CredentialFolder { get; set; }
+        public string CredentialFile { get; set; }
+        public bool IsRelativeToUserHomeDir { get; set; }
+    }
+    
+#if !WINDOWS_ONLY_BUILD
+
     /// <summary>
     /// Linux implementation of the credential store.
     /// 
@@ -25,13 +36,6 @@ namespace Microsoft.SqlTools.ServiceLayer.Credentials.Linux
     /// </summary>
     internal class LinuxCredentialStore : ICredentialStore
     {
-        internal struct StoreConfig
-        {
-            public string CredentialFolder { get; set; }
-            public string CredentialFile { get; set; }
-            public bool IsRelativeToUserHomeDir { get; set; }
-        }
-
         private string credentialFolderPath;
         private string credentialFileName;
         private FileTokenStorage storage;
@@ -228,4 +232,6 @@ namespace Microsoft.SqlTools.ServiceLayer.Credentials.Linux
             }
         }
     }
-}
+#endif
+
+}

src/Microsoft.SqlTools.ServiceLayer/Credentials/OSX/OSXCredentialStore.cs

@@ -10,6 +10,9 @@ using Microsoft.SqlTools.ServiceLayer.Utility;
 
 namespace Microsoft.SqlTools.ServiceLayer.Credentials.OSX
 {
+
+#if !WINDOWS_ONLY_BUILD
+
     /// <summary>
     /// OSX implementation of the credential store
     /// </summary>
@@ -155,4 +158,8 @@ namespace Microsoft.SqlTools.ServiceLayer.Credentials.OSX
             }
         }
     }
-}
+
+#endif
+
+}
+

src/Microsoft.SqlTools.ServiceLayer/project.json

@@ -5,6 +5,16 @@
     "debugType": "portable",
     "emitEntryPoint": true
   },
+  "configurations": {
+    "Integration": {
+      "buildOptions": {
+        "define": [
+          "WINDOWS_ONLY_BUILD"
+        ],
+        "emitEntryPoint": true
+      }
+    }
+  },
   "dependencies": {
     "Newtonsoft.Json": "9.0.1",
     "System.Data.Common": "4.1.0",

test/CodeCoverage/codecoverage.bat

@@ -17,7 +17,20 @@ dotnet build %WORKINGDIR%..\..\src\Microsoft.SqlTools.ServiceLayer\project.json
 
 REM run the tests through OpenCover and generate a report
 dotnet build %WORKINGDIR%..\..\test\Microsoft.SqlTools.ServiceLayer.Test\project.json %DOTNETCONFIG% 
-"%WORKINGDIR%packages\OpenCover.4.6.519\tools\OpenCover.Console.exe" -register:user -target:dotnet.exe -targetargs:"test %WORKINGDIR%..\Microsoft.SqlTools.ServiceLayer.Test\project.json %DOTNETCONFIG%" -oldstyle -filter:"+[Microsoft.SqlTools.*]* -[xunit*]*" -output:coverage.xml -searchdirs:%WORKINGDIR%..\Microsoft.SqlTools.ServiceLayer.Test\bin\Debug\netcoreapp1.0
+
+SET SQLTOOLSSERVICE_EXE=%WORKINGDIR%..\..\src\Microsoft.SqlTools.ServiceLayer\bin\Integration\netcoreapp1.0\win7-x64\Microsoft.SqlTools.ServiceLayer.exe
+SET SERVICECODECOVERAGE=TRUE
+SET CODECOVERAGETOOL="%WORKINGDIR%packages\OpenCover.4.6.519\tools\OpenCover.Console.exe"
+SET CODECOVERAGEOUTPUT=coverage.xml
+
+dotnet.exe test %WORKINGDIR%..\Microsoft.SqlTools.ServiceLayer.TestDriver\project.json %DOTNETCONFIG%"
+
+SET SERVICECODECOVERAGE=FALSE
+
+"%WORKINGDIR%packages\OpenCover.4.6.519\tools\OpenCover.Console.exe" -mergeoutput -register:user -target:dotnet.exe -targetargs:"test %WORKINGDIR%..\Microsoft.SqlTools.ServiceLayer.TestDriver\project.json %DOTNETCONFIG%" -oldstyle -filter:"+[Microsoft.SqlTools.*]* -[xunit*]*" -output:coverage.xml -searchdirs:%WORKINGDIR%..\Microsoft.SqlTools.ServiceLayer.TestDriver\bin\Debug\netcoreapp1.0
+
+"%WORKINGDIR%packages\OpenCover.4.6.519\tools\OpenCover.Console.exe" -mergeoutput -register:user -target:dotnet.exe -targetargs:"test %WORKINGDIR%..\Microsoft.SqlTools.ServiceLayer.Test\project.json %DOTNETCONFIG%" -oldstyle -filter:"+[Microsoft.SqlTools.*]* -[xunit*]*" -output:coverage.xml -searchdirs:%WORKINGDIR%..\Microsoft.SqlTools.ServiceLayer.Test\bin\Debug\netcoreapp1.0
+
 "%WORKINGDIR%packages\OpenCoverToCoberturaConverter.0.2.4.0\tools\OpenCoverToCoberturaConverter.exe"  -input:coverage.xml -output:outputCobertura.xml -sources:%WORKINGDIR%..\..\src\Microsoft.SqlTools.ServiceLayer
 "%WORKINGDIR%packages\ReportGenerator.2.4.5.0\tools\ReportGenerator.exe"  "-reports:coverage.xml" "-targetdir:%WORKINGDIR%\reports"
 

test/CodeCoverage/runintegration.bat

@@ -0,0 +1,4 @@
+set DOTNETCONFIG=-c Integration
+
+cmd /c npm install
+gulp

test/Microsoft.SqlTools.ServiceLayer.Test/Credentials/CredentialServiceTests.cs

@@ -24,7 +24,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Connection
     /// </summary>
     public class CredentialServiceTests : IDisposable
     {
-        private static readonly LinuxCredentialStore.StoreConfig config = new LinuxCredentialStore.StoreConfig()
+        private static readonly StoreConfig config = new StoreConfig()
         {
             CredentialFolder = ".testsecrets", 
             CredentialFile = "sqltestsecrets.json", 
@@ -61,6 +61,8 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Connection
         {
             credStore.DeletePassword(credentialId);
             credStore.DeletePassword(otherCredId);
+
+#if !WINDOWS_ONLY_BUILD
             if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
             {
                 string credsFolder = ((LinuxCredentialStore)credStore).CredentialFolderPath;
@@ -69,6 +71,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Connection
                     Directory.Delete(credsFolder, true);
                 }
             }
+#endif
         }
 
         [Fact]

test/Microsoft.SqlTools.ServiceLayer.Test/Credentials/Linux/LinuxInteropTests.cs

@@ -15,23 +15,25 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Credentials
         [Fact]
         public void GetEUidReturnsInt()
         {
+#if !WINDOWS_ONLY_BUILD           
             TestUtils.RunIfLinux(() =>
             {
                 Assert.NotNull(Interop.Sys.GetEUid());
             });
+#endif           
         }
 
         [Fact]
         public void GetHomeDirectoryFromPwFindsHomeDir()
         {
-
+#if !WINDOWS_ONLY_BUILD
             TestUtils.RunIfLinux(() =>
             {
                 string userDir = LinuxCredentialStore.GetHomeDirectoryFromPw();
                 Assert.StartsWith("/", userDir);
             });
+#endif
         }
-        
     }
 }
 

test/Microsoft.SqlTools.ServiceLayer.Test/project.json

@@ -8,7 +8,8 @@
     "Integration": {
       "buildOptions": {
         "define": [
-          "LIVE_CONNECTION_TESTS"
+          "LIVE_CONNECTION_TESTS",
+          "WINDOWS_ONLY_BUILD"
         ]
       }
     }

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs

@@ -9,6 +9,7 @@
 //
 
 using System;
+using System.IO;
 using System.Threading.Tasks;
 using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
 using Microsoft.SqlTools.ServiceLayer.Hosting.Protocol;
@@ -22,19 +23,55 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver
     /// </summary>
     public class ServiceTestDriver : TestDriverBase
     {
+
+        public const string ServiceCodeCoverageEnvironmentVariable = "SERVICECODECOVERAGE";
+
+        public const string CodeCoverageToolEnvironmentVariable = "CODECOVERAGETOOL";
+
+        public const string CodeCoverageOutputEnvironmentVariable = "CODECOVERAGEOUTPUT";        
+
         /// <summary>
         /// Environment variable that stores the path to the service host executable.
         /// </summary>
         public static string ServiceHostEnvironmentVariable
         {
             get { return "SQLTOOLSSERVICE_EXE"; }
-        }
+        }       
+
+        public bool IsCoverageRun { get; set; } 
 
         public ServiceTestDriver()
         {
             string serviceHostExecutable = Environment.GetEnvironmentVariable(ServiceHostEnvironmentVariable);
+            string serviceHostArguments = "--enable-logging";
 
-            var clientChannel = new StdioClientChannel(serviceHostExecutable, "--enable-logging");
+            //setup the service host for code coverage if the envvar is enabled
+            if (Environment.GetEnvironmentVariable(ServiceCodeCoverageEnvironmentVariable) == "True")
+            {
+                string coverageToolPath = Environment.GetEnvironmentVariable(CodeCoverageToolEnvironmentVariable);
+                if (!string.IsNullOrWhiteSpace(coverageToolPath))
+                {
+                    string serviceHostDirectory = Path.GetDirectoryName(serviceHostExecutable);
+                    if (string.IsNullOrWhiteSpace(serviceHostDirectory))
+                    {
+                        serviceHostDirectory = ".";
+                    }
+
+                    string coverageOutput = Environment.GetEnvironmentVariable(CodeCoverageOutputEnvironmentVariable);
+                    if (string.IsNullOrWhiteSpace(coverageOutput))
+                    {
+                        coverageOutput = "coverage.xml";
+                    }
+               
+                    serviceHostArguments = "-target:" + serviceHostExecutable + " -targetargs:" + serviceHostArguments 
+                        + " -register:user -oldstyle -filter:\"+[Microsoft.SqlTools.*]* -[xunit*]*\" -output:" + coverageOutput + " -searchdirs:" + serviceHostDirectory;
+                    serviceHostExecutable = coverageToolPath;
+
+                    this.IsCoverageRun = true;
+                }               
+            }
+
+            this.clientChannel = new StdioClientChannel(serviceHostExecutable, serviceHostArguments);
             this.protocolClient = new ProtocolEndpoint(clientChannel, MessageProtocolType.LanguageServer);
         }
 

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/TestDriverBase.cs

@@ -10,8 +10,10 @@
 
 using System;
 using System.Collections.Concurrent;
+using System.Diagnostics;
 using System.Threading.Tasks;
 using Microsoft.SqlTools.ServiceLayer.Hosting.Protocol;
+using Microsoft.SqlTools.ServiceLayer.Hosting.Protocol.Channel;
 using Microsoft.SqlTools.ServiceLayer.Hosting.Protocol.Contracts;
 using Microsoft.SqlTools.ServiceLayer.Utility;
 
@@ -24,12 +26,29 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver
     {
         protected ProtocolEndpoint protocolClient;
 
+        protected StdioClientChannel clientChannel;
+
         private ConcurrentDictionary<string, AsyncQueue<object>> eventQueuePerType =
             new ConcurrentDictionary<string, AsyncQueue<object>>();
 
         private ConcurrentDictionary<string, AsyncQueue<object>> requestQueuePerType =
             new ConcurrentDictionary<string, AsyncQueue<object>>();
 
+        public Process ServiceProcess
+        {
+            get
+            {
+                try
+                {
+                    return Process.GetProcessById(clientChannel.ProcessId);
+                }
+                catch
+                {
+                    return null;
+                }
+            }
+        }
+
         public Task<TResult> SendRequest<TParams, TResult>(
             RequestType<TParams, TResult> requestType, 
             TParams requestParams)

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/QueryExecutionTests.cs

@@ -3,8 +3,11 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 //
 
+using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility;
 using Xunit;
@@ -16,40 +19,49 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests
         [Fact]
         public async Task TestQueryingAfterCompletionRequests()
         {
-            string ownerUri = System.IO.Path.GetTempFileName();
+            try
-            string query = "SELECT * FROM sys.objects";
+            {
-            List<Task> tasks = new List<Task>();
+                string ownerUri = System.IO.Path.GetTempFileName();
+                string query = "SELECT * FROM sys.objects";
+                List<Task> tasks = new List<Task>();
 
-            await Connect(ownerUri, ConnectionTestUtils.AzureTestServerConnection);
+                await Connect(ownerUri, ConnectionTestUtils.AzureTestServerConnection);
-            Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10)));
-            var queryTask = RunQuery(ownerUri, query);
-            tasks.Add(queryTask);
-            await Task.WhenAll(tasks);
 
-            Assert.NotNull(queryTask.Result);
+
-            Assert.NotNull(queryTask.Result.BatchSummaries);
+                Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10)));
+                var queryTask = RunQuery(ownerUri, query);
+                tasks.Add(queryTask);
+                await Task.WhenAll(tasks);
+               
+                Assert.NotNull(queryTask.Result);
+                Assert.NotNull(queryTask.Result.BatchSummaries);
             
-            await Connect(ownerUri, ConnectionTestUtils.DataToolsTelemetryAzureConnection);
+                await Connect(ownerUri, ConnectionTestUtils.DataToolsTelemetryAzureConnection);
-            tasks.Clear();
+                tasks.Clear();
-            Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10)));
+                Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10)));
-            queryTask = RunQuery(ownerUri, query);
+                queryTask = RunQuery(ownerUri, query);
-            tasks.Add(queryTask);
+                tasks.Add(queryTask);
-            await Task.WhenAll(tasks);
+                await Task.WhenAll(tasks);
             
-            Assert.NotNull(queryTask.Result);
+                Assert.NotNull(queryTask.Result);
-            Assert.NotNull(queryTask.Result.BatchSummaries);
+                Assert.NotNull(queryTask.Result.BatchSummaries);
 
-            await Connect(ownerUri, ConnectionTestUtils.SqlDataToolsAzureConnection);
+                await Connect(ownerUri, ConnectionTestUtils.SqlDataToolsAzureConnection);
-            tasks.Clear();
+                tasks.Clear();
-            Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10)));
+                Enumerable.Range(0, 10).ToList().ForEach(arg => tasks.Add(RequestCompletion(ownerUri, query, 0, 10)));
-            queryTask = RunQuery(ownerUri, query);
+                queryTask = RunQuery(ownerUri, query);
-            tasks.Add(queryTask);
+                tasks.Add(queryTask);
-            await Task.WhenAll(tasks);
+                await Task.WhenAll(tasks);
             
-            Assert.NotNull(queryTask.Result);
+                Assert.NotNull(queryTask.Result);
-            Assert.NotNull(queryTask.Result.BatchSummaries);
+                Assert.NotNull(queryTask.Result.BatchSummaries);
 
-            await Disconnect(ownerUri);
+                await Disconnect(ownerUri);
+            }
+            finally
+            {
+                WaitForExit();
+            }
         }
     }
 }

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Tests/TestBase.cs

@@ -4,6 +4,7 @@
 //
 
 using System;
+using System.Diagnostics;
 using System.Threading.Tasks;
 using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
 using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts;
@@ -18,15 +19,37 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests
     /// </summary>
     public class TestBase : IDisposable
     {
+        private bool isRunning = false;
+
         public TestBase()
         {
             Driver = new ServiceTestDriver();
             Driver.Start().Wait();
+            this.isRunning = true;
         }
 
         public void Dispose()
         {
-            Driver.Stop().Wait();
+            if (this.isRunning)
+            {
+                WaitForExit();
+            }
+        }
+
+        public void WaitForExit()
+        {
+            this.isRunning = false;            
+
+            if (!Driver.IsCoverageRun)
+            {
+                Driver.Stop().Wait();
+            }
+            else
+            {
+                var p = Process.Start("taskkill", "/IM Microsoft.SqlTools.ServiceLayer.exe /F");
+                p.WaitForExit();    
+                Driver.ServiceProcess?.WaitForExit();
+            }
         }
 
         /// <summary>
@@ -43,9 +66,9 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Tests
         /// <summary>
         /// Request a new connection to be created
         /// </summary>
-        /// <returns>True if the connection completed successfully</returns>
+        /// <returns>True if the connection completed successfully</returns>        
         protected async Task<bool> Connect(string ownerUri, ConnectParams connectParams)
-        {
+        { 
             connectParams.OwnerUri = ownerUri;
             var connectResult = await Driver.SendRequest(ConnectionRequest.Type, connectParams);
             if (connectResult)

test/Microsoft.SqlTools.ServiceLayer.TestDriver/project.json

@@ -23,7 +23,7 @@
       "imports": [
         "dotnet5.4",
         "portable-net451+win8"
-      ],
+      ]
     }
   },
   "runtimes": {