Adding completion extension loading and execution logic (#833)

* Adding ICompletionExtension interface

* Adding extension loading and execution logic

* Fixing compilation error in VS 2017

* Using MEF for completion extension discovery

* using await on GetCompletionItems

* Adding an integration test for completion extension and update the completion extension interface

* Update the completion extension test

* Fix issues based on review comments

* Remove try/cache based on review comments, fix a integration test.

* More changes based on review comments

* Fixing SendResult logic for completion extension loading

* Only load completion extension from the assembly passed in, add more comments in the test

* Adding right assert messages in the test.

* More fixes based on review comments

* Dropping ICompletionExtensionProvider, load assembly only if it's loaded at the first time or updated since last load.

* Fix based on the latest review comments

* Adding missing TSQL functions in default completion list

* Update jsonrpc documentation for completion/extLoad

test/CompletionExtSample/CompletionExt.cs

@@ -0,0 +1,81 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.Composition;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Linq;
+using Microsoft.SqlTools.ServiceLayer.Connection;
+using Microsoft.SqlTools.ServiceLayer.LanguageServices.Completion;
+using Microsoft.SqlTools.ServiceLayer.LanguageServices.Completion.Extension;
+using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts;
+
+namespace Microsoft.SqlTools.Test.CompletionExtension
+{
+
+    [Export(typeof(ICompletionExtension))]
+    public class CompletionExt : ICompletionExtension
+    {
+        public string Name => "CompletionExt";
+
+        private string modelPath;
+
+        public CompletionExt()
+        {
+        }
+
+        void IDisposable.Dispose()
+        {
+        }
+
+        async Task<CompletionItem[]> ICompletionExtension.HandleCompletionAsync(ConnectionInfo connInfo, ScriptDocumentInfo scriptDocumentInfo, CompletionItem[] completions, CancellationToken token)
+        {
+            if (completions == null || completions == null || completions.Length == 0)
+            {
+                return completions;
+            }
+            return await Run(completions, token);
+        }
+
+        async Task ICompletionExtension.Initialize(IReadOnlyDictionary<string, object> properties, CancellationToken token)
+        {
+            modelPath = (string)properties["modelPath"];
+            await LoadModel(token).ConfigureAwait(false);
+            return;
+        }
+
+        private async Task LoadModel(CancellationToken token)
+        {
+            //loading model logic here
+            await Task.Delay(2000).ConfigureAwait(false); //for testing
+            token.ThrowIfCancellationRequested();
+            Console.WriteLine("Model loaded from: " + modelPath);
+        }
+
+        private async Task<CompletionItem[]> Run(CompletionItem[] completions, CancellationToken token)
+        {
+            Console.WriteLine("Enter ExecuteAsync");
+
+            var sortedItems = completions.OrderBy(item => item.SortText);
+            sortedItems.First().Preselect = true;
+            foreach(var item in sortedItems)
+            {
+                item.Command = new Command
+                {
+                    CommandStr = "vsintellicode.completionItemSelected",
+                    Arguments = new object[] { new Dictionary<string, string> { { "IsCommit", "True" } } }
+                };
+            }
+
+            //code to augment the default completion list
+            await Task.Delay(20); // for testing
+            token.ThrowIfCancellationRequested();
+            Console.WriteLine("Exit ExecuteAsync");
+            return sortedItems.ToArray();
+        }
+    }
+}

test/CompletionExtSample/Microsoft.SqlTools.Test.CompletionExtension.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="../../Common.props" />
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.2</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Microsoft.SqlTools.ServiceLayer\Microsoft.SqlTools.ServiceLayer.csproj" />
+  </ItemGroup>
+
+</Project>

test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/LanguageServer/LanguageServiceTests.cs

@@ -3,14 +3,21 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 //
 
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.SqlTools.ServiceLayer.IntegrationTests.Utility;
 using Microsoft.SqlTools.ServiceLayer.LanguageServices;
+using Microsoft.SqlTools.ServiceLayer.LanguageServices.Completion.Extension;
 using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts;
 using Microsoft.SqlTools.ServiceLayer.Test.Common;
+using Microsoft.SqlTools.ServiceLayer.UnitTests.ServiceHost;
 using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts;
+using Moq;
 using Xunit;
 
 namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.LanguageServer
@@ -75,9 +82,9 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.LanguageServer
             }
         }
 
-        // This test currently requires a live database connection to initialize
-        // SMO connected metadata provider.  Since we don't want a live DB dependency
-        // in the CI unit tests this scenario is currently disabled.
+        /// <summary>
+        /// This test tests auto completion
+        /// </summary>
         [Fact]
         public void AutoCompleteFindCompletions()
         {
@@ -90,11 +97,116 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.LanguageServer
             var completions = autoCompleteService.GetCompletionItems(
                 result.TextDocumentPosition,
                 result.ScriptFile,
-                result.ConnectionInfo);
+                result.ConnectionInfo).Result;
 
             Assert.True(completions.Length > 0);
         }
 
+        public static string AssemblyDirectory
+        {
+            get
+            {
+                string codeBase = Assembly.GetExecutingAssembly().CodeBase;
+                UriBuilder uri = new UriBuilder(codeBase);
+                string path = Uri.UnescapeDataString(uri.Path);
+                return Path.GetDirectoryName(path);
+            }
+        }
+
+        /// <summary>
+        /// This test tests completion extension interface in following aspects
+        /// 1. Loading a sample completion extension assembly
+        /// 2. Initializing a completion extension implementation
+        /// 3. Excuting an auto completion with extension enabled
+        /// </summary>
+        [Fact]
+        public async void AutoCompleteWithExtension()
+        {
+            var result = GetLiveAutoCompleteTestObjects();
+
+            result.TextDocumentPosition.Position.Character = 10;
+            result.ScriptFile = ScriptFileTests.GetTestScriptFile("select * f");
+            result.TextDocumentPosition.TextDocument.Uri = result.ScriptFile.FilePath;
+
+            var autoCompleteService = LanguageService.Instance;
+            var requestContext = new Mock<SqlTools.Hosting.Protocol.RequestContext<bool>>();
+            requestContext.Setup(x => x.SendResult(It.IsAny<bool>()))
+                .Returns(Task.FromResult(true));
+            requestContext.Setup(x => x.SendError(It.IsAny<string>(), 0))
+                .Returns(Task.FromResult(true));
+
+            //Create completion extension parameters
+            var extensionParams = new CompletionExtensionParams()
+            {
+                AssemblyPath = Path.Combine(AssemblyDirectory, "Microsoft.SqlTools.Test.CompletionExtension.dll"),
+                TypeName = "Microsoft.SqlTools.Test.CompletionExtension.CompletionExt",
+                Properties = new Dictionary<string, object> { { "modelPath", "testModel" } }
+            };
+
+            //load and initialize completion extension, expect a success
+            await autoCompleteService.HandleCompletionExtLoadRequest(extensionParams, requestContext.Object);
+
+            requestContext.Verify(x => x.SendResult(It.IsAny<bool>()), Times.Once);
+            requestContext.Verify(x => x.SendError(It.IsAny<string>(), 0), Times.Never);
+
+            //Try to load the same completion extension second time, expect an error sent
+            await autoCompleteService.HandleCompletionExtLoadRequest(extensionParams, requestContext.Object);
+
+            requestContext.Verify(x => x.SendResult(It.IsAny<bool>()), Times.Once);
+            requestContext.Verify(x => x.SendError(It.IsAny<string>(), 0), Times.Once);
+
+            //Try to load the completion extension with new modified timestamp, expect a success
+            var assemblyCopyPath = CopyFileWithNewModifiedTime(extensionParams.AssemblyPath);
+            extensionParams = new CompletionExtensionParams()
+            {
+                AssemblyPath = assemblyCopyPath,
+                TypeName = "Microsoft.SqlTools.Test.CompletionExtension.CompletionExt",
+                Properties = new Dictionary<string, object> { { "modelPath", "testModel" } }
+            };
+            //load and initialize completion extension
+            await autoCompleteService.HandleCompletionExtLoadRequest(extensionParams, requestContext.Object);
+
+            requestContext.Verify(x => x.SendResult(It.IsAny<bool>()), Times.Exactly(2));
+            requestContext.Verify(x => x.SendError(It.IsAny<string>(), 0), Times.Once);
+
+            ScriptParseInfo scriptInfo = new ScriptParseInfo { IsConnected = true };
+            autoCompleteService.ParseAndBind(result.ScriptFile, result.ConnectionInfo);
+            scriptInfo.ConnectionKey = autoCompleteService.BindingQueue.AddConnectionContext(result.ConnectionInfo);
+
+            //Invoke auto completion with extension enabled
+            var completions = autoCompleteService.GetCompletionItems(
+                result.TextDocumentPosition,
+                result.ScriptFile,
+                result.ConnectionInfo).Result;
+
+            //Validate completion list is not empty
+            Assert.True(completions != null && completions.Length > 0, "The completion list is null or empty!");
+            //Validate the first completion item in the list is preselected
+            Assert.True(completions[0].Preselect.HasValue && completions[0].Preselect.Value, "Preselect is not set properly in the first completion item by the completion extension!");
+            //Validate the Command object attached to the completion item by the extension
+            Assert.True(completions[0].Command != null && completions[0].Command.CommandStr == "vsintellicode.completionItemSelected", "Command is not set properly in the first completion item by the completion extension!");
+
+            //clean up the temp file
+            File.Delete(assemblyCopyPath);
+        }
+
+        /// <summary>
+        /// Make a copy of a file and update the last modified time
+        /// </summary>
+        /// <param name="filePath"></param>
+        /// <returns></returns>
+        private string CopyFileWithNewModifiedTime(string filePath)
+        {
+            var tempPath = Path.Combine(Path.GetTempPath(), Path.GetFileName(filePath));
+            if (File.Exists(tempPath))
+            {
+                File.Delete(tempPath);
+            }
+            File.Copy(filePath, tempPath);
+            File.SetLastWriteTimeUtc(tempPath, DateTime.UtcNow);
+            return tempPath;
+        }
+
         /// <summary>
         /// Verify that GetSignatureHelp returns not null when the provided TextDocumentPosition
         /// has an associated ScriptParseInfo and the provided query has a function that should
@@ -191,7 +303,7 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.LanguageServer
                     };
 
                 // First check that we don't have any items in the completion list as expected
-                var initialCompletionItems = langService.GetCompletionItems(
+                var initialCompletionItems = await langService.GetCompletionItems(
                     textDocumentPosition, connectionInfoResult.ScriptFile, connectionInfoResult.ConnectionInfo);
 
                 Assert.True(initialCompletionItems.Length == 0, $"Should not have any completion items initially. Actual : [{string.Join(',', initialCompletionItems.Select(ci => ci.Label))}]");
@@ -205,7 +317,7 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.LanguageServer
                     new TestEventContext());
 
                 // Now we should expect to see the item show up in the completion list
-                var afterTableCreationCompletionItems = langService.GetCompletionItems(
+                var afterTableCreationCompletionItems = await langService.GetCompletionItems(
                     textDocumentPosition, connectionInfoResult.ScriptFile, connectionInfoResult.ConnectionInfo);
 
                 Assert.True(afterTableCreationCompletionItems.Length == 1, $"Should only have a single completion item after rebuilding Intellisense cache. Actual : [{string.Join(',', initialCompletionItems.Select(ci => ci.Label))}]");

test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/Microsoft.SqlTools.ServiceLayer.IntegrationTests.csproj

@@ -27,6 +27,7 @@
     <ProjectReference Include="../Microsoft.SqlTools.ServiceLayer.Test.Common/Microsoft.SqlTools.ServiceLayer.Test.Common.csproj" />
     <ProjectReference Include="../../src/Microsoft.SqlTools.ManagedBatchParser/Microsoft.SqlTools.ManagedBatchParser.csproj" />
     <ProjectReference Include="../Microsoft.SqlTools.ServiceLayer.UnitTests/Microsoft.SqlTools.ServiceLayer.UnitTests.csproj" />
+    <ProjectReference Include="..\CompletionExtSample\Microsoft.SqlTools.Test.CompletionExtension.csproj" />
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="System.Net.Http" Version="4.3.1" />

test/Microsoft.SqlTools.ServiceLayer.UnitTests/LanguageServer/AutocompleteTests.cs

@@ -117,7 +117,7 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.LanguageServer
         {
             InitializeTestObjects();
             textDocument.TextDocument.Uri = "somethinggoeshere";
-            Assert.True(langService.GetCompletionItems(textDocument, scriptFile.Object, null).Length > 0);
+            Assert.True(langService.GetCompletionItems(textDocument, scriptFile.Object, null).Result.Length > 0);
         }
 
         [Fact]

test/Microsoft.SqlTools.ServiceLayer.UnitTests/ServiceHost/ScriptFileTests.cs

@@ -22,7 +22,7 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.ServiceHost
             "SELECT * FROM sys.objects as o2" + Environment.NewLine +
             "SELECT * FROM sys.objects as o3" + Environment.NewLine;
 
-        internal static ScriptFile GetTestScriptFile(string initialText = null)
+        public static ScriptFile GetTestScriptFile(string initialText = null)
         {
             if (initialText == null)
             {