Fix an issue with queue deadlocks causing test failures (#77)

src/Microsoft.SqlTools.ServiceLayer/LanguageServices/BindingQueue.cs

@@ -7,6 +7,7 @@ using System;
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.SqlTools.ServiceLayer.Utility;
 
 namespace Microsoft.SqlTools.ServiceLayer.LanguageServices
 {    
@@ -183,52 +184,62 @@ namespace Microsoft.SqlTools.ServiceLayer.LanguageServices
                             continue;
                         }
 
-                        IBindingContext bindingContext = GetOrCreateBindingContext(queueItem.Key);
+                        try
-                        if (bindingContext == null)                        
                         {
-                            queueItem.ItemProcessed.Set();
+                            IBindingContext bindingContext = GetOrCreateBindingContext(queueItem.Key);
-                            continue;
+                            if (bindingContext == null)                        
-                        }
-                    
-                        // prefer the queue item binding item, otherwise use the context default timeout
-                        int bindTimeout = queueItem.BindingTimeout ?? bindingContext.BindingTimeout;
-                        
-                        // handle the case a previous binding operation is still running
-                        if (!bindingContext.BindingLocked.WaitOne(bindTimeout))
-                        {
-                            queueItem.ResultsTask = Task.Run(() => 
-                            {
-                                var timeoutTask = queueItem.TimeoutOperation(bindingContext);
-                                timeoutTask.ContinueWith((obj) => queueItem.ItemProcessed.Set());                                
-                                return timeoutTask.Result;
-                            });
-
-                            continue;
-                        }
-
-                        // execute the binding operation
-                        CancellationTokenSource cancelToken = new CancellationTokenSource();
-                        queueItem.ResultsTask = queueItem.BindOperation(
-                                bindingContext,
-                                cancelToken.Token);
-
-                        // set notification events once the binding operation task completes
-                        queueItem.ResultsTask.ContinueWith((obj) => 
                             {
                                 queueItem.ItemProcessed.Set();
-                                bindingContext.BindingLocked.Set();
+                                continue;
-                            });
-
-                        // check if the binding tasks completed within the binding timeout
-                        if (!queueItem.ResultsTask.Wait(bindTimeout))
-                        {
-                            // if the task didn't complete then call the timeout callback
-                            if (queueItem.TimeoutOperation != null)
-                            {                            
-                                cancelToken.Cancel();
-                                queueItem.ResultsTask = queueItem.TimeoutOperation(bindingContext);
-                                queueItem.ResultsTask.ContinueWith((obj) => queueItem.ItemProcessed.Set());                                
                             }
+                        
+                            // prefer the queue item binding item, otherwise use the context default timeout
+                            int bindTimeout = queueItem.BindingTimeout ?? bindingContext.BindingTimeout;
+                            
+                            // handle the case a previous binding operation is still running
+                            if (!bindingContext.BindingLocked.WaitOne(bindTimeout))
+                            {
+                                queueItem.ResultsTask = Task.Run(() => 
+                                {                                
+                                    var timeoutTask = queueItem.TimeoutOperation(bindingContext);
+                                    timeoutTask.ContinueWith((obj) => queueItem.ItemProcessed.Set());                                
+                                    return timeoutTask.Result;
+                                });
+
+                                continue;
+                            }
+
+                            // execute the binding operation
+                            CancellationTokenSource cancelToken = new CancellationTokenSource();                        
+                            queueItem.ResultsTask = queueItem.BindOperation(
+                                    bindingContext,
+                                    cancelToken.Token);                            
+
+                            // set notification events once the binding operation task completes
+                            queueItem.ResultsTask.ContinueWith((obj) => 
+                                {   
+                                    queueItem.ItemProcessed.Set();
+                                    bindingContext.BindingLocked.Set();
+                                });
+
+                            // check if the binding tasks completed within the binding timeout
+                            if (!queueItem.ResultsTask.Wait(bindTimeout))
+                            {
+                                // if the task didn't complete then call the timeout callback
+                                if (queueItem.TimeoutOperation != null)
+                                {                            
+                                    cancelToken.Cancel();
+                                    queueItem.ResultsTask = queueItem.TimeoutOperation(bindingContext);
+                                    queueItem.ResultsTask.ContinueWith((obj) => queueItem.ItemProcessed.Set());                                
+                                }
+                            }                            
+                        }                        
+                        catch (Exception ex)
+                        {
+                            // catch and log any exceptions raised in the binding calls
+                            // set item processed to avoid deadlocks 
+                            Logger.Write(LogLevel.Error, "Binding queue threw exception " + ex.ToString());
+                            queueItem.ItemProcessed.Set();
                         }
 
                         // if a queue processing cancellation was requested then exit the loop

test/Microsoft.SqlTools.ServiceLayer.Test/LanguageServer/AutocompleteTests.cs

@@ -120,5 +120,20 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.LanguageServices
             // verify that send result was called with a completion array
             requestContext.Verify(m => m.SendResult(It.IsAny<CompletionItem[]>()), Times.Once());
         }
+
+        /// <summary>
+        /// Test the service initialization code path and verify nothing throws
+        /// </summary>
+        [Fact]
+        public async void UpdateLanguageServiceOnConnection()
+        {    
+            InitializeTestObjects();
+
+            AutoCompleteHelper.WorkspaceServiceInstance = workspaceService.Object;
+
+            ConnectionInfo connInfo = TestObjects.GetTestConnectionInfo();
+
+            await LanguageService.Instance.UpdateLanguageServiceOnConnection(connInfo);
+        }
     }
 }

test/Microsoft.SqlTools.ServiceLayer.Test/LanguageServer/LanguageServiceTests.cs

@@ -163,40 +163,6 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.LanguageServices
             Assert.True(LanguageService.ConnectionServiceInstance == null);
         }        
 
-        /// <summary>
-        /// Test the service initialization code path and verify nothing throws
-        /// </summary>
-        [Fact]
-        public async void UpdateLanguageServiceOnConnection()
-        {            
-            string ownerUri = "file://my/sample/file.sql";
-            var connectionService = TestObjects.GetTestConnectionService();
-            var connectionResult =
-                await connectionService
-                .Connect(new ConnectParams()
-                {
-                    OwnerUri = ownerUri,
-                    Connection = TestObjects.GetTestConnectionDetails()
-                });
-
-            // set up file for returning the query
-            var fileMock = new Mock<ScriptFile>();
-            fileMock.SetupGet(file => file.Contents).Returns(Common.StandardQuery);
-            fileMock.SetupGet(file => file.ClientFilePath).Returns(ownerUri);
-
-            // set up workspace mock
-            var workspaceService = new Mock<WorkspaceService<SqlToolsSettings>>();
-            workspaceService.Setup(service => service.Workspace.GetFile(It.IsAny<string>()))
-                .Returns(fileMock.Object);
-
-            AutoCompleteHelper.WorkspaceServiceInstance = workspaceService.Object;
-
-            ConnectionInfo connInfo = null;
-            connectionService.TryFindConnection(ownerUri, out connInfo);
-            
-            await LanguageService.Instance.UpdateLanguageServiceOnConnection(connInfo);
-        }
-
         /// <summary>
         /// Test the service initialization code path and verify nothing throws
         /// </summary>

test/Microsoft.SqlTools.ServiceLayer.Test/QueryExecution/Common.cs

@@ -249,11 +249,6 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.QueryExecution
 
             connInfo = Common.CreateTestConnectionInfo(null, false);
 
-            var srvConn = GetServerConnection(connInfo);
-            var displayInfoProvider = new MetadataDisplayInfoProvider();
-            var metadataProvider = SmoMetadataProvider.CreateConnectedProvider(srvConn);
-            var binder = BinderProvider.CreateBinder(metadataProvider);
-
             LanguageService.Instance.ScriptParseInfoMap.Add(textDocument.TextDocument.Uri,  new ScriptParseInfo());
 
             scriptFile = new ScriptFile {ClientFilePath = textDocument.TextDocument.Uri};