ckaczor/sqltoolsservice
1
0
Fork 0
mirror of https://github.com/ckaczor/sqltoolsservice.git synced 2026-01-29 01:25:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fix OE and Binding Queue reliability bugs (#702)

Browse Source
This commit is contained in:
Matt Irvine
2018-10-05 14:14:36 -07:00
committed by GitHub
parent 6f69f7e303
commit f45155aa4a
11 changed files with 200 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
src/Microsoft.SqlTools.ServiceLayer/LanguageServices/BindingQueue.cs
View File

@@ -39,6 +39,8 @@ namespace Microsoft.SqlTools.ServiceLayer.LanguageServices
/// </summary>
internal Dictionary<string, IBindingContext> BindingContextMap { get; set; }
internal Dictionary<IBindingContext, Task> BindingContextTasks { get; set; } = new Dictionary<IBindingContext, Task>();
/// <summary>
/// Constructor for a binding queue instance
/// </summary>
@@ -145,7 +147,9 @@ namespace Microsoft.SqlTools.ServiceLayer.LanguageServices
{
if (!this.BindingContextMap.ContainsKey(key))
{
this.BindingContextMap.Add(key, new T());
var bindingContext = new T();
this.BindingContextMap.Add(key, bindingContext);
this.BindingContextTasks.Add(bindingContext, Task.Run(() => null));
}
return this.BindingContextMap[key];
@@ -190,11 +194,17 @@ namespace Microsoft.SqlTools.ServiceLayer.LanguageServices
var bindingContext = this.BindingContextMap[key];
if (bindingContext.ServerConnection != null && bindingContext.ServerConnection.IsOpen)
{
bindingContext.ServerConnection.Disconnect();
// Disconnecting can take some time so run it in a separate task so that it doesn't block removal
Task.Run(() =>
{
bindingContext.ServerConnection.Cancel();
bindingContext.ServerConnection.Disconnect();
});
}
// remove key from the map
this.BindingContextMap.Remove(key);
this.BindingContextTasks.Remove(bindingContext);
}
}
}
@@ -286,86 +296,119 @@ namespace Microsoft.SqlTools.ServiceLayer.LanguageServices
continue;
}
bool lockTaken = false;
try
{
// prefer the queue item binding item, otherwise use the context default timeout
int bindTimeout = queueItem.BindingTimeout ?? bindingContext.BindingTimeout;
var bindingContextTask = this.BindingContextTasks[bindingContext];
// handle the case a previous binding operation is still running
if (!bindingContext.BindingLock.WaitOne(queueItem.WaitForLockTimeout ?? 0))
{
queueItem.Result = queueItem.TimeoutOperation != null
? queueItem.TimeoutOperation(bindingContext)
: null;
// Run in the binding context task in case this task has to wait for a previous binding operation
this.BindingContextTasks[bindingContext] = bindingContextTask.ContinueWith((task) =>
{
bool lockTaken = false;
try
{
// prefer the queue item binding item, otherwise use the context default timeout
int bindTimeout = queueItem.BindingTimeout ?? bindingContext.BindingTimeout;
continue;
}
bindingContext.BindingLock.Reset();
lockTaken = true;
// execute the binding operation
object result = null;
CancellationTokenSource cancelToken = new CancellationTokenSource();
// run the operation in a separate thread
var bindTask = Task.Run(() =>
{
try
// handle the case a previous binding operation is still running
if (!bindingContext.BindingLock.WaitOne(queueItem.WaitForLockTimeout ?? 0))
{
result = queueItem.BindOperation(
bindingContext,
cancelToken.Token);
}
catch (Exception ex)
{
Logger.Write(TraceEventType.Error, "Unexpected exception on the binding queue: " + ex.ToString());
if (queueItem.ErrorHandler != null)
try
{
result = queueItem.ErrorHandler(ex);
Logger.Write(TraceEventType.Warning, "Binding queue operation timed out waiting for previous operation to finish");
queueItem.Result = queueItem.TimeoutOperation != null
? queueItem.TimeoutOperation(bindingContext)
: null;
}
catch (Exception ex)
{
Logger.Write(TraceEventType.Error, "Exception running binding queue lock timeout handler: " + ex.ToString());
}
finally
{
queueItem.ItemProcessed.Set();
}
}
});
// check if the binding tasks completed within the binding timeout
if (bindTask.Wait(bindTimeout))
{
queueItem.Result = result;
}
else
{
cancelToken.Cancel();
// if the task didn't complete then call the timeout callback
if (queueItem.TimeoutOperation != null)
{
queueItem.Result = queueItem.TimeoutOperation(bindingContext);
return;
}
lockTaken = false;
bindingContext.BindingLock.Reset();
bindTask
.ContinueWith((a) => bindingContext.BindingLock.Set())
.ContinueWithOnFaulted(t => Logger.Write(TraceEventType.Error, "Binding queue threw exception " + t.Exception.ToString()));
lockTaken = true;
// execute the binding operation
object result = null;
CancellationTokenSource cancelToken = new CancellationTokenSource();
// run the operation in a separate thread
var bindTask = Task.Run(() =>
{
try
{
result = queueItem.BindOperation(
bindingContext,
cancelToken.Token);
}
catch (Exception ex)
{
Logger.Write(TraceEventType.Error, "Unexpected exception on the binding queue: " + ex.ToString());
if (queueItem.ErrorHandler != null)
{
result = queueItem.ErrorHandler(ex);
}
}
});
Task.Run(() =>
{
try
{
// check if the binding tasks completed within the binding timeout
if (bindTask.Wait(bindTimeout))
{
queueItem.Result = result;
}
else
{
cancelToken.Cancel();
// if the task didn't complete then call the timeout callback
if (queueItem.TimeoutOperation != null)
{
queueItem.Result = queueItem.TimeoutOperation(bindingContext);
}
bindTask.ContinueWithOnFaulted(t => Logger.Write(TraceEventType.Error, "Binding queue threw exception " + t.Exception.ToString()));
// Give the task a chance to cancel before moving on to the next operation
Task.WaitAny(bindTask, Task.Delay(bindingContext.BindingTimeout));
}
}
catch (Exception ex)
{
Logger.Write(TraceEventType.Error, "Binding queue task completion threw exception " + ex.ToString());
}
finally
{
// set item processed to avoid deadlocks
if (lockTaken)
{
bindingContext.BindingLock.Set();
}
queueItem.ItemProcessed.Set();
}
});
}
}
catch (Exception ex)
{
// catch and log any exceptions raised in the binding calls
// set item processed to avoid deadlocks
Logger.Write(TraceEventType.Error, "Binding queue threw exception " + ex.ToString());
}
finally
{
if (lockTaken)
catch (Exception ex)
{
bindingContext.BindingLock.Set();
// catch and log any exceptions raised in the binding calls
// set item processed to avoid deadlocks
Logger.Write(TraceEventType.Error, "Binding queue threw exception " + ex.ToString());
// set item processed to avoid deadlocks
if (lockTaken)
{
bindingContext.BindingLock.Set();
}
queueItem.ItemProcessed.Set();
}
queueItem.ItemProcessed.Set();
}
}, TaskContinuationOptions.None);
// if a queue processing cancellation was requested then exit the loop
if (token.IsCancellationRequested)