// // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // using System; using System.Diagnostics; using System.Globalization; using Microsoft.SqlTools.Utility; namespace Microsoft.SqlTools.ServiceLayer.Connection.ReliableConnection { internal sealed class RetryPolicyDefaults { /// /// The default number of retry attempts. /// public const int DefaulSchemaRetryCount = 6; /// /// The default number of retry attempts for create database. /// public const int DefaultCreateDatabaseRetryCount = 5; /// /// The default amount of time defining an interval between retries. /// public static readonly TimeSpan DefaultSchemaMinInterval = TimeSpan.FromSeconds(2.75); /// /// The default factor to use when determining exponential backoff between retries. /// public const double DefaultBackoffIntervalFactor = 2.0; /// /// The default maximum time between retries. /// public static readonly TimeSpan DefaultMaxRetryInterval = TimeSpan.FromSeconds(60); /// /// The default number of retry attempts. /// public static readonly int DefaultDataCommandRetryCount = 5; /// /// The default number of retry attempts for a connection related error /// public static readonly int DefaultConnectionRetryCount = 6; /// /// The default amount of time defining an interval between retries. /// public static readonly TimeSpan DefaultDataMinInterval = TimeSpan.FromSeconds(1.0); /// /// The default amount of time defining a time increment between retry attempts in the progressive delay policy. /// public static readonly TimeSpan DefaultProgressiveRetryIncrement = TimeSpan.FromMilliseconds(500); } /// /// Implements a collection of the RetryPolicyInfo elements holding retry policy settings. /// public sealed class RetryPolicyFactory { /// /// Returns a default policy that does no retries, it just invokes action exactly once. /// public static readonly RetryPolicy NoRetryPolicy = RetryPolicyFactory.CreateNoRetryPolicy(); /// /// Returns a default policy that does no retries, it just invokes action exactly once. /// public static readonly RetryPolicy PrimaryKeyViolationRetryPolicy = RetryPolicyFactory.CreatePrimaryKeyCommandRetryPolicy(); /// /// Implements a strategy that ignores any transient errors. /// Internal for testing purposes only /// public sealed class TransientErrorIgnoreStrategy : RetryPolicy.IErrorDetectionStrategy { private static readonly TransientErrorIgnoreStrategy _instance = new TransientErrorIgnoreStrategy(); public static TransientErrorIgnoreStrategy Instance { get { return _instance; } } public bool CanRetry(Exception ex) { return false; } public bool ShouldIgnoreError(Exception ex) { return false; } } /// /// Creates and returns a default Retry Policy for Schema based operations. /// /// An instance of class. public static RetryPolicy CreateDefaultSchemaCommandRetryPolicy(bool useRetry, int retriesPerPhase = RetryPolicyDefaults.DefaulSchemaRetryCount) { RetryPolicy policy; if (useRetry) { policy = new RetryPolicy.ExponentialDelayRetryPolicy( RetryPolicy.SqlAzureTemporaryErrorDetectionStrategy.Instance, retriesPerPhase, RetryPolicyDefaults.DefaultBackoffIntervalFactor, RetryPolicyDefaults.DefaultSchemaMinInterval, RetryPolicyDefaults.DefaultMaxRetryInterval); policy.FastFirstRetry = false; } else { policy = CreateNoRetryPolicy(); } return policy; } /// /// Creates and returns a default Retry Policy for Schema based connection operations. /// /// The RetryOccured event is wired to raise an RaiseAmbientRetryMessage message for a connection retry. /// An instance of class. public static RetryPolicy CreateSchemaConnectionRetryPolicy(int retriesPerPhase) { RetryPolicy policy = new RetryPolicy.ExponentialDelayRetryPolicy( RetryPolicy.SqlAzureTemporaryErrorDetectionStrategy.Instance, retriesPerPhase, RetryPolicyDefaults.DefaultBackoffIntervalFactor, RetryPolicyDefaults.DefaultSchemaMinInterval, RetryPolicyDefaults.DefaultMaxRetryInterval); policy.RetryOccurred += DataConnectionFailureRetry; return policy; } /// /// Creates and returns a default Retry Policy for Schema based command operations. /// /// The RetryOccured event is wired to raise an RaiseAmbientRetryMessage message for a command retry. /// An instance of class. public static RetryPolicy CreateSchemaCommandRetryPolicy(int retriesPerPhase) { RetryPolicy policy = new RetryPolicy.ExponentialDelayRetryPolicy( RetryPolicy.SqlAzureTemporaryErrorDetectionStrategy.Instance, retriesPerPhase, RetryPolicyDefaults.DefaultBackoffIntervalFactor, RetryPolicyDefaults.DefaultSchemaMinInterval, RetryPolicyDefaults.DefaultMaxRetryInterval); policy.FastFirstRetry = false; policy.RetryOccurred += CommandFailureRetry; return policy; } /// /// Creates and returns a Retry Policy for database creation operations. /// /// Errors to ignore if they occur after first retry /// /// The RetryOccured event is wired to raise an RaiseAmbientRetryMessage message for a command retry. /// The IgnoreErrorOccurred event is wired to raise an RaiseAmbientIgnoreMessage message for ignore. /// /// An instance of class. public static RetryPolicy CreateDatabaseCommandRetryPolicy(params int[] ignorableErrorNumbers) { RetryPolicy.SqlAzureTemporaryAndIgnorableErrorDetectionStrategy errorDetectionStrategy = new RetryPolicy.SqlAzureTemporaryAndIgnorableErrorDetectionStrategy(ignorableErrorNumbers); // 30, 60, 60, 60, 60 second retries RetryPolicy policy = new RetryPolicy.ExponentialDelayRetryPolicy( errorDetectionStrategy, RetryPolicyDefaults.DefaultCreateDatabaseRetryCount /* maxRetryCount */, RetryPolicyDefaults.DefaultBackoffIntervalFactor, TimeSpan.FromSeconds(30) /* minInterval */, TimeSpan.FromSeconds(60) /* maxInterval */); policy.FastFirstRetry = false; policy.RetryOccurred += CreateDatabaseCommandFailureRetry; policy.IgnoreErrorOccurred += CreateDatabaseCommandFailureIgnore; return policy; } /// /// Creates and returns an "ignoreable" command Retry Policy. /// /// Errors to ignore if they occur after first retry /// /// The RetryOccured event is wired to raise an RaiseAmbientRetryMessage message for a command retry. /// The IgnoreErrorOccurred event is wired to raise an RaiseAmbientIgnoreMessage message for ignore. /// /// An instance of class. public static RetryPolicy CreateElementCommandRetryPolicy(params int[] ignorableErrorNumbers) { Debug.Assert(ignorableErrorNumbers != null); RetryPolicy.SqlAzureTemporaryAndIgnorableErrorDetectionStrategy errorDetectionStrategy = new RetryPolicy.SqlAzureTemporaryAndIgnorableErrorDetectionStrategy(ignorableErrorNumbers); RetryPolicy policy = new RetryPolicy.ExponentialDelayRetryPolicy( errorDetectionStrategy, RetryPolicyDefaults.DefaulSchemaRetryCount, RetryPolicyDefaults.DefaultBackoffIntervalFactor, RetryPolicyDefaults.DefaultSchemaMinInterval, RetryPolicyDefaults.DefaultMaxRetryInterval); policy.FastFirstRetry = false; policy.RetryOccurred += ElementCommandFailureRetry; policy.IgnoreErrorOccurred += ElementCommandFailureIgnore; return policy; } /// /// Creates and returns an "primary key violation" command Retry Policy. /// /// Errors to ignore if they occur after first retry /// /// The RetryOccured event is wired to raise an RaiseAmbientRetryMessage message for a command retry. /// The IgnoreErrorOccurred event is wired to raise an RaiseAmbientIgnoreMessage message for ignore. /// /// An instance of class. public static RetryPolicy CreatePrimaryKeyCommandRetryPolicy() { RetryPolicy.SqlAzureTemporaryAndIgnorableErrorDetectionStrategy errorDetectionStrategy = new RetryPolicy.SqlAzureTemporaryAndIgnorableErrorDetectionStrategy(SqlErrorNumbers.PrimaryKeyViolationErrorNumber); RetryPolicy policy = new RetryPolicy.ExponentialDelayRetryPolicy( errorDetectionStrategy, RetryPolicyDefaults.DefaulSchemaRetryCount, RetryPolicyDefaults.DefaultBackoffIntervalFactor, RetryPolicyDefaults.DefaultSchemaMinInterval, RetryPolicyDefaults.DefaultMaxRetryInterval); policy.FastFirstRetry = true; policy.RetryOccurred += CommandFailureRetry; policy.IgnoreErrorOccurred += CommandFailureIgnore; return policy; } /// /// Creates a Policy that will never allow retries to occur. /// /// public static RetryPolicy CreateNoRetryPolicy() { return new RetryPolicy.FixedDelayPolicy(TransientErrorIgnoreStrategy.Instance, 0, TimeSpan.Zero); } /// /// Creates a Policy that is optimized for data-related script update operations. /// This is extremely error tolerant and uses a Time based delay policy that backs /// off until some overall length of delay has occurred. It is not as long-running /// as the ConnectionManager data transfer retry policy since that's intended for bulk upload /// of large amounts of data, whereas this is for individual batch scripts executed by the /// batch execution engine. /// /// public static RetryPolicy CreateDataScriptUpdateRetryPolicy() { return new RetryPolicy.TimeBasedRetryPolicy( RetryPolicy.DataTransferErrorDetectionStrategy.Instance, TimeSpan.FromMinutes(7), TimeSpan.FromMinutes(7), 0.1, TimeSpan.FromMilliseconds(250), TimeSpan.FromSeconds(30), 1.5); } /// /// Returns the default retry policy dedicated to handling exceptions with SQL connections /// /// The RetryPolicy policy public static RetryPolicy CreateFastDataRetryPolicy() { RetryPolicy retryPolicy = new RetryPolicy.FixedDelayPolicy( RetryPolicy.NetworkConnectivityErrorDetectionStrategy.Instance, RetryPolicyDefaults.DefaultDataCommandRetryCount, TimeSpan.FromMilliseconds(5)); retryPolicy.FastFirstRetry = true; retryPolicy.RetryOccurred += DataConnectionFailureRetry; return retryPolicy; } /// /// Returns the default retry policy dedicated to handling exceptions with SQL connections. /// No logging or other message handler is attached to the policy /// /// The RetryPolicy policy public static RetryPolicy CreateDefaultSchemaConnectionRetryPolicy() { return CreateDefaultConnectionRetryPolicy(); } /// /// Returns the default retry policy dedicated to handling exceptions with SQL connections. /// Adds an event handler to log and notify listeners of data connection retries /// /// The RetryPolicy policy public static RetryPolicy CreateDefaultDataConnectionRetryPolicy() { RetryPolicy retryPolicy = CreateDefaultConnectionRetryPolicy(); retryPolicy.RetryOccurred += DataConnectionFailureRetry; return retryPolicy; } /// /// Returns the default retry policy dedicated to handling exceptions with SQL connections /// /// The RetryPolicy policy public static RetryPolicy CreateDefaultConnectionRetryPolicy() { // Note: No longer use Ado.net Connection Pooling and hence do not need TimeBasedRetryPolicy to // conform to the backoff requirements in this case RetryPolicy retryPolicy = new RetryPolicy.ExponentialDelayRetryPolicy( RetryPolicy.NetworkConnectivityErrorDetectionStrategy.Instance, RetryPolicyDefaults.DefaultConnectionRetryCount, RetryPolicyDefaults.DefaultBackoffIntervalFactor, RetryPolicyDefaults.DefaultSchemaMinInterval, RetryPolicyDefaults.DefaultMaxRetryInterval); retryPolicy.FastFirstRetry = true; return retryPolicy; } /// /// Returns the default retry policy dedicated to handling retryable conditions with data transfer SQL commands. /// /// The RetryPolicy policy public static RetryPolicy CreateDefaultDataSqlCommandRetryPolicy() { RetryPolicy retryPolicy = new RetryPolicy.ExponentialDelayRetryPolicy( RetryPolicy.SqlAzureTemporaryErrorDetectionStrategy.Instance, RetryPolicyDefaults.DefaultDataCommandRetryCount, RetryPolicyDefaults.DefaultBackoffIntervalFactor, RetryPolicyDefaults.DefaultDataMinInterval, RetryPolicyDefaults.DefaultMaxRetryInterval); retryPolicy.FastFirstRetry = true; retryPolicy.RetryOccurred += CommandFailureRetry; return retryPolicy; } /// /// Returns the default retry policy dedicated to handling retryable conditions with data transfer SQL commands. /// /// The RetryPolicy policy public static RetryPolicy CreateDefaultDataTransferRetryPolicy() { RetryPolicy retryPolicy = new RetryPolicy.TimeBasedRetryPolicy( RetryPolicy.DataTransferErrorDetectionStrategy.Instance, TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(240), 0.1, TimeSpan.FromMilliseconds(250), TimeSpan.FromMinutes(2), 2); retryPolicy.FastFirstRetry = true; retryPolicy.RetryOccurred += CommandFailureRetry; return retryPolicy; } /// /// Returns the retry policy to handle data migration for column encryption. /// /// The RetryPolicy policy public static RetryPolicy CreateColumnEncryptionTransferRetryPolicy() { RetryPolicy retryPolicy = new RetryPolicy.TimeBasedRetryPolicy( RetryPolicy.DataTransferErrorDetectionStrategy.Instance, TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(5), 0.1, TimeSpan.FromMilliseconds(250), TimeSpan.FromMinutes(2), 2); retryPolicy.FastFirstRetry = true; retryPolicy.RetryOccurred += CommandFailureRetry; return retryPolicy; } public static void DataConnectionFailureRetry(RetryState retryState) { Logger.Write(TraceEventType.Information, string.Format(CultureInfo.InvariantCulture, "Connection retry number {0}. Delaying {1} ms before retry. Exception: {2}", retryState.RetryCount, retryState.Delay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture), retryState.LastError.ToString())); RetryPolicyUtils.RaiseAmbientRetryMessage(retryState, SqlSchemaModelErrorCodes.ServiceActions.ConnectionRetry); } public static void CommandFailureRetry(RetryState retryState, string commandKeyword) { Logger.Write(TraceEventType.Information, string.Format( CultureInfo.InvariantCulture, "{0} retry number {1}. Delaying {2} ms before retry. Exception: {3}", commandKeyword, retryState.RetryCount, retryState.Delay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture), retryState.LastError.ToString())); RetryPolicyUtils.RaiseAmbientRetryMessage(retryState, SqlSchemaModelErrorCodes.ServiceActions.CommandRetry); } public static void CommandFailureIgnore(RetryState retryState, string commandKeyword) { Logger.Write(TraceEventType.Information, string.Format( CultureInfo.InvariantCulture, "{0} retry number {1}. Ignoring failure. Exception: {2}", commandKeyword, retryState.RetryCount, retryState.LastError.ToString())); RetryPolicyUtils.RaiseAmbientIgnoreMessage(retryState, SqlSchemaModelErrorCodes.ServiceActions.CommandRetry); } public static void CommandFailureRetry(RetryState retryState) { CommandFailureRetry(retryState, "Command"); } public static void CommandFailureIgnore(RetryState retryState) { CommandFailureIgnore(retryState, "Command"); } public static void CreateDatabaseCommandFailureRetry(RetryState retryState) { CommandFailureRetry(retryState, "Database Command"); } public static void CreateDatabaseCommandFailureIgnore(RetryState retryState) { CommandFailureIgnore(retryState, "Database Command"); } public static void ElementCommandFailureRetry(RetryState retryState) { CommandFailureRetry(retryState, "Element Command"); } public static void ElementCommandFailureIgnore(RetryState retryState) { CommandFailureIgnore(retryState, "Element Command"); } } }