From a4fd3b03ff2df52e41b8122767edd30680760c02 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 7 May 2020 09:06:10 -0700 Subject: [PATCH] Added support for retrying SQL transient errors. (#22) --- THIRDPARTYNOTICES.md | 63 ++++++ scripts/Publish-LocalChanges.ps1 | 10 +- .../Extensions/SqlExceptionExtensionsTests.cs | 42 ++++ .../Client/RetrySqlCommandWrapperTests.cs | 145 +++++++++++++ ...erTransientFaultRetryPolicyFactoryTests.cs | 97 +++++++++ ...icrosoft.Health.SqlServer.UnitTests.csproj | 4 +- .../SqlExceptionFactory.cs | 123 +++++++++++ .../AssemblyInfo.cs | 1 + .../SqlServerDataStoreConfiguration.cs | 5 + ...rTransientFaultRetryPolicyConfiguration.cs | 35 ++++ .../Extensions/SqlExceptionExtensions.cs | 83 ++++++++ .../Client/IPollyRetryLoggerFactory.cs | 22 ++ ...lServerTransientFaultRetryPolicyFactory.cs | 21 ++ .../Client/PollyRetryLoggerFactory.cs | 44 ++++ .../Features/Client/RetrySqlCommandWrapper.cs | 49 +++++ .../Client/RetrySqlCommandWrapperFactory.cs | 34 ++++ .../Features/Client/SqlCommandWrapper.cs | 192 ++++++++++++++++++ .../Client/SqlCommandWrapperFactory.cs | 28 +++ .../Features/Client/SqlConnectionWrapper.cs | 14 +- .../Client/SqlConnectionWrapperFactory.cs | 10 +- ...lServerTransientFaultRetryPolicyFactory.cs | 54 +++++ .../Microsoft.Health.SqlServer.csproj | 4 +- .../SqlServerBaseRegistrationExtensions.cs | 15 ++ .../Sql/CreateProcedureVisitor.cs | 5 +- .../Sql/SqlModelGenerator.cs | 10 +- 25 files changed, 1092 insertions(+), 18 deletions(-) create mode 100644 src/Microsoft.Health.SqlServer.UnitTests/Extensions/SqlExceptionExtensionsTests.cs create mode 100644 src/Microsoft.Health.SqlServer.UnitTests/Features/Client/RetrySqlCommandWrapperTests.cs create mode 100644 src/Microsoft.Health.SqlServer.UnitTests/Features/Client/SqlServerTransientFaultRetryPolicyFactoryTests.cs create mode 100644 src/Microsoft.Health.SqlServer.UnitTests/SqlExceptionFactory.cs create mode 100644 src/Microsoft.Health.SqlServer/Configs/SqlServerTransientFaultRetryPolicyConfiguration.cs create mode 100644 src/Microsoft.Health.SqlServer/Extensions/SqlExceptionExtensions.cs create mode 100644 src/Microsoft.Health.SqlServer/Features/Client/IPollyRetryLoggerFactory.cs create mode 100644 src/Microsoft.Health.SqlServer/Features/Client/ISqlServerTransientFaultRetryPolicyFactory.cs create mode 100644 src/Microsoft.Health.SqlServer/Features/Client/PollyRetryLoggerFactory.cs create mode 100644 src/Microsoft.Health.SqlServer/Features/Client/RetrySqlCommandWrapper.cs create mode 100644 src/Microsoft.Health.SqlServer/Features/Client/RetrySqlCommandWrapperFactory.cs create mode 100644 src/Microsoft.Health.SqlServer/Features/Client/SqlCommandWrapper.cs create mode 100644 src/Microsoft.Health.SqlServer/Features/Client/SqlCommandWrapperFactory.cs create mode 100644 src/Microsoft.Health.SqlServer/Features/Client/SqlServerTransientFaultRetryPolicyFactory.cs diff --git a/THIRDPARTYNOTICES.md b/THIRDPARTYNOTICES.md index 3254bd23..e1811812 100644 --- a/THIRDPARTYNOTICES.md +++ b/THIRDPARTYNOTICES.md @@ -81,6 +81,69 @@ This file is based on or incorporates material from the projects listed below (T > > [ http://www.opensource.org/licenses/bsd-license.php ]> +## Polly 7.2.0 +* Component Source: https://github.com/App-vNext/Polly +* Component Copyright and License: + > New BSD License + > = + > Copyright (c) 2015-2018, App vNext + > All rights reserved. + > + > Redistribution and use in source and binary forms, with or without + > modification, are permitted provided that the following conditions are met: + > * Redistributions of source code must retain the above copyright + > notice, this list of conditions and the following disclaimer. + > * Redistributions in binary form must reproduce the above copyright + > notice, this list of conditions and the following disclaimer in the + > documentation and/or other materials provided with the distribution. + > * Neither the name of App vNext nor the + > names of its contributors may be used to endorse or promote products + > derived from this software without specific prior written permission. + > + > THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + > ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + > WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + > DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + > DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + > (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + > LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + > ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + > (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + > SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +## Polly.Contrib.WaitAndRetry +* Component Source: https://github.com/Polly-Contrib/Polly.Contrib.WaitAndRetry +* Component Copyright and License: + > BSD 3-Clause License + > + > Copyright (c) 2019, Polly.Contrib + > All rights reserved. + > + > Redistribution and use in source and binary forms, with or without + > modification, are permitted provided that the following conditions are met: + > + > * Redistributions of source code must retain the above copyright notice, this + > list of conditions and the following disclaimer. + > + > * Redistributions in binary form must reproduce the above copyright notice, + > this list of conditions and the following disclaimer in the documentation + > and/or other materials provided with the distribution. + > + > * Neither the name of the copyright holder nor the names of its + > contributors may be used to endorse or promote products derived from + > this software without specific prior written permission. + > + > THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + > AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + > IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + > DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + > FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + > DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + > SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + > CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + > OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + > OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ## StyleCop.Analyzers 1.1.118 * Component Source: https://github.com/DotNetAnalyzers/StyleCopAnalyzers/ * Component Copyright and License: diff --git a/scripts/Publish-LocalChanges.ps1 b/scripts/Publish-LocalChanges.ps1 index 1efeb13e..346bfcc9 100644 --- a/scripts/Publish-LocalChanges.ps1 +++ b/scripts/Publish-LocalChanges.ps1 @@ -90,7 +90,7 @@ Push-Location $RepoRootPath try { $branchName = &git rev-parse --abbrev-ref HEAD $branchNameParts = $branchName.Split("/") - $versionSuffix = "$($branchNameParts[$branchNameParts.Length - 1])-$(Get-Date -Format yyyyMMdd-HHmmss)-preview" + $version = "1.0.0-$($branchNameParts[$branchNameParts.Length - 1])-$(Get-Date -Format yyyyMMdd-HHmmss)-preview" # Find all projects in the components. $projects = Get-ChildItem -Recurse -Include *.csproj -Exclude *Tests.csproj $RepoRootPath @@ -100,10 +100,10 @@ try { $projects | ForEach-Object { if ($SkipBuild) { - &dotnet pack -o $tempPath --no-build --version-suffix $versionSuffix --include-symbols $_.FullName + &dotnet pack -o $tempPath --no-build --include-symbols $_.FullName /p:PackageVersion=$version } else { - &dotnet pack -o $tempPath --version-suffix $versionSuffix --include-symbols $_.FullName + &dotnet pack -o $tempPath --include-symbols $_.FullName /p:PackageVersion=$version } } @@ -142,8 +142,8 @@ try { $project = [xml](Get-Content $_) $project.SelectNodes("Project/ItemGroup/PackageReference") | - Where-Object { $_.Include -match "Microsoft.Health.*" -and $_.Include -ne "Microsoft.Health.Extensions.BuildTimeCodeGenerator" } | - ForEach-Object { $_.Version = "1.0.0-$versionSuffix" } + Where-Object { $_.Include -match "Microsoft.Health.*" } | + ForEach-Object { $_.Version = $version } $writer = New-Object System.IO.StreamWriter($_, $false, $utf8WithBom) diff --git a/src/Microsoft.Health.SqlServer.UnitTests/Extensions/SqlExceptionExtensionsTests.cs b/src/Microsoft.Health.SqlServer.UnitTests/Extensions/SqlExceptionExtensionsTests.cs new file mode 100644 index 00000000..57fc52dd --- /dev/null +++ b/src/Microsoft.Health.SqlServer.UnitTests/Extensions/SqlExceptionExtensionsTests.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Data.SqlClient; +using Microsoft.Health.SqlServer.Extensions; +using Xunit; + +namespace Microsoft.Health.SqlServer.UnitTests.Extensions +{ + public class SqlExceptionExtensionsTests + { + [Theory] + [InlineData(10928)] + [InlineData(10929)] + [InlineData(10053)] + [InlineData(10054)] + [InlineData(10060)] + [InlineData(18401)] + [InlineData(40197)] + [InlineData(40540)] + [InlineData(40613)] + [InlineData(40143)] + [InlineData(233)] + [InlineData(64)] + public void GivenATransientException_WhenCheckedIfExceptionIsTransient_ThenTrueShouldBeReturned(int number) + { + SqlException sqlException = SqlExceptionFactory.Create(number); + + Assert.True(sqlException.IsTransient()); + } + + [Fact] + public void GivenANonTransientException_WhenCheckedIfExceptionIsTransient_ThenFalseShouldBeReturned() + { + SqlException sqlException = SqlExceptionFactory.Create(10001); + + Assert.False(sqlException.IsTransient()); + } + } +} diff --git a/src/Microsoft.Health.SqlServer.UnitTests/Features/Client/RetrySqlCommandWrapperTests.cs b/src/Microsoft.Health.SqlServer.UnitTests/Features/Client/RetrySqlCommandWrapperTests.cs new file mode 100644 index 00000000..5a6d53bc --- /dev/null +++ b/src/Microsoft.Health.SqlServer.UnitTests/Features/Client/RetrySqlCommandWrapperTests.cs @@ -0,0 +1,145 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Data; +using System.Data.SqlClient; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.SqlServer.Extensions; +using Microsoft.Health.SqlServer.Features.Client; +using NSubstitute; +using NSubstitute.ExceptionExtensions; +using Polly; +using Xunit; + +namespace Microsoft.Health.SqlServer.UnitTests.Features.Client +{ + public class RetrySqlCommandWrapperTests + { + private static readonly CancellationToken DefaultCancellationToken = new CancellationTokenSource().Token; + + private readonly SqlCommandWrapper _sqlCommandWrapper = Substitute.For(new SqlCommand()); + private readonly IAsyncPolicy _asyncPolicy = Policy + .Handle(sqlException => sqlException.IsTransient()) + .WaitAndRetryAsync(new TimeSpan[] { TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero }); + + private readonly RetrySqlCommandWrapper _retrySqlCommandWrapper; + + public RetrySqlCommandWrapperTests() + { + _retrySqlCommandWrapper = new RetrySqlCommandWrapper(_sqlCommandWrapper, _asyncPolicy); + } + + [Fact] + public async Task GivenATransientException_WhenNonQueryIsExecuted_ThenItShouldRetry() + { + _sqlCommandWrapper.ExecuteNonQueryAsync(DefaultCancellationToken).Throws(CreateTransientException()); + + await ExecuteAndValidateExecuteNonQueryAsync(4); + } + + [Fact] + public async Task GivenANonTransientException_WhenNonQueryIsExecuted_ThenItShouldNotRetry() + { + _sqlCommandWrapper.ExecuteNonQueryAsync(DefaultCancellationToken).Returns( + _ => throw CreateTransientException(), + _ => throw CreateNonTransientException()); + + await ExecuteAndValidateExecuteNonQueryAsync(2); + } + + private async Task ExecuteAndValidateExecuteNonQueryAsync(int expectedNumberOfCalls) + { + await Assert.ThrowsAsync(() => _retrySqlCommandWrapper.ExecuteNonQueryAsync(DefaultCancellationToken)); + + await _sqlCommandWrapper.Received(expectedNumberOfCalls).ExecuteNonQueryAsync(DefaultCancellationToken); + } + + [Fact] + public async Task GivenATransientException_WhenReaderIsExecuted_ThenItShouldRetry() + { + _sqlCommandWrapper.ExecuteReaderAsync(DefaultCancellationToken).Throws(CreateTransientException()); + + await ExecuteAndValidateExecuteReaderAsync(4); + } + + [Fact] + public async Task GivenANonTransientException_WhenReaderIsExecuted_ThenItShouldNotRetry() + { + _sqlCommandWrapper.ExecuteReaderAsync(DefaultCancellationToken).Returns( + _ => throw CreateNonTransientException()); + + await ExecuteAndValidateExecuteReaderAsync(1); + } + + private async Task ExecuteAndValidateExecuteReaderAsync(int expectedNumberOfCalls) + { + await Assert.ThrowsAsync(() => _retrySqlCommandWrapper.ExecuteReaderAsync(DefaultCancellationToken)); + + await _sqlCommandWrapper.Received(expectedNumberOfCalls).ExecuteReaderAsync(DefaultCancellationToken); + } + + [Fact] + public async Task GivenATransientException_WhenReaderWithBehaviorIsExecuted_ThenItShouldRetry() + { + CommandBehavior behavior = CommandBehavior.SingleRow; + + _sqlCommandWrapper.ExecuteReaderAsync(behavior, DefaultCancellationToken).Throws(CreateTransientException()); + + await ExecuteAndValidateExecuteReaderAsync(behavior, 4); + } + + [Fact] + public async Task GivenANonTransientException_WhenReaderWithBehaviorIsExecuted_ThenItShouldNotRetry() + { + CommandBehavior behavior = CommandBehavior.SchemaOnly; + + _sqlCommandWrapper.ExecuteReaderAsync(behavior, DefaultCancellationToken).Returns( + _ => throw CreateNonTransientException()); + + await ExecuteAndValidateExecuteReaderAsync(behavior, 1); + } + + private async Task ExecuteAndValidateExecuteReaderAsync(CommandBehavior behavior, int expectedNumberOfCalls) + { + await Assert.ThrowsAsync(() => _retrySqlCommandWrapper.ExecuteReaderAsync(behavior, DefaultCancellationToken)); + + await _sqlCommandWrapper.Received(expectedNumberOfCalls).ExecuteReaderAsync(behavior, DefaultCancellationToken); + } + + [Fact] + public async Task GivenATransientException_WhenScalarIsExecuted_ThenItShouldRetry() + { + _sqlCommandWrapper.ExecuteScalarAsync(DefaultCancellationToken).Throws(CreateTransientException()); + + await ExecuteAndValidateExecuteScalarAsync(4); + } + + [Fact] + public async Task GivenANonTransientException_WhenScalarIsExecuted_ThenItShouldNotRetry() + { + _sqlCommandWrapper.ExecuteScalarAsync(DefaultCancellationToken).Returns( + _ => throw CreateTransientException(), + _ => throw CreateTransientException(), + _ => throw CreateNonTransientException()); + + await ExecuteAndValidateExecuteScalarAsync(3); + } + + private async Task ExecuteAndValidateExecuteScalarAsync(int expectedNumberOfCalls) + { + await Assert.ThrowsAsync(() => _retrySqlCommandWrapper.ExecuteScalarAsync(DefaultCancellationToken)); + + await _sqlCommandWrapper.Received(expectedNumberOfCalls).ExecuteScalarAsync(DefaultCancellationToken); + } + + private static SqlException CreateTransientException() + => SqlExceptionFactory.Create(10928); + + private static SqlException CreateNonTransientException() + => SqlExceptionFactory.Create(50404); + } +} diff --git a/src/Microsoft.Health.SqlServer.UnitTests/Features/Client/SqlServerTransientFaultRetryPolicyFactoryTests.cs b/src/Microsoft.Health.SqlServer.UnitTests/Features/Client/SqlServerTransientFaultRetryPolicyFactoryTests.cs new file mode 100644 index 00000000..a8c1cd47 --- /dev/null +++ b/src/Microsoft.Health.SqlServer.UnitTests/Features/Client/SqlServerTransientFaultRetryPolicyFactoryTests.cs @@ -0,0 +1,97 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Data.SqlClient; +using System.Threading.Tasks; +using Microsoft.Health.SqlServer.Configs; +using Microsoft.Health.SqlServer.Features.Client; +using NSubstitute; +using Polly; +using Xunit; + +namespace Microsoft.Health.SqlServer.UnitTests.Features.Client +{ + public class SqlServerTransientFaultRetryPolicyFactoryTests + { + private readonly SqlServerDataStoreConfiguration _sqlServerDataStoreConfiguration = new SqlServerDataStoreConfiguration(); + private readonly IPollyRetryLoggerFactory _pollyRetryLoggerFactory = Substitute.For(); + + private readonly SqlServerTransientFaultRetryPolicyFactory _sqlServerTransientFaultRetryPolicyFactory; + + private readonly IAsyncPolicy _asyncPolicy; + private readonly List _capturedRetries = new List(); + + public SqlServerTransientFaultRetryPolicyFactoryTests() + { + _sqlServerDataStoreConfiguration.TransientFaultRetryPolicy = new SqlServerTransientFaultRetryPolicyConfiguration() + { + InitialDelay = TimeSpan.FromMilliseconds(200), + RetryCount = 3, + Factor = 3, + FastFirst = true, + }; + + Action onRetryCapture = (exception, sleepDuration, retryCount, _) => + { + _capturedRetries.Add(sleepDuration); + }; + + _pollyRetryLoggerFactory.Create().Returns(onRetryCapture); + + _sqlServerTransientFaultRetryPolicyFactory = new SqlServerTransientFaultRetryPolicyFactory( + _sqlServerDataStoreConfiguration, + _pollyRetryLoggerFactory); + + _asyncPolicy = _sqlServerTransientFaultRetryPolicyFactory.Create(); + } + + [Fact] + public async Task GivenATransientException_WhenRetryPolicyIsUsed_ThenItShouldRetry() + { + await Assert.ThrowsAsync(() => + _asyncPolicy.ExecuteAsync(() => Task.Run(() => throw SqlExceptionFactory.CreateTransientException()))); + + ValidateCapturedRetries(); + } + + [Fact] + public async Task GivenANonTransientException_WhenRetryPolicyIsUsed_ThenItShouldNotRetry() + { + await Assert.ThrowsAsync(() => + _asyncPolicy.ExecuteAsync(() => Task.Run(() => throw SqlExceptionFactory.CreateNonTransientException()))); + + Assert.Empty(_capturedRetries); + } + + [Fact] + public async Task GivenATimeoutException_WhenRetryPolicyIsUsed_ThenItShouldRetry() + { + await Assert.ThrowsAsync(() => + _asyncPolicy.ExecuteAsync(() => Task.Run(() => throw new TimeoutException()))); + + ValidateCapturedRetries(); + } + + [Fact] + public async Task GivenOtherException_WhenRetryPolicyIsUsed_ThenItShouldNotRetry() + { + await Assert.ThrowsAsync(() => + _asyncPolicy.ExecuteAsync(() => Task.Run(() => throw new Exception()))); + + Assert.Empty(_capturedRetries); + } + + private void ValidateCapturedRetries() + { + Assert.Collection( + _capturedRetries, + item => Assert.Equal(TimeSpan.Zero, item), + item => Assert.Equal(TimeSpan.FromMilliseconds(200), item), + item => Assert.Equal(TimeSpan.FromMilliseconds(600), item)); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.Health.SqlServer.UnitTests/Microsoft.Health.SqlServer.UnitTests.csproj b/src/Microsoft.Health.SqlServer.UnitTests/Microsoft.Health.SqlServer.UnitTests.csproj index c7c23a19..5b4faaf1 100644 --- a/src/Microsoft.Health.SqlServer.UnitTests/Microsoft.Health.SqlServer.UnitTests.csproj +++ b/src/Microsoft.Health.SqlServer.UnitTests/Microsoft.Health.SqlServer.UnitTests.csproj @@ -1,4 +1,4 @@ - + netcoreapp3.1 @@ -9,6 +9,8 @@ + + diff --git a/src/Microsoft.Health.SqlServer.UnitTests/SqlExceptionFactory.cs b/src/Microsoft.Health.SqlServer.UnitTests/SqlExceptionFactory.cs new file mode 100644 index 00000000..c2665f80 --- /dev/null +++ b/src/Microsoft.Health.SqlServer.UnitTests/SqlExceptionFactory.cs @@ -0,0 +1,123 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Data.SqlClient; +using System.Linq.Expressions; +using System.Reflection; + +namespace Microsoft.Health.SqlServer.UnitTests +{ + /// + /// Provides functionality to create an instance of . + /// + /// + /// Because SqlException is marked as sealed with no public constructor, we need to + /// use reflection to be able to create an instance of it for unit testing. + /// + public static class SqlExceptionFactory + { + private static readonly Func SqlErrorCollectionConstructorDelegate; + private static readonly Action SqlErrorCollectionAddDelegate; + private static readonly Func SqlErrorConstructorDelegate; + private static readonly Func SqlExceptionConstructorDelegate; + + static SqlExceptionFactory() + { + // () => new SqlErrorCollection(); + ConstructorInfo constructorInfo = typeof(SqlErrorCollection).GetConstructor( + BindingFlags.NonPublic | BindingFlags.Instance, + binder: null, + types: new Type[0], + modifiers: null); + + SqlErrorCollectionConstructorDelegate = Expression.Lambda>( + Expression.New(constructorInfo)).Compile(); + + // (sqlErrorCollection, sqlError) => sqlErrorCollection.Add(SqlErrorCollection.Add(sqlError)); + MethodInfo methodInfo = typeof(SqlErrorCollection).GetMethod( + "Add", + BindingFlags.NonPublic | BindingFlags.Instance, + binder: null, + types: new Type[] { typeof(SqlError) }, + modifiers: null); + + ParameterExpression sqlErrorCollection = Expression.Parameter(typeof(SqlErrorCollection), "sqlErrorCollection"); + ParameterExpression sqlError = Expression.Parameter(typeof(SqlError), "sqlError"); + + SqlErrorCollectionAddDelegate = Expression.Lambda>( + Expression.Call(sqlErrorCollection, methodInfo, sqlError), + new[] { sqlErrorCollection, sqlError }).Compile(); + + // infoNumber => new SqlError(infoNumber, (byte)0, (byte)0, null, null, null, 0, null); + constructorInfo = typeof(SqlError).GetConstructor( + BindingFlags.NonPublic | BindingFlags.Instance, + binder: null, + types: new[] { typeof(int), typeof(byte), typeof(byte), typeof(string), typeof(string), typeof(string), typeof(int), typeof(Exception) }, + modifiers: null); + + ParameterExpression infoNumber = Expression.Parameter(typeof(int), "infoNumber"); + ConstantExpression zeroByte = Expression.Constant((byte)0, typeof(byte)); + ConstantExpression nullString = Expression.Constant(null, typeof(string)); + + SqlErrorConstructorDelegate = Expression.Lambda>( + Expression.New( + constructorInfo, + infoNumber, + zeroByte, + zeroByte, + nullString, + nullString, + nullString, + Expression.Constant(0, typeof(int)), + Expression.Constant(null, typeof(Exception))), + infoNumber).Compile(); + + // (message, errorCollection, innerException, clientConnectionId) => new SqlException(message, errorCollection, innerException, clientConnectionId); + constructorInfo = typeof(SqlException).GetConstructor( + BindingFlags.NonPublic | BindingFlags.Instance, + binder: null, + types: new[] { typeof(string), typeof(SqlErrorCollection), typeof(Exception), typeof(Guid) }, + modifiers: null); + + ParameterExpression message = Expression.Parameter(typeof(string), "message"); + ParameterExpression errorCollection = Expression.Parameter(typeof(SqlErrorCollection), "errorCollection"); + ParameterExpression innerException = Expression.Parameter(typeof(Exception), "innerException"); + ParameterExpression clientConnectionId = Expression.Parameter(typeof(Guid), "clientConnectionId"); + + SqlExceptionConstructorDelegate = Expression.Lambda>( + Expression.New(constructorInfo, message, errorCollection, innerException, clientConnectionId), + message, + errorCollection, + innerException, + clientConnectionId).Compile(); + } + + /// + /// Creates a new instance of . + /// + /// The info number. + /// The error message. + /// An instance of . + public static SqlException Create(int number, string errorMessage = "Simulated exception.") + { + SqlError sqlError = SqlErrorConstructorDelegate(number); + + SqlErrorCollection sqlErrorCollection = SqlErrorCollectionConstructorDelegate(); + + SqlErrorCollectionAddDelegate(sqlErrorCollection, sqlError); + + SqlException sqlException = SqlExceptionConstructorDelegate(errorMessage, sqlErrorCollection, null, Guid.NewGuid()); + + return sqlException; + } + + public static SqlException CreateTransientException() + => Create(10060); + + public static SqlException CreateNonTransientException() + => Create(50404); + } +} diff --git a/src/Microsoft.Health.SqlServer/AssemblyInfo.cs b/src/Microsoft.Health.SqlServer/AssemblyInfo.cs index e1fee0e4..0d4a04de 100644 --- a/src/Microsoft.Health.SqlServer/AssemblyInfo.cs +++ b/src/Microsoft.Health.SqlServer/AssemblyInfo.cs @@ -7,5 +7,6 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.Health.SqlServer.UnitTests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] [assembly: NeutralResourcesLanguage("en-us")] diff --git a/src/Microsoft.Health.SqlServer/Configs/SqlServerDataStoreConfiguration.cs b/src/Microsoft.Health.SqlServer/Configs/SqlServerDataStoreConfiguration.cs index f3ca08d6..68388de7 100644 --- a/src/Microsoft.Health.SqlServer/Configs/SqlServerDataStoreConfiguration.cs +++ b/src/Microsoft.Health.SqlServer/Configs/SqlServerDataStoreConfiguration.cs @@ -24,5 +24,10 @@ public class SqlServerDataStoreConfiguration /// Allows the experimental schema initializer to attempt to create the database if not present. /// public bool AllowDatabaseCreation { get; set; } + + /// + /// Configuration for transient fault retry policy. + /// + public SqlServerTransientFaultRetryPolicyConfiguration TransientFaultRetryPolicy { get; set; } = new SqlServerTransientFaultRetryPolicyConfiguration(); } } diff --git a/src/Microsoft.Health.SqlServer/Configs/SqlServerTransientFaultRetryPolicyConfiguration.cs b/src/Microsoft.Health.SqlServer/Configs/SqlServerTransientFaultRetryPolicyConfiguration.cs new file mode 100644 index 00000000..b9318ec4 --- /dev/null +++ b/src/Microsoft.Health.SqlServer/Configs/SqlServerTransientFaultRetryPolicyConfiguration.cs @@ -0,0 +1,35 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.SqlServer.Configs +{ + /// + /// Configuration for transient fault retry policy. + /// + public class SqlServerTransientFaultRetryPolicyConfiguration + { + /// + /// The duration value for the wait before the first retry. + /// + public TimeSpan InitialDelay { get; set; } = TimeSpan.FromMilliseconds(100); + + /// + /// The maximum number of retries to use, in addition to the original call. + /// + public int RetryCount { get; set; } = 3; + + /// + /// The exponent to multiply each subsequent duration by. + /// + public int Factor { get; set; } = 2; + + /// + /// Whether the first retry will be immediate or not. + /// + public bool FastFirst { get; set; } = true; + } +} diff --git a/src/Microsoft.Health.SqlServer/Extensions/SqlExceptionExtensions.cs b/src/Microsoft.Health.SqlServer/Extensions/SqlExceptionExtensions.cs new file mode 100644 index 00000000..eec0a4c9 --- /dev/null +++ b/src/Microsoft.Health.SqlServer/Extensions/SqlExceptionExtensions.cs @@ -0,0 +1,83 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Data.SqlClient; + +namespace Microsoft.Health.SqlServer.Extensions +{ + public static class SqlExceptionExtensions + { + /// + /// Determines whether the exception is transient. + /// + /// The exception to check. + /// true if the exception is transient; otherwise, false. + /// Inspired by https://github.com/Azure/elastic-db-tools/blob/master/Src/ElasticScale.Client/ElasticScale.Common/TransientFaultHandling/Implementation/SqlDatabaseTransientErrorDetectionStrategy.cs. + /// + public static bool IsTransient(this SqlException exception) + { + // Enumerate through all errors found in the exception. + foreach (SqlError err in exception.Errors) + { + switch (err.Number) + { + // SQL Error Code: 10928 + // Resource ID: %d. The %s limit for the database is %d and has been reached. + case 10928: + // SQL Error Code: 10929 + // Resource ID: %d. The %s minimum guarantee is %d, maximum limit is %d and the current usage for the database is %d. + // However, the server is currently too busy to support requests greater than %d for this database. + case 10929: + // SQL Error Code: 10053 + // A transport-level error has occurred when receiving results from the server. + // An established connection was aborted by the software in your host machine. + case 10053: + // SQL Error Code: 10054 + // A transport-level error has occurred when sending the request to the server. + // (provider: TCP Provider, error: 0 - An existing connection was forcibly closed by the remote host.) + case 10054: + // SQL Error Code: 10060 + // A network-related or instance-specific error occurred while establishing a connection to SQL Server. + // The server was not found or was not accessible. Verify that the instance name is correct and that SQL Server + // is configured to allow remote connections. (provider: TCP Provider, error: 0 - A connection attempt failed + // because the connected party did not properly respond after a period of time, or established connection failed + // because connected host has failed to respond.)"} + case 10060: + // SQL Error Code: 18401 + // Login failed for user '%s'. Reason: Server is in script upgrade mode. Only administrator can connect at this time. + // Devnote: this can happen when SQL is going through recovery (e.g. after failover) + case 18401: + // SQL Error Code: 40197 + // The service has encountered an error processing your request. Please try again. + case 40197: + // SQL Error Code: 40540 + // The service has encountered an error processing your request. Please try again. + case 40540: + // SQL Error Code: 40613 + // Database XXXX on server YYYY is not currently available. Please retry the connection later. If the problem persists, contact customer + // support, and provide them the session tracing ID of ZZZZZ. + case 40613: + // SQL Error Code: 40143 + // The service has encountered an error processing your request. Please try again. + case 40143: + // SQL Error Code: 233 + // The client was unable to establish a connection because of an error during connection initialization process before login. + // Possible causes include the following: the client tried to connect to an unsupported version of SQL Server; the server was too busy + // to accept new connections; or there was a resource limitation (insufficient memory or maximum allowed connections) on the server. + // (provider: TCP Provider, error: 0 - An existing connection was forcibly closed by the remote host.) + case 233: + // SQL Error Code: 64 + // A connection was successfully established with the server, but then an error occurred during the login process. + // (provider: TCP Provider, error: 0 - The specified network name is no longer available.) + case 64: + return true; + } + } + + return false; + } + } +} diff --git a/src/Microsoft.Health.SqlServer/Features/Client/IPollyRetryLoggerFactory.cs b/src/Microsoft.Health.SqlServer/Features/Client/IPollyRetryLoggerFactory.cs new file mode 100644 index 00000000..6c991bc6 --- /dev/null +++ b/src/Microsoft.Health.SqlServer/Features/Client/IPollyRetryLoggerFactory.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Polly; + +namespace Microsoft.Health.SqlServer.Features.Client +{ + /// + /// Provides functionality for creating a logger for the retry policy. + /// + internal interface IPollyRetryLoggerFactory + { + /// + /// Creates a logger. + /// + /// A logger delegate. + Action Create(); + } +} diff --git a/src/Microsoft.Health.SqlServer/Features/Client/ISqlServerTransientFaultRetryPolicyFactory.cs b/src/Microsoft.Health.SqlServer/Features/Client/ISqlServerTransientFaultRetryPolicyFactory.cs new file mode 100644 index 00000000..8b075d32 --- /dev/null +++ b/src/Microsoft.Health.SqlServer/Features/Client/ISqlServerTransientFaultRetryPolicyFactory.cs @@ -0,0 +1,21 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Polly; + +namespace Microsoft.Health.SqlServer.Features.Client +{ + /// + /// Provides functionality to create retry policy for handling transient errors. + /// + public interface ISqlServerTransientFaultRetryPolicyFactory + { + /// + /// Creates a retry policy. + /// + /// A object. + IAsyncPolicy Create(); + } +} diff --git a/src/Microsoft.Health.SqlServer/Features/Client/PollyRetryLoggerFactory.cs b/src/Microsoft.Health.SqlServer/Features/Client/PollyRetryLoggerFactory.cs new file mode 100644 index 00000000..555e8cf2 --- /dev/null +++ b/src/Microsoft.Health.SqlServer/Features/Client/PollyRetryLoggerFactory.cs @@ -0,0 +1,44 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using EnsureThat; +using Microsoft.Extensions.Logging; +using Polly; + +namespace Microsoft.Health.SqlServer.Features.Client +{ + /// + /// Provides functionality for creating a logger for the retry policy. + /// + internal class PollyRetryLoggerFactory : IPollyRetryLoggerFactory + { + private static readonly Action LogRetryDelegate = + LoggerMessage.Define( + LogLevel.Warning, + default, + "The operation failed. Will retry in '{SleepDuration}'. Retried {RetryCount} of time(s) so far."); + + private readonly ILoggerFactory _loggerFactory; + + public PollyRetryLoggerFactory(ILoggerFactory loggerFactory) + { + EnsureArg.IsNotNull(loggerFactory, nameof(loggerFactory)); + + _loggerFactory = loggerFactory; + } + + /// + public Action Create() + { + ILogger logger = _loggerFactory.CreateLogger("PollyRetryLogger"); + + return (exception, sleepDuration, retryCount, context) => + { + LogRetryDelegate(logger, sleepDuration, retryCount, exception); + }; + } + } +} diff --git a/src/Microsoft.Health.SqlServer/Features/Client/RetrySqlCommandWrapper.cs b/src/Microsoft.Health.SqlServer/Features/Client/RetrySqlCommandWrapper.cs new file mode 100644 index 00000000..7269292f --- /dev/null +++ b/src/Microsoft.Health.SqlServer/Features/Client/RetrySqlCommandWrapper.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Data; +using System.Data.SqlClient; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Polly; + +namespace Microsoft.Health.SqlServer.Features.Client +{ + /// + /// A wrapper around to provide automatic retries for transient errors. + /// + internal class RetrySqlCommandWrapper : SqlCommandWrapper + { + private readonly SqlCommandWrapper _sqlCommandWrapper; + private readonly IAsyncPolicy _retryPolicy; + + public RetrySqlCommandWrapper(SqlCommandWrapper sqlCommandWrapper, IAsyncPolicy retryPolicy) + : base(sqlCommandWrapper) + { + EnsureArg.IsNotNull(sqlCommandWrapper, nameof(sqlCommandWrapper)); + EnsureArg.IsNotNull(retryPolicy, nameof(retryPolicy)); + + _sqlCommandWrapper = sqlCommandWrapper; + _retryPolicy = retryPolicy; + } + + /// + public override Task ExecuteNonQueryAsync(CancellationToken cancellationToken) + => _retryPolicy.ExecuteAsync(() => _sqlCommandWrapper.ExecuteNonQueryAsync(cancellationToken)); + + /// + public override Task ExecuteScalarAsync(CancellationToken cancellationToken) + => _retryPolicy.ExecuteAsync(() => _sqlCommandWrapper.ExecuteScalarAsync(cancellationToken)); + + /// + public override Task ExecuteReaderAsync(CancellationToken cancellationToken) + => _retryPolicy.ExecuteAsync(() => _sqlCommandWrapper.ExecuteReaderAsync(cancellationToken)); + + /// + public override Task ExecuteReaderAsync(CommandBehavior behavior, CancellationToken cancellationToken) + => _retryPolicy.ExecuteAsync(() => _sqlCommandWrapper.ExecuteReaderAsync(behavior, cancellationToken)); + } +} diff --git a/src/Microsoft.Health.SqlServer/Features/Client/RetrySqlCommandWrapperFactory.cs b/src/Microsoft.Health.SqlServer/Features/Client/RetrySqlCommandWrapperFactory.cs new file mode 100644 index 00000000..7abca42b --- /dev/null +++ b/src/Microsoft.Health.SqlServer/Features/Client/RetrySqlCommandWrapperFactory.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Data.SqlClient; +using EnsureThat; +using Polly; + +namespace Microsoft.Health.SqlServer.Features.Client +{ + /// + /// Provides functionality to create an instance of . + /// + internal class RetrySqlCommandWrapperFactory : SqlCommandWrapperFactory + { + private readonly IAsyncPolicy _retryPolicy; + + public RetrySqlCommandWrapperFactory(ISqlServerTransientFaultRetryPolicyFactory sqlTransientFaultRetryPolicyFactory) + { + EnsureArg.IsNotNull(sqlTransientFaultRetryPolicyFactory, nameof(sqlTransientFaultRetryPolicyFactory)); + + _retryPolicy = sqlTransientFaultRetryPolicyFactory.Create(); + } + + /// + public override SqlCommandWrapper Create(SqlCommand sqlCommand) + { + return new RetrySqlCommandWrapper( + base.Create(sqlCommand), + _retryPolicy); + } + } +} diff --git a/src/Microsoft.Health.SqlServer/Features/Client/SqlCommandWrapper.cs b/src/Microsoft.Health.SqlServer/Features/Client/SqlCommandWrapper.cs new file mode 100644 index 00000000..a6504353 --- /dev/null +++ b/src/Microsoft.Health.SqlServer/Features/Client/SqlCommandWrapper.cs @@ -0,0 +1,192 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Data; +using System.Data.Sql; +using System.Data.SqlClient; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; + +namespace Microsoft.Health.SqlServer.Features.Client +{ + /// + /// A wrapper around to allow extensibility. + /// + public class SqlCommandWrapper : IDisposable + { + private readonly SqlCommand _sqlCommand; + + public SqlCommandWrapper(SqlCommand sqlCommand) + { + EnsureArg.IsNotNull(sqlCommand, nameof(sqlCommand)); + + _sqlCommand = sqlCommand; + } + + protected SqlCommandWrapper(SqlCommandWrapper sqlCommandWrapper) + { + EnsureArg.IsNotNull(sqlCommandWrapper, nameof(sqlCommandWrapper)); + + _sqlCommand = sqlCommandWrapper._sqlCommand; + } + + /// + /// . + /// + + public CommandType CommandType + { + get => _sqlCommand.CommandType; + set => _sqlCommand.CommandType = value; + } + + /// + /// . + /// + + public int CommandTimeout + { + get => _sqlCommand.CommandTimeout; + set => _sqlCommand.CommandTimeout = value; + } + + /// + /// . + /// + public string CommandText + { + get => _sqlCommand.CommandText; + set => _sqlCommand.CommandText = value; + } + + /// + /// . + /// + + public SqlConnection Connection + { + get => _sqlCommand.Connection; + set => _sqlCommand.Connection = value; + } + + /// + /// . + /// + public SqlNotificationRequest Notification + { + get => _sqlCommand.Notification; + set => _sqlCommand.Notification = value; + } + + /// + /// . + /// + public SqlParameterCollection Parameters => _sqlCommand.Parameters; + + /// + /// . + /// + public SqlTransaction Transaction + { + get => _sqlCommand.Transaction; + set => _sqlCommand.Transaction = value; + } + + /// + /// . + /// + public virtual void Cancel() + => _sqlCommand.Cancel(); + + /// + /// . + /// + /// A object. + public virtual SqlParameter CreateParameter() + => _sqlCommand.CreateParameter(); + + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// . + /// + /// The cancellation token. + /// A task representing the asynchronous operation. + public virtual Task ExecuteNonQueryAsync(CancellationToken cancellationToken) + => _sqlCommand.ExecuteNonQueryAsync(cancellationToken); + + /// + /// . + /// + /// The cancellation token. + /// A task representing the asynchronous operation. + public virtual Task ExecuteScalarAsync(CancellationToken cancellationToken) + => _sqlCommand.ExecuteScalarAsync(cancellationToken); + + public virtual void Prepare() + => _sqlCommand.Prepare(); + + /// + /// . + /// + /// The cancellation token. + /// A task representing the asynchronous operation. + public virtual Task PrepareAsync(CancellationToken cancellationToken) + => _sqlCommand.PrepareAsync(cancellationToken); + + /// + /// . + /// + /// A object. + public virtual SqlDataReader ExecuteReader() + => _sqlCommand.ExecuteReader(); + + /// + /// . + /// + /// One of the values. + /// A object. + public virtual SqlDataReader ExecuteReader(CommandBehavior behavior) + => _sqlCommand.ExecuteReader(behavior); + + /// + /// . + /// + /// The cancellation token. + /// A task representing the asynchronous operation. + public virtual Task ExecuteReaderAsync(CancellationToken cancellationToken) + => _sqlCommand.ExecuteReaderAsync(cancellationToken); + + /// + /// . + /// + /// One of the values. + /// The cancellation token. + /// A task representing the asynchronous operation. + public virtual Task ExecuteReaderAsync(CommandBehavior behavior, CancellationToken cancellationToken) + => _sqlCommand.ExecuteReaderAsync(behavior, cancellationToken); + + /// + /// . + /// + public virtual void ResetCommandTimeout() + => _sqlCommand.ResetCommandTimeout(); + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + _sqlCommand.Dispose(); + } + } + } +} diff --git a/src/Microsoft.Health.SqlServer/Features/Client/SqlCommandWrapperFactory.cs b/src/Microsoft.Health.SqlServer/Features/Client/SqlCommandWrapperFactory.cs new file mode 100644 index 00000000..1a84425e --- /dev/null +++ b/src/Microsoft.Health.SqlServer/Features/Client/SqlCommandWrapperFactory.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Data.SqlClient; +using EnsureThat; + +namespace Microsoft.Health.SqlServer.Features.Client +{ + /// + /// Provides functionality to create an instance of . + /// + public class SqlCommandWrapperFactory + { + /// + /// Creates an instance of . + /// + /// The underlying instance. + /// A newly created . + public virtual SqlCommandWrapper Create(SqlCommand sqlCommand) + { + EnsureArg.IsNotNull(sqlCommand, nameof(sqlCommand)); + + return new SqlCommandWrapper(sqlCommand); + } + } +} diff --git a/src/Microsoft.Health.SqlServer/Features/Client/SqlConnectionWrapper.cs b/src/Microsoft.Health.SqlServer/Features/Client/SqlConnectionWrapper.cs index 6998d8b7..f4a5c13e 100644 --- a/src/Microsoft.Health.SqlServer/Features/Client/SqlConnectionWrapper.cs +++ b/src/Microsoft.Health.SqlServer/Features/Client/SqlConnectionWrapper.cs @@ -16,14 +16,21 @@ public class SqlConnectionWrapper : IDisposable { private readonly bool _enlistInTransactionIfPresent; private readonly SqlTransactionHandler _sqlTransactionHandler; + private readonly SqlCommandWrapperFactory _sqlCommandWrapperFactory; - public SqlConnectionWrapper(SqlServerDataStoreConfiguration configuration, SqlTransactionHandler sqlTransactionHandler, bool enlistInTransactionIfPresent) + public SqlConnectionWrapper( + SqlServerDataStoreConfiguration configuration, + SqlTransactionHandler sqlTransactionHandler, + SqlCommandWrapperFactory sqlCommandWrapperFactory, + bool enlistInTransactionIfPresent) { EnsureArg.IsNotNull(configuration, nameof(configuration)); EnsureArg.IsNotNull(sqlTransactionHandler, nameof(sqlTransactionHandler)); + EnsureArg.IsNotNull(sqlCommandWrapperFactory, nameof(sqlCommandWrapperFactory)); _sqlTransactionHandler = sqlTransactionHandler; _enlistInTransactionIfPresent = enlistInTransactionIfPresent; + _sqlCommandWrapperFactory = sqlCommandWrapperFactory; if (_enlistInTransactionIfPresent && sqlTransactionHandler.SqlTransactionScope?.SqlConnection != null) { @@ -59,12 +66,13 @@ public SqlConnectionWrapper(SqlServerDataStoreConfiguration configuration, SqlTr public SqlTransaction SqlTransaction { get; } - public SqlCommand CreateSqlCommand() + public SqlCommandWrapper CreateSqlCommand() { SqlCommand sqlCommand = SqlConnection.CreateCommand(); + sqlCommand.Transaction = SqlTransaction; - return sqlCommand; + return _sqlCommandWrapperFactory.Create(sqlCommand); } protected virtual void Dispose(bool disposing) diff --git a/src/Microsoft.Health.SqlServer/Features/Client/SqlConnectionWrapperFactory.cs b/src/Microsoft.Health.SqlServer/Features/Client/SqlConnectionWrapperFactory.cs index 3ea80911..d8f25755 100644 --- a/src/Microsoft.Health.SqlServer/Features/Client/SqlConnectionWrapperFactory.cs +++ b/src/Microsoft.Health.SqlServer/Features/Client/SqlConnectionWrapperFactory.cs @@ -13,19 +13,25 @@ public class SqlConnectionWrapperFactory { private readonly SqlServerDataStoreConfiguration _configuration; private readonly SqlTransactionHandler _sqlTransactionHandler; + private readonly SqlCommandWrapperFactory _sqlCommandWrapperFactory; - public SqlConnectionWrapperFactory(SqlServerDataStoreConfiguration configuration, SqlTransactionHandler sqlTransactionHandler) + public SqlConnectionWrapperFactory( + SqlServerDataStoreConfiguration configuration, + SqlTransactionHandler sqlTransactionHandler, + SqlCommandWrapperFactory sqlCommandWrapperFactory) { EnsureArg.IsNotNull(configuration, nameof(configuration)); EnsureArg.IsNotNull(sqlTransactionHandler, nameof(sqlTransactionHandler)); + EnsureArg.IsNotNull(sqlCommandWrapperFactory, nameof(sqlCommandWrapperFactory)); _configuration = configuration; _sqlTransactionHandler = sqlTransactionHandler; + _sqlCommandWrapperFactory = sqlCommandWrapperFactory; } public SqlConnectionWrapper ObtainSqlConnectionWrapper(bool enlistInTransaction = false) { - return new SqlConnectionWrapper(_configuration, _sqlTransactionHandler, enlistInTransaction); + return new SqlConnectionWrapper(_configuration, _sqlTransactionHandler, _sqlCommandWrapperFactory, enlistInTransaction); } } } \ No newline at end of file diff --git a/src/Microsoft.Health.SqlServer/Features/Client/SqlServerTransientFaultRetryPolicyFactory.cs b/src/Microsoft.Health.SqlServer/Features/Client/SqlServerTransientFaultRetryPolicyFactory.cs new file mode 100644 index 00000000..c90bf9f3 --- /dev/null +++ b/src/Microsoft.Health.SqlServer/Features/Client/SqlServerTransientFaultRetryPolicyFactory.cs @@ -0,0 +1,54 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Data.SqlClient; +using EnsureThat; +using Microsoft.Health.SqlServer.Configs; +using Microsoft.Health.SqlServer.Extensions; +using Polly; +using Polly.Contrib.WaitAndRetry; + +namespace Microsoft.Health.SqlServer.Features.Client +{ + /// + /// Provides functionality to create retry policy for handling transient errors. + /// + internal class SqlServerTransientFaultRetryPolicyFactory : ISqlServerTransientFaultRetryPolicyFactory + { + private readonly IAsyncPolicy _retryPolicy; + + public SqlServerTransientFaultRetryPolicyFactory( + SqlServerDataStoreConfiguration sqlServerDataStoreConfiguration, + IPollyRetryLoggerFactory pollyRetryLoggerFactory) + { + EnsureArg.IsNotNull(sqlServerDataStoreConfiguration, nameof(sqlServerDataStoreConfiguration)); + EnsureArg.IsNotNull(pollyRetryLoggerFactory, nameof(pollyRetryLoggerFactory)); + + SqlServerTransientFaultRetryPolicyConfiguration transientFaultRetryPolicyConfiguration = sqlServerDataStoreConfiguration.TransientFaultRetryPolicy; + + IEnumerable sleepDurations = Backoff.ExponentialBackoff( + transientFaultRetryPolicyConfiguration.InitialDelay, + transientFaultRetryPolicyConfiguration.RetryCount, + transientFaultRetryPolicyConfiguration.Factor, + transientFaultRetryPolicyConfiguration.FastFirst); + + PolicyBuilder policyBuilder = Policy + .Handle(sqlException => sqlException.IsTransient()) + .Or(); + + Action onRetryLogger = pollyRetryLoggerFactory.Create(); + + _retryPolicy = policyBuilder.WaitAndRetryAsync( + sleepDurations, + onRetry: onRetryLogger); + } + + /// + public IAsyncPolicy Create() + => _retryPolicy; + } +} diff --git a/src/Microsoft.Health.SqlServer/Microsoft.Health.SqlServer.csproj b/src/Microsoft.Health.SqlServer/Microsoft.Health.SqlServer.csproj index fc3b364b..efd6978f 100644 --- a/src/Microsoft.Health.SqlServer/Microsoft.Health.SqlServer.csproj +++ b/src/Microsoft.Health.SqlServer/Microsoft.Health.SqlServer.csproj @@ -1,4 +1,4 @@ - + @@ -11,6 +11,8 @@ + + diff --git a/src/Microsoft.Health.SqlServer/Registration/SqlServerBaseRegistrationExtensions.cs b/src/Microsoft.Health.SqlServer/Registration/SqlServerBaseRegistrationExtensions.cs index 33f07a53..23742c2b 100644 --- a/src/Microsoft.Health.SqlServer/Registration/SqlServerBaseRegistrationExtensions.cs +++ b/src/Microsoft.Health.SqlServer/Registration/SqlServerBaseRegistrationExtensions.cs @@ -50,6 +50,21 @@ public static IServiceCollection AddSqlServerBase( .AsSelf() .AsImplementedInterfaces(); + services.Add() + .Singleton() + .AsSelf() + .AsImplementedInterfaces(); + + services.Add() + .Singleton() + .AsSelf() + .AsImplementedInterfaces(); + + services.Add() + .Singleton() + .AsSelf() + .AsService(); + services.Add() .Scoped() .AsSelf() diff --git a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateProcedureVisitor.cs b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateProcedureVisitor.cs index 23826df5..9ec20103 100644 --- a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateProcedureVisitor.cs +++ b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateProcedureVisitor.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.Data; -using System.Data.SqlClient; using System.Linq; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; @@ -141,7 +140,7 @@ private MethodDeclarationSyntax AddPopulateCommandMethod(CreateProcedureStatemen .AddModifiers(Token(SyntaxKind.PublicKeyword)) // first parameter is the SqlCommand - .AddParameterListParameters(Parameter(Identifier(CommandParameterName)).WithType(typeof(SqlCommand).ToTypeSyntax(useGlobalAlias: true))) + .AddParameterListParameters(Parameter(Identifier(CommandParameterName)).WithType(ParseTypeName("SqlCommandWrapper"))) // Add a parameter for each stored procedure parameter .AddParameterListParameters(node.Parameters.Select(selector: p => @@ -218,7 +217,7 @@ private MemberDeclarationSyntax AddPopulateCommandMethodForTableValuedParameters .AddModifiers(Token(SyntaxKind.PublicKeyword)) // first parameter is the SqlCommand - .AddParameterListParameters(Parameter(Identifier(CommandParameterName)).WithType(typeof(SqlCommand).ToTypeSyntax(useGlobalAlias: true))) + .AddParameterListParameters(Parameter(Identifier(CommandParameterName)).WithType(ParseTypeName("SqlCommandWrapper"))) // Add a parameter for each non-TVP .AddParameterListParameters(nonTableParameters.Select(selector: p => diff --git a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlModelGenerator.cs b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlModelGenerator.cs index ace53a8d..b254cefd 100644 --- a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlModelGenerator.cs +++ b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlModelGenerator.cs @@ -46,9 +46,13 @@ public SqlModelGenerator(string[] args) .OrderBy(m => m, MemberSorting.Comparer) .ToArray()); - var usings = SyntaxFactory.UsingDirective(SyntaxFactory.ParseName("Microsoft.Health.SqlServer.Features.Schema.Model")); - - return (classDeclaration, new[] { usings }); + return ( + classDeclaration, + new[] + { + UsingDirective(ParseName("Microsoft.Health.SqlServer.Features.Client")), + UsingDirective(ParseName("Microsoft.Health.SqlServer.Features.Schema.Model")), + }); } private TSqlFragment ParseSqlFile()