-
Notifications
You must be signed in to change notification settings - Fork 37
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added support for retrying SQL transient errors. (#22)
- Loading branch information
Showing
25 changed files
with
1,092 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
42 changes: 42 additions & 0 deletions
42
src/Microsoft.Health.SqlServer.UnitTests/Extensions/SqlExceptionExtensionsTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| // ------------------------------------------------------------------------------------------------- | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. | ||
| // ------------------------------------------------------------------------------------------------- | ||
|
|
||
| using System.Data.SqlClient; | ||
| using Microsoft.Health.SqlServer.Extensions; | ||
| using Xunit; | ||
|
|
||
| namespace Microsoft.Health.SqlServer.UnitTests.Extensions | ||
| { | ||
| public class SqlExceptionExtensionsTests | ||
| { | ||
| [Theory] | ||
| [InlineData(10928)] | ||
| [InlineData(10929)] | ||
| [InlineData(10053)] | ||
| [InlineData(10054)] | ||
| [InlineData(10060)] | ||
| [InlineData(18401)] | ||
| [InlineData(40197)] | ||
| [InlineData(40540)] | ||
| [InlineData(40613)] | ||
| [InlineData(40143)] | ||
| [InlineData(233)] | ||
| [InlineData(64)] | ||
| public void GivenATransientException_WhenCheckedIfExceptionIsTransient_ThenTrueShouldBeReturned(int number) | ||
| { | ||
| SqlException sqlException = SqlExceptionFactory.Create(number); | ||
|
|
||
| Assert.True(sqlException.IsTransient()); | ||
| } | ||
|
|
||
| [Fact] | ||
| public void GivenANonTransientException_WhenCheckedIfExceptionIsTransient_ThenFalseShouldBeReturned() | ||
| { | ||
| SqlException sqlException = SqlExceptionFactory.Create(10001); | ||
|
|
||
| Assert.False(sqlException.IsTransient()); | ||
| } | ||
| } | ||
| } |
145 changes: 145 additions & 0 deletions
145
src/Microsoft.Health.SqlServer.UnitTests/Features/Client/RetrySqlCommandWrapperTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,145 @@ | ||
| // ------------------------------------------------------------------------------------------------- | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. | ||
| // ------------------------------------------------------------------------------------------------- | ||
|
|
||
| using System; | ||
| using System.Data; | ||
| using System.Data.SqlClient; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Health.SqlServer.Extensions; | ||
| using Microsoft.Health.SqlServer.Features.Client; | ||
| using NSubstitute; | ||
| using NSubstitute.ExceptionExtensions; | ||
| using Polly; | ||
| using Xunit; | ||
|
|
||
| namespace Microsoft.Health.SqlServer.UnitTests.Features.Client | ||
| { | ||
| public class RetrySqlCommandWrapperTests | ||
| { | ||
| private static readonly CancellationToken DefaultCancellationToken = new CancellationTokenSource().Token; | ||
|
|
||
| private readonly SqlCommandWrapper _sqlCommandWrapper = Substitute.For<SqlCommandWrapper>(new SqlCommand()); | ||
| private readonly IAsyncPolicy _asyncPolicy = Policy | ||
| .Handle<SqlException>(sqlException => sqlException.IsTransient()) | ||
| .WaitAndRetryAsync(new TimeSpan[] { TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero }); | ||
|
|
||
| private readonly RetrySqlCommandWrapper _retrySqlCommandWrapper; | ||
|
|
||
| public RetrySqlCommandWrapperTests() | ||
| { | ||
| _retrySqlCommandWrapper = new RetrySqlCommandWrapper(_sqlCommandWrapper, _asyncPolicy); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenATransientException_WhenNonQueryIsExecuted_ThenItShouldRetry() | ||
| { | ||
| _sqlCommandWrapper.ExecuteNonQueryAsync(DefaultCancellationToken).Throws(CreateTransientException()); | ||
|
|
||
| await ExecuteAndValidateExecuteNonQueryAsync(4); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenANonTransientException_WhenNonQueryIsExecuted_ThenItShouldNotRetry() | ||
| { | ||
| _sqlCommandWrapper.ExecuteNonQueryAsync(DefaultCancellationToken).Returns<int>( | ||
| _ => throw CreateTransientException(), | ||
| _ => throw CreateNonTransientException()); | ||
|
|
||
| await ExecuteAndValidateExecuteNonQueryAsync(2); | ||
| } | ||
|
|
||
| private async Task ExecuteAndValidateExecuteNonQueryAsync(int expectedNumberOfCalls) | ||
| { | ||
| await Assert.ThrowsAsync<SqlException>(() => _retrySqlCommandWrapper.ExecuteNonQueryAsync(DefaultCancellationToken)); | ||
|
|
||
| await _sqlCommandWrapper.Received(expectedNumberOfCalls).ExecuteNonQueryAsync(DefaultCancellationToken); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenATransientException_WhenReaderIsExecuted_ThenItShouldRetry() | ||
| { | ||
| _sqlCommandWrapper.ExecuteReaderAsync(DefaultCancellationToken).Throws(CreateTransientException()); | ||
|
|
||
| await ExecuteAndValidateExecuteReaderAsync(4); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenANonTransientException_WhenReaderIsExecuted_ThenItShouldNotRetry() | ||
| { | ||
| _sqlCommandWrapper.ExecuteReaderAsync(DefaultCancellationToken).Returns<SqlDataReader>( | ||
| _ => throw CreateNonTransientException()); | ||
|
|
||
| await ExecuteAndValidateExecuteReaderAsync(1); | ||
| } | ||
|
|
||
| private async Task ExecuteAndValidateExecuteReaderAsync(int expectedNumberOfCalls) | ||
| { | ||
| await Assert.ThrowsAsync<SqlException>(() => _retrySqlCommandWrapper.ExecuteReaderAsync(DefaultCancellationToken)); | ||
|
|
||
| await _sqlCommandWrapper.Received(expectedNumberOfCalls).ExecuteReaderAsync(DefaultCancellationToken); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenATransientException_WhenReaderWithBehaviorIsExecuted_ThenItShouldRetry() | ||
| { | ||
| CommandBehavior behavior = CommandBehavior.SingleRow; | ||
|
|
||
| _sqlCommandWrapper.ExecuteReaderAsync(behavior, DefaultCancellationToken).Throws(CreateTransientException()); | ||
|
|
||
| await ExecuteAndValidateExecuteReaderAsync(behavior, 4); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenANonTransientException_WhenReaderWithBehaviorIsExecuted_ThenItShouldNotRetry() | ||
| { | ||
| CommandBehavior behavior = CommandBehavior.SchemaOnly; | ||
|
|
||
| _sqlCommandWrapper.ExecuteReaderAsync(behavior, DefaultCancellationToken).Returns<SqlDataReader>( | ||
| _ => throw CreateNonTransientException()); | ||
|
|
||
| await ExecuteAndValidateExecuteReaderAsync(behavior, 1); | ||
| } | ||
|
|
||
| private async Task ExecuteAndValidateExecuteReaderAsync(CommandBehavior behavior, int expectedNumberOfCalls) | ||
| { | ||
| await Assert.ThrowsAsync<SqlException>(() => _retrySqlCommandWrapper.ExecuteReaderAsync(behavior, DefaultCancellationToken)); | ||
|
|
||
| await _sqlCommandWrapper.Received(expectedNumberOfCalls).ExecuteReaderAsync(behavior, DefaultCancellationToken); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenATransientException_WhenScalarIsExecuted_ThenItShouldRetry() | ||
| { | ||
| _sqlCommandWrapper.ExecuteScalarAsync(DefaultCancellationToken).Throws(CreateTransientException()); | ||
|
|
||
| await ExecuteAndValidateExecuteScalarAsync(4); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenANonTransientException_WhenScalarIsExecuted_ThenItShouldNotRetry() | ||
| { | ||
| _sqlCommandWrapper.ExecuteScalarAsync(DefaultCancellationToken).Returns( | ||
| _ => throw CreateTransientException(), | ||
| _ => throw CreateTransientException(), | ||
| _ => throw CreateNonTransientException()); | ||
|
|
||
| await ExecuteAndValidateExecuteScalarAsync(3); | ||
| } | ||
|
|
||
| private async Task ExecuteAndValidateExecuteScalarAsync(int expectedNumberOfCalls) | ||
| { | ||
| await Assert.ThrowsAsync<SqlException>(() => _retrySqlCommandWrapper.ExecuteScalarAsync(DefaultCancellationToken)); | ||
|
|
||
| await _sqlCommandWrapper.Received(expectedNumberOfCalls).ExecuteScalarAsync(DefaultCancellationToken); | ||
| } | ||
|
|
||
| private static SqlException CreateTransientException() | ||
| => SqlExceptionFactory.Create(10928); | ||
|
|
||
| private static SqlException CreateNonTransientException() | ||
| => SqlExceptionFactory.Create(50404); | ||
| } | ||
| } |
97 changes: 97 additions & 0 deletions
97
...lth.SqlServer.UnitTests/Features/Client/SqlServerTransientFaultRetryPolicyFactoryTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| // ------------------------------------------------------------------------------------------------- | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. | ||
| // ------------------------------------------------------------------------------------------------- | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Data.SqlClient; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Health.SqlServer.Configs; | ||
| using Microsoft.Health.SqlServer.Features.Client; | ||
| using NSubstitute; | ||
| using Polly; | ||
| using Xunit; | ||
|
|
||
| namespace Microsoft.Health.SqlServer.UnitTests.Features.Client | ||
| { | ||
| public class SqlServerTransientFaultRetryPolicyFactoryTests | ||
| { | ||
| private readonly SqlServerDataStoreConfiguration _sqlServerDataStoreConfiguration = new SqlServerDataStoreConfiguration(); | ||
| private readonly IPollyRetryLoggerFactory _pollyRetryLoggerFactory = Substitute.For<IPollyRetryLoggerFactory>(); | ||
|
|
||
| private readonly SqlServerTransientFaultRetryPolicyFactory _sqlServerTransientFaultRetryPolicyFactory; | ||
|
|
||
| private readonly IAsyncPolicy _asyncPolicy; | ||
| private readonly List<TimeSpan> _capturedRetries = new List<TimeSpan>(); | ||
|
|
||
| public SqlServerTransientFaultRetryPolicyFactoryTests() | ||
| { | ||
| _sqlServerDataStoreConfiguration.TransientFaultRetryPolicy = new SqlServerTransientFaultRetryPolicyConfiguration() | ||
| { | ||
| InitialDelay = TimeSpan.FromMilliseconds(200), | ||
| RetryCount = 3, | ||
| Factor = 3, | ||
| FastFirst = true, | ||
| }; | ||
|
|
||
| Action<Exception, TimeSpan, int, Context> onRetryCapture = (exception, sleepDuration, retryCount, _) => | ||
| { | ||
| _capturedRetries.Add(sleepDuration); | ||
| }; | ||
|
|
||
| _pollyRetryLoggerFactory.Create().Returns(onRetryCapture); | ||
|
|
||
| _sqlServerTransientFaultRetryPolicyFactory = new SqlServerTransientFaultRetryPolicyFactory( | ||
| _sqlServerDataStoreConfiguration, | ||
| _pollyRetryLoggerFactory); | ||
|
|
||
| _asyncPolicy = _sqlServerTransientFaultRetryPolicyFactory.Create(); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenATransientException_WhenRetryPolicyIsUsed_ThenItShouldRetry() | ||
| { | ||
| await Assert.ThrowsAsync<SqlException>(() => | ||
| _asyncPolicy.ExecuteAsync(() => Task.Run(() => throw SqlExceptionFactory.CreateTransientException()))); | ||
|
|
||
| ValidateCapturedRetries(); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenANonTransientException_WhenRetryPolicyIsUsed_ThenItShouldNotRetry() | ||
| { | ||
| await Assert.ThrowsAsync<SqlException>(() => | ||
| _asyncPolicy.ExecuteAsync(() => Task.Run(() => throw SqlExceptionFactory.CreateNonTransientException()))); | ||
|
|
||
| Assert.Empty(_capturedRetries); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenATimeoutException_WhenRetryPolicyIsUsed_ThenItShouldRetry() | ||
| { | ||
| await Assert.ThrowsAsync<TimeoutException>(() => | ||
| _asyncPolicy.ExecuteAsync(() => Task.Run(() => throw new TimeoutException()))); | ||
|
|
||
| ValidateCapturedRetries(); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task GivenOtherException_WhenRetryPolicyIsUsed_ThenItShouldNotRetry() | ||
| { | ||
| await Assert.ThrowsAsync<Exception>(() => | ||
| _asyncPolicy.ExecuteAsync(() => Task.Run(() => throw new Exception()))); | ||
|
|
||
| Assert.Empty(_capturedRetries); | ||
| } | ||
|
|
||
| private void ValidateCapturedRetries() | ||
| { | ||
| Assert.Collection( | ||
| _capturedRetries, | ||
| item => Assert.Equal(TimeSpan.Zero, item), | ||
| item => Assert.Equal(TimeSpan.FromMilliseconds(200), item), | ||
| item => Assert.Equal(TimeSpan.FromMilliseconds(600), item)); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.