From b2ecf638fdb1b98f5c7c670c90c83832645df3ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20St=C3=BChmer?= Date: Sun, 27 Sep 2026 08:58:30 +0200 Subject: [PATCH] test(integration): stop one failed SQL Server connect from failing every test SqlClient's connection pool blocking period cached a single failed connect to the SQL Server container and rethrew it for every CREATE DATABASE on the shared master connection string, failing 96 net10.0 tests at once. Disable the blocking period for the test connection strings, retry failed connects with backoff, stop retrying once the server has proven unreachable, and add the container output to the failure message. --- .../Services/SqlServerContainerFixture.cs | 28 +++++- .../SqlServerDatabaseServiceFixture.cs | 85 +++++++++++++------ 2 files changed, 87 insertions(+), 26 deletions(-) diff --git a/tests/NetEvolve.Pulse.Tests.Integration/Internals/Services/SqlServerContainerFixture.cs b/tests/NetEvolve.Pulse.Tests.Integration/Internals/Services/SqlServerContainerFixture.cs index 7dab8f14..364f7dba 100644 --- a/tests/NetEvolve.Pulse.Tests.Integration/Internals/Services/SqlServerContainerFixture.cs +++ b/tests/NetEvolve.Pulse.Tests.Integration/Internals/Services/SqlServerContainerFixture.cs @@ -15,10 +15,36 @@ public sealed class SqlServerContainerFixture : IAsyncDisposable, IAsyncInitiali private MsSqlContainer? _container; + // Pool Blocking Period=NeverBlock: by default SqlClient caches a failed open for 5 s to 1 min and rethrows it for + // every open on the same pool. All test databases are created through this one connection string, so a single + // failed connect would otherwise fail every fixture that initializes in that window. public string ConnectionString => ( _container ?? throw new InvalidOperationException("The SQL Server container has not been started.") - ).GetConnectionString() + ";MultipleActiveResultSets=True;"; + ).GetConnectionString() + ";MultipleActiveResultSets=True;Pool Blocking Period=NeverBlock;"; + + // The tail of the container output, to tell a crashed SQL Server (microsoft/mssql-docker#974) from a network issue. + internal async Task GetDiagnosticsAsync() + { + if (_container is null) + { + return "The SQL Server container has not been started."; + } + + try + { + var (stdout, stderr) = await _container.GetLogsAsync().ConfigureAwait(false); + var tail = string.Join( + Environment.NewLine, + $"{stdout}{stderr}".Split('\n', StringSplitOptions.RemoveEmptyEntries).TakeLast(40) + ); + return $"SQL Server container {_container.Id} output (last 40 lines):{Environment.NewLine}{tail}"; + } + catch (Exception ex) + { + return $"Reading the SQL Server container output failed: {ex.Message}"; + } + } public ValueTask DisposeAsync() => _container?.DisposeAsync() ?? ValueTask.CompletedTask; diff --git a/tests/NetEvolve.Pulse.Tests.Integration/Internals/Services/SqlServerDatabaseServiceFixture.cs b/tests/NetEvolve.Pulse.Tests.Integration/Internals/Services/SqlServerDatabaseServiceFixture.cs index 49923efa..7acaf156 100644 --- a/tests/NetEvolve.Pulse.Tests.Integration/Internals/Services/SqlServerDatabaseServiceFixture.cs +++ b/tests/NetEvolve.Pulse.Tests.Integration/Internals/Services/SqlServerDatabaseServiceFixture.cs @@ -21,32 +21,39 @@ public sealed class SqlServerDatabaseServiceFixture : IServiceFixture // ("Could not obtain exclusive lock on database 'model'"), so creation is serialized per test session. private const int ModelLockErrorNumber = 1807; + private const int MaxAttempts = 5; + private static readonly SemaphoreSlim CreateDatabaseLock = new(1, 1); + // Set once connecting has failed MaxAttempts times. Creation is serialized, so without it every remaining fixture + // would wait through the same connect timeouts against an unreachable server. + private static Exception? _serverUnreachable; + public async Task InitializeAsync() { await CreateDatabaseLock.WaitAsync().ConfigureAwait(false); try { - // Bounded retry in case 'model' is briefly held by SQL Server itself (e.g. right after startup). - for (var attempt = 1; attempt < 5; attempt++) + if (_serverUnreachable is not null) { - try - { - await CreateDatabaseAsync().ConfigureAwait(false); - return; - } - catch (SqlException ex) when (ex.Number == ModelLockErrorNumber) - { - await Task.Delay(TimeSpan.FromMilliseconds(500 * attempt)).ConfigureAwait(false); - } + throw new InvalidOperationException( + $"Failed to create SQL Server test database '{DatabaseName}': SQL Server was unreachable earlier in this test session.", + _serverUnreachable + ); } - await CreateDatabaseAsync().ConfigureAwait(false); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to create SQL Server test database '{DatabaseName}'.", ex); + try + { + await CreateDatabaseAsync(Container.ConnectionString, DatabaseName).ConfigureAwait(false); + } + catch (Exception ex) + { + var diagnostics = await Container.GetDiagnosticsAsync().ConfigureAwait(false); + throw new InvalidOperationException( + $"Failed to create SQL Server test database '{DatabaseName}'.{Environment.NewLine}{diagnostics}", + ex + ); + } } finally { @@ -54,21 +61,49 @@ public async Task InitializeAsync() } } - private async Task CreateDatabaseAsync() + private static async Task CreateDatabaseAsync(string connectionString, string databaseName) { - var con = new SqlConnection(Container.ConnectionString); - await using (con.ConfigureAwait(false)) + // Bounded retry: the connect fails transiently (managed SNI error 35 on Linux, which SqlClient does not treat + // as transient, see dotnet/SqlClient#4665), or 'model' is briefly held by SQL Server itself (e.g. right after + // startup). + for (var attempt = 1; attempt <= MaxAttempts; attempt++) { - await con.OpenAsync().ConfigureAwait(false); - - var cmd = con.CreateCommand(); - await using (cmd.ConfigureAwait(false)) + var con = new SqlConnection(connectionString); + await using (con.ConfigureAwait(false)) { + try + { + await con.OpenAsync().ConfigureAwait(false); + } + catch (SqlException ex) + { + if (attempt == MaxAttempts) + { + _serverUnreachable = ex; + throw; + } + + await Task.Delay(TimeSpan.FromSeconds(attempt)).ConfigureAwait(false); + continue; + } + + try + { + var cmd = con.CreateCommand(); + await using (cmd.ConfigureAwait(false)) + { #pragma warning disable CA2100, S2077 // Review SQL queries for security vulnerabilities; DatabaseName is test-controlled, not user input - cmd.CommandText = $"CREATE DATABASE [{DatabaseName}]"; + cmd.CommandText = $"CREATE DATABASE [{databaseName}]"; #pragma warning restore CA2100, S2077 - _ = await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); + _ = await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); + return; + } + } + catch (SqlException ex) when (ex.Number == ModelLockErrorNumber && attempt < MaxAttempts) + { + await Task.Delay(TimeSpan.FromMilliseconds(500 * attempt)).ConfigureAwait(false); + } } } }