Improve exception/error handling

RAIC.Extensions.Configuration.EntityFrameworkCore.SqlServer/README.md

@@ -7,8 +7,8 @@ This library enhances `RAIC.Extensions.Configuration.EntityFrameworkCore` with s
 
 ## Goals
 1. No polling!
-1. Updates happen in background via worker service (`IHostedService`)
-1. Only update settings which change rather than reloading all of them
+1. Updates happen in background via a [hosted service](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/host/hosted-services) implementation
+1. Only update settings which change rather than updating them all
 
 
 ## Requirements
@@ -17,13 +17,16 @@ This library enhances `RAIC.Extensions.Configuration.EntityFrameworkCore` with s
 ## Gotchas
 * Won't work with Azure SQL until Microsoft adds/enables Service Broker support
 * Setting values cannot be `null` (as signified by the `RequiredAttribute` on `ISetting.Value`)
+* Consider adding `ConnectRetryCount` and `ConnectRetryInterval` [parameters](https://learn.microsoft.com/en-us/dotnet/framework/data/adonet/whats-new#sqlclient-data-provider) to your connection string if not already present
 
 ## Known Issues
 * Not tested under load
+* Transient failure detection logic is not well tested given the challenges in reproducing these issues
 
 ## Configuration Options
-There is a single property which can be configured (cf. the `SqlServerNotificationConfigurationReloaderOptions` POCO)
+There are two properties which can be configured (cf. the `SqlServerNotificationConfigurationReloaderOptions` POCO)
 1. `ConnectionString` - the full connection string for the SQL Server instance
+1. `TransientErrors` - the list of `SqlError.Number` values that will be treated as transient if present in a thrown `SqlException`'s `Errors` collection while listening for or processing notifications (ie. reconnection will be attempted if any are present)
 
 ## Setup
 For `SqlServerNotificationConfigurationReloader` to work it requires Change Tracking to be enable on the `Settings` table (and therefore also on the database itself)
@@ -37,7 +40,7 @@ ALTER TABLE dbo.Settings
 ENABLE CHANGE_TRACKING;
 ```
 
-Reccommend adding your SQL to the migration which adds the `Settings` table/view (or a new migration if that table/view already exists).
+Recommend adding your SQL to the migration which adds the `Settings` table/view (or a new migration if that table/view already exists).
 
 ## Usage Example
 

RAIC.Extensions.Configuration.EntityFrameworkCore.SqlServer/SqlServerNotificationConfigurationReloader.cs

@@ -12,6 +12,9 @@ public class SqlServerNotificationConfigurationReloaderOptions // must be public
 {
     [Required]
     public required string ConnectionString { get; set; }
+
+    [Required]
+    public required HashSet<int> TransientErrors { get; set; } = [2, 53, 121, 10060, 11001];
 }
 
 
@@ -40,35 +43,50 @@ internal class SqlServerNotificationConfigurationReloader<TDbContext, TSetting>
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        var settingsQuery = await Initialise(stoppingToken);
 
-        do
+        try
         {
+            var settingsQuery = await Initialise(stoppingToken);
+
             do
             {
-                var (taskCompletionSource, cancelTokenRegistration) = await ListenForNotifications(settingsQuery, stoppingToken);
-                using (cancelTokenRegistration)
+                try
                 {
-                    var lastVersion = await GetChangeTrackingVersion(stoppingToken);
-
-                    try
+                    do
                     {
-                        await taskCompletionSource.Task;
+                        var notificationCompletion = await ListenForNotifications(settingsQuery, stoppingToken);
+                        using var tcsCancelRegistration = stoppingToken.Register(() => notificationCompletion.TrySetCanceled());
+
+                        var lastVersion = await GetChangeTrackingVersion(stoppingToken);
+
+                        await notificationCompletion.Task;
                         UpdateConfiguration(lastVersion);
                     }
-                    catch (Exception e) when (e is not TaskCanceledException)
-                    {
-                        _logger?.LogWarning(e, "Exception while listening for notifications on query '{query}'", settingsQuery);
-                        break;
-                    }
+                    while (true); // each notification is one-and-done, so must keep re-regesitering for notifications indefinitely
+                }
+                catch (SqlException e) when (e.InnerException is SqlException inner && _options.TransientErrors.Overlaps(inner.Errors.OfType<SqlError>().Select(e => e.Number)))
+                {
+                    _logger?.LogWarning(e, "Transient exception during notification setup process or the processing of notifications");
+                }
+                catch (SqlNotificationException e) // only ever thown while waiting for notification task to complete, must (?) be dropped connection of some sort
+                {
+                    _logger?.LogWarning(e, "Transient exception while listening for notifications");
+                }
+                catch (Exception e) when (e is not OperationCanceledException)
+                {
+                    _logger?.LogError(e, "Exception while setting up, listening for or processing notifications");
+                    throw;
                 }
             }
-            while (true); // each notification is one-and-done, so must keep re-regesitering indefinitely
-        }
-        while (await ReconnectOnDependencyException(stoppingToken));
+            while (await TryReconnect(stoppingToken));
 
-        _logger?.LogWarning("Giving up listening for notifications on query '{query}' because reconnection failed. Configuration updates from database will no longer occur", settingsQuery);
-        SqlDependency.Stop(_options.ConnectionString);
+            _logger?.LogWarning("Giving up listening for notifications on query '{query}' because reconnection failed. Configuration updates from database will no longer occur", settingsQuery);
+            SqlDependency.Stop(_options.ConnectionString);
+        }
+        catch (OperationCanceledException e)
+        {
+            _logger?.LogInformation(e, "Exiting due to signal from stopping token");
+        }
     }
 
 
@@ -88,7 +106,7 @@ internal class SqlServerNotificationConfigurationReloader<TDbContext, TSetting>
 
         var settingsQuery = _dbContext.Settings.ToQueryString();
 
-        _logger?.LogInformation("Listening for notifications on query '{query}'", settingsQuery);
+        _logger?.LogInformation(@"Query upon which notifications will be listened for: ""{query}""", settingsQuery);
 
         await openTask;
 
@@ -96,8 +114,10 @@ internal class SqlServerNotificationConfigurationReloader<TDbContext, TSetting>
     }
 
 
-    private async Task<(TaskCompletionSource, CancellationTokenRegistration)> ListenForNotifications(string settingsQuery, CancellationToken stoppingToken)
+    private async Task<TaskCompletionSource> ListenForNotifications(string settingsQuery, CancellationToken stoppingToken)
     {
+        // Can't reuse the command built here, as once it gets associated with a SqlDependency it can't be used with another,
+        // and SqlDependencies can't be used more than once either... hence there's no point "preparing" the command either
         using var command = new SqlCommand()
         {
             Connection = (SqlConnection)_dbContext.Database.GetDbConnection(),
@@ -108,22 +128,22 @@ internal class SqlServerNotificationConfigurationReloader<TDbContext, TSetting>
         var dependency = new SqlDependency(command);
         var tcs = new TaskCompletionSource();
         dependency.OnChange += OnChange;
-        var tcsCancelRegistration = stoppingToken.Register(() => tcs.TrySetCanceled());
 
         try
         {
             await command.ExecuteNonQueryAsync(stoppingToken);
         }
-        catch (Exception e) when (e is not TaskCanceledException)
+        catch (Exception e) when (e is not OperationCanceledException)
         {
             _logger?.LogError(e, "Exception while attempting to register query dependency");
             throw;
         }
 
-        return (tcs, tcsCancelRegistration);
+        return tcs;
 
         void OnChange(object sender, SqlNotificationEventArgs args)
         {
+            dependency.OnChange -= OnChange;
             switch (args.Info)
             {
                 case SqlNotificationInfo.Insert:
@@ -135,7 +155,7 @@ internal class SqlServerNotificationConfigurationReloader<TDbContext, TSetting>
                     return;
                 case SqlNotificationInfo.Error:
                     _logger?.LogWarning("SqlDependency '{info}' from {type}@{source}", args.Info, args.Type, args.Source);
-                    tcs.TrySetException(new Exception($"SqlDependency {args.Info} from {args.Type}@{args.Source}"));
+                    tcs.TrySetException(new SqlNotificationException($"SqlDependency {args.Info} from {args.Type}@{args.Source}"));
                     return;
                 default:
                     _logger?.LogWarning("Ignoring '{info}' from {type}@{source} received from SqlDependency", args.Info, args.Type, args.Source);
@@ -148,7 +168,7 @@ internal class SqlServerNotificationConfigurationReloader<TDbContext, TSetting>
     private async Task<long> GetChangeTrackingVersion(CancellationToken stoppingToken)
     {
         var query = "SELECT CHANGE_TRACKING_CURRENT_VERSION() as value";
-        return await _dbContext.Database.SqlQueryRaw<long>(query).FirstOrDefaultAsync(stoppingToken);
+        return await _dbContext.Database.SqlQueryRaw<long>(query).SingleAsync(stoppingToken);
     }
 
 
@@ -198,19 +218,15 @@ internal class SqlServerNotificationConfigurationReloader<TDbContext, TSetting>
     }
 
 
-    private async Task<bool> ReconnectOnDependencyException(CancellationToken stoppingToken)
+    private async Task<bool> TryReconnect(CancellationToken stoppingToken)
     {
-        // explicitly close connection to force subsequent OpenConnectionAsync() to actually try to open the connection instead of it possibly being a no-op
-        await _dbContext.Database.CloseConnectionAsync();
-        try
+        // Unlike the postgres version of this there's no retry logic here - SqlClient does its own retries internally so it is not needed (?)
+        if (await _dbContext.Database.CanConnectAsync(stoppingToken))
         {
             await _dbContext.Database.OpenConnectionAsync(stoppingToken);
             return true;
         }
-        catch (Exception e)
-        {
-            _logger?.LogWarning(e, "Exception while attempting to reconnect to database. Configuration updates from database will no longer occur");
-        }
+
         return false;
     }
 
@@ -263,4 +279,15 @@ internal class SqlServerNotificationConfigurationReloader<TDbContext, TSetting>
         base.Dispose();
         _onChangeHandler?.Dispose();
     }
+
+    private class SqlNotificationException : Exception
+    {
+        public SqlNotificationException(string? message) : base(message)
+        {
+        }
+
+        public SqlNotificationException(string? message, Exception? innerException) : base(message, innerException)
+        {
+        }
+    }
 }