fix(eventing): back off outbox retries and make dead-letters recoverable

src/BuildingBlocks/Eventing/EventingOptions.cs

@@ -20,6 +20,17 @@ public sealed class EventingOptions
     /// </summary>
     public int OutboxMaxRetries { get; set; } = 5;
 
+    /// <summary>
+    /// Base delay (seconds) for exponential retry backoff after a failed dispatch. The n-th retry
+    /// waits <c>base * 2^(n-1)</c>, capped at <see cref="OutboxRetryMaxDelaySeconds"/>.
+    /// </summary>
+    public int OutboxRetryBaseDelaySeconds { get; set; } = 30;
+
+    /// <summary>
+    /// Upper bound (seconds) on the exponential retry backoff.
+    /// </summary>
+    public int OutboxRetryMaxDelaySeconds { get; set; } = 3600;
+
     /// <summary>
     /// Whether inbox-based idempotent handling is enabled.
     /// </summary>

src/BuildingBlocks/Eventing/Outbox/EfCoreOutboxStore.cs

@@ -1,6 +1,7 @@
 using FSH.Framework.Eventing.Abstractions;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 
 namespace FSH.Framework.Eventing.Outbox;
 
@@ -15,17 +16,21 @@ public sealed class EfCoreOutboxStore<TDbContext> : IOutboxStore
     private readonly IEventSerializer _serializer;
     private readonly ILogger<EfCoreOutboxStore<TDbContext>> _logger;
     private readonly TimeProvider _timeProvider;
+    private readonly EventingOptions _options;
 
     public EfCoreOutboxStore(
         TDbContext dbContext,
         IEventSerializer serializer,
         ILogger<EfCoreOutboxStore<TDbContext>> logger,
-        TimeProvider timeProvider)
+        TimeProvider timeProvider,
+        IOptions<EventingOptions> options)
     {
+        ArgumentNullException.ThrowIfNull(options);
         _dbContext = dbContext;
         _serializer = serializer;
         _logger = logger;
         _timeProvider = timeProvider;
+        _options = options.Value;
     }
 
     public async Task AddAsync(IIntegrationEvent @event, CancellationToken ct = default)
@@ -51,8 +56,9 @@ public async Task AddAsync(IIntegrationEvent @event, CancellationToken ct = defa
 
     public async Task<IReadOnlyList<OutboxMessage>> GetPendingBatchAsync(int batchSize, CancellationToken ct = default)
     {
+        var now = _timeProvider.GetUtcNow().UtcDateTime;
         return await _dbContext.Set<OutboxMessage>()
-            .Where(m => !m.IsDead && m.ProcessedOnUtc == null)
+            .Where(m => !m.IsDead && m.ProcessedOnUtc == null && (m.NextRetryAt == null || m.NextRetryAt <= now))
             .OrderBy(m => m.CreatedOnUtc)
             .Take(batchSize)
             .ToListAsync(ct)
@@ -75,11 +81,62 @@ public async Task MarkAsFailedAsync(OutboxMessage message, string error, bool is
         message.RetryCount++;
         message.LastError = error;
         message.IsDead = isDead;
+        // Space out retries with exponential backoff so a persistently failing message doesn't
+        // re-fire every dispatch cycle. A dead message won't be retried, so leave it eligible-null.
+        message.NextRetryAt = isDead ? null : _timeProvider.GetUtcNow().UtcDateTime.Add(BackoffFor(message.RetryCount));
         _dbContext.Set<OutboxMessage>().Update(message);
 
-        _logger.LogWarning("Outbox message {MessageId} failed. RetryCount={RetryCount}, IsDead={IsDead}, Error={Error}",
-            message.Id, message.RetryCount, message.IsDead, error);
-
         await _dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
     }
+
+    public async Task<IReadOnlyList<OutboxMessage>> GetDeadLetteredAsync(int max, CancellationToken ct = default)
+    {
+        if (max <= 0) max = 100;
+        return await _dbContext.Set<OutboxMessage>()
+            .Where(m => m.IsDead)
+            .OrderBy(m => m.CreatedOnUtc)
+            .Take(max)
+            .ToListAsync(ct)
+            .ConfigureAwait(false);
+    }
+
+    public async Task<int> RedriveDeadLettersAsync(IReadOnlyCollection<Guid>? ids, CancellationToken ct = default)
+    {
+        var query = _dbContext.Set<OutboxMessage>().Where(m => m.IsDead);
+        if (ids is { Count: > 0 })
+        {
+            query = query.Where(m => ids.Contains(m.Id));
+        }
+
+        var dead = await query.ToListAsync(ct).ConfigureAwait(false);
+        foreach (var message in dead)
+        {
+            message.IsDead = false;
+            message.RetryCount = 0;
+            message.LastError = null;
+            message.NextRetryAt = null;
+        }
+
+        if (dead.Count > 0)
+        {
+            await _dbContext.SaveChangesAsync(ct).ConfigureAwait(false);
+            Telemetry.EventingTelemetry.OutboxRedriven.Add(dead.Count);
+            if (_logger.IsEnabled(LogLevel.Information))
+            {
+                _logger.LogInformation("Redrove {Count} dead-lettered outbox message(s) for another attempt.", dead.Count);
+            }
+        }
+
+        return dead.Count;
+    }
+
+    private TimeSpan BackoffFor(int retryCount)
+    {
+        var baseSeconds = _options.OutboxRetryBaseDelaySeconds > 0 ? _options.OutboxRetryBaseDelaySeconds : 30;
+        var maxSeconds = _options.OutboxRetryMaxDelaySeconds > 0 ? _options.OutboxRetryMaxDelaySeconds : 3600;
+        // retryCount is 1 after the first failure → first backoff is exactly baseSeconds.
+        var exponent = Math.Min(retryCount - 1, 30); // clamp so the shift can't overflow
+        var seconds = Math.Min((double)baseSeconds * Math.Pow(2, exponent), maxSeconds);
+        return TimeSpan.FromSeconds(seconds);
+    }
 }

src/BuildingBlocks/Eventing/Outbox/IOutboxStore.cs

@@ -14,4 +14,14 @@ public interface IOutboxStore
     Task MarkAsProcessedAsync(OutboxMessage message, CancellationToken ct = default);
 
     Task MarkAsFailedAsync(OutboxMessage message, string error, bool isDead, CancellationToken ct = default);
+
+    /// <summary>Lists dead-lettered (exhausted) messages so an operator can inspect them.</summary>
+    Task<IReadOnlyList<OutboxMessage>> GetDeadLetteredAsync(int max, CancellationToken ct = default);
+
+    /// <summary>
+    /// Resets dead-lettered messages for another dispatch attempt (clears the dead flag, retry
+    /// count, last error and backoff). Pass specific ids, or <c>null</c> to redrive them all.
+    /// Returns the number of messages redriven.
+    /// </summary>
+    Task<int> RedriveDeadLettersAsync(IReadOnlyCollection<Guid>? ids, CancellationToken ct = default);
 }

src/BuildingBlocks/Eventing/Outbox/OutboxDispatcher.cs

@@ -1,4 +1,5 @@
 using FSH.Framework.Eventing.Abstractions;
+using FSH.Framework.Eventing.Telemetry;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -80,6 +81,7 @@ public async Task DispatchAsync(CancellationToken ct = default)
                 if (isDead)
                 {
                     deadLetterCount++;
+                    EventingTelemetry.OutboxDeadLettered.Add(1);
                 }
 
                 if (isDead)

src/BuildingBlocks/Eventing/Outbox/OutboxMessage.cs

@@ -33,6 +33,13 @@ public class OutboxMessage : IGlobalEntity
     public string? LastError { get; set; }
 
     public bool IsDead { get; set; }
+
+    /// <summary>
+    /// Earliest UTC time the message is eligible for another dispatch attempt. Set to a
+    /// backed-off future time after a failure so retries space out instead of re-firing every
+    /// dispatch cycle; <c>null</c> means immediately eligible (fresh message or after a redrive).
+    /// </summary>
+    public DateTime? NextRetryAt { get; set; }
 }
 
 public sealed class OutboxMessageConfiguration : IEntityTypeConfiguration<OutboxMessage>

src/BuildingBlocks/Eventing/Telemetry/EventingTelemetry.cs

@@ -0,0 +1,27 @@
+using System.Diagnostics.Metrics;
+
+namespace FSH.Framework.Eventing.Telemetry;
+
+/// <summary>
+/// OpenTelemetry metrics for the eventing building block. Register with
+/// <c>metrics.AddMeter(EventingTelemetry.MeterName)</c> in the OTel setup.
+/// </summary>
+public static class EventingTelemetry
+{
+    /// <summary>Name of the <see cref="Meter"/> used for eventing metrics.</summary>
+    public const string MeterName = "FSH.Eventing";
+
+    internal static readonly Meter Meter = new(MeterName);
+
+    /// <summary>Outbox messages that exhausted their retries and were dead-lettered.</summary>
+    internal static readonly Counter<long> OutboxDeadLettered = Meter.CreateCounter<long>(
+        "fsh.eventing.outbox.deadlettered",
+        unit: "{message}",
+        description: "Number of outbox messages moved to dead-letter after exhausting retries.");
+
+    /// <summary>Dead-lettered outbox messages reset for another dispatch attempt.</summary>
+    internal static readonly Counter<long> OutboxRedriven = Meter.CreateCounter<long>(
+        "fsh.eventing.outbox.redriven",
+        unit: "{message}",
+        description: "Number of dead-lettered outbox messages redriven for another attempt.");
+}

src/BuildingBlocks/Web/Observability/OpenTelemetry/Extensions.cs

@@ -105,6 +105,10 @@ private static void ConfigureMetricsAndTracing(
                 // Auditing pipeline metrics (published, dropped, flush, dead-letter).
                 metrics.AddMeter("FSH.Modules.Auditing");
 
+                // Eventing outbox metrics (dead-letter, redrive) — string literal matches
+                // EventingTelemetry.MeterName; Web does not reference the Eventing project.
+                metrics.AddMeter("FSH.Eventing");
+
                 foreach (var meterName in options.Metrics.MeterNames ?? Array.Empty<string>())
                 {
                     metrics.AddMeter(meterName);