From 080a5ac0de608dc08548ebede56423fdc1642506 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 13:21:07 -0500 Subject: [PATCH 01/20] Add Elasticsearch 9 compatibility testing --- build/docker/elasticsearch/9.x/Dockerfile | 4 +++ .../Extensions/ElasticsearchExtensions.cs | 14 ++++++--- src/Exceptionless.AppHost/Program.cs | 31 ++++++++++++++++--- .../Exceptionless.Tests/AppWebHostFactory.cs | 20 ++++++++++-- 4 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 build/docker/elasticsearch/9.x/Dockerfile diff --git a/build/docker/elasticsearch/9.x/Dockerfile b/build/docker/elasticsearch/9.x/Dockerfile new file mode 100644 index 0000000000..8658600712 --- /dev/null +++ b/build/docker/elasticsearch/9.x/Dockerfile @@ -0,0 +1,4 @@ +# https://www.docker.elastic.co/ +FROM docker.elastic.co/elasticsearch/elasticsearch:9.4.2 + +RUN elasticsearch-plugin install -b mapper-size diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index ec5c5e4031..4f6cc8564c 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -60,7 +60,11 @@ public static IResourceBuilder AddElasticsearch(this IDis .PublishAsConnectionString(); } - public static IResourceBuilder WithKibana(this IResourceBuilder builder, Action>? configureContainer = null, string? containerName = null) + public static IResourceBuilder WithKibana( + this IResourceBuilder builder, + Action>? configureContainer = null, + string? containerName = null, + int? port = null) { ArgumentNullException.ThrowIfNull(builder); @@ -79,7 +83,7 @@ public static IResourceBuilder WithKibana(this IResourceB var resourceBuilder = builder.ApplicationBuilder.AddResource(resource) .WithImage(ElasticsearchContainerImageTags.KibanaImage, ElasticsearchContainerImageTags.Tag) .WithImageRegistry(ElasticsearchContainerImageTags.KibanaRegistry) - .WithHttpEndpoint(targetPort: KibanaPort, name: containerName) + .WithHttpEndpoint(targetPort: KibanaPort, port: port, name: containerName) .WithUrlForEndpoint(containerName, u => u.DisplayText = "Kibana") .WithEnvironment("xpack.security.enabled", "false") .WithEnvironment(ctx => @@ -134,9 +138,11 @@ public async Task CheckHealthAsync(HealthCheckContext context using var settings = new ElasticsearchClientSettings(new Uri(connectionString)); var client = new ElasticsearchClient(settings); - var response = await client.PingAsync(cancellationToken); + var response = await client.Cluster.HealthAsync( + request => request.WaitForStatus(Elastic.Clients.Elasticsearch.HealthStatus.Yellow), + cancellationToken); return response.IsValidResponse ? HealthCheckResult.Healthy() - : new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch ping failed: {response.DebugInformation}"); + : new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch cluster health check failed: {response.DebugInformation}"); } } diff --git a/src/Exceptionless.AppHost/Program.cs b/src/Exceptionless.AppHost/Program.cs index b12b48aefe..6e62e7fc45 100644 --- a/src/Exceptionless.AppHost/Program.cs +++ b/src/Exceptionless.AppHost/Program.cs @@ -13,6 +13,12 @@ bool servicesOnly = HasArgument("--services-only"); bool ciE2E = HasArgument("--ci-e2e"); bool includeDevTools = !ciE2E; +int elasticsearchPort = GetPort("Elasticsearch:Port", 9200); +string elasticsearchImageTag = builder.Configuration["Elasticsearch:ImageTag"] ?? ElasticsearchContainerImageTags.Tag; +string elasticsearchContainerName = builder.Configuration["Elasticsearch:ContainerName"] ?? "Exceptionless-Elasticsearch"; +string elasticsearchDataVolume = builder.Configuration["Elasticsearch:DataVolume"] ?? "exceptionless.data.v1"; +string kibanaContainerName = builder.Configuration["Elasticsearch:KibanaContainerName"] ?? "Exceptionless-Kibana"; +int kibanaPort = GetPort("Elasticsearch:KibanaPort", 5601); int oldAppHttpPort = worktreePorts?.OldAppHttp ?? 7120; int oldAppPort = worktreePorts?.OldAppHttps ?? 7121; int oldAppLiveReloadPort = worktreePorts?.OldAppLiveReload ?? 35729; @@ -24,8 +30,9 @@ string exceptionlessServerUrl = worktreePorts?.ApiHttpsUrl ?? $"https://api-ex.dev.localhost:{DefaultApiHttpsPort}"; const string SharedEmailConnectionString = "smtp://localhost:1026"; -var elastic = builder.AddElasticsearch("Elasticsearch", port: 9200) - .WithDataVolume("exceptionless.data.v1") +var elastic = builder.AddElasticsearch("Elasticsearch", port: elasticsearchPort) + .WithImageTag(elasticsearchImageTag) + .WithDataVolume(elasticsearchDataVolume) .WithEndpointProxySupport(false); var storage = builder.AddAzureStorage("Storage") @@ -70,15 +77,17 @@ var ownedElastic = elastic; elastic = ownedElastic .WithLifetime(ContainerLifetime.Persistent) - .WithContainerName("Exceptionless-Elasticsearch"); + .WithContainerName(elasticsearchContainerName); if (!servicesOnly && includeDevTools) { elastic = elastic.WithKibana(b => b + .WithImageTag(elasticsearchImageTag) .WithLifetime(ContainerLifetime.Persistent) .WithEndpointProxySupport(false) - .WithContainerName("Exceptionless-Kibana") - .WithParentRelationship(ownedElastic)); + .WithContainerName(kibanaContainerName) + .WithParentRelationship(ownedElastic), + port: kibanaPort); } var ownedCache = cache; @@ -242,3 +251,15 @@ await builder.Build().RunAsync(); bool HasArgument(string name) => args.Any(arg => StringComparer.OrdinalIgnoreCase.Equals(arg, name) || StringComparer.OrdinalIgnoreCase.Equals(arg, name.TrimStart('-'))); + +int GetPort(string key, int defaultValue) +{ + string? value = builder.Configuration[key]; + if (String.IsNullOrWhiteSpace(value)) + return defaultValue; + + if (!Int32.TryParse(value, out int port) || port is < 1 or > 65535) + throw new InvalidOperationException($"Configuration value '{key}' must be a valid TCP port."); + + return port; +} diff --git a/tests/Exceptionless.Tests/AppWebHostFactory.cs b/tests/Exceptionless.Tests/AppWebHostFactory.cs index c08f21687f..6ca9651305 100644 --- a/tests/Exceptionless.Tests/AppWebHostFactory.cs +++ b/tests/Exceptionless.Tests/AppWebHostFactory.cs @@ -21,7 +21,7 @@ namespace Exceptionless.Tests; public class AppWebHostFactory : WebApplicationFactory, IAsyncLifetime { - private const string SharedElasticsearchUrl = "http://localhost:9200"; + private static readonly string SharedElasticsearchUrl = GetSharedElasticsearchUrl(); private static readonly TimeSpan SharedElasticsearchStartupTimeout = TimeSpan.FromMinutes(3); private static int s_counter = -1; private static readonly Lazy> s_sharedAppHost = new(StartSharedAppHostAsync, LazyThreadSafetyMode.ExecutionAndPublication); @@ -58,16 +58,30 @@ private static async Task StartSharedAppHostAsync() return app; } + private static string GetSharedElasticsearchUrl() + { + const int defaultPort = 9200; + string? configuredPort = Environment.GetEnvironmentVariable("Elasticsearch__Port"); + if (String.IsNullOrWhiteSpace(configuredPort)) + return $"http://localhost:{defaultPort}"; + + if (!Int32.TryParse(configuredPort, out int port) || port is < 1 or > 65535) + throw new InvalidOperationException("Environment variable 'Elasticsearch__Port' must be a valid TCP port."); + + return $"http://localhost:{port}"; + } + private static async Task WaitForElasticsearchAsync(Uri elasticsearchUri) { - using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(1) }; + using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(2) }; var deadline = TimeProvider.System.GetUtcNow() + SharedElasticsearchStartupTimeout; + var healthUri = new Uri(elasticsearchUri, "/_cluster/health?wait_for_status=yellow&timeout=1s"); while (TimeProvider.System.GetUtcNow() < deadline) { try { - using var response = await client.GetAsync(elasticsearchUri); + using var response = await client.GetAsync(healthUri); if (response.StatusCode == HttpStatusCode.OK) return; } From 6d55ff62f52fa6186ca31c41f80c125652ee4e9a Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 14:16:07 -0500 Subject: [PATCH 02/20] Upgrade Elasticsearch stack to 9.4.2 --- .github/workflows/elasticsearch-docker-8.yml | 2 +- .github/workflows/elasticsearch-docker-9.yml | 46 +++++++++++++++++++ Dockerfile | 2 +- docker/docker-compose.apm.yml | 8 ++-- docker/docker-compose.dev.yml | 4 +- docker/docker-compose.yml | 4 +- k8s/elastic-monitor.yaml | 8 ++-- k8s/ex-dev-elasticsearch.yaml | 6 +-- k8s/ex-prod-elasticsearch.yaml | 6 +-- k8s/exceptionless/values.yaml | 2 +- samples/docker-compose.all-in-one.yml | 2 +- samples/docker-compose.yml | 4 +- .../Extensions/ElasticsearchExtensions.cs | 4 +- 13 files changed, 72 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/elasticsearch-docker-9.yml diff --git a/.github/workflows/elasticsearch-docker-8.yml b/.github/workflows/elasticsearch-docker-8.yml index ef74055e42..872667f22f 100644 --- a/.github/workflows/elasticsearch-docker-8.yml +++ b/.github/workflows/elasticsearch-docker-8.yml @@ -43,4 +43,4 @@ jobs: working-directory: build/docker/elasticsearch/8.x run: | VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile) - docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION --tag exceptionless/elasticsearch:latest + docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION diff --git a/.github/workflows/elasticsearch-docker-9.yml b/.github/workflows/elasticsearch-docker-9.yml new file mode 100644 index 0000000000..a16bd76f4b --- /dev/null +++ b/.github/workflows/elasticsearch-docker-9.yml @@ -0,0 +1,46 @@ +name: Elasticsearch 9.x Docker Image CI + +on: + push: + paths: + - "build/docker/elasticsearch/9.x/**" + - ".github/workflows/elasticsearch-docker-9.yml" + +jobs: + build: + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') != true + + steps: + - uses: actions/checkout@v7 + - name: Setup .NET Core + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.301 + - name: Build Reason + env: + GITHUB_EVENT: ${{ toJson(github) }} + run: "echo ref: ${{github.ref}} event: ${{github.event_name}}" + - name: Build Version + run: | + dotnet tool install --global minver-cli --version 7.0.0 + version=$(minver --tag-prefix v) + echo "MINVERVERSIONOVERRIDE=$version" >> $GITHUB_ENV + echo "VERSION=$version" >> $GITHUB_ENV + echo "### Version: $version" >> $GITHUB_STEP_SUMMARY + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + platforms: linux/amd64,linux/arm64 + - name: Build custom Elasticsearch 9.x docker image + working-directory: build/docker/elasticsearch/9.x + run: | + VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile) + docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION --tag exceptionless/elasticsearch:latest diff --git a/Dockerfile b/Dockerfile index 4904cdf6f9..7d8a3587dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,7 @@ ENTRYPOINT ["/app/app-docker-entrypoint.sh"] # completely self-contained -FROM exceptionless/elasticsearch:8.19.15 AS exceptionless +FROM exceptionless/elasticsearch:9.4.2 AS exceptionless WORKDIR /app COPY --from=job-publish /app/src/Exceptionless.Job/out ./ diff --git a/docker/docker-compose.apm.yml b/docker/docker-compose.apm.yml index bd55972eb9..9862cacd7d 100644 --- a/docker/docker-compose.apm.yml +++ b/docker/docker-compose.apm.yml @@ -2,7 +2,7 @@ version: "2.2" services: setup: - image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15 + image: docker.elastic.co/elasticsearch/elasticsearch:9.4.2 volumes: - certs:/usr/share/elasticsearch/config/certs user: "0" @@ -53,7 +53,7 @@ services: depends_on: setup: condition: service_healthy - image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15 + image: docker.elastic.co/elasticsearch/elasticsearch:9.4.2 volumes: - certs:/usr/share/elasticsearch/config/certs - esdata:/usr/share/elasticsearch/data @@ -98,7 +98,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.4.2 volumes: - certs:/usr/share/kibana/config/certs ports: @@ -124,7 +124,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/apm/apm-server:8.19.15 + image: docker.elastic.co/apm/apm-server:9.4.2 volumes: - certs:/usr/share/apm-server/certs ports: diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index a18d81b4cf..dd24a0dbbf 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -50,7 +50,7 @@ services: - appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:8.19.15 + image: exceptionless/elasticsearch:9.4.2 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -74,7 +74,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.4.2 ports: - 5601:5601 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c81594d41f..0c3db36ba7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch: - image: exceptionless/elasticsearch:8.19.15 + image: exceptionless/elasticsearch:9.4.2 environment: node.name: elasticsearch cluster.name: exceptionless @@ -26,7 +26,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.4.2 environment: xpack.security.enabled: "false" ports: diff --git a/k8s/elastic-monitor.yaml b/k8s/elastic-monitor.yaml index 943cde867d..c64e2a25b9 100644 --- a/k8s/elastic-monitor.yaml +++ b/k8s/elastic-monitor.yaml @@ -4,7 +4,7 @@ metadata: name: elastic-monitor namespace: elastic-system spec: - version: 8.19.15 + version: 9.4.2 podDisruptionBudget: {} nodeSets: - name: main @@ -228,7 +228,7 @@ metadata: name: kibana-monitor namespace: elastic-system spec: - version: 8.19.15 + version: 9.4.2 count: 1 http: tls: @@ -364,7 +364,7 @@ metadata: name: fleet-server namespace: elastic-system spec: - version: 8.19.15 + version: 9.4.2 kibanaRef: name: kibana-monitor elasticsearchRefs: @@ -388,7 +388,7 @@ metadata: name: elastic-agent namespace: elastic-system spec: - version: 8.19.15 + version: 9.4.2 kibanaRef: name: kibana-monitor fleetServerRef: diff --git a/k8s/ex-dev-elasticsearch.yaml b/k8s/ex-dev-elasticsearch.yaml index 46f14cc5eb..c93ffa74ed 100644 --- a/k8s/ex-dev-elasticsearch.yaml +++ b/k8s/ex-dev-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 8.19.15 - image: exceptionless/elasticsearch:8.19.15 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.4.2 + image: exceptionless/elasticsearch:9.4.2 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch secureSettings: - secretName: ex-dev-snapshots http: @@ -68,7 +68,7 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 8.19.15 + version: 9.4.2 count: 1 elasticsearchRef: name: ex-dev diff --git a/k8s/ex-prod-elasticsearch.yaml b/k8s/ex-prod-elasticsearch.yaml index f159ae3046..65df870c7d 100644 --- a/k8s/ex-prod-elasticsearch.yaml +++ b/k8s/ex-prod-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 8.19.15 - image: exceptionless/elasticsearch:8.19.15 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.4.2 + image: exceptionless/elasticsearch:9.4.2 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch monitoring: metrics: elasticsearchRefs: @@ -79,7 +79,7 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 8.19.15 + version: 9.4.2 count: 1 elasticsearchRef: name: ex-prod diff --git a/k8s/exceptionless/values.yaml b/k8s/exceptionless/values.yaml index db1c202926..1619443801 100644 --- a/k8s/exceptionless/values.yaml +++ b/k8s/exceptionless/values.yaml @@ -36,7 +36,7 @@ elasticsearch: connectionString: image: repository: exceptionless/elasticsearch - tag: 8.19.15 + tag: 9.4.2 pullPolicy: IfNotPresent redis: diff --git a/samples/docker-compose.all-in-one.yml b/samples/docker-compose.all-in-one.yml index 5b1caf2d42..573d41f74d 100644 --- a/samples/docker-compose.all-in-one.yml +++ b/samples/docker-compose.all-in-one.yml @@ -20,7 +20,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.4.2 ports: - 5601:5601 diff --git a/samples/docker-compose.yml b/samples/docker-compose.yml index d73e0518c9..4991f36e34 100644 --- a/samples/docker-compose.yml +++ b/samples/docker-compose.yml @@ -44,7 +44,7 @@ services: - ex_appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:8.19.15 + image: exceptionless/elasticsearch:9.4.2 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -58,7 +58,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.4.2 ports: - 5601:5601 diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index 4f6cc8564c..f7584f2843 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -13,7 +13,7 @@ public static class ElasticsearchBuilderExtensions private const int KibanaPort = 5601; /// - /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 8.19.15 tag of the Elasticsearch container image + /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 9.4.2 tag of the Elasticsearch container image /// /// The . /// The name of the resource. This name will be used as the connection string name when referenced in a dependency. @@ -125,7 +125,7 @@ internal static class ElasticsearchContainerImageTags public const string Image = "exceptionless/elasticsearch"; public const string KibanaRegistry = "docker.elastic.co"; public const string KibanaImage = "kibana/kibana"; - public const string Tag = "8.19.15"; + public const string Tag = "9.4.2"; } internal sealed class ElasticsearchConnectionHealthCheck(Func connectionStringFactory) : IHealthCheck From b3ebf7effe8361c263cbd327293a3b8f4fe3b999 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 19:40:17 -0500 Subject: [PATCH 03/20] Address Elasticsearch upgrade review feedback --- .github/workflows/elasticsearch-docker-9.yml | 8 ++++++- .../Extensions/ElasticsearchExtensions.cs | 12 ++++++++--- .../Exceptionless.Tests/AppWebHostFactory.cs | 21 ++++++++++++++++++- .../AppWebHostFactoryTests.cs | 18 ++++++++++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/.github/workflows/elasticsearch-docker-9.yml b/.github/workflows/elasticsearch-docker-9.yml index a16bd76f4b..b8c90d86ed 100644 --- a/.github/workflows/elasticsearch-docker-9.yml +++ b/.github/workflows/elasticsearch-docker-9.yml @@ -41,6 +41,12 @@ jobs: platforms: linux/amd64,linux/arm64 - name: Build custom Elasticsearch 9.x docker image working-directory: build/docker/elasticsearch/9.x + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile) - docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION --tag exceptionless/elasticsearch:latest + TAGS=(--tag "exceptionless/elasticsearch:$VERSION") + if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then + TAGS+=(--tag "exceptionless/elasticsearch:latest") + fi + docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . "${TAGS[@]}" diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index f7584f2843..b96df31f93 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -141,8 +141,14 @@ public async Task CheckHealthAsync(HealthCheckContext context var response = await client.Cluster.HealthAsync( request => request.WaitForStatus(Elastic.Clients.Elasticsearch.HealthStatus.Yellow), cancellationToken); - return response.IsValidResponse - ? HealthCheckResult.Healthy() - : new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch cluster health check failed: {response.DebugInformation}"); + bool isReady = response.IsValidResponse + && !response.TimedOut + && response.Status is Elastic.Clients.Elasticsearch.HealthStatus.Yellow or Elastic.Clients.Elasticsearch.HealthStatus.Green; + if (isReady) + return HealthCheckResult.Healthy(); + + return new HealthCheckResult( + context.Registration.FailureStatus, + $"Elasticsearch cluster health check failed. Timed out: {response.TimedOut}; status: {response.Status}. {response.DebugInformation}"); } } diff --git a/tests/Exceptionless.Tests/AppWebHostFactory.cs b/tests/Exceptionless.Tests/AppWebHostFactory.cs index 6ca9651305..8f6fbdbd75 100644 --- a/tests/Exceptionless.Tests/AppWebHostFactory.cs +++ b/tests/Exceptionless.Tests/AppWebHostFactory.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using System.Net; +using System.Text.Json; using Aspire.Hosting; using Aspire.Hosting.Testing; using Exceptionless.Core; @@ -82,12 +83,16 @@ private static async Task WaitForElasticsearchAsync(Uri elasticsearchUri) try { using var response = await client.GetAsync(healthUri); - if (response.StatusCode == HttpStatusCode.OK) + using var document = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync()); + if (IsElasticsearchReady(response.StatusCode, document.RootElement)) return; } catch (HttpRequestException) { } + catch (JsonException) + { + } catch (TaskCanceledException) { } @@ -98,6 +103,20 @@ private static async Task WaitForElasticsearchAsync(Uri elasticsearchUri) throw new TimeoutException("Timed out waiting for the shared Elasticsearch container to be ready."); } + internal static bool IsElasticsearchReady(HttpStatusCode statusCode, JsonElement health) + { + if (statusCode != HttpStatusCode.OK) + return false; + + bool requestCompleted = health.TryGetProperty("timed_out", out var timedOut) + && timedOut.ValueKind == JsonValueKind.False; + bool clusterReady = health.TryGetProperty("status", out var status) + && status.ValueKind == JsonValueKind.String + && status.GetString() is "yellow" or "green"; + + return requestCompleted && clusterReady; + } + protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.UseEnvironment(Environments.Development); diff --git a/tests/Exceptionless.Tests/AppWebHostFactoryTests.cs b/tests/Exceptionless.Tests/AppWebHostFactoryTests.cs index 7fef487faa..539785b681 100644 --- a/tests/Exceptionless.Tests/AppWebHostFactoryTests.cs +++ b/tests/Exceptionless.Tests/AppWebHostFactoryTests.cs @@ -1,4 +1,6 @@ +using System.Net; using System.Text; +using System.Text.Json; using Foundatio.Storage; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -7,6 +9,22 @@ namespace Exceptionless.Tests; public sealed class AppWebHostFactoryTests { + [Theory] + [InlineData(HttpStatusCode.OK, """{"timed_out":false,"status":"yellow"}""", true)] + [InlineData(HttpStatusCode.OK, """{"timed_out":false,"status":"green"}""", true)] + [InlineData(HttpStatusCode.OK, """{"timed_out":true,"status":"yellow"}""", false)] + [InlineData(HttpStatusCode.OK, """{"timed_out":false,"status":"red"}""", false)] + [InlineData(HttpStatusCode.OK, """{"timed_out":false,"status":1}""", false)] + [InlineData(HttpStatusCode.ServiceUnavailable, """{"timed_out":false,"status":"yellow"}""", false)] + public void IsElasticsearchReady_ClusterHealthResponse_ReturnsExpectedResult(HttpStatusCode statusCode, string json, bool expected) + { + using var document = JsonDocument.Parse(json); + + bool isReady = AppWebHostFactory.IsElasticsearchReady(statusCode, document.RootElement); + + Assert.Equal(expected, isReady); + } + [Fact] public async Task ConfigureWebHost_MultipleFactories_IsolatesFileStorageByAppScope() { From 2631987875570e5c692c28b52c9affc9db37c141 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 20:00:11 -0500 Subject: [PATCH 04/20] Protect Elasticsearch release image tags --- .github/workflows/elasticsearch-docker-9.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/elasticsearch-docker-9.yml b/.github/workflows/elasticsearch-docker-9.yml index b8c90d86ed..db5910edcf 100644 --- a/.github/workflows/elasticsearch-docker-9.yml +++ b/.github/workflows/elasticsearch-docker-9.yml @@ -45,8 +45,9 @@ jobs: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile) - TAGS=(--tag "exceptionless/elasticsearch:$VERSION") if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then - TAGS+=(--tag "exceptionless/elasticsearch:latest") + TAGS=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest") + else + TAGS=(--tag "exceptionless/elasticsearch:$VERSION-$GITHUB_SHA") fi docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . "${TAGS[@]}" From 34f32349b74d316367cfe6fac2ba9b0ba899243d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 20:15:54 -0500 Subject: [PATCH 05/20] chore: update Elasticsearch stack to 9.4.4 --- Dockerfile | 2 +- build/docker/elasticsearch/9.x/Dockerfile | 2 +- docker/docker-compose.apm.yml | 8 ++++---- docker/docker-compose.dev.yml | 4 ++-- docker/docker-compose.yml | 4 ++-- k8s/elastic-monitor.yaml | 8 ++++---- k8s/ex-dev-elasticsearch.yaml | 6 +++--- k8s/ex-prod-elasticsearch.yaml | 6 +++--- k8s/exceptionless/values.yaml | 2 +- samples/docker-compose.all-in-one.yml | 2 +- samples/docker-compose.yml | 4 ++-- .../Extensions/ElasticsearchExtensions.cs | 4 ++-- 12 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7d8a3587dd..c16d2b1137 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,7 @@ ENTRYPOINT ["/app/app-docker-entrypoint.sh"] # completely self-contained -FROM exceptionless/elasticsearch:9.4.2 AS exceptionless +FROM exceptionless/elasticsearch:9.4.4 AS exceptionless WORKDIR /app COPY --from=job-publish /app/src/Exceptionless.Job/out ./ diff --git a/build/docker/elasticsearch/9.x/Dockerfile b/build/docker/elasticsearch/9.x/Dockerfile index 8658600712..e27831a5c2 100644 --- a/build/docker/elasticsearch/9.x/Dockerfile +++ b/build/docker/elasticsearch/9.x/Dockerfile @@ -1,4 +1,4 @@ # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:9.4.2 +FROM docker.elastic.co/elasticsearch/elasticsearch:9.4.4 RUN elasticsearch-plugin install -b mapper-size diff --git a/docker/docker-compose.apm.yml b/docker/docker-compose.apm.yml index 9862cacd7d..b797b60a68 100644 --- a/docker/docker-compose.apm.yml +++ b/docker/docker-compose.apm.yml @@ -2,7 +2,7 @@ version: "2.2" services: setup: - image: docker.elastic.co/elasticsearch/elasticsearch:9.4.2 + image: docker.elastic.co/elasticsearch/elasticsearch:9.4.4 volumes: - certs:/usr/share/elasticsearch/config/certs user: "0" @@ -53,7 +53,7 @@ services: depends_on: setup: condition: service_healthy - image: docker.elastic.co/elasticsearch/elasticsearch:9.4.2 + image: docker.elastic.co/elasticsearch/elasticsearch:9.4.4 volumes: - certs:/usr/share/elasticsearch/config/certs - esdata:/usr/share/elasticsearch/data @@ -98,7 +98,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/kibana/kibana:9.4.2 + image: docker.elastic.co/kibana/kibana:9.4.4 volumes: - certs:/usr/share/kibana/config/certs ports: @@ -124,7 +124,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/apm/apm-server:9.4.2 + image: docker.elastic.co/apm/apm-server:9.4.4 volumes: - certs:/usr/share/apm-server/certs ports: diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index dd24a0dbbf..604325d68c 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -50,7 +50,7 @@ services: - appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:9.4.2 + image: exceptionless/elasticsearch:9.4.4 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -74,7 +74,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.4.2 + image: docker.elastic.co/kibana/kibana:9.4.4 ports: - 5601:5601 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0c3db36ba7..417e3751b5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch: - image: exceptionless/elasticsearch:9.4.2 + image: exceptionless/elasticsearch:9.4.4 environment: node.name: elasticsearch cluster.name: exceptionless @@ -26,7 +26,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.4.2 + image: docker.elastic.co/kibana/kibana:9.4.4 environment: xpack.security.enabled: "false" ports: diff --git a/k8s/elastic-monitor.yaml b/k8s/elastic-monitor.yaml index c64e2a25b9..eb409fcc8a 100644 --- a/k8s/elastic-monitor.yaml +++ b/k8s/elastic-monitor.yaml @@ -4,7 +4,7 @@ metadata: name: elastic-monitor namespace: elastic-system spec: - version: 9.4.2 + version: 9.4.4 podDisruptionBudget: {} nodeSets: - name: main @@ -228,7 +228,7 @@ metadata: name: kibana-monitor namespace: elastic-system spec: - version: 9.4.2 + version: 9.4.4 count: 1 http: tls: @@ -364,7 +364,7 @@ metadata: name: fleet-server namespace: elastic-system spec: - version: 9.4.2 + version: 9.4.4 kibanaRef: name: kibana-monitor elasticsearchRefs: @@ -388,7 +388,7 @@ metadata: name: elastic-agent namespace: elastic-system spec: - version: 9.4.2 + version: 9.4.4 kibanaRef: name: kibana-monitor fleetServerRef: diff --git a/k8s/ex-dev-elasticsearch.yaml b/k8s/ex-dev-elasticsearch.yaml index c93ffa74ed..ee61778a6f 100644 --- a/k8s/ex-dev-elasticsearch.yaml +++ b/k8s/ex-dev-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 9.4.2 - image: exceptionless/elasticsearch:9.4.2 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.4.4 + image: exceptionless/elasticsearch:9.4.4 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch secureSettings: - secretName: ex-dev-snapshots http: @@ -68,7 +68,7 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 9.4.2 + version: 9.4.4 count: 1 elasticsearchRef: name: ex-dev diff --git a/k8s/ex-prod-elasticsearch.yaml b/k8s/ex-prod-elasticsearch.yaml index 65df870c7d..88457e15d1 100644 --- a/k8s/ex-prod-elasticsearch.yaml +++ b/k8s/ex-prod-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 9.4.2 - image: exceptionless/elasticsearch:9.4.2 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.4.4 + image: exceptionless/elasticsearch:9.4.4 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch monitoring: metrics: elasticsearchRefs: @@ -79,7 +79,7 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 9.4.2 + version: 9.4.4 count: 1 elasticsearchRef: name: ex-prod diff --git a/k8s/exceptionless/values.yaml b/k8s/exceptionless/values.yaml index 1619443801..4027592194 100644 --- a/k8s/exceptionless/values.yaml +++ b/k8s/exceptionless/values.yaml @@ -36,7 +36,7 @@ elasticsearch: connectionString: image: repository: exceptionless/elasticsearch - tag: 9.4.2 + tag: 9.4.4 pullPolicy: IfNotPresent redis: diff --git a/samples/docker-compose.all-in-one.yml b/samples/docker-compose.all-in-one.yml index 573d41f74d..0560b023cf 100644 --- a/samples/docker-compose.all-in-one.yml +++ b/samples/docker-compose.all-in-one.yml @@ -20,7 +20,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.4.2 + image: docker.elastic.co/kibana/kibana:9.4.4 ports: - 5601:5601 diff --git a/samples/docker-compose.yml b/samples/docker-compose.yml index 4991f36e34..1d57db836e 100644 --- a/samples/docker-compose.yml +++ b/samples/docker-compose.yml @@ -44,7 +44,7 @@ services: - ex_appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:9.4.2 + image: exceptionless/elasticsearch:9.4.4 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -58,7 +58,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.4.2 + image: docker.elastic.co/kibana/kibana:9.4.4 ports: - 5601:5601 diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index b96df31f93..e348e42e74 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -13,7 +13,7 @@ public static class ElasticsearchBuilderExtensions private const int KibanaPort = 5601; /// - /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 9.4.2 tag of the Elasticsearch container image + /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 9.4.4 tag of the Elasticsearch container image /// /// The . /// The name of the resource. This name will be used as the connection string name when referenced in a dependency. @@ -125,7 +125,7 @@ internal static class ElasticsearchContainerImageTags public const string Image = "exceptionless/elasticsearch"; public const string KibanaRegistry = "docker.elastic.co"; public const string KibanaImage = "kibana/kibana"; - public const string Tag = "9.4.2"; + public const string Tag = "9.4.4"; } internal sealed class ElasticsearchConnectionHealthCheck(Func connectionStringFactory) : IHealthCheck From d5070d7d6cdf513ed7e3700c76aeda64bdd537ef Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 20:34:33 -0500 Subject: [PATCH 06/20] Test Elasticsearch branch images safely --- .github/workflows/build.yaml | 38 ++++++++++++++++++++ .github/workflows/elasticsearch-docker-9.yml | 3 +- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1d0725d975..ab412f4f6e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -83,6 +83,7 @@ jobs: timeout-minutes: 30 outputs: version: ${{ steps.version.outputs.version }} + elasticsearch_image_tag: ${{ steps.elasticsearch_image.outputs.tag }} should_publish: ${{ (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'dev-preview')) && secrets.DOCKER_USERNAME != '' && secrets.DOCKER_PASSWORD != '' }} is_prod_deploy: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name != 'pull_request' }} is_dev_deploy: ${{ (github.event_name == 'repository_dispatch' && github.event.action == 'preview') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'dev-preview')) }} @@ -129,9 +130,43 @@ jobs: echo "version=$version" >> $GITHUB_OUTPUT echo "### $version" >> $GITHUB_STEP_SUMMARY + - name: Resolve Elasticsearch image + id: elasticsearch_image + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + version=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile) + tag=$version + + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] && + ! git diff --quiet "$PR_BASE_SHA"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_sha=$(sha256sum build/docker/elasticsearch/9.x/Dockerfile | cut -d " " -f 1) + tag="$version-sha256-$image_sha" + image="exceptionless/elasticsearch:$tag" + + for attempt in {1..30}; do + if docker manifest inspect "$image" > /dev/null 2>&1; then + break + fi + + if [[ "$attempt" -eq 30 ]]; then + echo "::error::Timed out waiting for $image to be published." + exit 1 + fi + + sleep 10 + done + fi + + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "### Elasticsearch image: exceptionless/elasticsearch:$tag" >> "$GITHUB_STEP_SUMMARY" + test-api: + needs: version runs-on: ubuntu-latest timeout-minutes: 30 + env: + Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }} steps: - name: Checkout @@ -223,8 +258,11 @@ jobs: run: echo "npm run test:integration" test-e2e: + needs: version runs-on: ubuntu-latest timeout-minutes: 45 + env: + Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }} steps: - name: Checkout diff --git a/.github/workflows/elasticsearch-docker-9.yml b/.github/workflows/elasticsearch-docker-9.yml index db5910edcf..fe9a697c63 100644 --- a/.github/workflows/elasticsearch-docker-9.yml +++ b/.github/workflows/elasticsearch-docker-9.yml @@ -48,6 +48,7 @@ jobs: if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then TAGS=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest") else - TAGS=(--tag "exceptionless/elasticsearch:$VERSION-$GITHUB_SHA") + IMAGE_SHA=$(sha256sum Dockerfile | cut -d " " -f 1) + TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA") fi docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . "${TAGS[@]}" From 609860389b089dca8794a2dd1d62d3be6831cf4c Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 21:00:30 -0500 Subject: [PATCH 07/20] Address Elasticsearch upgrade migration feedback --- .github/workflows/build.yaml | 9 +++++ .github/workflows/elasticsearch-docker-9.yml | 7 ++-- docker/docker-compose.dev.yml | 2 ++ docker/docker-compose.yml | 2 ++ .../upgrading-self-hosted-instance.md | 35 +++++++++++++++++++ 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ab412f4f6e..c3eb94db08 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -134,12 +134,21 @@ jobs: id: elasticsearch_image env: PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PUSH_BEFORE_SHA: ${{ github.event.before }} run: | version=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile) tag=$version + image_changed=false if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] && ! git diff --quiet "$PR_BASE_SHA"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + elif [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == "refs/heads/main" ]] && + ! git diff --quiet "$PUSH_BEFORE_SHA"..HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + fi + + if [[ "$image_changed" == "true" ]]; then image_sha=$(sha256sum build/docker/elasticsearch/9.x/Dockerfile | cut -d " " -f 1) tag="$version-sha256-$image_sha" image="exceptionless/elasticsearch:$tag" diff --git a/.github/workflows/elasticsearch-docker-9.yml b/.github/workflows/elasticsearch-docker-9.yml index fe9a697c63..6fe30d0bb5 100644 --- a/.github/workflows/elasticsearch-docker-9.yml +++ b/.github/workflows/elasticsearch-docker-9.yml @@ -45,10 +45,9 @@ jobs: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile) + IMAGE_SHA=$(sha256sum Dockerfile | cut -d " " -f 1) + TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA") if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then - TAGS=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest") - else - IMAGE_SHA=$(sha256sum Dockerfile | cut -d " " -f 1) - TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA") + TAGS+=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest") fi docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . "${TAGS[@]}" diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 604325d68c..6752cf1a07 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -59,6 +59,8 @@ services: - 9200:9200 - 9300:9300 volumes: + # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9. + # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance - esdata7:/usr/share/elasticsearch/data healthcheck: test: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 417e3751b5..11b3f2a705 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,6 +11,8 @@ services: ports: - 9200:9200 volumes: + # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9. + # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance - esdata:/usr/share/elasticsearch/data healthcheck: test: diff --git a/docs/docs/self-hosting/upgrading-self-hosted-instance.md b/docs/docs/self-hosting/upgrading-self-hosted-instance.md index dd0974989f..2335c9e421 100644 --- a/docs/docs/self-hosting/upgrading-self-hosted-instance.md +++ b/docs/docs/self-hosting/upgrading-self-hosted-instance.md @@ -8,6 +8,41 @@ title: "Upgrading" **If you are upgrading from v1 or [v2](https://github.com/exceptionless/Exceptionless/releases/tag/v2.0.0) you will need to upgrade to [v3.0](https://github.com/exceptionless/Exceptionless/releases/tag/v3.0.0) before upgrading to the latest release.** +## Upgrading from v8 to v9 + +Exceptionless v9 uses Elasticsearch 9. Do not point an Elasticsearch 9 node at an existing data volume until the cluster has been prepared with Elasticsearch 8.19. Elasticsearch 9 can fail to start when incompatible indices created before Elasticsearch 8 remain. + +Use this upgrade path for an existing self-hosted installation: + +1. Take a current Elasticsearch snapshot or other verified backup and test that it can be restored. Elasticsearch does not support downgrading a data directory after it has been upgraded. +2. Upgrade Elasticsearch and Kibana to the latest 8.19.x patch release first, using the existing data volume. Do not start Elasticsearch 9 yet. +3. Stop the Exceptionless app and job services, but leave Elasticsearch and Kibana 8.19 running. This prevents writes while legacy indices are reindexed. +4. Open Kibana's **Upgrade Assistant** and resolve every critical issue. Reindex every active Exceptionless index created before Elasticsearch 8. Delete only indices you have confirmed are no longer needed; do not mark active Exceptionless indices as read-only. +5. If this data volume previously ran Elasticsearch 7, temporarily disable the GeoIP downloader while still on Elasticsearch 8.19. Elasticsearch deletes its downloaded `.geoip_databases` system index when this setting is disabled; Exceptionless data is not affected. + + ```bash + curl -fsS -X PUT "http://localhost:9200/_cluster/settings" \ + -H "Content-Type: application/json" \ + -d '{"persistent":{"ingest.geoip.downloader.enabled":false}}' + ``` + +6. Confirm that the deprecation API reports no critical issues: + + ```bash + curl -fsS "http://localhost:9200/_migration/deprecations?pretty" + ``` + +7. Stop Elasticsearch and Kibana 8.19 without deleting their data volume. Update the Elasticsearch and Kibana images to the v9 versions, start them, and verify cluster health before restarting the Exceptionless app and jobs. In Docker Compose, do not run `docker compose down -v` because `-v` deletes the data volume. +8. After Elasticsearch 9 is healthy, restore the default GeoIP downloader behavior: + + ```bash + curl -fsS -X PUT "http://localhost:9200/_cluster/settings" \ + -H "Content-Type: application/json" \ + -d '{"persistent":{"ingest.geoip.downloader.enabled":null}}' + ``` + +See Elastic's [prepare-to-upgrade guide](https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade) for the supported 8.x to 9.x upgrade requirements and Upgrade Assistant details. + ## Upgrading from v7.1 to v8 We simplified the self hosting process by integrating the UI into the existing app images. As such `exceptionless/ui` docker images are deprecated and we recommend using `exceptionless/app`. From e2afb7cbe5c56a3a2db3bc9b4f887ad3ebc56154 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 21:21:54 -0500 Subject: [PATCH 08/20] Fix Elasticsearch candidate image isolation --- .github/workflows/build.yaml | 2 +- .github/workflows/elasticsearch-docker-9.yml | 7 ++--- src/Exceptionless.AppHost/Program.cs | 3 +- .../AppHostConfigurationTests.cs | 30 +++++++++++++++++++ 4 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 tests/Exceptionless.Tests/AppHostConfigurationTests.cs diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c3eb94db08..e884ca2132 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -149,7 +149,7 @@ jobs: fi if [[ "$image_changed" == "true" ]]; then - image_sha=$(sha256sum build/docker/elasticsearch/9.x/Dockerfile | cut -d " " -f 1) + image_sha=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1) tag="$version-sha256-$image_sha" image="exceptionless/elasticsearch:$tag" diff --git a/.github/workflows/elasticsearch-docker-9.yml b/.github/workflows/elasticsearch-docker-9.yml index 6fe30d0bb5..a911616691 100644 --- a/.github/workflows/elasticsearch-docker-9.yml +++ b/.github/workflows/elasticsearch-docker-9.yml @@ -40,14 +40,13 @@ jobs: with: platforms: linux/amd64,linux/arm64 - name: Build custom Elasticsearch 9.x docker image - working-directory: build/docker/elasticsearch/9.x env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | - VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile) - IMAGE_SHA=$(sha256sum Dockerfile | cut -d " " -f 1) + VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile) + IMAGE_SHA=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1) TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA") if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then TAGS+=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest") fi - docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . "${TAGS[@]}" + docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x "${TAGS[@]}" diff --git a/src/Exceptionless.AppHost/Program.cs b/src/Exceptionless.AppHost/Program.cs index 6e62e7fc45..ef33ee8333 100644 --- a/src/Exceptionless.AppHost/Program.cs +++ b/src/Exceptionless.AppHost/Program.cs @@ -15,6 +15,7 @@ bool includeDevTools = !ciE2E; int elasticsearchPort = GetPort("Elasticsearch:Port", 9200); string elasticsearchImageTag = builder.Configuration["Elasticsearch:ImageTag"] ?? ElasticsearchContainerImageTags.Tag; +string kibanaImageTag = builder.Configuration["Elasticsearch:KibanaImageTag"] ?? ElasticsearchContainerImageTags.Tag; string elasticsearchContainerName = builder.Configuration["Elasticsearch:ContainerName"] ?? "Exceptionless-Elasticsearch"; string elasticsearchDataVolume = builder.Configuration["Elasticsearch:DataVolume"] ?? "exceptionless.data.v1"; string kibanaContainerName = builder.Configuration["Elasticsearch:KibanaContainerName"] ?? "Exceptionless-Kibana"; @@ -82,7 +83,7 @@ if (!servicesOnly && includeDevTools) { elastic = elastic.WithKibana(b => b - .WithImageTag(elasticsearchImageTag) + .WithImageTag(kibanaImageTag) .WithLifetime(ContainerLifetime.Persistent) .WithEndpointProxySupport(false) .WithContainerName(kibanaContainerName) diff --git a/tests/Exceptionless.Tests/AppHostConfigurationTests.cs b/tests/Exceptionless.Tests/AppHostConfigurationTests.cs new file mode 100644 index 0000000000..455dda2859 --- /dev/null +++ b/tests/Exceptionless.Tests/AppHostConfigurationTests.cs @@ -0,0 +1,30 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Testing; +using Xunit; + +namespace Exceptionless.Tests; + +public class AppHostConfigurationTests +{ + [Fact] + public async Task CreateAsync_WithSeparateElasticsearchAndKibanaOverrides_UsesIndependentImageTags() + { + const string elasticsearchImageTag = "9.4.4-sha256-candidate"; + const string kibanaImageTag = "9.4.4"; + var appHost = await DistributedApplicationTestingBuilder.CreateAsync( + [ + $"--Elasticsearch:ImageTag={elasticsearchImageTag}", + $"--Elasticsearch:KibanaImageTag={kibanaImageTag}" + ], + TestContext.Current.CancellationToken); + + var elasticsearch = Assert.Single(appHost.Resources.OfType()); + var kibana = Assert.Single(appHost.Resources.OfType()); + var elasticsearchImage = Assert.Single(elasticsearch.Annotations.OfType()); + var kibanaImage = Assert.Single(kibana.Annotations.OfType()); + + Assert.Equal(elasticsearchImageTag, elasticsearchImage.Tag); + Assert.Equal(kibanaImageTag, kibanaImage.Tag); + } +} From 9a27375c245e9294220f63afd8f2dcccfa9b19d4 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 28 Jul 2026 21:23:35 -0500 Subject: [PATCH 09/20] Wait for uncached Elasticsearch image builds --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e884ca2132..b30f975fa6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -153,12 +153,12 @@ jobs: tag="$version-sha256-$image_sha" image="exceptionless/elasticsearch:$tag" - for attempt in {1..30}; do + for attempt in {1..150}; do if docker manifest inspect "$image" > /dev/null 2>&1; then break fi - if [[ "$attempt" -eq 30 ]]; then + if [[ "$attempt" -eq 150 ]]; then echo "::error::Timed out waiting for $image to be published." exit 1 fi From 750a4aa0b781cbdba6abd169f2d9db8f9f799668 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 4 Aug 2026 08:59:49 -0500 Subject: [PATCH 10/20] chore: update Elasticsearch stack to 9.5.0 --- Dockerfile | 2 +- build/docker/elasticsearch/9.x/Dockerfile | 2 +- docker/docker-compose.apm.yml | 8 ++++---- docker/docker-compose.dev.yml | 4 ++-- docker/docker-compose.yml | 4 ++-- k8s/elastic-monitor.yaml | 8 ++++---- k8s/ex-dev-elasticsearch.yaml | 6 +++--- k8s/ex-prod-elasticsearch.yaml | 6 +++--- k8s/exceptionless/values.yaml | 2 +- samples/docker-compose.all-in-one.yml | 2 +- samples/docker-compose.yml | 4 ++-- .../Extensions/ElasticsearchExtensions.cs | 4 ++-- tests/Exceptionless.Tests/AppHostConfigurationTests.cs | 4 ++-- 13 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index c16d2b1137..dbbf22e082 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,7 @@ ENTRYPOINT ["/app/app-docker-entrypoint.sh"] # completely self-contained -FROM exceptionless/elasticsearch:9.4.4 AS exceptionless +FROM exceptionless/elasticsearch:9.5.0 AS exceptionless WORKDIR /app COPY --from=job-publish /app/src/Exceptionless.Job/out ./ diff --git a/build/docker/elasticsearch/9.x/Dockerfile b/build/docker/elasticsearch/9.x/Dockerfile index e27831a5c2..cc03ebce5a 100644 --- a/build/docker/elasticsearch/9.x/Dockerfile +++ b/build/docker/elasticsearch/9.x/Dockerfile @@ -1,4 +1,4 @@ # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:9.4.4 +FROM docker.elastic.co/elasticsearch/elasticsearch:9.5.0 RUN elasticsearch-plugin install -b mapper-size diff --git a/docker/docker-compose.apm.yml b/docker/docker-compose.apm.yml index b797b60a68..220098afad 100644 --- a/docker/docker-compose.apm.yml +++ b/docker/docker-compose.apm.yml @@ -2,7 +2,7 @@ version: "2.2" services: setup: - image: docker.elastic.co/elasticsearch/elasticsearch:9.4.4 + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.0 volumes: - certs:/usr/share/elasticsearch/config/certs user: "0" @@ -53,7 +53,7 @@ services: depends_on: setup: condition: service_healthy - image: docker.elastic.co/elasticsearch/elasticsearch:9.4.4 + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.0 volumes: - certs:/usr/share/elasticsearch/config/certs - esdata:/usr/share/elasticsearch/data @@ -98,7 +98,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/kibana/kibana:9.4.4 + image: docker.elastic.co/kibana/kibana:9.5.0 volumes: - certs:/usr/share/kibana/config/certs ports: @@ -124,7 +124,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/apm/apm-server:9.4.4 + image: docker.elastic.co/apm/apm-server:9.5.0 volumes: - certs:/usr/share/apm-server/certs ports: diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 6752cf1a07..2b8518034b 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -50,7 +50,7 @@ services: - appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:9.4.4 + image: exceptionless/elasticsearch:9.5.0 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -76,7 +76,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.4.4 + image: docker.elastic.co/kibana/kibana:9.5.0 ports: - 5601:5601 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 11b3f2a705..b1706174b1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch: - image: exceptionless/elasticsearch:9.4.4 + image: exceptionless/elasticsearch:9.5.0 environment: node.name: elasticsearch cluster.name: exceptionless @@ -28,7 +28,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.4.4 + image: docker.elastic.co/kibana/kibana:9.5.0 environment: xpack.security.enabled: "false" ports: diff --git a/k8s/elastic-monitor.yaml b/k8s/elastic-monitor.yaml index eb409fcc8a..f1c47ad253 100644 --- a/k8s/elastic-monitor.yaml +++ b/k8s/elastic-monitor.yaml @@ -4,7 +4,7 @@ metadata: name: elastic-monitor namespace: elastic-system spec: - version: 9.4.4 + version: 9.5.0 podDisruptionBudget: {} nodeSets: - name: main @@ -228,7 +228,7 @@ metadata: name: kibana-monitor namespace: elastic-system spec: - version: 9.4.4 + version: 9.5.0 count: 1 http: tls: @@ -364,7 +364,7 @@ metadata: name: fleet-server namespace: elastic-system spec: - version: 9.4.4 + version: 9.5.0 kibanaRef: name: kibana-monitor elasticsearchRefs: @@ -388,7 +388,7 @@ metadata: name: elastic-agent namespace: elastic-system spec: - version: 9.4.4 + version: 9.5.0 kibanaRef: name: kibana-monitor fleetServerRef: diff --git a/k8s/ex-dev-elasticsearch.yaml b/k8s/ex-dev-elasticsearch.yaml index ee61778a6f..6a8b1fd851 100644 --- a/k8s/ex-dev-elasticsearch.yaml +++ b/k8s/ex-dev-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 9.4.4 - image: exceptionless/elasticsearch:9.4.4 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.5.0 + image: exceptionless/elasticsearch:9.5.0 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch secureSettings: - secretName: ex-dev-snapshots http: @@ -68,7 +68,7 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 9.4.4 + version: 9.5.0 count: 1 elasticsearchRef: name: ex-dev diff --git a/k8s/ex-prod-elasticsearch.yaml b/k8s/ex-prod-elasticsearch.yaml index 88457e15d1..de7848bce1 100644 --- a/k8s/ex-prod-elasticsearch.yaml +++ b/k8s/ex-prod-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 9.4.4 - image: exceptionless/elasticsearch:9.4.4 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.5.0 + image: exceptionless/elasticsearch:9.5.0 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch monitoring: metrics: elasticsearchRefs: @@ -79,7 +79,7 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 9.4.4 + version: 9.5.0 count: 1 elasticsearchRef: name: ex-prod diff --git a/k8s/exceptionless/values.yaml b/k8s/exceptionless/values.yaml index 4027592194..2e576bcd32 100644 --- a/k8s/exceptionless/values.yaml +++ b/k8s/exceptionless/values.yaml @@ -36,7 +36,7 @@ elasticsearch: connectionString: image: repository: exceptionless/elasticsearch - tag: 9.4.4 + tag: 9.5.0 pullPolicy: IfNotPresent redis: diff --git a/samples/docker-compose.all-in-one.yml b/samples/docker-compose.all-in-one.yml index 0560b023cf..622e74830e 100644 --- a/samples/docker-compose.all-in-one.yml +++ b/samples/docker-compose.all-in-one.yml @@ -20,7 +20,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.4.4 + image: docker.elastic.co/kibana/kibana:9.5.0 ports: - 5601:5601 diff --git a/samples/docker-compose.yml b/samples/docker-compose.yml index 1d57db836e..7fcb4cc0de 100644 --- a/samples/docker-compose.yml +++ b/samples/docker-compose.yml @@ -44,7 +44,7 @@ services: - ex_appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:9.4.4 + image: exceptionless/elasticsearch:9.5.0 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -58,7 +58,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.4.4 + image: docker.elastic.co/kibana/kibana:9.5.0 ports: - 5601:5601 diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index e348e42e74..4022e89583 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -13,7 +13,7 @@ public static class ElasticsearchBuilderExtensions private const int KibanaPort = 5601; /// - /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 9.4.4 tag of the Elasticsearch container image + /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 9.5.0 tag of the Elasticsearch container image /// /// The . /// The name of the resource. This name will be used as the connection string name when referenced in a dependency. @@ -125,7 +125,7 @@ internal static class ElasticsearchContainerImageTags public const string Image = "exceptionless/elasticsearch"; public const string KibanaRegistry = "docker.elastic.co"; public const string KibanaImage = "kibana/kibana"; - public const string Tag = "9.4.4"; + public const string Tag = "9.5.0"; } internal sealed class ElasticsearchConnectionHealthCheck(Func connectionStringFactory) : IHealthCheck diff --git a/tests/Exceptionless.Tests/AppHostConfigurationTests.cs b/tests/Exceptionless.Tests/AppHostConfigurationTests.cs index 455dda2859..5ec833ceaa 100644 --- a/tests/Exceptionless.Tests/AppHostConfigurationTests.cs +++ b/tests/Exceptionless.Tests/AppHostConfigurationTests.cs @@ -10,8 +10,8 @@ public class AppHostConfigurationTests [Fact] public async Task CreateAsync_WithSeparateElasticsearchAndKibanaOverrides_UsesIndependentImageTags() { - const string elasticsearchImageTag = "9.4.4-sha256-candidate"; - const string kibanaImageTag = "9.4.4"; + const string elasticsearchImageTag = "9.5.0-sha256-candidate"; + const string kibanaImageTag = "9.5.0"; var appHost = await DistributedApplicationTestingBuilder.CreateAsync( [ $"--Elasticsearch:ImageTag={elasticsearchImageTag}", From 6543a8aa63ed58c83959a6efe66e1935fcae07b1 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 5 Aug 2026 09:08:08 -0500 Subject: [PATCH 11/20] Fix dispatched Elasticsearch candidate selection --- .github/workflows/build.yaml | 7 +++++++ .github/workflows/preview-command.yml | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b30f975fa6..e2c897f234 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -134,6 +134,7 @@ jobs: id: elasticsearch_image env: PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PREVIEW_BASE_SHA: ${{ github.event.client_payload.base_sha }} PUSH_BEFORE_SHA: ${{ github.event.before }} run: | version=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile) @@ -146,6 +147,12 @@ jobs: elif [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == "refs/heads/main" ]] && ! git diff --quiet "$PUSH_BEFORE_SHA"..HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then image_changed=true + elif [[ "$GITHUB_EVENT_NAME" == "repository_dispatch" ]] && + ! git diff --quiet "${PREVIEW_BASE_SHA:-origin/main}"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + elif [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]] && + ! git diff --quiet origin/main...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true fi if [[ "$image_changed" == "true" ]]; then diff --git a/.github/workflows/preview-command.yml b/.github/workflows/preview-command.yml index fe94875393..e4dd285c7f 100644 --- a/.github/workflows/preview-command.yml +++ b/.github/workflows/preview-command.yml @@ -77,6 +77,7 @@ jobs: const headRepository = pullRequest.head.repo.full_name; const headRef = pullRequest.head.ref; const headSha = pullRequest.head.sha; + const baseSha = pullRequest.base.sha; const headLabel = pullRequest.head.label; const headShortSha = headSha.slice(0, 12); @@ -95,6 +96,7 @@ jobs: core.setOutput("head-ref", headRef); core.setOutput("head-label", headLabel); core.setOutput("head-sha", headSha); + core.setOutput("base-sha", baseSha); core.setOutput("head-short-sha", headShortSha); const previewLabel = "dev-preview"; @@ -142,6 +144,7 @@ jobs: HEAD_REF: ${{ steps.preview.outputs.head-ref }} HEAD_LABEL: ${{ steps.preview.outputs.head-label }} HEAD_SHA: ${{ steps.preview.outputs.head-sha }} + BASE_SHA: ${{ steps.preview.outputs.base-sha }} with: script: | await github.rest.repos.createDispatchEvent({ @@ -152,7 +155,8 @@ jobs: pr_number: Number(process.env.PR_NUMBER), head_ref: process.env.HEAD_REF, head_label: process.env.HEAD_LABEL, - head_sha: process.env.HEAD_SHA + head_sha: process.env.HEAD_SHA, + base_sha: process.env.BASE_SHA } }); From 81b47cf45d544392fb3025f799d7208704dfdeb8 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 5 Aug 2026 09:13:34 -0500 Subject: [PATCH 12/20] Build fork Elasticsearch candidates locally --- .github/workflows/build.yaml | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e2c897f234..9f946514e7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -84,6 +84,7 @@ jobs: outputs: version: ${{ steps.version.outputs.version }} elasticsearch_image_tag: ${{ steps.elasticsearch_image.outputs.tag }} + build_elasticsearch_image: ${{ steps.elasticsearch_image.outputs.build_locally }} should_publish: ${{ (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'dev-preview')) && secrets.DOCKER_USERNAME != '' && secrets.DOCKER_PASSWORD != '' }} is_prod_deploy: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name != 'pull_request' }} is_dev_deploy: ${{ (github.event_name == 'repository_dispatch' && github.event.action == 'preview') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'dev-preview')) }} @@ -134,12 +135,14 @@ jobs: id: elasticsearch_image env: PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} PREVIEW_BASE_SHA: ${{ github.event.client_payload.base_sha }} PUSH_BEFORE_SHA: ${{ github.event.before }} run: | version=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile) tag=$version image_changed=false + build_locally=false if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] && ! git diff --quiet "$PR_BASE_SHA"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then @@ -160,21 +163,26 @@ jobs: tag="$version-sha256-$image_sha" image="exceptionless/elasticsearch:$tag" - for attempt in {1..150}; do - if docker manifest inspect "$image" > /dev/null 2>&1; then - break - fi + if [[ "$GITHUB_EVENT_NAME" == "pull_request" && "$PR_HEAD_REPOSITORY" != "$GITHUB_REPOSITORY" ]]; then + build_locally=true + else + for attempt in {1..150}; do + if docker manifest inspect "$image" > /dev/null 2>&1; then + break + fi - if [[ "$attempt" -eq 150 ]]; then - echo "::error::Timed out waiting for $image to be published." - exit 1 - fi + if [[ "$attempt" -eq 150 ]]; then + echo "::error::Timed out waiting for $image to be published." + exit 1 + fi - sleep 10 - done + sleep 10 + done + fi fi echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "build_locally=$build_locally" >> "$GITHUB_OUTPUT" echo "### Elasticsearch image: exceptionless/elasticsearch:$tag" >> "$GITHUB_STEP_SUMMARY" test-api: @@ -190,6 +198,10 @@ jobs: with: ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }} + - name: Build fork Elasticsearch candidate locally + if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }} + run: docker build --tag "exceptionless/elasticsearch:${{ needs.version.outputs.elasticsearch_image_tag }}" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x + - name: Setup .NET Core uses: actions/setup-dotnet@v6 with: @@ -284,6 +296,10 @@ jobs: - name: Checkout uses: actions/checkout@v6 + - name: Build fork Elasticsearch candidate locally + if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }} + run: docker build --tag "exceptionless/elasticsearch:${{ needs.version.outputs.elasticsearch_image_tag }}" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x + - name: Setup .NET Core uses: actions/setup-dotnet@v6 with: From 5618cb9b083bf6d0b3ad87f149ef84d173e4834a Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 12 Aug 2026 12:32:52 -0500 Subject: [PATCH 13/20] Fix Elasticsearch 9 all-in-one package installation --- Dockerfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index dbbf22e082..7b0c1b3d43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -113,21 +113,21 @@ COPY ./build/supervisord.conf /etc/ USER root # install dotnet and supervisor -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - supervisor \ +RUN microdnf install -y \ wget \ dos2unix \ ca-certificates \ + python3-pip \ \ # .NET dependencies - libc6 \ - libgcc-s1 \ - libicu74 \ - libssl3 \ - libstdc++6 \ + glibc \ + libgcc \ + libicu \ + openssl-libs \ + libstdc++ \ tzdata \ - && rm -rf /var/lib/apt/lists/* \ + && pip3 install --no-cache-dir supervisor==4.3.0 \ + && microdnf clean all \ && dos2unix /app/docker-entrypoint.sh ENV discovery.type=single-node \ From 14ee90485aad46bef25911f811490768a666cee1 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 12 Aug 2026 12:37:13 -0500 Subject: [PATCH 14/20] Install archive support in Elasticsearch runtime --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7b0c1b3d43..580f31de86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -125,6 +125,7 @@ RUN microdnf install -y \ libicu \ openssl-libs \ libstdc++ \ + tar \ tzdata \ && pip3 install --no-cache-dir supervisor==4.3.0 \ && microdnf clean all \ From 40265bff0a748927a2da8aa03f1b1f3380d61913 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 12 Aug 2026 12:37:58 -0500 Subject: [PATCH 15/20] Install gzip for the .NET runtime archive --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 580f31de86..f6f015c18a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -121,6 +121,7 @@ RUN microdnf install -y \ \ # .NET dependencies glibc \ + gzip \ libgcc \ libicu \ openssl-libs \ From f80df4048d92a89e2c437d1d5c9df4a65fd3820a Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 12 Aug 2026 12:47:26 -0500 Subject: [PATCH 16/20] Check out preview heads for E2E tests --- .github/workflows/build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9f946514e7..69db68ae5a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -295,6 +295,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }} - name: Build fork Elasticsearch candidate locally if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }} From 9d1dfbb05905071708805c422feabe5d2bb74fcf Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 6 Sep 2026 15:28:09 -0500 Subject: [PATCH 17/20] Document staged Elasticsearch 9 production migration --- docs/elasticsearch-9-production-migration.md | 134 +++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 docs/elasticsearch-9-production-migration.md diff --git a/docs/elasticsearch-9-production-migration.md b/docs/elasticsearch-9-production-migration.md new file mode 100644 index 0000000000..f9a4742b38 --- /dev/null +++ b/docs/elasticsearch-9-production-migration.md @@ -0,0 +1,134 @@ +# Elasticsearch 9 production migration plan + +Status: proposed; no production changes have been made. Inventory observations below are from September 6, 2026. This is an operator-run migration, not an application-startup migration. + +## Separate the three changes + +| Change | Release boundary | Required data work | +| --- | --- | --- | +| Run Elasticsearch 9 with existing application queries | Base PR [#2416](https://github.com/exceptionless/Exceptionless/pull/2416) alone | Resolve unsupported pre-8 indexes before starting 9; do not rewrite all 8-created indexes | +| Recreate indexes in the current server's index format | Explicit maintenance after the server upgrade stabilizes | Selected retained indexes, independently of application schema versions | +| Enable ES\|QL stack/event filtering and cursor-based stack summaries | Experimental PR [#2511](https://github.com/exceptionless/Exceptionless/pull/2511), separately deployed | Convert the canonical stack index to lookup mode, validate mixed-generation event sources, and benchmark production-scale queries | + +An Elasticsearch server version, `index.version.created`, the repository's schema version, and `index.mode` are different things. A force merge or server restart is not an index recreation. Do not bump the daily event schema version just to trigger a cluster-wide rewrite. + +The base PR retains the application query/index schema behavior and the Elasticsearch 8 client compatibility bridge; it must run independently of the experimental query services. There is no application-level legacy/JOIN switch. Deploy the base application release first, then the JOIN release only after its prerequisites pass. + +Elastic permits the supported previous-major index format on the next major. The documented LOOKUP JOIN constraint applies to the lookup-side index, not a blanket requirement to recreate every source event index. Confirm the exact expression joins used by the experiment against real 8-created event partitions on the target server before treating this as a production guarantee. + +### Local evidence on the rebased PRs + +Both solution builds passed with zero warnings/errors. The base PR independently passed all 829 API endpoint tests against the isolated custom Elasticsearch 9.5.0 image. + +A separate disposable cluster created `migration-events-v1` on 8.19.15 (`index.version.created=8537000`), then started 9.5.0 on that same fixture volume without recreating the event index. A newly created `migration-stacks-v2` had lookup mode and creation version `9107000`. The following expression join returned the expected `[1, "stack-1"]` row with `is_partial=false`: + +```esql +FROM migration-events-v1 +| LOOKUP JOIN migration-stacks-v2 ON stack_id == id AND is_deleted == false AND QSTR("status:open") +| WHERE id IS NOT NULL +| STATS total = COUNT(*) BY stack_id +| SORT total DESC, stack_id ASC +| LIMIT 26 +``` + +This proves mixed-generation support for that query shape, not production mapping coverage, throughput, or readiness. The restored production-data rehearsal remains required. + +## Observed production topology and missing sizing evidence + +- Production is green on Elasticsearch 8.19.15: four ready data/ingest/master nodes, each with 18 GiB container memory, a 9 GiB JVM heap, and a 600 GiB premium managed disk claim. +- Total provisioned storage is 2,400 GiB. This is **not** measured used storage or available migration headroom. Replica copies, shard placement, watermarks, growth, and recovery reserve all matter. +- The deployed ECK operator is 3.3.2. Production Kibana and the separate monitoring Elasticsearch cluster are on 8.19.15. +- The available Kubernetes identity can read resource metadata and logs, but cannot open an Elasticsearch port-forward or service proxy. Index creation versions, document counts, store sizes, shard distribution, and actual per-node free disk have **not** been measured. No duration or capacity promise is possible yet. + +Obtain an existing read-only Elasticsearch connection or have an operator export the following. Use monitoring/metadata privileges, not write or reindex privileges. Run requests against the intended cluster and record the cluster UUID; do not put credentials or complete source documents in reports. + +```http +GET / +GET /_cluster/health +GET /_cluster/settings?include_defaults=true&flat_settings=true +GET /_nodes/stats/fs,jvm,indices?filter_path=nodes.*.name,nodes.*.fs.total,nodes.*.jvm.mem,nodes.*.indices.indexing,nodes.*.indices.search,nodes.*.indices.merges +GET /_cat/allocation?format=json&bytes=b +GET /_cat/shards?format=json&bytes=b&h=index,shard,prirep,state,docs,store,node +GET /_cat/indices?format=json&bytes=b&expand_wildcards=all&h=index,health,status,pri,rep,docs.count,docs.deleted,pri.store.size,store.size +GET /_all/_settings?expand_wildcards=all&flat_settings=true&filter_path=*.settings.index.version.*,*.settings.index.mode,*.settings.index.number_of_*,*.settings.index.blocks.*,*.settings.index.lifecycle.*,*.settings.index.default_pipeline,*.settings.index.final_pipeline +GET /_all/_alias?expand_wildcards=all +GET /_migration/deprecations +GET /_snapshot/_all +GET /_slm/stats +``` + +System-index metadata may require an operator's separate access. Inventory those indexes through Upgrade Assistant rather than assuming application preflight covers them. Obtain latest successful snapshot details and a restore-test record; a configured repository alone does not prove a usable backup. + +Build a manifest with one row per concrete index: canonical aliases, application owner, creation version, mode, date partition, primary bytes, total bytes, exact live document count for migration candidates, shard/replica counts, retention deadline, last observed writes, pipelines, and chosen action. CAT document counts can include nested documents; use `_count` for copy verification. Capture mappings for selected pilot/copy candidates, including `_source` availability and dynamic fields. + +Measure normal and peak ingestion, search latency, disk growth, merge I/O, queue age, and recovery throughput over a representative workload window. Confirm configured maximum retention and actual cleanup behavior; do not assume an old daily partition is immutable or already expired. + +## Phase 1: rehearsal and pre-upgrade work on 8.19 + +1. Pin the exact approved target patch and container digest. The PR currently pins 9.5.0; Elastic's current documented release is 9.5.3. Review intervening security fixes, known issues, plugins, client behavior, ECK support, and release-date upgrade compatibility before approving production. Test the same image that will be deployed, including the Exceptionless plugins. Do not silently substitute a production image during execution. +2. Patch Elasticsearch and Kibana to the latest approved 8.19 release first. Run Upgrade Assistant and resolve critical deprecations. Inventory all application, retained schema/error, hidden, monitoring, and system indexes. +3. Reindex any writable pre-8 application indexes **on Elasticsearch 8** before starting 9. Delete expired data only under the existing retention policy and explicit operator approval. Archive/read-only options are not replacements for writable Exceptionless indexes. Let Elastic's tooling own system-index migrations. +4. Restore a current snapshot to an isolated rehearsal cluster with matching topology/settings. Verify restore permissions, encryption keys, repository access, and recovery time. Restrict network access and apply production-data handling controls. +5. Run the base application on the rehearsed 8 cluster, upgrade that cluster to the target 9 patch, and rerun ingest, event/stack queries, stack status changes, jobs, saved views, aliases, retention, and deletion checks. Include existing records with old/missing fields and all retained creation versions. +6. Separately create a lookup stack index in the rehearsal environment and test the experimental expression JOIN against **unchanged 8-created event indexes**. Compare status/deleted-stack filtering, tenant isolation, counts/charts, date boundaries, forward/backward cursors, and hydrated result identity. This is a correctness gate, not a throughput benchmark. + +No production load tests or reindex experiments are authorized by this plan. + +## Phase 2: server upgrade with the base PR only + +1. Deploy and soak the base-compatible application independently of the server change while keeping production infrastructure pinned to 8.19. Coordinate GitOps/release manifests so an application deployment cannot unintentionally apply the major-version infrastructure change. +2. Approve rollback RPO/RTO and the treatment of writes accepted after the backup boundary. Take a fresh successful snapshot and verify the restore procedure. If replay of post-snapshot events is required, demonstrate durable queue retention/replay and idempotency first; do not assume the current pipeline can recreate every stack/status mutation. +3. Upgrade the monitoring cluster and supporting components in the supported order before the monitored production cluster where required. Keep Kibana matched to its Elasticsearch version. Follow the ECK rolling-upgrade procedure; do not hand-delete pods or change shard allocation independently of the operator without an approved runbook. +4. Upgrade one production node at a time. Because all four nodes have the same roles, confirm voting quorum and recovery capacity with one node unavailable. Wait for each node's shard recovery and health before proceeding. Halt on sustained unassigned shards, disk watermark pressure, write/search errors, queue growth, or latency outside the agreed SLO. +5. Validate the base application against the upgraded cluster and keep the JOIN release undeployed. Do not run compatibility reindexing during this initial stabilization window. + +Rollback is **not** a downgrade of the upgraded disks or reverting the ECK version field. Recover on an older-version cluster from the pre-upgrade snapshot, with the approved replay/data-loss procedure. Preserve that recovery path until the upgrade has been accepted. + +## Phase 3: upgrade index formats without rewriting the world at once + +Prefer new daily event partitions created naturally on 9 and let eligible old partitions expire through verified retention. That upgrades the active data progressively with no bulk copy. Reindex only retained partitions that will outlive the agreed migration deadline or need a demonstrated format-specific feature. All remaining long-lived non-event indexes need an explicit current-format migration plan too. + +Blake's [Foundatio.Repositories PR #307](https://github.com/FoundatioFx/Foundatio.Repositories/pull/307), reviewed at `aec4fb78652c1dae8d08085e5895e28fdf10a2a5`, is a promising operator primitive, but is still open and is not integrated into the base PR's package. It separates compatibility upgrades from normal schema/configuration startup. It preserves canonical aliases while replacing physical indexes and exposes inspection/recovery for interrupted operations. + +Important constraints of that implementation: + +- It fences writes for the entire copy and requires writers, consumers, retention/index maintenance, and alias managers to be stopped. It is not a zero-downtime CDC or dual-write solution. +- It copies one exact index into `reindexed-v9-` using `_create_from` (an Elastic Technical Preview API), verifies the task, counts, mappings/settings, and alias topology, then atomically deletes the source and assigns its aliases to the target. +- Its copy task is unsliced and throttled. Do not size it assuming shard-parallel slicing. Partial/ambiguous operations require its evidence-based inspection/recovery, not blind retry or manual unblock. +- It rejects non-standard modes and several managed topologies. It preserves creation settings rather than changing them, so it does **not** implement the standard-to-lookup stack conversion. +- Canonical names become aliases. Prove exact-ID reads/patches, routing, mapping discovery, daily alias maintenance, old-schema discovery, retention deletion, and operational scripts still work. Restart/drain clients whose concurrency tokens refer to the retired physical index. + +Before adoption, finish review/release of #307, consume the approved package in a separate maintenance change, and test its failure/recovery cases with Exceptionless. Do not hide it in `ConfigureIndexesAsync` or add an unconditional global schema bump. + +For each candidate: + +1. Start with a small representative partition, then a larger/high-field-count partition. Measure sustained copy throughput **with** the intended production workload and throttle on the rehearsal cluster. +2. Reserve capacity for the complete target plus configured replica restoration, temporary segment/merge overhead, ingestion growth, and a node-recovery margin. Check fit per node and per shard against configured watermarks, not just aggregate free bytes. Provision additional capacity before copying if needed. +3. Stop all affected writers and index managers; drain in-flight work. Historical event dates can still receive late ingestion, deletions, and cleanup. A timestamp alone does not establish safety. If writers cannot be paused per partition with proven routing and queueing, the library's current safe procedure implies a broader maintenance outage. Decide that tradeoff explicitly before scheduling a massive copy. +4. Recompute the preflight; snapshot; copy only the approved concrete index with conservative throttling. Persist task identity, progress, baseline counts, aliases/settings, and recovery evidence outside the process. +5. Verify completion, exact counts, representative content/queries, replica recovery, alias identity, and application read/write/delete behavior. Release the write pause only after these gates pass. Then proceed to the next index; keep concurrency at one initially. +6. Abort/pause on the agreed disk/latency/queue limits. Use the library's inspection and verified cancellation path. A completed atomic cutover cannot be undone by canceling the task; snapshot restore is the recovery boundary. + +Sizing worksheet: `copy duration ≈ primary source bytes / measured effective source-byte throughput`, plus refresh, validation, replica recovery, and cutover. Alternatively use exact documents divided by measured documents/second for matching document distributions. Estimate the **write-pause duration**, queue accumulation (`arrival rate × pause duration`), and catch-up time separately. Do not estimate from raw disk bandwidth or promise a universal 2× free-space rule. + +## Phase 4: canonical stack lookup conversion and JOIN rollout + +The experiment defines the canonical stack schema as version 2 with `index.mode=lookup` and one primary shard. Do not deploy it and allow ordinary startup to initiate an unplanned production schema migration. + +1. Measure the current stack primary size, document cardinality, growth, update rate, largest tenant, and heap needed by representative joins. The single lookup primary is a hard capacity/throughput constraint; replicas can distribute reads but do not shard primary writes. Establish whether this design fits the forecast, not just today's sample. +2. Use a dedicated, reviewed schema-conversion maintenance operation that creates the experiment's exact lookup mapping/settings and copies existing stack documents without changing IDs or relationships. It must handle partial copies, missing/default fields, retained aliases, and rollback. #307's format upgrade is not this operation. Avoid copying stacks twice merely to reach current format and then lookup mode. +3. Pause ingestion consumers and all stack-mutating APIs/jobs/maintenance, drain in-flight writes, snapshot, copy and verify, then perform an explicit atomic alias cutover. Rehearse full outage/recovery behavior; do not assume the existing generic schema reindexer's catch-up pass proves no missed deletes or concurrent status changes. +4. Before allowing traffic, verify the actual lookup mode, mapping, one-primary setting, complete stack IDs/counts, aliases, replica health, and existing-record semantics. Deploy the experimental application only after migration success. Keep the base release available as an application rollback candidate, but rehearse its writes against the new mapping; application rollback does not revert the index conversion. +5. Benchmark representative tenant/time-range/skew combinations for event status filtering and stack grouping/count/charts/paging. Capture p50/p95/p99 latency, CPU, heap/breakers, I/O, concurrent ingestion impact, and cursor correctness under changes. Cursor pagination does not remove the cost of filtering/grouping the qualifying event population, and it is not a point-in-time snapshot. +6. Require correctness and SLO acceptance before the JOIN production rollout. If the canonical one-primary stack index does not fit, stop and redesign the lookup topology; do not deploy on the strength of tiny local benchmarks. + +## Decisions required before scheduling + +- Read-only index/disk/snapshot inventory and representative workload measurements. +- Approved source/target patches, image digests, ECK/component compatibility, and rehearsal evidence. +- Retention-based completion deadline versus retained event partitions that must be copied. +- Additional capacity and acceptable per-index/global write outage, including queue/replay limits. +- Adoption of #307 plus a separate reviewed lookup-mode migration implementation. +- Restore-tested RPO/RTO, cutover/abort thresholds, and named operator/approval owner for each stage. + +References: Elastic's [upgrade preparation](https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade), [rolling upgrade and rollback guidance](https://www.elastic.co/docs/deploy-manage/upgrade/deployment-or-cluster/elasticsearch), [LOOKUP JOIN constraints](https://www.elastic.co/docs/reference/query-languages/esql/esql-lookup-join), and [snapshot compatibility](https://www.elastic.co/docs/deploy-manage/tools/snapshot-and-restore). From 18f2eb24b21b633d6e30895b499b3fa09ee901a4 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 6 Sep 2026 18:34:46 -0500 Subject: [PATCH 18/20] Clarify stack JOIN prototype is experimental only --- docs/elasticsearch-9-production-migration.md | 26 +++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/elasticsearch-9-production-migration.md b/docs/elasticsearch-9-production-migration.md index f9a4742b38..044595a521 100644 --- a/docs/elasticsearch-9-production-migration.md +++ b/docs/elasticsearch-9-production-migration.md @@ -2,17 +2,19 @@ Status: proposed; no production changes have been made. Inventory observations below are from September 6, 2026. This is an operator-run migration, not an application-startup migration. +**Stack PR #2511 is an experiment only, not the production implementation or a planned production release.** Its queries, endpoint changes, and lookup schema are proof-of-concept evidence, not an approved architecture. The proper stack/event query refactor still needs to be designed and implemented in the Foundatio.Repositories PR, then integrated into Exceptionless through a separately reviewed application change. Neither the Elasticsearch 9 server upgrade nor index-format maintenance depends on shipping #2511. + ## Separate the three changes | Change | Release boundary | Required data work | | --- | --- | --- | | Run Elasticsearch 9 with existing application queries | Base PR [#2416](https://github.com/exceptionless/Exceptionless/pull/2416) alone | Resolve unsupported pre-8 indexes before starting 9; do not rewrite all 8-created indexes | | Recreate indexes in the current server's index format | Explicit maintenance after the server upgrade stabilizes | Selected retained indexes, independently of application schema versions | -| Enable ES\|QL stack/event filtering and cursor-based stack summaries | Experimental PR [#2511](https://github.com/exceptionless/Exceptionless/pull/2511), separately deployed | Convert the canonical stack index to lookup mode, validate mixed-generation event sources, and benchmark production-scale queries | +| Properly refactor stack/event queries and stack pagination | Future repositories-led implementation and separately reviewed Exceptionless integration; [#2511](https://github.com/exceptionless/Exceptionless/pull/2511) is experimental evidence only | Determine index/migration requirements from the approved repository design; validate mixed-generation event sources and production-scale queries | An Elasticsearch server version, `index.version.created`, the repository's schema version, and `index.mode` are different things. A force merge or server restart is not an index recreation. Do not bump the daily event schema version just to trigger a cluster-wide rewrite. -The base PR retains the application query/index schema behavior and the Elasticsearch 8 client compatibility bridge; it must run independently of the experimental query services. There is no application-level legacy/JOIN switch. Deploy the base application release first, then the JOIN release only after its prerequisites pass. +The base PR retains the application query/index schema behavior and the Elasticsearch 8 client compatibility bridge; it must run independently of the experimental query services. There is no application-level legacy/JOIN switch. Deploy the base application release independently. Do not deploy the experiment; any future query release must use the approved repositories implementation and satisfy its own migration and validation gates. Elastic permits the supported previous-major index format on the next major. The documented LOOKUP JOIN constraint applies to the lookup-side index, not a blanket requirement to recreate every source event index. Confirm the exact expression joins used by the experiment against real 8-created event partitions on the target server before treating this as a production guarantee. @@ -70,7 +72,7 @@ Measure normal and peak ingestion, search latency, disk growth, merge I/O, queue 3. Reindex any writable pre-8 application indexes **on Elasticsearch 8** before starting 9. Delete expired data only under the existing retention policy and explicit operator approval. Archive/read-only options are not replacements for writable Exceptionless indexes. Let Elastic's tooling own system-index migrations. 4. Restore a current snapshot to an isolated rehearsal cluster with matching topology/settings. Verify restore permissions, encryption keys, repository access, and recovery time. Restrict network access and apply production-data handling controls. 5. Run the base application on the rehearsed 8 cluster, upgrade that cluster to the target 9 patch, and rerun ingest, event/stack queries, stack status changes, jobs, saved views, aliases, retention, and deletion checks. Include existing records with old/missing fields and all retained creation versions. -6. Separately create a lookup stack index in the rehearsal environment and test the experimental expression JOIN against **unchanged 8-created event indexes**. Compare status/deleted-stack filtering, tenant isolation, counts/charts, date boundaries, forward/backward cursors, and hydrated result identity. This is a correctness gate, not a throughput benchmark. +6. As separate experimental research, create a lookup stack index in the rehearsal environment and test the experimental expression JOIN against **unchanged 8-created event indexes**. Compare status/deleted-stack filtering, tenant isolation, counts/charts, date boundaries, forward/backward cursors, and hydrated result identity. These findings inform the repositories PR; they neither approve the experiment for production nor block the independent base upgrade. Repeat correctness and performance validation against the eventual repository implementation. No production load tests or reindex experiments are authorized by this plan. @@ -80,7 +82,7 @@ No production load tests or reindex experiments are authorized by this plan. 2. Approve rollback RPO/RTO and the treatment of writes accepted after the backup boundary. Take a fresh successful snapshot and verify the restore procedure. If replay of post-snapshot events is required, demonstrate durable queue retention/replay and idempotency first; do not assume the current pipeline can recreate every stack/status mutation. 3. Upgrade the monitoring cluster and supporting components in the supported order before the monitored production cluster where required. Keep Kibana matched to its Elasticsearch version. Follow the ECK rolling-upgrade procedure; do not hand-delete pods or change shard allocation independently of the operator without an approved runbook. 4. Upgrade one production node at a time. Because all four nodes have the same roles, confirm voting quorum and recovery capacity with one node unavailable. Wait for each node's shard recovery and health before proceeding. Halt on sustained unassigned shards, disk watermark pressure, write/search errors, queue growth, or latency outside the agreed SLO. -5. Validate the base application against the upgraded cluster and keep the JOIN release undeployed. Do not run compatibility reindexing during this initial stabilization window. +5. Validate the base application against the upgraded cluster with its existing queries. Do not deploy #2511 or run compatibility reindexing during this initial stabilization window. Rollback is **not** a downgrade of the upgraded disks or reverting the ECK version field. Recover on an older-version cluster from the pre-upgrade snapshot, with the approved replay/data-loss procedure. Preserve that recovery path until the upgrade has been accepted. @@ -100,6 +102,8 @@ Important constraints of that implementation: Before adoption, finish review/release of #307, consume the approved package in a separate maintenance change, and test its failure/recovery cases with Exceptionless. Do not hide it in `ConfigureIndexesAsync` or add an unconditional global schema bump. +The reviewed #307 scope is index-format maintenance. This plan does not claim that it already contains the proper stack query refactor. That query design and implementation remain work to resolve in the repositories PR, independently of the reindex primitive. + For each candidate: 1. Start with a small representative partition, then a larger/high-field-count partition. Measure sustained copy throughput **with** the intended production workload and throttle on the rehearsal cluster. @@ -111,14 +115,18 @@ For each candidate: Sizing worksheet: `copy duration ≈ primary source bytes / measured effective source-byte throughput`, plus refresh, validation, replica recovery, and cutover. Alternatively use exact documents divided by measured documents/second for matching document distributions. Estimate the **write-pause duration**, queue accumulation (`arrival rate × pause duration`), and catch-up time separately. Do not estimate from raw disk bandwidth or promise a universal 2× free-space rule. -## Phase 4: canonical stack lookup conversion and JOIN rollout +## Phase 4: future repositories-led query refactor, not deployment of #2511 + +First settle the production query design in the Foundatio.Repositories PR: repository-level filtering/JOIN composition, grouped stack queries and counts, sorting/cursor semantics, result contracts, and index lifecycle support. Exceptionless should consume that capability rather than promote the experiment's application-side ES|QL service into the production architecture. Retain the intended endpoint ownership: stacks come from stacks endpoints, events from events endpoints, without application-side filtering joins. + +Review and release that repository implementation, then create a separately reviewed Exceptionless integration and migration plan. #2511 is only a source of feasibility evidence and regression scenarios. Passing its tests is not approval to merge or deploy it as the production refactor. -The experiment defines the canonical stack schema as version 2 with `index.mode=lookup` and one primary shard. Do not deploy it and allow ordinary startup to initiate an unplanned production schema migration. +The experiment defines stack schema version 2 with `index.mode=lookup` and one primary shard. These are provisional choices, not production migration instructions. The following requirements apply **only if the approved repositories design retains that lookup topology**; revise them if the design changes. Do not initiate this conversion through ordinary startup. 1. Measure the current stack primary size, document cardinality, growth, update rate, largest tenant, and heap needed by representative joins. The single lookup primary is a hard capacity/throughput constraint; replicas can distribute reads but do not shard primary writes. Establish whether this design fits the forecast, not just today's sample. -2. Use a dedicated, reviewed schema-conversion maintenance operation that creates the experiment's exact lookup mapping/settings and copies existing stack documents without changing IDs or relationships. It must handle partial copies, missing/default fields, retained aliases, and rollback. #307's format upgrade is not this operation. Avoid copying stacks twice merely to reach current format and then lookup mode. +2. Use a dedicated, reviewed schema-conversion maintenance operation that creates the approved repository implementation's mapping/settings and copies existing stack documents without changing IDs or relationships. It must handle partial copies, missing/default fields, retained aliases, and rollback. #307's currently reviewed format upgrade is not this operation. Avoid copying stacks twice merely to reach current format and then lookup mode. 3. Pause ingestion consumers and all stack-mutating APIs/jobs/maintenance, drain in-flight writes, snapshot, copy and verify, then perform an explicit atomic alias cutover. Rehearse full outage/recovery behavior; do not assume the existing generic schema reindexer's catch-up pass proves no missed deletes or concurrent status changes. -4. Before allowing traffic, verify the actual lookup mode, mapping, one-primary setting, complete stack IDs/counts, aliases, replica health, and existing-record semantics. Deploy the experimental application only after migration success. Keep the base release available as an application rollback candidate, but rehearse its writes against the new mapping; application rollback does not revert the index conversion. +4. Before allowing traffic, verify the actual lookup mode, mapping, one-primary setting, complete stack IDs/counts, aliases, replica health, and existing-record semantics. Deploy only the separately approved repository-backed application implementation after migration success, never the experimental PR. Keep the base release available as an application rollback candidate, but rehearse its writes against the new mapping; application rollback does not revert the index conversion. 5. Benchmark representative tenant/time-range/skew combinations for event status filtering and stack grouping/count/charts/paging. Capture p50/p95/p99 latency, CPU, heap/breakers, I/O, concurrent ingestion impact, and cursor correctness under changes. Cursor pagination does not remove the cost of filtering/grouping the qualifying event population, and it is not a point-in-time snapshot. 6. Require correctness and SLO acceptance before the JOIN production rollout. If the canonical one-primary stack index does not fit, stop and redesign the lookup topology; do not deploy on the strength of tiny local benchmarks. @@ -128,7 +136,7 @@ The experiment defines the canonical stack schema as version 2 with `index.mode= - Approved source/target patches, image digests, ECK/component compatibility, and rehearsal evidence. - Retention-based completion deadline versus retained event partitions that must be copied. - Additional capacity and acceptable per-index/global write outage, including queue/replay limits. -- Adoption of #307 plus a separate reviewed lookup-mode migration implementation. +- Adoption of #307's reindex capability where needed; completion of the proper stack/event query refactor in the repositories PR, followed by separately reviewed Exceptionless integration and any required schema migration. #2511 is not a production deliverable. - Restore-tested RPO/RTO, cutover/abort thresholds, and named operator/approval owner for each stage. References: Elastic's [upgrade preparation](https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade), [rolling upgrade and rollback guidance](https://www.elastic.co/docs/deploy-manage/upgrade/deployment-or-cluster/elasticsearch), [LOOKUP JOIN constraints](https://www.elastic.co/docs/reference/query-languages/esql/esql-lookup-join), and [snapshot compatibility](https://www.elastic.co/docs/deploy-manage/tools/snapshot-and-restore). From ce201fddc008681f1599ff258a42aedbdfbafc9e Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 6 Sep 2026 18:43:14 -0500 Subject: [PATCH 19/20] Upgrade Elastic Stack 9 base to 9.5.3 --- .github/workflows/elasticsearch-docker-8.yml | 12 +++++++++--- Dockerfile | 2 +- build/docker/elasticsearch/8.x/Dockerfile | 3 +-- build/docker/elasticsearch/9.x/Dockerfile | 2 +- docker/docker-compose.apm.yml | 8 ++++---- docker/docker-compose.dev.yml | 4 ++-- docker/docker-compose.yml | 4 ++-- docs/elasticsearch-9-production-migration.md | 2 +- k8s/elastic-monitor.yaml | 8 ++++---- k8s/ex-dev-elasticsearch.yaml | 6 +++--- k8s/ex-prod-elasticsearch.yaml | 6 +++--- k8s/exceptionless/values.yaml | 2 +- samples/docker-compose.all-in-one.yml | 6 ++++-- samples/docker-compose.yml | 4 ++-- .../Extensions/ElasticsearchExtensions.cs | 4 ++-- .../Exceptionless.Tests/AppHostConfigurationTests.cs | 4 ++-- 16 files changed, 42 insertions(+), 35 deletions(-) diff --git a/.github/workflows/elasticsearch-docker-8.yml b/.github/workflows/elasticsearch-docker-8.yml index 872667f22f..770c497fad 100644 --- a/.github/workflows/elasticsearch-docker-8.yml +++ b/.github/workflows/elasticsearch-docker-8.yml @@ -40,7 +40,13 @@ jobs: with: platforms: linux/amd64,linux/arm64 - name: Build custom Elasticsearch 8.x docker image - working-directory: build/docker/elasticsearch/8.x + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | - VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile) - docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION + VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/8.x/Dockerfile) + IMAGE_SHA=$(git ls-files -- build/docker/elasticsearch/8.x .github/workflows/elasticsearch-docker-8.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1) + TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA") + if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then + TAGS+=(--tag "exceptionless/elasticsearch:$VERSION") + fi + docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file build/docker/elasticsearch/8.x/Dockerfile build/docker/elasticsearch/8.x "${TAGS[@]}" diff --git a/Dockerfile b/Dockerfile index f6f015c18a..b3c4bdc53c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,7 @@ ENTRYPOINT ["/app/app-docker-entrypoint.sh"] # completely self-contained -FROM exceptionless/elasticsearch:9.5.0 AS exceptionless +FROM exceptionless/elasticsearch:9.5.3 AS exceptionless WORKDIR /app COPY --from=job-publish /app/src/Exceptionless.Job/out ./ diff --git a/build/docker/elasticsearch/8.x/Dockerfile b/build/docker/elasticsearch/8.x/Dockerfile index bbab4cc3bc..3be7362ff5 100644 --- a/build/docker/elasticsearch/8.x/Dockerfile +++ b/build/docker/elasticsearch/8.x/Dockerfile @@ -1,5 +1,4 @@ # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:8.19.15 +FROM docker.elastic.co/elasticsearch/elasticsearch:8.19.21 RUN elasticsearch-plugin install -b mapper-size - diff --git a/build/docker/elasticsearch/9.x/Dockerfile b/build/docker/elasticsearch/9.x/Dockerfile index cc03ebce5a..75eb1700b9 100644 --- a/build/docker/elasticsearch/9.x/Dockerfile +++ b/build/docker/elasticsearch/9.x/Dockerfile @@ -1,4 +1,4 @@ # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:9.5.0 +FROM docker.elastic.co/elasticsearch/elasticsearch:9.5.3 RUN elasticsearch-plugin install -b mapper-size diff --git a/docker/docker-compose.apm.yml b/docker/docker-compose.apm.yml index 220098afad..c9cdfa81f7 100644 --- a/docker/docker-compose.apm.yml +++ b/docker/docker-compose.apm.yml @@ -2,7 +2,7 @@ version: "2.2" services: setup: - image: docker.elastic.co/elasticsearch/elasticsearch:9.5.0 + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.3 volumes: - certs:/usr/share/elasticsearch/config/certs user: "0" @@ -53,7 +53,7 @@ services: depends_on: setup: condition: service_healthy - image: docker.elastic.co/elasticsearch/elasticsearch:9.5.0 + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.3 volumes: - certs:/usr/share/elasticsearch/config/certs - esdata:/usr/share/elasticsearch/data @@ -98,7 +98,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/kibana/kibana:9.5.0 + image: docker.elastic.co/kibana/kibana:9.5.3 volumes: - certs:/usr/share/kibana/config/certs ports: @@ -124,7 +124,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/apm/apm-server:9.5.0 + image: docker.elastic.co/apm/apm-server:9.5.3 volumes: - certs:/usr/share/apm-server/certs ports: diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 2b8518034b..5a1d298220 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -50,7 +50,7 @@ services: - appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:9.5.0 + image: exceptionless/elasticsearch:9.5.3 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -76,7 +76,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.5.0 + image: docker.elastic.co/kibana/kibana:9.5.3 ports: - 5601:5601 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b1706174b1..101ea52335 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch: - image: exceptionless/elasticsearch:9.5.0 + image: exceptionless/elasticsearch:9.5.3 environment: node.name: elasticsearch cluster.name: exceptionless @@ -28,7 +28,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.5.0 + image: docker.elastic.co/kibana/kibana:9.5.3 environment: xpack.security.enabled: "false" ports: diff --git a/docs/elasticsearch-9-production-migration.md b/docs/elasticsearch-9-production-migration.md index 044595a521..3bf7750e7c 100644 --- a/docs/elasticsearch-9-production-migration.md +++ b/docs/elasticsearch-9-production-migration.md @@ -67,7 +67,7 @@ Measure normal and peak ingestion, search latency, disk growth, merge I/O, queue ## Phase 1: rehearsal and pre-upgrade work on 8.19 -1. Pin the exact approved target patch and container digest. The PR currently pins 9.5.0; Elastic's current documented release is 9.5.3. Review intervening security fixes, known issues, plugins, client behavior, ECK support, and release-date upgrade compatibility before approving production. Test the same image that will be deployed, including the Exceptionless plugins. Do not silently substitute a production image during execution. +1. Pin the exact approved target patch and container digest. The base PR now targets 9.5.3, and the separate 8.x rollout PR targets 8.19.21. These releases were published September 3 and September 2, 2026, respectively, and are less than two weeks old at this review: allow for elevated early-release risk in staging/soak acceptance. Review security fixes, known issues, plugins, client behavior, ECK support, and release-date upgrade compatibility before approving production. Test the same image that will be deployed, including the Exceptionless plugins. Earlier 9.5.0 test evidence above must not be treated as validation of 9.5.3. Do not silently substitute a production image during execution. 2. Patch Elasticsearch and Kibana to the latest approved 8.19 release first. Run Upgrade Assistant and resolve critical deprecations. Inventory all application, retained schema/error, hidden, monitoring, and system indexes. 3. Reindex any writable pre-8 application indexes **on Elasticsearch 8** before starting 9. Delete expired data only under the existing retention policy and explicit operator approval. Archive/read-only options are not replacements for writable Exceptionless indexes. Let Elastic's tooling own system-index migrations. 4. Restore a current snapshot to an isolated rehearsal cluster with matching topology/settings. Verify restore permissions, encryption keys, repository access, and recovery time. Restrict network access and apply production-data handling controls. diff --git a/k8s/elastic-monitor.yaml b/k8s/elastic-monitor.yaml index f1c47ad253..7da6e53e19 100644 --- a/k8s/elastic-monitor.yaml +++ b/k8s/elastic-monitor.yaml @@ -4,7 +4,7 @@ metadata: name: elastic-monitor namespace: elastic-system spec: - version: 9.5.0 + version: 9.5.3 podDisruptionBudget: {} nodeSets: - name: main @@ -228,7 +228,7 @@ metadata: name: kibana-monitor namespace: elastic-system spec: - version: 9.5.0 + version: 9.5.3 count: 1 http: tls: @@ -364,7 +364,7 @@ metadata: name: fleet-server namespace: elastic-system spec: - version: 9.5.0 + version: 9.5.3 kibanaRef: name: kibana-monitor elasticsearchRefs: @@ -388,7 +388,7 @@ metadata: name: elastic-agent namespace: elastic-system spec: - version: 9.5.0 + version: 9.5.3 kibanaRef: name: kibana-monitor fleetServerRef: diff --git a/k8s/ex-dev-elasticsearch.yaml b/k8s/ex-dev-elasticsearch.yaml index 6a8b1fd851..532328e54e 100644 --- a/k8s/ex-dev-elasticsearch.yaml +++ b/k8s/ex-dev-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 9.5.0 - image: exceptionless/elasticsearch:9.5.0 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.5.3 + image: exceptionless/elasticsearch:9.5.3 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch secureSettings: - secretName: ex-dev-snapshots http: @@ -68,7 +68,7 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 9.5.0 + version: 9.5.3 count: 1 elasticsearchRef: name: ex-dev diff --git a/k8s/ex-prod-elasticsearch.yaml b/k8s/ex-prod-elasticsearch.yaml index de7848bce1..155cddc58c 100644 --- a/k8s/ex-prod-elasticsearch.yaml +++ b/k8s/ex-prod-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 9.5.0 - image: exceptionless/elasticsearch:9.5.0 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.5.3 + image: exceptionless/elasticsearch:9.5.3 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch monitoring: metrics: elasticsearchRefs: @@ -79,7 +79,7 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 9.5.0 + version: 9.5.3 count: 1 elasticsearchRef: name: ex-prod diff --git a/k8s/exceptionless/values.yaml b/k8s/exceptionless/values.yaml index 2e576bcd32..0b55332862 100644 --- a/k8s/exceptionless/values.yaml +++ b/k8s/exceptionless/values.yaml @@ -36,7 +36,7 @@ elasticsearch: connectionString: image: repository: exceptionless/elasticsearch - tag: 9.5.0 + tag: 9.5.3 pullPolicy: IfNotPresent redis: diff --git a/samples/docker-compose.all-in-one.yml b/samples/docker-compose.all-in-one.yml index 622e74830e..b4148c5f69 100644 --- a/samples/docker-compose.all-in-one.yml +++ b/samples/docker-compose.all-in-one.yml @@ -19,10 +19,12 @@ services: # Runs Kibana for working with Elasticsearch data directly. This is normally not needed and takes up resources when running. kibana: depends_on: - - elasticsearch - image: docker.elastic.co/kibana/kibana:9.5.0 + - exceptionless + image: docker.elastic.co/kibana/kibana:9.5.3 ports: - 5601:5601 + environment: + ELASTICSEARCH_HOSTS: http://exceptionless:9200 volumes: ex_esdata: diff --git a/samples/docker-compose.yml b/samples/docker-compose.yml index 7fcb4cc0de..b6101dd41d 100644 --- a/samples/docker-compose.yml +++ b/samples/docker-compose.yml @@ -44,7 +44,7 @@ services: - ex_appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:9.5.0 + image: exceptionless/elasticsearch:9.5.3 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -58,7 +58,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:9.5.0 + image: docker.elastic.co/kibana/kibana:9.5.3 ports: - 5601:5601 diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index 4022e89583..ed25ed161d 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -13,7 +13,7 @@ public static class ElasticsearchBuilderExtensions private const int KibanaPort = 5601; /// - /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 9.5.0 tag of the Elasticsearch container image + /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 9.5.3 tag of the Elasticsearch container image /// /// The . /// The name of the resource. This name will be used as the connection string name when referenced in a dependency. @@ -125,7 +125,7 @@ internal static class ElasticsearchContainerImageTags public const string Image = "exceptionless/elasticsearch"; public const string KibanaRegistry = "docker.elastic.co"; public const string KibanaImage = "kibana/kibana"; - public const string Tag = "9.5.0"; + public const string Tag = "9.5.3"; } internal sealed class ElasticsearchConnectionHealthCheck(Func connectionStringFactory) : IHealthCheck diff --git a/tests/Exceptionless.Tests/AppHostConfigurationTests.cs b/tests/Exceptionless.Tests/AppHostConfigurationTests.cs index 5ec833ceaa..04087739c9 100644 --- a/tests/Exceptionless.Tests/AppHostConfigurationTests.cs +++ b/tests/Exceptionless.Tests/AppHostConfigurationTests.cs @@ -10,8 +10,8 @@ public class AppHostConfigurationTests [Fact] public async Task CreateAsync_WithSeparateElasticsearchAndKibanaOverrides_UsesIndependentImageTags() { - const string elasticsearchImageTag = "9.5.0-sha256-candidate"; - const string kibanaImageTag = "9.5.0"; + const string elasticsearchImageTag = "9.5.3-sha256-candidate"; + const string kibanaImageTag = "9.5.3"; var appHost = await DistributedApplicationTestingBuilder.CreateAsync( [ $"--Elasticsearch:ImageTag={elasticsearchImageTag}", From d6a093e24ab80ddca6d2d88ff85b1fbe18d1c8b1 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 6 Sep 2026 18:58:12 -0500 Subject: [PATCH 20/20] Document quiesced all-in-one Elasticsearch upgrade --- .../upgrading-self-hosted-instance.md | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/docs/self-hosting/upgrading-self-hosted-instance.md b/docs/docs/self-hosting/upgrading-self-hosted-instance.md index 2335c9e421..ff3d7cb665 100644 --- a/docs/docs/self-hosting/upgrading-self-hosted-instance.md +++ b/docs/docs/self-hosting/upgrading-self-hosted-instance.md @@ -16,7 +16,7 @@ Use this upgrade path for an existing self-hosted installation: 1. Take a current Elasticsearch snapshot or other verified backup and test that it can be restored. Elasticsearch does not support downgrading a data directory after it has been upgraded. 2. Upgrade Elasticsearch and Kibana to the latest 8.19.x patch release first, using the existing data volume. Do not start Elasticsearch 9 yet. -3. Stop the Exceptionless app and job services, but leave Elasticsearch and Kibana 8.19 running. This prevents writes while legacy indices are reindexed. +3. Stop the Exceptionless app and job services, but leave Elasticsearch and Kibana 8.19 running. This prevents writes while legacy indices are reindexed. All-in-one installations must use the Elasticsearch-only procedure below; killing the app process is insufficient because its supervisor restarts it. 4. Open Kibana's **Upgrade Assistant** and resolve every critical issue. Reindex every active Exceptionless index created before Elasticsearch 8. Delete only indices you have confirmed are no longer needed; do not mark active Exceptionless indices as read-only. 5. If this data volume previously ran Elasticsearch 7, temporarily disable the GeoIP downloader while still on Elasticsearch 8.19. Elasticsearch deletes its downloaded `.geoip_databases` system index when this setting is disabled; Exceptionless data is not affected. @@ -43,6 +43,33 @@ Use this upgrade path for an existing self-hosted installation: See Elastic's [prepare-to-upgrade guide](https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade) for the supported 8.x to 9.x upgrade requirements and Upgrade Assistant details. +### All-in-one: keep Elasticsearch running without the app + +For `samples/docker-compose.all-in-one.yml`, run these commands from the existing deployment directory with its existing Compose project name and environment. Do not create a new project or change the volume mapping: that can silently select an empty data volume. Take and restore-test the snapshot first. Stop external Exceptionless jobs, ingestion consumers, and other writers too. + +1. Pin `exceptionless` to an approved **8.x all-in-one application image containing Elasticsearch 8.19.21**, and `kibana` to `docker.elastic.co/kibana/kibana:8.19.21`. Do not use `latest` or an Elasticsearch 9 image during preparation. Keep the original volume, security settings, and resource limits. +2. Stop both existing containers without removing their volumes: + + ```powershell + docker compose -f docker-compose.all-in-one.yml stop kibana exceptionless + ``` + +3. Start only Elasticsearch from the all-in-one image in a dedicated foreground terminal. Overriding the entrypoint bypasses the supervisor entirely, so neither the app nor its in-process jobs start. `--service-ports` and `--use-aliases` preserve the service's ports and Kibana's `exceptionless` hostname; `--no-deps` prevents other services from starting. + + ```powershell + docker compose -f docker-compose.all-in-one.yml run --rm --no-deps --service-ports --use-aliases --entrypoint /usr/local/bin/docker-entrypoint.sh exceptionless eswrapper + ``` + + Verify `GET /` reports 8.19.21 and the expected cluster UUID, and check cluster health, index counts, and representative records before doing maintenance. Confirm no application process is running. Never start the regular `exceptionless` service while this maintenance container holds its data volume. +4. In another terminal, start only Kibana and complete steps 4–6 above, keeping all application writers stopped: + + ```powershell + docker compose -f docker-compose.all-in-one.yml up -d --no-deps kibana + ``` + +5. Stop Kibana, then press Ctrl+C in the maintenance terminal and wait for Elasticsearch to exit cleanly. Pin the approved v9 all-in-one image and matching Kibana version. Repeat the Elasticsearch-only command to perform the 9 upgrade and verify health before any app startup; perform step 8 above against this node. +6. Stop that Elasticsearch-only container cleanly. Only after the migration checks pass, start the regular all-in-one service and matching Kibana with `docker compose -f docker-compose.all-in-one.yml up -d`. Resume external writers after application verification. Never use `down -v`, run two nodes against the same volume, or restart an 8.x image against a volume already opened by 9. + ## Upgrading from v7.1 to v8 We simplified the self hosting process by integrating the UI into the existing app images. As such `exceptionless/ui` docker images are deprecated and we recommend using `exceptionless/app`.