diff --git a/CHANGELOG.md b/CHANGELOG.md index 51793e63..3f7453b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## master / unreleased * [FEATURE] Add remote-write OTLP endpoint in nginx #667 +* [FEATURE] Ingester: support zone-awareness #668 * [DEPENDENCY] update kiwigrid/k8s-sidecar docker tag to v2.11.2 #663 ## 3.3.8 / 2026-08-17 diff --git a/Chart.lock b/Chart.lock index ac519239..d7476d37 100644 --- a/Chart.lock +++ b/Chart.lock @@ -14,5 +14,8 @@ dependencies: - name: memcached repository: https://charts.bitnami.com/bitnami version: 6.14.0 -digest: sha256:af0c109667e9402918877431f9e269c447c030d398c8e1ade6f7a0171a856c8f -generated: "2026-04-21T11:51:34.160601+09:00" +- name: rollout-operator + repository: https://grafana.github.io/helm-charts + version: 0.51.1 +digest: sha256:d0ad2a749ae02f92ad6864ce4afc174cf10d050dbb6ac5263fab2f53b29686ab +generated: "2026-09-10T14:14:02.592218194+02:00" diff --git a/Chart.yaml b/Chart.yaml index 65205d7c..dd9809fb 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -41,3 +41,7 @@ dependencies: version: 6.14.0 repository: https://charts.bitnami.com/bitnami condition: memcached-parquet-labels.enabled + - name: rollout-operator + version: 0.51.1 + repository: https://grafana.github.io/helm-charts + condition: rollout_operator.enabled diff --git a/charts/rollout-operator-0.51.1.tgz b/charts/rollout-operator-0.51.1.tgz new file mode 100644 index 00000000..2bb83da1 Binary files /dev/null and b/charts/rollout-operator-0.51.1.tgz differ diff --git a/docs/guides/migrate_to_zone_aware.markdown b/docs/guides/migrate_to_zone_aware.markdown new file mode 100644 index 00000000..29e0508d --- /dev/null +++ b/docs/guides/migrate_to_zone_aware.markdown @@ -0,0 +1,84 @@ +--- +layout: page +title: Migrate to zone-aware ingesters +parent: Guides +has_children: false +has_toc: false +--- + +# Migrate to zone-aware ingesters +{: .no_toc } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + +## Overview + +This migration guide shows how to migrate to zone-aware ingesters without downtime or data loss. +The general process is the following: New stateful sets are created, the write traffic is routed to them, the read traffic is routed to them, the old stateful set is disabled. +During the migration, it is ensured that at most one ingester is unavailable at the time, and that an ingester's data is always written to persistent storage before it is shut down. + +The chart makes use of the [rollout-operator](https://github.com/grafana/rollout-operator) to coordinate rollouts of the stateful sets. This will automatically set the stateful set's update strategy to `OnDelete`. + +## Prerequisites + +Make sure to set the following settings before starting the migration: +- Ingesters are deployed as a stateful set (deployment is currently not supported) +- Autoscaling is disabled (autoscaling is not supported for zone-aware ingesters) +- `podManagementPolicy` is "OrderedReady" (default), not "Parallel" (OrderedReady creates pods consecutively when scaling up or down) +- `frontend_address` is set in the ruler config (make the ruler read from the queriers, not directly from the ingesters. Otherwise, recording and alerting rules may not be evaluated correctly during migration) + +## Migration steps + +**Important**: Always continue the next step only when all pods are in the ready state. + +1. Before starting the migration, you should ensure that the querier uses all ingesters during the migration. This means that shuffle sharding should be disabled and sharding by all labels should be enabled. + It is sufficient to set these settings on the querier using `querier.extraArgs`. Set `distributor.sharding-strategy` to `default` and `distributor.shard-by-all-labels` to `"true"` there. + Warning: This may increase resource usage of the queriers. + +1. Set `ingester.zoneAwareReplication.enabled=true`, `ingester.zoneAwareReplication.migration.enabled=true`, `ingester.zoneAwareReplication.zones` to the desired zones but with `replicas=0`. Set `rollout_operator.enabled=true`. Upgrade the chart. + ```yaml + ingester: + zoneAwareReplication: + enabled: true + migration: + enabled: true + zones: + - name: zone-a + replicas: 0 + nodeSelector: + topology.kubernetes.io/zone: zone-a + - name: zone-b + replicas: 0 + nodeSelector: + topology.kubernetes.io/zone: zone-b + - name: zone-c + replicas: 0 + nodeSelector: + topology.kubernetes.io/zone: zone-c + ``` + The stateful sets will be scaled up in the next steps and not created at once to ensure that at most one ingester is unavailable at a time. + +1. In `ingester.zoneAwareReplication.zones`, set `replicas` to the desired replicas for **the first** zone, the install the Helm chart. + +1. Repeat the process for the other zones. + +1. Enable zone-awareness on the write path by setting `ingester.zoneAwareReplication.migration.writePath=true` and install the Helm chart. This makes the distributors ship data to the new ingesters while the queriers still use all ingesters. Wait for `querier.query_store_after` so that the data that is still on the old ingesters can be queried from the object storage. If `query_store_after` is unset, wait at least `3 x bucket_store.sync_interval` (default 3x15m). + This also disables the distributors from writing to the old ingesters. + +1. Enable zone-awareness on the read path by setting `ingester.zoneAwareReplication.migration.readPath=true` and install the Helm chart. This makes the queriers use the new ingesters. + This also disables the queriers from reading from the old ingesters. + +1. Set `ingester.replicas` to 0. This will scale down the stateful set, one replica at a time, so that the ring remains healthy. + +1. Remove all values below `ingester.zoneAwareReplication.migration`. This will delete the old stateful set. + +1. If you have set any querier arguments in the first step, wait `-querier.shuffle-sharding-ingesters-lookback-period` before removing `querier.extraArgs`. + +## Faster rollouts + +With zone-awareness enabled, it is possible to roll all ingesters in a zone simultaneously. +If you want to benefit from these faster rollouts, set `ingester.zoneAwareReplication.maxUnavailable` to the number of replicas per zone and set `ingester.statefulSet.podManagementPolicy` to "Parallel". This will require recreating the stateful sets. Use `kubectl delete sts <...> --cascade=orphan` to delete only the stateful set, not the pods. diff --git a/templates/distributor/distributor-dep.yaml b/templates/distributor/distributor-dep.yaml index 5470274e..82cbed63 100644 --- a/templates/distributor/distributor-dep.yaml +++ b/templates/distributor/distributor-dep.yaml @@ -54,6 +54,15 @@ spec: args: - "-target=distributor" - "-config.file=/etc/cortex/cortex.yaml" + {{- /* enable zone-awareness if it's enabled and not in migration or in migration with writePath enabled */ -}} + {{- if and .Values.ingester.zoneAwareReplication.enabled + (or (not .Values.ingester.zoneAwareReplication.migration.enabled) + .Values.ingester.zoneAwareReplication.migration.writePath) }} + - "-distributor.zone-awareness-enabled" + {{- if .Values.ingester.zoneAwareReplication.migration.enabled }} + - "-distributor.excluded-zones=default" + {{- end }} + {{- end }} {{- range $key, $value := .Values.distributor.extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} diff --git a/templates/ingester/_helpers-ingester.tpl b/templates/ingester/_helpers-ingester.tpl index 47053273..8d57fd0a 100644 --- a/templates/ingester/_helpers-ingester.tpl +++ b/templates/ingester/_helpers-ingester.tpl @@ -21,3 +21,20 @@ ingester selector labels {{ include "cortex.selectorLabels" . }} app.kubernetes.io/component: ingester {{- end }} + +{{/* +*/}} +{{- define "cortex.ingesterZoneAwareReplicationMap" -}} +{{- $zoneMap := dict }} +{{- if .Values.ingester.zoneAwareReplication.enabled }} +{{- range $zone := .Values.ingester.zoneAwareReplication.zones }} +{{- $_ := set $zone "stsSuffix" (printf "-%s" $zone.name) }} +{{- $zoneMap := set $zoneMap $zone.name $zone }} +{{- end }} +{{- end }} +{{- if or (not .Values.ingester.zoneAwareReplication.enabled) (and .Values.ingester.zoneAwareReplication.enabled .Values.ingester.zoneAwareReplication.migration.enabled) }} +{{- $defaultZone := dict "stsSuffix" "" "name" "default" "nodeSelector" .Values.ingester.nodeSelector "replicas" .Values.ingester.replicas }} +{{- $zoneMap := set $zoneMap "default" $defaultZone }} +{{- end }} +{{- toYaml $zoneMap }} +{{- end }} diff --git a/templates/ingester/ingester-dep.yaml b/templates/ingester/ingester-dep.yaml index 7d270101..564062bd 100644 --- a/templates/ingester/ingester-dep.yaml +++ b/templates/ingester/ingester-dep.yaml @@ -1,4 +1,7 @@ {{- if and (not .Values.ingester.statefulSet.enabled) .Values.ingester.enabled -}} +{{- if .Values.ingester.zoneAwareReplication.enabled }} +{{- fail "Zone-aware replication is currently not supported with an ingester deployment. Use a stateful set instead." }} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/templates/ingester/ingester-hpa.yaml b/templates/ingester/ingester-hpa.yaml index 1bc6e6a5..da700ac0 100644 --- a/templates/ingester/ingester-hpa.yaml +++ b/templates/ingester/ingester-hpa.yaml @@ -1,4 +1,7 @@ {{- if and .Values.ingester.enabled .Values.ingester.autoscaling.enabled -}} +{{- if .Values.ingester.zoneAwareReplication.enabled }} +{{- fail "Zone-aware replication is currently not supported with autoscaling." }} +{{- end }} {{- with .Values.ingester.autoscaling -}} apiVersion: {{ include "cortex.hpaVersion" $ }} kind: HorizontalPodAutoscaler diff --git a/templates/ingester/ingester-poddisruptionbudget.yaml b/templates/ingester/ingester-poddisruptionbudget.yaml index cfa2a29f..7a565b0e 100644 --- a/templates/ingester/ingester-poddisruptionbudget.yaml +++ b/templates/ingester/ingester-poddisruptionbudget.yaml @@ -1,4 +1,7 @@ -{{- if and (gt (int .Values.ingester.replicas) 1) (.Values.ingester.podDisruptionBudget) (.Values.ingester.enabled) }} +{{- if and .Values.ingester.enabled + .Values.ingester.podDisruptionBudget + (or (gt (int .Values.ingester.replicas) 1) + (.Values.ingester.zoneAwareReplication.enabled)) }} apiVersion: {{ include "cortex.pdbVersion" . }} kind: PodDisruptionBudget metadata: diff --git a/templates/ingester/ingester-statefulset.yaml b/templates/ingester/ingester-statefulset.yaml index b6d37ae2..8f5fa6da 100644 --- a/templates/ingester/ingester-statefulset.yaml +++ b/templates/ingester/ingester-statefulset.yaml @@ -1,23 +1,42 @@ {{- if and .Values.ingester.statefulSet.enabled .Values.ingester.enabled -}} +{{- $zoneMap := include "cortex.ingesterZoneAwareReplicationMap" $ | fromYaml }} +{{- $nonDefaultZones := without (keys $zoneMap) "default" }} +{{- range $zoneName, $zone := $zoneMap }} +{{- with $ }} apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ include "cortex.ingesterFullname" . }} + name: {{ include "cortex.ingesterFullname" . }}{{ $zone.stsSuffix }} namespace: {{ .Release.Namespace }} labels: {{- include "cortex.ingesterLabels" . | nindent 4 }} app.kubernetes.io/part-of: memberlist + {{- if .Values.ingester.zoneAwareReplication.enabled }} + rollout-group: {{ include "cortex.ingesterFullname" . }} + {{- end }} annotations: - {{- toYaml .Values.ingester.annotations | nindent 4 }} + {{- if .Values.ingester.zoneAwareReplication.enabled }} + {{- with .Values.ingester.zoneAwareReplication.maxUnavailable }} + rollout-max-unavailable: {{ quote . }} + {{- end }} + {{- end }} + {{- with .Values.ingester.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- if not .Values.ingester.autoscaling.enabled }} - replicas: {{ .Values.ingester.replicas }} + replicas: {{ $zone.replicas }} {{- end }} selector: matchLabels: {{- include "cortex.ingesterSelectorLabels" . | nindent 6 }} updateStrategy: - {{- toYaml .Values.ingester.statefulStrategy | nindent 4 }} + {{- /* rollout-operator requires strategy OnDelete */ -}} + {{- if ne $zoneName "default" }} + type: OnDelete + {{- else }} + {{- toYaml .Values.ingester.statefulStrategy | nindent 4 }} + {{- end }} podManagementPolicy: "{{ .Values.ingester.statefulSet.podManagementPolicy }}" serviceName: {{ template "cortex.fullname" . }}-ingester-headless {{- if .Values.ingester.persistentVolume.enabled }} @@ -52,6 +71,11 @@ spec: metadata: labels: {{- include "cortex.ingesterLabels" . | nindent 8 }} + {{- if .Values.ingester.zoneAwareReplication.enabled }} + # required for rollout-operator + zone: {{ $zoneName }} + name: {{ include "cortex.ingesterFullname" . }}{{ $zone.stsSuffix }} + {{- end }} app.kubernetes.io/part-of: memberlist {{- with .Values.ingester.podLabels }} {{- toYaml . | nindent 8 }} @@ -78,7 +102,7 @@ spec: {{- end }} {{- end }} nodeSelector: - {{- toYaml .Values.ingester.nodeSelector | nindent 8 }} + {{- toYaml $zone.nodeSelector | nindent 8 }} {{- if .Values.ingester.topologySpreadConstraints }} topologySpreadConstraints: {{- toYaml .Values.ingester.topologySpreadConstraints | nindent 8}} @@ -110,6 +134,14 @@ spec: args: - "-target=ingester" - "-config.file=/etc/cortex/cortex.yaml" + {{- if .Values.ingester.zoneAwareReplication.enabled }} + - "-ingester.availability-zone={{ $zoneName }}" + {{- if and .Values.ingester.zoneAwareReplication.migration.enabled + (eq $zoneName "default") }} + - "-blocks-storage.tsdb.flush-blocks-on-shutdown=true" + - "-ingester.unregister-on-shutdown=true" + {{- end }} + {{- end }} {{- include "cortex.memcached" . | nindent 12}} {{- range $key, $value := .Values.ingester.extraArgs }} - "-{{ $key }}={{ $value }}" @@ -160,4 +192,7 @@ spec: lifecycle: {{- toYaml . | nindent 12 }} {{- end }} +--- +{{- end -}} +{{- end -}} {{- end -}} diff --git a/templates/querier/querier-dep.yaml b/templates/querier/querier-dep.yaml index ee1f7493..18dfb2e4 100644 --- a/templates/querier/querier-dep.yaml +++ b/templates/querier/querier-dep.yaml @@ -52,6 +52,15 @@ spec: args: - "-target=querier" - "-config.file=/etc/cortex/cortex.yaml" + {{- /* enable zone-awareness if it's enabled and not in migration or in migration with readPath enabled */ -}} + {{- if and .Values.ingester.zoneAwareReplication.enabled + (or (not .Values.ingester.zoneAwareReplication.migration.enabled) + .Values.ingester.zoneAwareReplication.migration.readPath) }} + - "-distributor.zone-awareness-enabled" + {{- if .Values.ingester.zoneAwareReplication.migration.enabled }} + - "-distributor.excluded-zones=default" + {{- end }} + {{- end }} {{- if .Values.query_scheduler.enabled }} - "-querier.scheduler-address={{ template "cortex.querySchedulerFullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:{{ .Values.config.server.grpc_listen_port }}" {{- end }} diff --git a/templates/ruler/ruler-dep.yaml b/templates/ruler/ruler-dep.yaml index 6e9f36fd..28764ea2 100644 --- a/templates/ruler/ruler-dep.yaml +++ b/templates/ruler/ruler-dep.yaml @@ -121,6 +121,15 @@ spec: args: - "-target=ruler" - "-config.file=/etc/cortex/cortex.yaml" + {{- /* enable zone-awareness if it's enabled and not in migration or in migration with writePath enabled */ -}} + {{- if and .Values.ingester.zoneAwareReplication.enabled + (or (not .Values.ingester.zoneAwareReplication.migration.enabled) + .Values.ingester.zoneAwareReplication.migration.writePath) }} + - "-distributor.zone-awareness-enabled" + {{- if .Values.ingester.zoneAwareReplication.migration.enabled }} + - "-distributor.excluded-zones=default" + {{- end }} + {{- end }} {{- if and .Values.alertmanager.enabled (not .Values.config.ruler.alertmanager_url) }} {{- if .Values.config.ruler.enable_alertmanager_discovery }} - "-ruler.alertmanager-url=http://_http-metrics._tcp.{{ template "cortex.alertmanagerFullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}/api/prom/alertmanager/" diff --git a/values.yaml b/values.yaml index 975a5469..39611289 100644 --- a/values.yaml +++ b/values.yaml @@ -449,6 +449,28 @@ ingester: # -- ref: https://cortexmetrics.io/docs/guides/ingesters-scaling-up-and-down/#scaling-down and https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies for scaledown details podManagementPolicy: OrderedReady + zoneAwareReplication: + enabled: false + migration: + enabled: false + writePath: false + readPath: false + + # maxUnavailable: 1 + zones: [] + # - name: zone-a + # replicas: 0 + # nodeSelector: + # topology.kubernetes.io/zone: zone-a + # - name: zone-b + # replicas: 0 + # nodeSelector: + # topology.kubernetes.io/zone: zone-b + # - name: zone-c + # replicas: 0 + # nodeSelector: + # topology.kubernetes.io/zone: zone-c + service: annotations: {} labels: {} @@ -1822,3 +1844,10 @@ memberlist: service: annotations: {} labels: {} + +rollout_operator: + enabled: false + webhooks: + enabled: false + crds: + enabled: false