From 8f35889827bb47f03d2910efee6ca365b1acf441 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Thu, 10 Sep 2026 16:39:28 +0900 Subject: [PATCH 1/2] Add zone awareness support for ingester Signed-off-by: SungJin1212 --- CHANGELOG.md | 1 + README.md | 3 + ci/test-zone-aware-values.yaml | 188 ++++++++++++++++++ templates/distributor/distributor-dep.yaml | 3 + templates/ingester/ingester-dep.yaml | 5 +- templates/ingester/ingester-hpa.yaml | 2 +- .../ingester-poddisruptionbudget.yaml | 2 +- templates/ingester/ingester-statefulset.yaml | 5 +- templates/ingester/ingester-svc-headless.yaml | 2 +- templates/ingester/ingester-zone-dep.yaml | 145 ++++++++++++++ templates/ingester/ingester-zone-hpa.yaml | 36 ++++ .../ingester-zone-poddisruptionbudget.yaml | 23 +++ .../ingester/ingester-zone-statefulset.yaml | 177 +++++++++++++++++ .../ingester/ingester-zone-svc-headless.yaml | 29 +++ templates/querier/querier-dep.yaml | 3 + templates/ruler/ruler-dep.yaml | 3 + values.yaml | 25 +++ 17 files changed, 647 insertions(+), 5 deletions(-) create mode 100644 ci/test-zone-aware-values.yaml create mode 100644 templates/ingester/ingester-zone-dep.yaml create mode 100644 templates/ingester/ingester-zone-hpa.yaml create mode 100644 templates/ingester/ingester-zone-poddisruptionbudget.yaml create mode 100644 templates/ingester/ingester-zone-statefulset.yaml create mode 100644 templates/ingester/ingester-zone-svc-headless.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 010e7ca3..bf3bca14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## master / unreleased +* [FEATURE] Support zone awareness on Ingester. #632 * [DEPENDENCY] update kiwigrid/k8s-sidecar docker tag to v2.11.2 #663 ## 3.3.8 / 2026-08-17 diff --git a/README.md b/README.md index cf19b914..72e655ad 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,9 @@ Kubernetes: `^1.19.0-0` | ingester.​terminationGracePeriodSeconds | int | `240` | | | ingester.​tolerations | list | `[]` | | | ingester.​topologySpreadConstraints | list | `[]` | | +| ingester.​zoneAwareness.​enabled | bool | `false` | If true, ingesters are deployed as one StatefulSet (or Deployment) per entry in `zones`, each started with `-ingester.availability-zone=`, and ingesters, distributors, queriers and rulers get `-distributor.zone-awareness-enabled`. Replicas of a series are spread across zones, so run at least as many zones as the replication factor to survive a zone outage, and the same number of replicas in every zone to keep the load balanced. | +| ingester.​zoneAwareness.​migration.​enabled | bool | `false` | If true, the zone-less ingester workload is kept alongside the zone workloads and `-distributor.zone-awareness-enabled` is set to false on every component. | +| ingester.​zoneAwareness.​zones | list | `[{"name":"zone-a","nodeSelector":{},"replicas":1},{"name":"zone-b","nodeSelector":{},"replicas":1},{"name":"zone-c","nodeSelector":{},"replicas":1}]` | Zones to deploy ingesters into. `name` is required. `replicas`, `nodeSelector`, `affinity`, `tolerations` and `annotations` fall back to the top-level ingester values when unset; `podLabels` and `podAnnotations` are merged into them. | | ingress.​annotations | object | `{}` | | | ingress.​enabled | bool | `false` | | | ingress.​hosts[0].​host | string | `"chart-example.local"` | | diff --git a/ci/test-zone-aware-values.yaml b/ci/test-zone-aware-values.yaml new file mode 100644 index 00000000..6be1a99c --- /dev/null +++ b/ci/test-zone-aware-values.yaml @@ -0,0 +1,188 @@ +ingress: + enabled: true + +config: + ingester: + lifecycler: + join_after: 30s + final_sleep: 30s + tokens_file_path: "/data/tokens" + ring: + replication_factor: 1 + kvstore: + store: "memberlist" + + blocks_storage: + backend: "filesystem" + tsdb: + dir: "/data/tsdb" + bucket_store: + sync_dir: "/data/tsdb-sync" + filesystem: + dir: "/data/store" + + distributor: + ring: + kvstore: + store: memberlist + + ruler: + ring: + kvstore: + store: memberlist + + ruler_storage: + backend: "filesystem" + filesystem: + dir: "/data/store" + + alertmanager_storage: + backend: "filesystem" + filesystem: + dir: "/data/store" + + memberlist: + join_members: + - '{{ include "cortex.fullname" $ }}-memberlist' + +ingester: + replicas: 1 + statefulSet: + enabled: true + zoneAwareness: + enabled: true + zones: + - name: zone-a + replicas: 1 + - name: zone-b + replicas: 1 + - name: zone-c + replicas: 1 + extraVolumes: + - name: tmp-test + emptyDir: {} + extraVolumeMounts: + - name: tmp-test + mountPath: /tmp-test +compactor: + replicas: 1 + extraVolumes: + - name: tmp-test + emptyDir: {} + extraVolumeMounts: + - name: tmp-test + mountPath: /tmp-test +distributor: + replicas: 1 + autoscaling: + enabled: true + minReplicas: 1 + extraVolumes: + - name: tmp-test + emptyDir: {} + extraVolumeMounts: + - name: tmp-test + mountPath: /tmp-test +querier: + replicas: 1 + autoscaling: + enabled: true + extraVolumes: + - name: tmp-test + emptyDir: {} + extraVolumeMounts: + - name: tmp-test + mountPath: /tmp-test +query_frontend: + replicas: 1 + extraVolumes: + - name: tmp-test + emptyDir: {} + extraVolumeMounts: + - name: tmp-test + mountPath: /tmp-test +nginx: + replicas: 1 + autoscaling: + enabled: true + extraVolumes: + - name: tmp-test + emptyDir: {} + extraVolumeMounts: + - name: tmp-test + mountPath: /tmp-test + config: + httpSnippet: |- + # http snippet + mainSnippet: |- + # main snippet + serverSnippet: |- + # server snippet +runtimeconfigmap: + annotations: + foo: bar +alertmanager: + replicas: 3 + statefulSet: + enabled: true + extraVolumes: + - name: tmp-test + emptyDir: {} + extraVolumeMounts: + - name: tmp-test + mountPath: /tmp-test + sidecar: + enabled: true +ruler: + replicas: 1 + enabled: true + extraVolumes: + - name: tmp-test + emptyDir: {} + extraVolumeMounts: + - name: tmp-test + mountPath: /tmp-test + sidecar: + enabled: true +store_gateway: + startupProbe: null + replicas: 1 + extraVolumes: + - name: tmp-test + emptyDir: {} + extraVolumeMounts: + - name: tmp-test + mountPath: /tmp-test + +memcached-frontend: + enabled: true + architecture: standalone + replicaCount: 1 + resources: + requests: + cpu: 0 + memory: 0 +memcached-blocks: + enabled: true + architecture: standalone + replicaCount: 1 + resources: + requests: + cpu: 0 + memory: 0 +memcached-blocks-index: + enabled: true + architecture: standalone + replicaCount: 1 + resources: + requests: + cpu: 0 + memory: 0 +memcached-blocks-metadata: + enabled: true + architecture: standalone + replicaCount: 1 + resources: + requests: + cpu: 0 + memory: 0 diff --git a/templates/distributor/distributor-dep.yaml b/templates/distributor/distributor-dep.yaml index 5470274e..4b907ce2 100644 --- a/templates/distributor/distributor-dep.yaml +++ b/templates/distributor/distributor-dep.yaml @@ -54,6 +54,9 @@ spec: args: - "-target=distributor" - "-config.file=/etc/cortex/cortex.yaml" + {{- if .Values.ingester.zoneAwareness.enabled }} + - -distributor.zone-awareness-enabled={{ not .Values.ingester.zoneAwareness.migration.enabled }} + {{- end }} {{- range $key, $value := .Values.distributor.extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} diff --git a/templates/ingester/ingester-dep.yaml b/templates/ingester/ingester-dep.yaml index 7d270101..4df3a1a5 100644 --- a/templates/ingester/ingester-dep.yaml +++ b/templates/ingester/ingester-dep.yaml @@ -1,4 +1,4 @@ -{{- if and (not .Values.ingester.statefulSet.enabled) .Values.ingester.enabled -}} +{{- if and (not .Values.ingester.statefulSet.enabled) .Values.ingester.enabled (or (not .Values.ingester.zoneAwareness.enabled) .Values.ingester.zoneAwareness.migration.enabled) -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -54,6 +54,9 @@ spec: args: - "-target=ingester" - "-config.file=/etc/cortex/cortex.yaml" + {{- if .Values.ingester.zoneAwareness.enabled }} + - "-distributor.zone-awareness-enabled={{ not .Values.ingester.zoneAwareness.migration.enabled }}" + {{- end }} {{- include "cortex.memcached" . | nindent 12}} {{- range $key, $value := .Values.ingester.extraArgs }} - "-{{ $key }}={{ $value }}" diff --git a/templates/ingester/ingester-hpa.yaml b/templates/ingester/ingester-hpa.yaml index 1bc6e6a5..a0d4a58a 100644 --- a/templates/ingester/ingester-hpa.yaml +++ b/templates/ingester/ingester-hpa.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.ingester.enabled .Values.ingester.autoscaling.enabled -}} +{{- if and .Values.ingester.enabled .Values.ingester.autoscaling.enabled (or (not .Values.ingester.zoneAwareness.enabled) .Values.ingester.zoneAwareness.migration.enabled) -}} {{- with .Values.ingester.autoscaling -}} apiVersion: {{ include "cortex.hpaVersion" $ }} kind: HorizontalPodAutoscaler diff --git a/templates/ingester/ingester-poddisruptionbudget.yaml b/templates/ingester/ingester-poddisruptionbudget.yaml index cfa2a29f..218cca3a 100644 --- a/templates/ingester/ingester-poddisruptionbudget.yaml +++ b/templates/ingester/ingester-poddisruptionbudget.yaml @@ -1,4 +1,4 @@ -{{- if and (gt (int .Values.ingester.replicas) 1) (.Values.ingester.podDisruptionBudget) (.Values.ingester.enabled) }} +{{- if and (gt (int .Values.ingester.replicas) 1) (.Values.ingester.podDisruptionBudget) (.Values.ingester.enabled) (or (not .Values.ingester.zoneAwareness.enabled) .Values.ingester.zoneAwareness.migration.enabled) }} apiVersion: {{ include "cortex.pdbVersion" . }} kind: PodDisruptionBudget metadata: diff --git a/templates/ingester/ingester-statefulset.yaml b/templates/ingester/ingester-statefulset.yaml index b6d37ae2..bdef6504 100644 --- a/templates/ingester/ingester-statefulset.yaml +++ b/templates/ingester/ingester-statefulset.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.ingester.statefulSet.enabled .Values.ingester.enabled -}} +{{- if and .Values.ingester.statefulSet.enabled .Values.ingester.enabled (or (not .Values.ingester.zoneAwareness.enabled) .Values.ingester.zoneAwareness.migration.enabled) -}} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -110,6 +110,9 @@ spec: args: - "-target=ingester" - "-config.file=/etc/cortex/cortex.yaml" + {{- if .Values.ingester.zoneAwareness.enabled }} + - "-distributor.zone-awareness-enabled={{ not .Values.ingester.zoneAwareness.migration.enabled }}" + {{- end }} {{- include "cortex.memcached" . | nindent 12}} {{- range $key, $value := .Values.ingester.extraArgs }} - "-{{ $key }}={{ $value }}" diff --git a/templates/ingester/ingester-svc-headless.yaml b/templates/ingester/ingester-svc-headless.yaml index 0f698179..cc7ee924 100644 --- a/templates/ingester/ingester-svc-headless.yaml +++ b/templates/ingester/ingester-svc-headless.yaml @@ -1,4 +1,4 @@ -{{- if .Values.ingester.enabled -}} +{{- if and .Values.ingester.enabled (or (not .Values.ingester.zoneAwareness.enabled) .Values.ingester.zoneAwareness.migration.enabled) -}} apiVersion: v1 kind: Service metadata: diff --git a/templates/ingester/ingester-zone-dep.yaml b/templates/ingester/ingester-zone-dep.yaml new file mode 100644 index 00000000..b84e67a3 --- /dev/null +++ b/templates/ingester/ingester-zone-dep.yaml @@ -0,0 +1,145 @@ +{{- if and .Values.ingester.enabled (not .Values.ingester.statefulSet.enabled) .Values.ingester.zoneAwareness.enabled -}} +{{- range $zone := .Values.ingester.zoneAwareness.zones }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cortex.ingesterFullname" $ }}-{{ $zone.name }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "cortex.ingesterLabels" $ | nindent 4 }} + app.kubernetes.io/part-of: memberlist + zone: {{ $zone.name }} + annotations: + {{- toYaml ($zone.annotations | default $.Values.ingester.annotations) | nindent 4 }} +spec: + {{- if not $.Values.ingester.autoscaling.enabled }} + replicas: {{ if hasKey $zone "replicas" }}{{ $zone.replicas }}{{ else }}{{ $.Values.ingester.replicas }}{{ end }} + {{- end }} + selector: + matchLabels: + {{- include "cortex.ingesterSelectorLabels" $ | nindent 6 }} + zone: {{ $zone.name }} + strategy: + {{- toYaml $.Values.ingester.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "cortex.ingesterLabels" $ | nindent 8 }} + app.kubernetes.io/part-of: memberlist + zone: {{ $zone.name }} + {{- with $.Values.ingester.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $zone.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "cortex.configChecksum" $ }} + {{- with $.Values.ingester.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $zone.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ $.Values.ingester.serviceAccount.name | default (include "cortex.serviceAccountName" $) }} + {{- if $.Values.ingester.priorityClassName }} + priorityClassName: {{ $.Values.ingester.priorityClassName }} + {{- end }} + {{- if $.Values.ingester.securityContext.enabled }} + securityContext: {{- omit $.Values.ingester.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + initContainers: + {{- toYaml $.Values.ingester.initContainers | nindent 8 }} + {{- if $.Values.image.pullSecrets }} + imagePullSecrets: + {{- range $.Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + containers: + - name: ingester + image: "{{ $.Values.image.repository }}:{{ default $.Chart.AppVersion $.Values.image.tag }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + args: + - "-target=ingester" + - "-config.file=/etc/cortex/cortex.yaml" + - "-ingester.availability-zone={{ $zone.name }}" + - "-distributor.zone-awareness-enabled={{ not $.Values.ingester.zoneAwareness.migration.enabled }}" + {{- include "cortex.memcached" $ | nindent 12 }} + {{- range $key, $value := $.Values.ingester.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if $.Values.ingester.extraVolumeMounts }} + {{- toYaml $.Values.ingester.extraVolumeMounts | nindent 12 }} + {{- end }} + - name: config + mountPath: /etc/cortex + - name: runtime-config + mountPath: /etc/cortex-runtime-config + - name: storage + mountPath: "/data" + {{- with $.Values.ingester.persistentVolume.subPath }} + subPath: {{ . }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ $.Values.config.server.http_listen_port }} + protocol: TCP + - name: grpc + containerPort: {{ $.Values.config.server.grpc_listen_port }} + protocol: TCP + - name: gossip + containerPort: {{ $.Values.config.memberlist.bind_port }} + protocol: TCP + {{- if $.Values.ingester.startupProbe }} + startupProbe: + {{- toYaml $.Values.ingester.startupProbe | nindent 12 }} + {{- end }} + {{- if $.Values.ingester.livenessProbe }} + livenessProbe: + {{- toYaml $.Values.ingester.livenessProbe | nindent 12 }} + {{- end }} + readinessProbe: + {{- toYaml $.Values.ingester.readinessProbe | nindent 12 }} + resources: + {{- toYaml $.Values.ingester.resources | nindent 12 }} + {{- if $.Values.ingester.containerSecurityContext.enabled }} + securityContext: {{- omit $.Values.ingester.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if $.Values.ingester.env }} + env: + {{- toYaml $.Values.ingester.env | nindent 12 }} + {{- end }} + {{- with $.Values.ingester.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with $.Values.ingester.extraContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + {{- toYaml ($zone.nodeSelector | default $.Values.ingester.nodeSelector) | nindent 8 }} + {{- if $.Values.ingester.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml $.Values.ingester.topologySpreadConstraints | nindent 8 }} + {{- end }} + affinity: + {{- toYaml ($zone.affinity | default $.Values.ingester.affinity) | nindent 8 }} + tolerations: + {{- toYaml ($zone.tolerations | default $.Values.ingester.tolerations) | nindent 8 }} + terminationGracePeriodSeconds: {{ $.Values.ingester.terminationGracePeriodSeconds }} + volumes: + {{- include "cortex.configVolume" $ | nindent 8 }} + - name: runtime-config + configMap: + name: {{ template "cortex.fullname" $ }}-runtime-config + - name: storage + emptyDir: {} + {{- if $.Values.ingester.extraVolumes }} + {{- toYaml $.Values.ingester.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/templates/ingester/ingester-zone-hpa.yaml b/templates/ingester/ingester-zone-hpa.yaml new file mode 100644 index 00000000..8d060e09 --- /dev/null +++ b/templates/ingester/ingester-zone-hpa.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.ingester.enabled .Values.ingester.autoscaling.enabled .Values.ingester.zoneAwareness.enabled -}} +{{- range $zone := .Values.ingester.zoneAwareness.zones }} +{{- with $.Values.ingester.autoscaling }} +--- +apiVersion: {{ include "cortex.hpaVersion" $ }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "cortex.ingesterFullname" $ }}-{{ $zone.name }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "cortex.ingesterLabels" $ | nindent 4 }} + zone: {{ $zone.name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: {{ if $.Values.ingester.statefulSet.enabled }}StatefulSet{{ else }}Deployment{{ end }} + name: {{ include "cortex.ingesterFullname" $ }}-{{ $zone.name }} + minReplicas: {{ .minReplicas }} + maxReplicas: {{ .maxReplicas }} + metrics: + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .targetMemoryUtilizationPercentage }} + {{- with .extraMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/templates/ingester/ingester-zone-poddisruptionbudget.yaml b/templates/ingester/ingester-zone-poddisruptionbudget.yaml new file mode 100644 index 00000000..992baaf7 --- /dev/null +++ b/templates/ingester/ingester-zone-poddisruptionbudget.yaml @@ -0,0 +1,23 @@ +{{- /* Per-zone PDBs are skipped during migration: the zone-less PDB still matches every ingester pod and overlapping PDBs block evictions. */ -}} +{{- if and .Values.ingester.enabled .Values.ingester.podDisruptionBudget .Values.ingester.zoneAwareness.enabled (not .Values.ingester.zoneAwareness.migration.enabled) -}} +{{- range $zone := .Values.ingester.zoneAwareness.zones }} +{{- $replicas := ternary $zone.replicas $.Values.ingester.replicas (hasKey $zone "replicas") }} +{{- if gt (int $replicas) 1 }} +--- +apiVersion: {{ include "cortex.pdbVersion" $ }} +kind: PodDisruptionBudget +metadata: + name: {{ include "cortex.ingesterFullname" $ }}-{{ $zone.name }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "cortex.ingesterLabels" $ | nindent 4 }} + zone: {{ $zone.name }} +spec: + selector: + matchLabels: + {{- include "cortex.ingesterSelectorLabels" $ | nindent 6 }} + zone: {{ $zone.name }} + {{- toYaml $.Values.ingester.podDisruptionBudget | nindent 2 }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/templates/ingester/ingester-zone-statefulset.yaml b/templates/ingester/ingester-zone-statefulset.yaml new file mode 100644 index 00000000..c4c72eef --- /dev/null +++ b/templates/ingester/ingester-zone-statefulset.yaml @@ -0,0 +1,177 @@ +{{- if and .Values.ingester.enabled .Values.ingester.statefulSet.enabled .Values.ingester.zoneAwareness.enabled -}} +{{- range $zone := .Values.ingester.zoneAwareness.zones }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "cortex.ingesterFullname" $ }}-{{ $zone.name }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "cortex.ingesterLabels" $ | nindent 4 }} + app.kubernetes.io/part-of: memberlist + zone: {{ $zone.name }} + annotations: + {{- toYaml ($zone.annotations | default $.Values.ingester.annotations) | nindent 4 }} +spec: + {{- if not $.Values.ingester.autoscaling.enabled }} + replicas: {{ if hasKey $zone "replicas" }}{{ $zone.replicas }}{{ else }}{{ $.Values.ingester.replicas }}{{ end }} + {{- end }} + selector: + matchLabels: + {{- include "cortex.ingesterSelectorLabels" $ | nindent 6 }} + zone: {{ $zone.name }} + updateStrategy: + {{- toYaml $.Values.ingester.statefulStrategy | nindent 4 }} + podManagementPolicy: "{{ $.Values.ingester.statefulSet.podManagementPolicy }}" + serviceName: {{ include "cortex.ingesterFullname" $ }}-headless-{{ $zone.name }} + {{- if $.Values.ingester.persistentVolume.enabled }} + {{- if semverCompare ">=1.23-0" $.Capabilities.KubeVersion.Version }} + {{- with $.Values.ingester.persistentVolume.retentionPolicy }} + persistentVolumeClaimRetentionPolicy: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: storage + {{- if $.Values.ingester.persistentVolume.annotations }} + annotations: + {{ toYaml $.Values.ingester.persistentVolume.annotations | nindent 10 }} + {{- end }} + spec: + {{- if $.Values.ingester.persistentVolume.storageClass }} + {{- if (eq "-" $.Values.ingester.persistentVolume.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $.Values.ingester.persistentVolume.storageClass }}" + {{- end }} + {{- end }} + accessModes: + {{ toYaml $.Values.ingester.persistentVolume.accessModes | nindent 10 }} + resources: + requests: + storage: "{{ $.Values.ingester.persistentVolume.size }}" + {{- end }} + template: + metadata: + labels: + {{- include "cortex.ingesterLabels" $ | nindent 8 }} + app.kubernetes.io/part-of: memberlist + zone: {{ $zone.name }} + {{- with $.Values.ingester.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $zone.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "cortex.configChecksum" $ }} + {{- with $.Values.ingester.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $zone.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ $.Values.ingester.serviceAccount.name | default (include "cortex.serviceAccountName" $) }} + {{- if $.Values.ingester.priorityClassName }} + priorityClassName: {{ $.Values.ingester.priorityClassName }} + {{- end }} + {{- if $.Values.ingester.securityContext.enabled }} + securityContext: {{- omit $.Values.ingester.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + initContainers: + {{- toYaml $.Values.ingester.initContainers | nindent 8 }} + {{- if $.Values.image.pullSecrets }} + imagePullSecrets: + {{- range $.Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + nodeSelector: + {{- toYaml ($zone.nodeSelector | default $.Values.ingester.nodeSelector) | nindent 8 }} + {{- if $.Values.ingester.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml $.Values.ingester.topologySpreadConstraints | nindent 8 }} + {{- end }} + affinity: + {{- toYaml ($zone.affinity | default $.Values.ingester.affinity) | nindent 8 }} + tolerations: + {{- toYaml ($zone.tolerations | default $.Values.ingester.tolerations) | nindent 8 }} + terminationGracePeriodSeconds: {{ $.Values.ingester.terminationGracePeriodSeconds }} + volumes: + {{- include "cortex.configVolume" $ | nindent 8 }} + - name: runtime-config + configMap: + name: {{ template "cortex.fullname" $ }}-runtime-config + {{- if not $.Values.ingester.persistentVolume.enabled }} + - name: storage + emptyDir: {} + {{- end }} + {{- if $.Values.ingester.extraVolumes }} + {{- toYaml $.Values.ingester.extraVolumes | nindent 8 }} + {{- end }} + containers: + {{- if $.Values.ingester.extraContainers }} + {{- toYaml $.Values.ingester.extraContainers | nindent 8 }} + {{- end }} + - name: ingester + image: "{{ $.Values.image.repository }}:{{ default $.Chart.AppVersion $.Values.image.tag }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + args: + - "-target=ingester" + - "-config.file=/etc/cortex/cortex.yaml" + - "-ingester.availability-zone={{ $zone.name }}" + - "-distributor.zone-awareness-enabled={{ not $.Values.ingester.zoneAwareness.migration.enabled }}" + {{- include "cortex.memcached" $ | nindent 12 }} + {{- range $key, $value := $.Values.ingester.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if $.Values.ingester.extraVolumeMounts }} + {{- toYaml $.Values.ingester.extraVolumeMounts | nindent 12 }} + {{- end }} + - name: config + mountPath: /etc/cortex + - name: runtime-config + mountPath: /etc/cortex-runtime-config + - name: storage + mountPath: "/data" + {{- with $.Values.ingester.persistentVolume.subPath }} + subPath: {{ . }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ $.Values.config.server.http_listen_port }} + protocol: TCP + - name: grpc + containerPort: {{ $.Values.config.server.grpc_listen_port }} + protocol: TCP + - name: gossip + containerPort: {{ $.Values.config.memberlist.bind_port }} + protocol: TCP + {{- if $.Values.ingester.startupProbe }} + startupProbe: + {{- toYaml $.Values.ingester.startupProbe | nindent 12 }} + {{- end }} + {{- if $.Values.ingester.livenessProbe }} + livenessProbe: + {{- toYaml $.Values.ingester.livenessProbe | nindent 12 }} + {{- end }} + readinessProbe: + {{- toYaml $.Values.ingester.readinessProbe | nindent 12 }} + resources: + {{- toYaml $.Values.ingester.resources | nindent 12 }} + {{- if $.Values.ingester.containerSecurityContext.enabled }} + securityContext: {{- omit $.Values.ingester.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if $.Values.ingester.env }} + env: + {{- toYaml $.Values.ingester.env | nindent 12 }} + {{- end }} + {{- with $.Values.ingester.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/templates/ingester/ingester-zone-svc-headless.yaml b/templates/ingester/ingester-zone-svc-headless.yaml new file mode 100644 index 00000000..57b552f2 --- /dev/null +++ b/templates/ingester/ingester-zone-svc-headless.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.ingester.enabled .Values.ingester.zoneAwareness.enabled -}} +{{- range $zone := .Values.ingester.zoneAwareness.zones }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cortex.ingesterFullname" $ }}-headless-{{ $zone.name }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "cortex.ingesterLabels" $ | nindent 4 }} + zone: {{ $zone.name }} + {{- with $.Values.ingester.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml $.Values.ingester.service.annotations | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: {{ $.Values.config.server.grpc_listen_port }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "cortex.ingesterSelectorLabels" $ | nindent 4 }} + zone: {{ $zone.name }} +{{- end }} +{{- end -}} diff --git a/templates/querier/querier-dep.yaml b/templates/querier/querier-dep.yaml index ee1f7493..3f13bb4d 100644 --- a/templates/querier/querier-dep.yaml +++ b/templates/querier/querier-dep.yaml @@ -59,6 +59,9 @@ spec: - "-querier.frontend-address={{ template "cortex.queryFrontendFullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:{{ .Values.config.server.grpc_listen_port }}" {{- end }} {{- include "cortex.memcached" . | nindent 12}} + {{- if .Values.ingester.zoneAwareness.enabled }} + - -distributor.zone-awareness-enabled={{ not .Values.ingester.zoneAwareness.migration.enabled }} + {{- end }} {{- range $key, $value := .Values.querier.extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} diff --git a/templates/ruler/ruler-dep.yaml b/templates/ruler/ruler-dep.yaml index 6e9f36fd..0a4374fa 100644 --- a/templates/ruler/ruler-dep.yaml +++ b/templates/ruler/ruler-dep.yaml @@ -129,6 +129,9 @@ spec: {{- end }} {{- end }} {{- include "cortex.memcached" . | nindent 12}} + {{- if .Values.ingester.zoneAwareness.enabled }} + - -distributor.zone-awareness-enabled={{ not .Values.ingester.zoneAwareness.migration.enabled }} + {{- end }} {{- range $key, $value := .Values.ruler.extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} diff --git a/values.yaml b/values.yaml index 975a5469..eb8f23a7 100644 --- a/values.yaml +++ b/values.yaml @@ -607,6 +607,31 @@ ingester: extraPorts: [] env: [] + zoneAwareness: + # -- If true, ingesters are deployed as one StatefulSet (or Deployment) per entry in `zones`, + # each started with `-ingester.availability-zone=`, and ingesters, distributors, + # queriers and rulers get `-distributor.zone-awareness-enabled`. Replicas of a series are + # spread across zones, so run at least as many zones as the replication factor to survive a + # zone outage, and the same number of replicas in every zone to keep the load balanced. + enabled: false + migration: + # -- If true, the zone-less ingester workload is kept alongside the zone workloads and + # `-distributor.zone-awareness-enabled` is set to false on every component. + enabled: false + # -- Zones to deploy ingesters into. `name` is required. `replicas`, `nodeSelector`, `affinity`, + # `tolerations` and `annotations` fall back to the top-level ingester values when unset; + # `podLabels` and `podAnnotations` are merged into them. + zones: + - name: zone-a + replicas: 1 + nodeSelector: {} + - name: zone-b + replicas: 1 + nodeSelector: {} + - name: zone-c + replicas: 1 + nodeSelector: {} + ruler: enabled: true replicas: 1 From e9713220a6bf99eab7eefad39d5e7376bb2a333f Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Thu, 10 Sep 2026 18:48:32 +0900 Subject: [PATCH 2/2] delete ingester zone hpa Signed-off-by: SungJin1212 --- templates/ingester/ingester-zone-hpa.yaml | 36 ----------------------- 1 file changed, 36 deletions(-) delete mode 100644 templates/ingester/ingester-zone-hpa.yaml diff --git a/templates/ingester/ingester-zone-hpa.yaml b/templates/ingester/ingester-zone-hpa.yaml deleted file mode 100644 index 8d060e09..00000000 --- a/templates/ingester/ingester-zone-hpa.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{- if and .Values.ingester.enabled .Values.ingester.autoscaling.enabled .Values.ingester.zoneAwareness.enabled -}} -{{- range $zone := .Values.ingester.zoneAwareness.zones }} -{{- with $.Values.ingester.autoscaling }} ---- -apiVersion: {{ include "cortex.hpaVersion" $ }} -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "cortex.ingesterFullname" $ }}-{{ $zone.name }} - namespace: {{ $.Release.Namespace }} - labels: - {{- include "cortex.ingesterLabels" $ | nindent 4 }} - zone: {{ $zone.name }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: {{ if $.Values.ingester.statefulSet.enabled }}StatefulSet{{ else }}Deployment{{ end }} - name: {{ include "cortex.ingesterFullname" $ }}-{{ $zone.name }} - minReplicas: {{ .minReplicas }} - maxReplicas: {{ .maxReplicas }} - metrics: - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .targetMemoryUtilizationPercentage }} - {{- with .extraMetrics }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .behavior }} - behavior: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} -{{- end }} -{{- end -}}