From 8b3a77d8a8db6c7a89db0de80103f30cca4d7d24 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Thu, 10 Sep 2026 09:45:34 +0200 Subject: [PATCH 01/11] ingester: support zone-awareness Signed-off-by: Timon Engelke --- CHANGELOG.md | 1 + templates/distributor/distributor-dep.yaml | 3 ++ templates/ingester/_helpers-ingester.tpl | 16 ++++++++ templates/ingester/ingester-statefulset.yaml | 40 ++++++++++++++++++-- templates/querier/querier-dep.yaml | 3 ++ templates/ruler/ruler-dep.yaml | 3 ++ values.yaml | 19 ++++++++++ 7 files changed, 81 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51793e63..3f7453b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## master / unreleased * [FEATURE] Add remote-write OTLP endpoint in nginx #667 +* [FEATURE] Ingester: support zone-awareness #668 * [DEPENDENCY] update kiwigrid/k8s-sidecar docker tag to v2.11.2 #663 ## 3.3.8 / 2026-08-17 diff --git a/templates/distributor/distributor-dep.yaml b/templates/distributor/distributor-dep.yaml index 5470274e..75837036 100644 --- a/templates/distributor/distributor-dep.yaml +++ b/templates/distributor/distributor-dep.yaml @@ -54,6 +54,9 @@ spec: args: - "-target=distributor" - "-config.file=/etc/cortex/cortex.yaml" + {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }} + - "-distributor.zone-awareness-enabled" + {{- end }} {{- range $key, $value := .Values.distributor.extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} diff --git a/templates/ingester/_helpers-ingester.tpl b/templates/ingester/_helpers-ingester.tpl index 47053273..9ced89e9 100644 --- a/templates/ingester/_helpers-ingester.tpl +++ b/templates/ingester/_helpers-ingester.tpl @@ -21,3 +21,19 @@ ingester selector labels {{ include "cortex.selectorLabels" . }} app.kubernetes.io/component: ingester {{- end }} + +{{/* +*/}} +{{- define "cortex.ingesterZoneAwareReplicationMap" -}} +{{- $zoneMap := dict }} +{{- range $zone := .Values.ingester.zoneAwareReplication.zones }} +{{- $_ := set $zone "stsSuffix" (printf "-%s" $zone.name) }} +{{- $zoneMap := set $zoneMap $zone.name $zone }} +{{- end }} +{{- end }} +{{- if or (not .Values.ingester.zoneAwareReplication.enabled) (and .Values.ingester.zoneAwareReplication.enabled .Values.ingester.zoneAwareReplication.migration) }} +{{- $defaultZone := dict "stsSuffix" "" "name" "default" "nodeSelector" .Values.ingester.nodeSelector "replicas" .Values.ingester.replicas }} +{{- $zoneMap := set $zoneMap "default" $defaultZone }} +{{- end }} +{{- toYaml $zoneMap }} +{{- end }} diff --git a/templates/ingester/ingester-statefulset.yaml b/templates/ingester/ingester-statefulset.yaml index b6d37ae2..9dcc561a 100644 --- a/templates/ingester/ingester-statefulset.yaml +++ b/templates/ingester/ingester-statefulset.yaml @@ -1,23 +1,38 @@ {{- if and .Values.ingester.statefulSet.enabled .Values.ingester.enabled -}} +{{- $zoneMap := include "cortex.ingesterZoneAwareReplicationMap" $ | fromYaml }} +{{- range $zoneName, $zone := $zoneMap }} +{{- with $ }} apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ include "cortex.ingesterFullname" . }} + name: {{ include "cortex.ingesterFullname" . }}{{ $zone.stsSuffix }} namespace: {{ .Release.Namespace }} labels: {{- include "cortex.ingesterLabels" . | nindent 4 }} app.kubernetes.io/part-of: memberlist + {{- with $zone.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $zone.annotations }} annotations: - {{- toYaml .Values.ingester.annotations | nindent 4 }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- if not .Values.ingester.autoscaling.enabled }} - replicas: {{ .Values.ingester.replicas }} + replicas: {{ $zone.replicas }} {{- end }} selector: matchLabels: {{- include "cortex.ingesterSelectorLabels" . | nindent 6 }} + {{- if .Values.ingester.zoneAwareReplication.enabled }} + zone: {{ $zoneName }} + {{- end }} updateStrategy: + {{- if ne $zoneName "default" }} + type: OnDelete + {{- else }} {{- toYaml .Values.ingester.statefulStrategy | nindent 4 }} + {{- end }} podManagementPolicy: "{{ .Values.ingester.statefulSet.podManagementPolicy }}" serviceName: {{ template "cortex.fullname" . }}-ingester-headless {{- if .Values.ingester.persistentVolume.enabled }} @@ -34,6 +49,10 @@ spec: annotations: {{ toYaml .Values.ingester.persistentVolume.annotations | nindent 10 }} {{- end }} + {{- with $zone.podLabels }} + labels: + {{- toYaml . | nindent 10 }} + {{- end }} spec: {{- if .Values.ingester.persistentVolume.storageClass }} {{- if (eq "-" .Values.ingester.persistentVolume.storageClass) }} @@ -52,6 +71,9 @@ spec: metadata: labels: {{- include "cortex.ingesterLabels" . | nindent 8 }} + {{- if .Values.ingester.zoneAwareReplication.enabled }} + zone: {{ $zoneName }} + {{- end }} app.kubernetes.io/part-of: memberlist {{- with .Values.ingester.podLabels }} {{- toYaml . | nindent 8 }} @@ -78,7 +100,7 @@ spec: {{- end }} {{- end }} nodeSelector: - {{- toYaml .Values.ingester.nodeSelector | nindent 8 }} + {{- toYaml $zone.nodeSelector | nindent 8 }} {{- if .Values.ingester.topologySpreadConstraints }} topologySpreadConstraints: {{- toYaml .Values.ingester.topologySpreadConstraints | nindent 8}} @@ -110,6 +132,13 @@ spec: args: - "-target=ingester" - "-config.file=/etc/cortex/cortex.yaml" + {{- if .Values.ingester.zoneAwareReplication.enabled }} + - "-ingester.availability-zone={{ $zoneName }}" + {{- if .Values.ingester.zoneAwareReplication.migration }} + - "-blocks-storage.tsdb.flush-blocks-on-shutdown=true" + - "-ingester.unregister-on-shutdown=true" + {{- end }} + {{- end }} {{- include "cortex.memcached" . | nindent 12}} {{- range $key, $value := .Values.ingester.extraArgs }} - "-{{ $key }}={{ $value }}" @@ -160,4 +189,7 @@ spec: lifecycle: {{- toYaml . | nindent 12 }} {{- end }} +--- +{{- end -}} +{{- end -}} {{- end -}} diff --git a/templates/querier/querier-dep.yaml b/templates/querier/querier-dep.yaml index ee1f7493..138533c8 100644 --- a/templates/querier/querier-dep.yaml +++ b/templates/querier/querier-dep.yaml @@ -52,6 +52,9 @@ spec: args: - "-target=querier" - "-config.file=/etc/cortex/cortex.yaml" + {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }} + - "-distributor.zone-awareness-enabled" + {{- end }} {{- if .Values.query_scheduler.enabled }} - "-querier.scheduler-address={{ template "cortex.querySchedulerFullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:{{ .Values.config.server.grpc_listen_port }}" {{- end }} diff --git a/templates/ruler/ruler-dep.yaml b/templates/ruler/ruler-dep.yaml index 6e9f36fd..3316ae53 100644 --- a/templates/ruler/ruler-dep.yaml +++ b/templates/ruler/ruler-dep.yaml @@ -121,6 +121,9 @@ spec: args: - "-target=ruler" - "-config.file=/etc/cortex/cortex.yaml" + {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }} + - "-distributor.zone-awareness-enabled" + {{- end }} {{- if and .Values.alertmanager.enabled (not .Values.config.ruler.alertmanager_url) }} {{- if .Values.config.ruler.enable_alertmanager_discovery }} - "-ruler.alertmanager-url=http://_http-metrics._tcp.{{ template "cortex.alertmanagerFullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}/api/prom/alertmanager/" diff --git a/values.yaml b/values.yaml index 975a5469..dd6c1a61 100644 --- a/values.yaml +++ b/values.yaml @@ -449,6 +449,23 @@ ingester: # -- ref: https://cortexmetrics.io/docs/guides/ingesters-scaling-up-and-down/#scaling-down and https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies for scaledown details podManagementPolicy: OrderedReady + zoneAwareReplication: + enabled: false + migration: false + zones: [] + # - name: eu02-1 + # nodeSelector: + # topology.kubernetes.io/zone: eu02-1 + # labels: {} + # - name: eu02-2 + # nodeSelector: + # topology.kubernetes.io/zone: eu02-2 + # labels: {} + # - name: eu02-3 + # nodeSelector: + # topology.kubernetes.io/zone: eu02-3 + # labels: {} + service: annotations: {} labels: {} @@ -470,6 +487,8 @@ ingester: # -- Additional Cortex container arguments, e.g. log.level (debug, info, warn, error) extraArgs: {} + extraLabels: {} + # -- Pod Labels podLabels: {} From 16f1635050fea1e5a8c71fd4b6c6ffe895a4dde9 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Thu, 10 Sep 2026 10:06:49 +0200 Subject: [PATCH 02/11] remove accidental "end" Signed-off-by: Timon Engelke --- templates/ingester/_helpers-ingester.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/ingester/_helpers-ingester.tpl b/templates/ingester/_helpers-ingester.tpl index 9ced89e9..53975fef 100644 --- a/templates/ingester/_helpers-ingester.tpl +++ b/templates/ingester/_helpers-ingester.tpl @@ -30,7 +30,6 @@ app.kubernetes.io/component: ingester {{- $_ := set $zone "stsSuffix" (printf "-%s" $zone.name) }} {{- $zoneMap := set $zoneMap $zone.name $zone }} {{- end }} -{{- end }} {{- if or (not .Values.ingester.zoneAwareReplication.enabled) (and .Values.ingester.zoneAwareReplication.enabled .Values.ingester.zoneAwareReplication.migration) }} {{- $defaultZone := dict "stsSuffix" "" "name" "default" "nodeSelector" .Values.ingester.nodeSelector "replicas" .Values.ingester.replicas }} {{- $zoneMap := set $zoneMap "default" $defaultZone }} From af1aa629e1e48fd044cdbfb1662a2ee1a9d0a2bc Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Mon, 14 Sep 2026 17:26:18 +0200 Subject: [PATCH 03/11] add rollout guide and adapt templates where necessary Signed-off-by: Timon Engelke --- Chart.lock | 7 ++- Chart.yaml | 4 ++ charts/rollout-operator-0.51.1.tgz | Bin 0 -> 10215 bytes docs/guides/migrate_to_zone_aware.markdown | 41 ++++++++++++++++++ templates/distributor/distributor-dep.yaml | 8 +++- templates/ingester/_helpers-ingester.tpl | 4 +- .../ingester-poddisruptionbudget.yaml | 5 ++- templates/ingester/ingester-statefulset.yaml | 28 ++++++------ templates/querier/querier-dep.yaml | 5 ++- templates/ruler/ruler-dep.yaml | 8 +++- values.yaml | 20 +++++++-- 11 files changed, 106 insertions(+), 24 deletions(-) create mode 100644 charts/rollout-operator-0.51.1.tgz create mode 100644 docs/guides/migrate_to_zone_aware.markdown diff --git a/Chart.lock b/Chart.lock index ac519239..d7476d37 100644 --- a/Chart.lock +++ b/Chart.lock @@ -14,5 +14,8 @@ dependencies: - name: memcached repository: https://charts.bitnami.com/bitnami version: 6.14.0 -digest: sha256:af0c109667e9402918877431f9e269c447c030d398c8e1ade6f7a0171a856c8f -generated: "2026-04-21T11:51:34.160601+09:00" +- name: rollout-operator + repository: https://grafana.github.io/helm-charts + version: 0.51.1 +digest: sha256:d0ad2a749ae02f92ad6864ce4afc174cf10d050dbb6ac5263fab2f53b29686ab +generated: "2026-09-10T14:14:02.592218194+02:00" diff --git a/Chart.yaml b/Chart.yaml index 65205d7c..dd9809fb 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -41,3 +41,7 @@ dependencies: version: 6.14.0 repository: https://charts.bitnami.com/bitnami condition: memcached-parquet-labels.enabled + - name: rollout-operator + version: 0.51.1 + repository: https://grafana.github.io/helm-charts + condition: rollout_operator.enabled diff --git a/charts/rollout-operator-0.51.1.tgz b/charts/rollout-operator-0.51.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..2bb83da1e600d63d7df3b7b61397ceccba1b1802 GIT binary patch literal 10215 zcmVDc zVQyr3R8em|NM&qo0PKD1a@@9-ZhrGA*0eGe$;V`0baQ&P=agc5XS>F6RFacjvs0Ng znnZUaCJ6=rTkhC7H4iaQI8Sm43jptZkuA$fcJqhbB5_#@SeFmL0w8!C^IZ8nqk_~5#l%>6+glCkfpkc_8eSdKiBW#ylP;P}g6;6*f}X++bIQ91T}2zeSY zg-JpY$$+M08q>%FNJ&D?OAkQMj7z4tSdF3A^8l8v;6X6NVr3{;W(xXRkQqq{G{wVv2>Ca7$ek;f8d`b%*{%aTv2f@G}c!suCYPHJf7)X}IEF_v1 zuR`YOFc^BDx`#gQvqt}89=^V_I=1Qm=zQ4H|IzVY|L@~zD);SFn9ZqFV~`6nIz1cD zzKo6s$0yP3%fayM%j40Aj!x+Dh)ho@c{Ce-d2$|wBNCEhGClwD%d$CJ^cI6?|7!yHZNd(h?K!V~hrH~ORX#^})99BFRplFiCL{S-do;Q#Al*D5X z-~fCdp712$Da2$-V?zsxW8l-DX{aD#f`%$yK|)e8rx8q7pfp(_`YaD1#OyCH_ZZ&( z=6P7^%RHG<0el8Lj^K)-v%jVaNfcBOmU@CNnYL|zV^SNu7c60F48wuvv4qU&xM2?0 z!VjHn&ZN?bA|Ae`B4FGD$nrS8t7z>L~8$g%lbCyIDNSQ@A}vZX&mAP8JQ#3cUIwC0lPme(afO4SV) zuUR_BzSW;I9%j;$>mp6LLR{$@T>qyTYLw^!kTNaGt}Wt~=AIfitFOExiD%0EH#oI=TcDt|NGzngMSwl zp>!a#O6$T6QwyFeohAv;Sn2X+F`4V}iV{4Qz|(kD^y!1ray$T*N=4!rl`DZIBUslo zPa{3BIXHlcMnn^k`PAB)g9H8g1tm)g6FUMjT|a?CtF1>UCQH{%fO;sf` z3h$(#GBxd{O{%kZnn6B;DM_P;ST=n_k}RfUT^PUvEvPJ=$5bOSL8Er}AO&eiLAN|I ziIvz^_HS53O<8~f#IdOajVYQ8EA_h7b)(WW!gj}2`!`2JLA9VEPeT?PLrvgoroPRj z>yDU;hd@%ulc}I_OiWii)YgMI5+u!tmQwf|jT1)(L^RT4HHxaFYb&aD!gBT1#S`5= zQx+K}ZPqVjOM7O5Cq_@S8tBqcuAisI@u12{3L)~p;<@EaW;~Tln>g%-ROZ5|5%N#p zFsY10&$uXho~84Cp3Mb`OzP5iY56KF&~ua8Hl>72OSn~kWBLypGTIc-BV!M=fGG}> zTZtCxnxw|=YLDu?6q)j+VR)XiNQ;rKr6&B=6I?`QhJeIXtJbi95zS)0!qS_j&y;e> zqpAV*G9F#5O?XJ(C_#?EIDehC3Ij48UDGfZOs$@1$$g^=6FZY%3!Z0V7!3vk?G(C` zg(M?W7BhtdCH)2<50)GoiqHoi9b;Y^*D~qXINkx;2Ul4hEAz$)TCuqL@ zPEw&^hb9Gid%-033@MxR$&2Se#5~tchm9KYbjIenmdvKYA375O6W@>dJc#L%#*bM# z)Kb{I5HF@qmWP~SZI*)rQnJ7ZMm#!rGZ2aj>L+Jlqj~OC4|DvjT(yP zv={&rjv>wC*p>CN$S*DbI~P=IFo(M8`6?D+Jv6GAnMQUqfY72)8iDSLY(uhCQL!YN zJNL8BZg)1t5~@unOW|t{QEoaO%;*RXX)q69cq)%PusL3t3>(W96sfjaVN@c^;uZEg zYO{2H1q)^A982n$CCv(%C`b5R@1`-I_7kGLXa6Y=(X=2deUh;ti4H7Hh9uM*{F|lD zcHGeEg7ep<6SJcMbiYkac8LSHrt~(myZwVAE7tRpD_sRa;8zsu;3tH+R6Hq!P!FTD zDrz|jG*~Wt@k8wha}viZ*VGyiYvb*JsCBSr49dl-?fM*EFGWHcOHqz>ga$O!j~woW zBv%G`SkeRdCYMU%;3-u%l%_C5Aa*>!A)YC^YJhB|-;2M&VA=%d`qV#14(%10K5Sa4MOc(04&Smnc^jA9>);#*Q}oK*&98Gl4va}seO1& zS7V^1N3MK?Vq`Uj=V|SOnHU<^-s#dZ!J|CHH4|^0N5F*@gVZ&kE%n~cI@uW*sl6Bq zti^^7>lZE~@b16Aipq;FYkRP2FTxP~Ru3TP9z#OKys8XtLZ`)OwXXx)=9M|w@ol)`%%y|qM}mznl^w#)YEM(e^%VH{Zs_ zi}`%+5WxL(SEa^UiyecHQ%{N`x)}r4FuY9x&{Q!&cspStLFTg=3mHw- z=BDcamj%S?XqLm2E3NGsnY9E=1Da1gF@H}_3t$zq^Zl3KzIt{Ys5k0E>)5vbdwMcx ztpA*ij?eb%Klkxm(Tu|w-oAyvto;9tClr4B&GQ_jjQe?TJVk9&>{jZLmC?blTGs*F5NSi*!a6&KW>6Gd;x%Kw$8 z)F;1tGqOSde_GHuqe2EM`>>_pE&6}jj{iCz?&H7i=Xv|K|G<0pCev!H8Y@k9YWhL{ zx8J@l@XKTfZ4V3eY-PZbUt+G{2!ij8L$9vT!8<*uPD405#-A*? z)+$kF*wsuyS@3{!d=+~Y*sXCbLrDH-rDcGG<+iiRW(^B5;aiX|Lf!NWr4n1Y0? zv=GRA>YEg{AyFi3TmIzQ@}Z|S_9u;yQ;xwKECQC+r)T6Y(o^XazuGbm5ZI;_G>f(K z?fuk4_WcT{ep3r8;;qTFVPl#P6mhFBCKdYaoe6CaS$5yes{ckoNj6>YReyA->)=ucPU8kDp1-7Y^)ia2Xv0)m2SB#&iS@&6=ATynEc-HI{f z&OOp>Fu#)}XE zj=jTGO%&0iMn#V`_|U|eKMzaO^;ny(&tJpm%cGV>eft)E<}8KY!=BB!$>f$%W4dv6 zkOxd$7lY67-77Lj?p@pC%RrGiyeoRBWfP8|ElMV%3x{(FD%|(~RgTg1c4uh5cF}hw zokv|#RH>k|^i9PSO<(^0)Cm3KJ{!h=Wqig5Mgwmd|DBzmpESpRXXk_c`0qZR>i!>+ zWwO56CuM=wH535>MYi>9#1+$CD^U;bY&>wxLL$d7#J{=%h&8{KjjjZSrdDK0 zC0FB{$sj9p`K?*}?VNQvFUR4Ttw9N*=zL`g)FrTV{#_OkMO_^Ud2^kMIUU2GeDhsO zmW0J*8dK9O&YoCOE0W~sto3z0#>>!xOP$Rw-_Y%k%51f+uPqboB#QcpYYqD$O zoAVOxmg`j8d2Dg%e)X8ciEP)BHzZlevY*($vmdtN4OMWug8FShEfM1hy7YN6q` z<)j)aUn|iC%YO*{Z7jaUq;`S#pjuu%Xt9&FQqCHokljQV(RLM9-TttST@UVAOUfE@ z+z2(4OLXUE`VnfC9O)LR2DVtiY9_}tsKZ@*l4ku%DIMh4c2{7ki}k+KdMY91WvXhN`TN!O+2-+o%F94i z-^HiC&rHCU@&EDh;JiNm9}Q1Oqy6~*KAvrM%}j+?<_KKyq@ATh0SsP0GOGq|?w|}u z{<|sgiQS=Mj@W3=C)8wZ1CXi?#%zo@#*4G`XBsNMDb0RJ#?aHyr`O7I+mYaVP8L+- z;{Drj8&rrjn^R&&Q)OCmTpvi32H^JN+xMH&sX38CC~7vttNRn@=0qNsw%@l=nsqCt z-+y=c^y1aC;uXmlZqsU{B3Z?~mU7xiwJiNlr}yz|VM%L0^y_TNkY{vUMmTtbhfxQ^SaEsfWL8joD-cZPLB8>qC>1=X0=qANS5kUCxR z=Hk{moVu6pE~6SVTc@iL`}-cAuJXbz6l`F7W9;jWO7%Nqx_r^`XN}In0W{)qZTj{G zg$-n#t*KP9h(4RmXsF~EUh>qaRAckW+qW%}YANcgG@sgy{$ZaD_%Gv;PnQgLczjg+ zHyoX|@!$C#|J}#)JHdZ7S^LxDzn!@KL*l^;8Q z8Q(h{5Lu|Yv8}p>B)Zo z*S$O+2mgIz&mY(o{k0_R&kq52X8Dha1DovaCj<9jaSs-MzhLp(^r=`~X_w!V-otTm^Ah}v_!G7%q zc2_*6_Z$v>2Lam_W%zaAU}ukNAi&H8nUn6P_D#&B!XI;L{=d=Y5r0KArg%=S4;a1I zuEJuHMkNh|v0G*DT|;00kvA&JF_ybsT&q><@Uv0PuRNvXh6tMRC}L9Nc%R2KkLE4T z976?%hO-?4tNbS5yB(F7-h}Fx#;rteT_0QRRWeZaMAfN&YBhfEXN&!xvNU4p{KMS9 z7W+RqJ3nsj|2aQD+1vm7c;3VQe`RFi59kJXOt0unm*EV`2BtlLiqCr=7G-V@{L`Mt z6m?w-~ISwp5wx!k(CO=AU1B#T3h?Ho+s94Jo0^SuD}i2IiH> z_U>)@%^tCXYu`?*Rf5_T5p$|8*M)L-g_y4iq3@Wl*dY?$Z_M;{S6QX95(wW6p0ANm z-~9Og+wrht|9N(LcR$xM4afht_<)%gGaWc~Hc5+V9Y)IEcD)bJZ7? z)~q4k;$)4Fe<%e_ql~ju6;pC+cPegc1b4?rxky*u`f=Nn0vbiv?)fOQHLm~oz)C9D zZd;EVt z&u!zsc9fXZe+AW1jX(2|NRC5 zTgU&yrv494&i3p7_wsyP{l9y~_lk(Y)~YbR+7O;<0uGxYf4(_88N*d=ePK0RRTk;o zrCRG#RhEk@E8X=qWAj(J_Wg+?oOaOv_jmmE+3Nq#N3H#TgZ=sc_wsxM{l9z1U-kI6 z^MLof z`hPqcHP3%KKRw;+|GhlFll}j{X3?~er+P#GP3)%M&4tX*Z;UbuQ_8tT((Z9K1bML|b^A6y9A{fslJb(I3dO;?*wz^)x zB1-xux!^y0!BU1iqWwR@ig`=_uby2z{pMMaM0Z!mX8j+Yog6p!|C|iY_v=6R@*Kd| z1{Js9GW?Ck3E(|no`=89eT_3Nz+4Tbjl&P?vj2F9;oIYWf6mk*pW=YaBJh4Q@sTz@ z<&CR>=N+`~nDGAPhhi8y2u_E=u&je6^FS`}idGE{rzD!wzD@0$#6C-BTqJ0HNc?}L zTE+D7EGEjAzvM*FBlwpeUSW7Y$;^I0#?vh|==6Ri`7U*occzVDIS7uw3}Gh@@y_hWG)#^*xZaC@g6>i*lb3HhP!el6-{LICQ_8Jg9Et4V@e7-sHAwo zFJ4Y+oP#3rHM+)(3f<%|r8Dg!NxE`3G8O~|o;R6HrbI5h1GtFHh05i9b9mV&{tYCG z+(ZNvYGOyHR$UnH>-6R-SXs$yXCo(*iQ%Ezn3^Ql@mHqf;oG9)9dTHeZAbt=g})So zo_~kG*b9sQ9o8wUsruD|t~VWW!SgtRDRtFQyqp&Auhc?RhHMhqz80P-VA9P8q;tqK zo(A4?mbM%8>23_Q>N zaJ}8)m8DAXC=Y1_1X6kfJTp?r1yHS~q2_@=&!C-%#$?sgt&tI-n4$PaOWozuufQEb zsAUQDLKsce(rbG@f+tr`kx7zjL1M-TXu>E~+DWCBU=77^e|zA04<0=D%1E-&Ax)#Z z&)O=uR3dX+-p81EXb0=l+qN4GN*Mu5OHPa|!rCfa9BK;~21Gzuoh4!8TuzSlYWlK@^V=n6|N4c~NCM4@ zbNt$H%+tbq{_lVP56Fs`x=X1e9!kUCPg5$@PnjU0Vj-3N5IgNBH!=ECRCZebC@{{U z=n@j&kX6N!F(pkmDXu+4CaBbI+OlOu?jcBC5l-7)=q!>-i)#H+hX3NGE8K25F%pR* zS*%R+ZttT8Z4CGfrn*fu^JuEykR;Qd#CZ(4Ai|iNroA`UID-}?qj9=?OSgZ+BKi;r ze9hFinZ(O}bP-fQAM#`>XdF9T5H#dzh<&QrdI-GGIJo-i;)!mWrG{u0NT6Rf>Vyk@ zS}t_m0X)}j#|xme;yLmwsLXgO+0=Rln#x>Q-8EFxH%uyrX>23w1YA_Djy5?+^1oru z|GfC-g=x^z0q1E%#co<+g;f)jpjD}QZ?p+Jf zN#n!>DFTf-Ne<-X!~w=?^dl`ytQn%vQhKw>N}VMXKbF3 z>=xC!CA4psR0tMnsg??L@*PZiy+RVZuEB3hq@BSsE05T3T9|Aq;|=0f(fiX?MU_p* zR&Tj%HbV8A<e4uxvW*+pjn{HSJbX<>z<4`(Wkg)b zuny2Q^P79rn|-uQ>s$vY+qv;DwVmu600tl!N#fH zL$}7&G%h-Br@G`3)KZvCYi)r}e~4D?2wWnQrG4FNH$|f}sYYb>jM zHEn0RZP+K?fg#oc2U`r-^A@^_w%hUBFycC+A)7HOwcl3@Dm093lvmElm|~)?8q)4= z*LWU3S!mKMTXAir19)!26(m(S-mvs)vxR1zm_nF@IZd^Bq!HxO0aE80i$P13z^)Cm z?bfaDJg&pJc3Ev$R(P=|hVn*vira0RV0W9M1G^MF(T$#WR#NOVt+r~);tq0c*=3G) zY4JMkuVB+gsc~rw)^42cpzR`7RHSAluS}@%%}Z?=rLXYDGw18qeS*e#)2d| zu=E&mx_GPaC*E0#Y)pn8R)5VQ%H7;JE*roh4dww1Po)9b)eA;DVM8k}ux*1Z@Y^^w z_94?@@RF;_G*>a(X6F^a%t;h-0+l%lG)%>K_Mshiy2&hPO|94=RxzCt)Z3sIw^Fy3 ze*%A`?gBkrfIMvVlDYn~JpaI;|H z!hV+ZwsVr-sB3Mwa*fxJ3dLr4aAwcQp-xwJ0NQeg@L#>ShDruI_mj)&JY+#ujhK3H zQPY>dD#cvd6vd#Tc?Nx}7XtVu$LSB4QguUV3PbFz^TB-b-@4hx>pjLdBSaeLLjD+?tN6*l6yD}h}L z`HDoY+>S(QmC?;9%Heb+U1x<+Irq3Bl|?JpW?g$jLQz?ZZ=~KYmG76z_e6e6Ru7dj7|8Q~%FL zqy7H>dwGg2T^7G-lqa`!2EtA_tHwDS%S zzH01Y-5XBZXH+=krvaY^THt_|cL0ZX2Fea_r(1~|VL|k88jL<@Jfwjcze)cM6rax# z2DpV+Ga|2y<)wE35l<}|!HH>tlP#ub4X10MY+?pN&4gd*aw-a0i_+2%74e(mMc55f z!47tJqyFE~Ua)QYe>QCH|2jJ!?Dv1&%X45ncztixaO;+&a%wAguAke}`8FThvaFrf;*QfPPHT}=ENL{xfFt)xaY5#^_O6@^ zz7!a5^MuEFQa!=LhuT=^8n1+OoK>#{|5@_%5>LPh%)OC8M+A|vNRyQ62Ps+C78BH%PpMQUR-T2^go8W@V>rCkGQ>=bw4uI?H>a(VZ zO~nA7%41k*e{n_Xj-6A{@!d{|jbYgFF<~i7@?;Ey)<+E0X`ZsuuDH>W$T)WWCt_XZ z(~dKYEJ^Sdeyf~TL}FSkR#t>&iQ9ebor zI?rPwTA%T&J+H7n0d7v&B8twrsI>k8Jh$g!u@ri*^C=Bg40Ec0#BmRJuCly5?U%&y zA7j#EyZ_&M4B%Ecur2=M>}*)~|Hq?~^0zpXtF3JV`;C2qT&cY=X zAx)L;zYk^R35mlT*G<%dHmWhxEG8x*E2?x0-Pbn$;+m>2B=&G%JqVZ>dNZSDLmO-y zhCMQgMrNAE3>`BbX{UDS+UAPR>6=?N5H^wdI2^P;%dJ?YQxIFtby1Jao|(hAjS!GD zst2Xm^>rrUA3UWO+7Mjw=xJdHzCuG_SI{LW-o^8JT>x_$e|bb;M~Y2q0qYJJyjAl+ zV-&HuKG=i}k>D;_xD~=QPoviaMfd}>*olSxC#cFrYJzRb*cuGjowFs?nw#+mUs!RT zF&alu$Te<(ja={&t{)fSyf_cJpqo3NxixJ>cV_QbNyd>C(1ZY30mjMy>gl783}wphTlhp^SbXsa)v8e-F4q>;IZree)3B-zYE)~$2WRe1zDcm8ZRoqwt zvke9WCDF>*@-leSh?7q1BIYnIvFsb`|4~}C(&^SSt(Ms6dSdQascl5D8wt!tl-_8N z^d3TQxab{qcC(!cMr@(`tGkm;_XfL-dVkwKv^2nXlO3y#K(`+VTn --cascade=orphan` to keep the pods). diff --git a/templates/distributor/distributor-dep.yaml b/templates/distributor/distributor-dep.yaml index 75837036..9aea2e9c 100644 --- a/templates/distributor/distributor-dep.yaml +++ b/templates/distributor/distributor-dep.yaml @@ -54,8 +54,14 @@ spec: args: - "-target=distributor" - "-config.file=/etc/cortex/cortex.yaml" - {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }} + {{- /* enable zone-awareness if it's enabled and not in migration or in migration with writePath enabled */ -}} + {{- if and .Values.ingester.zoneAwareReplication.enabled + (or (not .Values.ingester.zoneAwareReplication.migration.enabled) + .Values.ingester.zoneAwareReplication.migration.writePath) }} - "-distributor.zone-awareness-enabled" + {{- if .Values.ingester.zoneAwareReplication.migration.enabled }} + - "-distributor.excluded-zones=zone-default" + {{- end }} {{- end }} {{- range $key, $value := .Values.distributor.extraArgs }} - "-{{ $key }}={{ $value }}" diff --git a/templates/ingester/_helpers-ingester.tpl b/templates/ingester/_helpers-ingester.tpl index 53975fef..8d57fd0a 100644 --- a/templates/ingester/_helpers-ingester.tpl +++ b/templates/ingester/_helpers-ingester.tpl @@ -26,11 +26,13 @@ app.kubernetes.io/component: ingester */}} {{- define "cortex.ingesterZoneAwareReplicationMap" -}} {{- $zoneMap := dict }} +{{- if .Values.ingester.zoneAwareReplication.enabled }} {{- range $zone := .Values.ingester.zoneAwareReplication.zones }} {{- $_ := set $zone "stsSuffix" (printf "-%s" $zone.name) }} {{- $zoneMap := set $zoneMap $zone.name $zone }} {{- end }} -{{- if or (not .Values.ingester.zoneAwareReplication.enabled) (and .Values.ingester.zoneAwareReplication.enabled .Values.ingester.zoneAwareReplication.migration) }} +{{- end }} +{{- if or (not .Values.ingester.zoneAwareReplication.enabled) (and .Values.ingester.zoneAwareReplication.enabled .Values.ingester.zoneAwareReplication.migration.enabled) }} {{- $defaultZone := dict "stsSuffix" "" "name" "default" "nodeSelector" .Values.ingester.nodeSelector "replicas" .Values.ingester.replicas }} {{- $zoneMap := set $zoneMap "default" $defaultZone }} {{- end }} diff --git a/templates/ingester/ingester-poddisruptionbudget.yaml b/templates/ingester/ingester-poddisruptionbudget.yaml index cfa2a29f..7a565b0e 100644 --- a/templates/ingester/ingester-poddisruptionbudget.yaml +++ b/templates/ingester/ingester-poddisruptionbudget.yaml @@ -1,4 +1,7 @@ -{{- if and (gt (int .Values.ingester.replicas) 1) (.Values.ingester.podDisruptionBudget) (.Values.ingester.enabled) }} +{{- if and .Values.ingester.enabled + .Values.ingester.podDisruptionBudget + (or (gt (int .Values.ingester.replicas) 1) + (.Values.ingester.zoneAwareReplication.enabled)) }} apiVersion: {{ include "cortex.pdbVersion" . }} kind: PodDisruptionBudget metadata: diff --git a/templates/ingester/ingester-statefulset.yaml b/templates/ingester/ingester-statefulset.yaml index 9dcc561a..08a5172f 100644 --- a/templates/ingester/ingester-statefulset.yaml +++ b/templates/ingester/ingester-statefulset.yaml @@ -1,5 +1,6 @@ {{- if and .Values.ingester.statefulSet.enabled .Values.ingester.enabled -}} {{- $zoneMap := include "cortex.ingesterZoneAwareReplicationMap" $ | fromYaml }} +{{- $nonDefaultZones := without (keys $zoneMap) "default" }} {{- range $zoneName, $zone := $zoneMap }} {{- with $ }} apiVersion: apps/v1 @@ -10,13 +11,18 @@ metadata: labels: {{- include "cortex.ingesterLabels" . | nindent 4 }} app.kubernetes.io/part-of: memberlist - {{- with $zone.labels }} - {{- toYaml . | nindent 4 }} + {{- if .Values.ingester.zoneAwareReplication.enabled }} + rollout-group: {{ include "cortex.ingesterFullname" . }} {{- end }} - {{- with $zone.annotations }} annotations: + {{- if .Values.ingester.zoneAwareReplication.enabled }} + {{- with .Values.ingester.zoneAwareReplication.maxUnavailable }} + rollout-max-unavailable: {{ quote . }} + {{- end }} + {{- end }} + {{- with .Values.ingester.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: {{- if not .Values.ingester.autoscaling.enabled }} replicas: {{ $zone.replicas }} @@ -24,14 +30,12 @@ spec: selector: matchLabels: {{- include "cortex.ingesterSelectorLabels" . | nindent 6 }} - {{- if .Values.ingester.zoneAwareReplication.enabled }} - zone: {{ $zoneName }} - {{- end }} updateStrategy: + {{- /* rollout-operator requires strategy OnDelete */ -}} {{- if ne $zoneName "default" }} type: OnDelete {{- else }} - {{- toYaml .Values.ingester.statefulStrategy | nindent 4 }} + {{- toYaml .Values.ingester.statefulStrategy | nindent 4 }} {{- end }} podManagementPolicy: "{{ .Values.ingester.statefulSet.podManagementPolicy }}" serviceName: {{ template "cortex.fullname" . }}-ingester-headless @@ -49,10 +53,6 @@ spec: annotations: {{ toYaml .Values.ingester.persistentVolume.annotations | nindent 10 }} {{- end }} - {{- with $zone.podLabels }} - labels: - {{- toYaml . | nindent 10 }} - {{- end }} spec: {{- if .Values.ingester.persistentVolume.storageClass }} {{- if (eq "-" .Values.ingester.persistentVolume.storageClass) }} @@ -72,7 +72,9 @@ spec: labels: {{- include "cortex.ingesterLabels" . | nindent 8 }} {{- if .Values.ingester.zoneAwareReplication.enabled }} + # required for rollout-operator zone: {{ $zoneName }} + name: {{ include "cortex.ingesterFullname" . }}{{ $zone.stsSuffix }} {{- end }} app.kubernetes.io/part-of: memberlist {{- with .Values.ingester.podLabels }} @@ -134,7 +136,7 @@ spec: - "-config.file=/etc/cortex/cortex.yaml" {{- if .Values.ingester.zoneAwareReplication.enabled }} - "-ingester.availability-zone={{ $zoneName }}" - {{- if .Values.ingester.zoneAwareReplication.migration }} + {{- if .Values.ingester.zoneAwareReplication.migration.enabled }} - "-blocks-storage.tsdb.flush-blocks-on-shutdown=true" - "-ingester.unregister-on-shutdown=true" {{- end }} diff --git a/templates/querier/querier-dep.yaml b/templates/querier/querier-dep.yaml index 138533c8..1f2a31ea 100644 --- a/templates/querier/querier-dep.yaml +++ b/templates/querier/querier-dep.yaml @@ -52,7 +52,10 @@ spec: args: - "-target=querier" - "-config.file=/etc/cortex/cortex.yaml" - {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }} + {{- /* enable zone-awareness if it's enabled and not in migration or in migration with readPath enabled */ -}} + {{- if and .Values.ingester.zoneAwareReplication.enabled + (or (not .Values.ingester.zoneAwareReplication.migration.enabled) + .Values.ingester.zoneAwareReplication.migration.readPath) }} - "-distributor.zone-awareness-enabled" {{- end }} {{- if .Values.query_scheduler.enabled }} diff --git a/templates/ruler/ruler-dep.yaml b/templates/ruler/ruler-dep.yaml index 3316ae53..d8259c75 100644 --- a/templates/ruler/ruler-dep.yaml +++ b/templates/ruler/ruler-dep.yaml @@ -121,8 +121,14 @@ spec: args: - "-target=ruler" - "-config.file=/etc/cortex/cortex.yaml" - {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }} + {{- /* enable zone-awareness if it's enabled and not in migration or in migration with writePath enabled */ -}} + {{- if and .Values.ingester.zoneAwareReplication.enabled + (or (not .Values.ingester.zoneAwareReplication.migration.enabled) + .Values.ingester.zoneAwareReplication.migration.writePath) }} - "-distributor.zone-awareness-enabled" + {{- if .Values.ingester.zoneAwareReplication.migration.enabled }} + - "-distributor.excluded-zones=zone-default" + {{- end }} {{- end }} {{- if and .Values.alertmanager.enabled (not .Values.config.ruler.alertmanager_url) }} {{- if .Values.config.ruler.enable_alertmanager_discovery }} diff --git a/values.yaml b/values.yaml index dd6c1a61..ba6ae8b9 100644 --- a/values.yaml +++ b/values.yaml @@ -451,20 +451,25 @@ ingester: zoneAwareReplication: enabled: false - migration: false + migration: + enabled: false + writePath: false + readPath: false + + # maxUnavailable: 1 zones: [] # - name: eu02-1 + # replicas: 0 # nodeSelector: # topology.kubernetes.io/zone: eu02-1 - # labels: {} # - name: eu02-2 + # replicas: 0 # nodeSelector: # topology.kubernetes.io/zone: eu02-2 - # labels: {} # - name: eu02-3 + # replicas: 0 # nodeSelector: # topology.kubernetes.io/zone: eu02-3 - # labels: {} service: annotations: {} @@ -1841,3 +1846,10 @@ memberlist: service: annotations: {} labels: {} + +rollout_operator: + enabled: false + webhooks: + enabled: false + crds: + enabled: false From 713bd4fde1af506e6d67324a89353845c4923757 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Mon, 14 Sep 2026 17:29:36 +0200 Subject: [PATCH 04/11] remove accidentally added extraLabels Signed-off-by: Timon Engelke --- values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/values.yaml b/values.yaml index ba6ae8b9..a9c8189d 100644 --- a/values.yaml +++ b/values.yaml @@ -492,8 +492,6 @@ ingester: # -- Additional Cortex container arguments, e.g. log.level (debug, info, warn, error) extraArgs: {} - extraLabels: {} - # -- Pod Labels podLabels: {} From 8b894707561503311f24d0e68eef165706476b5b Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Mon, 14 Sep 2026 17:31:18 +0200 Subject: [PATCH 05/11] move shuffle shard disable to beginning Signed-off-by: Timon Engelke --- docs/guides/migrate_to_zone_aware.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guides/migrate_to_zone_aware.markdown b/docs/guides/migrate_to_zone_aware.markdown index 49cd007c..092852f9 100644 --- a/docs/guides/migrate_to_zone_aware.markdown +++ b/docs/guides/migrate_to_zone_aware.markdown @@ -18,14 +18,14 @@ Make sure to set the following settings before starting the migration: Always continue the next step only when all pods are back to "ready". +1. If you have shuffle-sharding enabled, turn it off for querier by setting `querier.extraArgs` to `-distributor.sharding-strategy=default`. This is required because otherwise, the new ingester instances will not be considered by the queriers. Warning: This may increase resource usage. + 1. Set `ingester.zoneAwareReplication.enabled=true`, `ingester.zoneAwareReplication.migration.enabled=true`, `ingester.zoneAwareReplication.zones` to the desired zones but with `replicas=0`. Set `rollout_operator.enabled=true`. Upgrade the chart. 1. In `ingester.zoneAwareReplication.zones`, set `replicas` to the desired replicas for **the first** zone, the install the Helm chart. Scaling up instead of creating directly the desired replicas is required to ensure that the replicas are created consecutively. 1. Repeat the process for the other zones. -1. If you have shuffle-sharding enabled, turn it off for querier by setting `querier.extraArgs` to `-distributor.sharding-strategy=default`. This is required because otherwise, the new ingester instances will not be considered by the queriers. Warning: This may increase resource usage. - 1. Enable zone-awareness on the write path by setting `ingester.zoneAwareReplication.migration.writePath=true` and install the Helm chart. This makes the distributors ship data to the new ingesters while the queriers still use all ingesters. Wait for `querier.query_store_after` so that all data that would be fetched by the queriers is on the new ingesters. This also disables the distributors from writing to the old ingesters. From 56107b80b09afbcc352b251b3a9211bd8146ffd9 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 15 Sep 2026 13:52:53 +0200 Subject: [PATCH 06/11] update migration guide Signed-off-by: Timon Engelke --- docs/guides/migrate_to_zone_aware.markdown | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/guides/migrate_to_zone_aware.markdown b/docs/guides/migrate_to_zone_aware.markdown index 092852f9..39276363 100644 --- a/docs/guides/migrate_to_zone_aware.markdown +++ b/docs/guides/migrate_to_zone_aware.markdown @@ -21,12 +21,13 @@ Always continue the next step only when all pods are back to "ready". 1. If you have shuffle-sharding enabled, turn it off for querier by setting `querier.extraArgs` to `-distributor.sharding-strategy=default`. This is required because otherwise, the new ingester instances will not be considered by the queriers. Warning: This may increase resource usage. 1. Set `ingester.zoneAwareReplication.enabled=true`, `ingester.zoneAwareReplication.migration.enabled=true`, `ingester.zoneAwareReplication.zones` to the desired zones but with `replicas=0`. Set `rollout_operator.enabled=true`. Upgrade the chart. + The stateful sets will be scaled up instead of created directly with the desired replicas to ensure that the replicas are created consecutively. -1. In `ingester.zoneAwareReplication.zones`, set `replicas` to the desired replicas for **the first** zone, the install the Helm chart. Scaling up instead of creating directly the desired replicas is required to ensure that the replicas are created consecutively. +1. In `ingester.zoneAwareReplication.zones`, set `replicas` to the desired replicas for **the first** zone, the install the Helm chart. 1. Repeat the process for the other zones. -1. Enable zone-awareness on the write path by setting `ingester.zoneAwareReplication.migration.writePath=true` and install the Helm chart. This makes the distributors ship data to the new ingesters while the queriers still use all ingesters. Wait for `querier.query_store_after` so that all data that would be fetched by the queriers is on the new ingesters. +1. Enable zone-awareness on the write path by setting `ingester.zoneAwareReplication.migration.writePath=true` and install the Helm chart. This makes the distributors ship data to the new ingesters while the queriers still use all ingesters. Wait for `querier.query_store_after` so that the data that is still on the old ingesters can be queried from the object storage. If `query_store_after` is unset, wait at least `3 x bucket_store.sync_interval` (default 3x15m). This also disables the distributors from writing to the old ingesters. 1. Enable zone-awareness on the read path by setting `ingester.zoneAwareReplication.migration.readPath=true` and install the Helm chart. This makes the queriers use the new ingesters. From c5f5d77654e8aa827e66753e11b199a5a1134984 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 15 Sep 2026 14:26:08 +0200 Subject: [PATCH 07/11] don't let new ingesters leave the ring on restart Signed-off-by: Timon Engelke --- templates/ingester/ingester-statefulset.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/ingester/ingester-statefulset.yaml b/templates/ingester/ingester-statefulset.yaml index 08a5172f..8f5fa6da 100644 --- a/templates/ingester/ingester-statefulset.yaml +++ b/templates/ingester/ingester-statefulset.yaml @@ -136,7 +136,8 @@ spec: - "-config.file=/etc/cortex/cortex.yaml" {{- if .Values.ingester.zoneAwareReplication.enabled }} - "-ingester.availability-zone={{ $zoneName }}" - {{- if .Values.ingester.zoneAwareReplication.migration.enabled }} + {{- if and .Values.ingester.zoneAwareReplication.migration.enabled + (eq $zoneName "default") }} - "-blocks-storage.tsdb.flush-blocks-on-shutdown=true" - "-ingester.unregister-on-shutdown=true" {{- end }} From 3abaead115c80e25dd86e5b545accf00b9ec1a74 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 15 Sep 2026 15:48:27 +0200 Subject: [PATCH 08/11] update documentation Signed-off-by: Timon Engelke --- docs/guides/migrate_to_zone_aware.markdown | 48 ++++++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/docs/guides/migrate_to_zone_aware.markdown b/docs/guides/migrate_to_zone_aware.markdown index 39276363..fff6924c 100644 --- a/docs/guides/migrate_to_zone_aware.markdown +++ b/docs/guides/migrate_to_zone_aware.markdown @@ -6,9 +6,24 @@ has_children: false has_toc: false --- +# Migrate to zone-aware ingesters +{: .no_toc } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + +## Overview + This migration guide shows how to migrate to zone-aware ingesters without downtime or data loss. The general process is the following: New stateful sets are created, the write traffic is routed to them, the read traffic is routed to them, the old stateful set is disabled. -The chart makes use of the [rollout-operator](https://github.com/grafana/rollout-operator) to coordinate rollouts of the stateful sets. This will automatically set the update strategy to `OnDelete`. +During the migration, it is ensured that at most one ingester is unavailable at the time, and that an ingester's data is always written to persistent storage before it is shut down. + +The chart makes use of the [rollout-operator](https://github.com/grafana/rollout-operator) to coordinate rollouts of the stateful sets. This will automatically set the stateful set's update strategy to `OnDelete`. + +## Prerequisites Make sure to set the following settings before starting the migration: - Ingesters are deployed as a stateful set (deployment is currently not supported) @@ -16,12 +31,34 @@ Make sure to set the following settings before starting the migration: - `podManagementPolicy` is "OrderedReady" (default), not "Parallel" (OrderedReady creates pods consecutively when scaling up or down) - `frontend_address` is set in the ruler config (make the ruler read from the queriers, not directly from the ingesters. Otherwise, recording and alerting rules may not be evaluated correctly during migration) -Always continue the next step only when all pods are back to "ready". +## Migration steps + +**Important**: Always continue the next step only when all pods are in the ready state. 1. If you have shuffle-sharding enabled, turn it off for querier by setting `querier.extraArgs` to `-distributor.sharding-strategy=default`. This is required because otherwise, the new ingester instances will not be considered by the queriers. Warning: This may increase resource usage. 1. Set `ingester.zoneAwareReplication.enabled=true`, `ingester.zoneAwareReplication.migration.enabled=true`, `ingester.zoneAwareReplication.zones` to the desired zones but with `replicas=0`. Set `rollout_operator.enabled=true`. Upgrade the chart. - The stateful sets will be scaled up instead of created directly with the desired replicas to ensure that the replicas are created consecutively. + ```yaml + ingester: + zoneAwareReplication: + enabled: true + migration: + enabled: true + zones: + - name: zone-a + replicas: 0 + nodeSelector: + topology.kubernetes.io/zone: zone-a + - name: zone-b + replicas: 0 + nodeSelector: + topology.kubernetes.io/zone: zone-b + - name: zone-c + replicas: 0 + nodeSelector: + topology.kubernetes.io/zone: zone-c + ``` + The stateful sets will be scaled up in the next steps and not created at once to ensure that at most one ingester is unavailable at a time. 1. In `ingester.zoneAwareReplication.zones`, set `replicas` to the desired replicas for **the first** zone, the install the Helm chart. @@ -39,4 +76,7 @@ Always continue the next step only when all pods are back to "ready". 1. If you have previously disabled shuffle-sharding, wait `-querier.shuffle-sharding-ingesters-lookback-period` before removing `querier.extraArgs`. -If you want to benefit from faster rollouts, set `ingester.zoneAwareReplication.maxUnavailable` to the number of replicas per zone and set `podManagementPolicy` to "Parallel". This will require recreating the stateful sets (use `kubectl delete sts <...> --cascade=orphan` to keep the pods). +## Faster rollouts + +With zone-awareness enabled, it is possible to roll all ingesters in a zone simultaneously. +If you want to benefit from these faster rollouts, set `ingester.zoneAwareReplication.maxUnavailable` to the number of replicas per zone and set `ingester.statefulSet.podManagementPolicy` to "Parallel". This will require recreating the stateful sets. Use `kubectl delete sts <...> --cascade=orphan` to delete only the stateful set, not the pods. From f2fcd4b49c2134917bc171d5c2013256c6675794 Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 15 Sep 2026 15:58:25 +0200 Subject: [PATCH 09/11] add explicit fails for ingester HPA and deployment Signed-off-by: Timon Engelke --- templates/ingester/ingester-dep.yaml | 3 +++ templates/ingester/ingester-hpa.yaml | 3 +++ values.yaml | 12 ++++++------ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/templates/ingester/ingester-dep.yaml b/templates/ingester/ingester-dep.yaml index 7d270101..564062bd 100644 --- a/templates/ingester/ingester-dep.yaml +++ b/templates/ingester/ingester-dep.yaml @@ -1,4 +1,7 @@ {{- if and (not .Values.ingester.statefulSet.enabled) .Values.ingester.enabled -}} +{{- if .Values.ingester.zoneAwareReplication.enabled }} +{{- fail "Zone-aware replication is currently not supported with an ingester deployment. Use a stateful set instead." }} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/templates/ingester/ingester-hpa.yaml b/templates/ingester/ingester-hpa.yaml index 1bc6e6a5..da700ac0 100644 --- a/templates/ingester/ingester-hpa.yaml +++ b/templates/ingester/ingester-hpa.yaml @@ -1,4 +1,7 @@ {{- if and .Values.ingester.enabled .Values.ingester.autoscaling.enabled -}} +{{- if .Values.ingester.zoneAwareReplication.enabled }} +{{- fail "Zone-aware replication is currently not supported with autoscaling." }} +{{- end }} {{- with .Values.ingester.autoscaling -}} apiVersion: {{ include "cortex.hpaVersion" $ }} kind: HorizontalPodAutoscaler diff --git a/values.yaml b/values.yaml index a9c8189d..39611289 100644 --- a/values.yaml +++ b/values.yaml @@ -458,18 +458,18 @@ ingester: # maxUnavailable: 1 zones: [] - # - name: eu02-1 + # - name: zone-a # replicas: 0 # nodeSelector: - # topology.kubernetes.io/zone: eu02-1 - # - name: eu02-2 + # topology.kubernetes.io/zone: zone-a + # - name: zone-b # replicas: 0 # nodeSelector: - # topology.kubernetes.io/zone: eu02-2 - # - name: eu02-3 + # topology.kubernetes.io/zone: zone-b + # - name: zone-c # replicas: 0 # nodeSelector: - # topology.kubernetes.io/zone: eu02-3 + # topology.kubernetes.io/zone: zone-c service: annotations: {} From a85519cf7a408ce160bafd1187b0a23bca81870a Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Tue, 15 Sep 2026 16:59:05 +0200 Subject: [PATCH 10/11] fix zone name in excluded-zones Signed-off-by: Timon Engelke --- templates/distributor/distributor-dep.yaml | 2 +- templates/ruler/ruler-dep.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/distributor/distributor-dep.yaml b/templates/distributor/distributor-dep.yaml index 9aea2e9c..82cbed63 100644 --- a/templates/distributor/distributor-dep.yaml +++ b/templates/distributor/distributor-dep.yaml @@ -60,7 +60,7 @@ spec: .Values.ingester.zoneAwareReplication.migration.writePath) }} - "-distributor.zone-awareness-enabled" {{- if .Values.ingester.zoneAwareReplication.migration.enabled }} - - "-distributor.excluded-zones=zone-default" + - "-distributor.excluded-zones=default" {{- end }} {{- end }} {{- range $key, $value := .Values.distributor.extraArgs }} diff --git a/templates/ruler/ruler-dep.yaml b/templates/ruler/ruler-dep.yaml index d8259c75..28764ea2 100644 --- a/templates/ruler/ruler-dep.yaml +++ b/templates/ruler/ruler-dep.yaml @@ -127,7 +127,7 @@ spec: .Values.ingester.zoneAwareReplication.migration.writePath) }} - "-distributor.zone-awareness-enabled" {{- if .Values.ingester.zoneAwareReplication.migration.enabled }} - - "-distributor.excluded-zones=zone-default" + - "-distributor.excluded-zones=default" {{- end }} {{- end }} {{- if and .Values.alertmanager.enabled (not .Values.config.ruler.alertmanager_url) }} From 18ecb260c0c998db7722e791a26c5e1a3c01beec Mon Sep 17 00:00:00 2001 From: Timon Engelke Date: Wed, 16 Sep 2026 13:53:58 +0200 Subject: [PATCH 11/11] update read path migration Signed-off-by: Timon Engelke --- docs/guides/migrate_to_zone_aware.markdown | 6 ++++-- templates/querier/querier-dep.yaml | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/guides/migrate_to_zone_aware.markdown b/docs/guides/migrate_to_zone_aware.markdown index fff6924c..29e0508d 100644 --- a/docs/guides/migrate_to_zone_aware.markdown +++ b/docs/guides/migrate_to_zone_aware.markdown @@ -35,7 +35,9 @@ Make sure to set the following settings before starting the migration: **Important**: Always continue the next step only when all pods are in the ready state. -1. If you have shuffle-sharding enabled, turn it off for querier by setting `querier.extraArgs` to `-distributor.sharding-strategy=default`. This is required because otherwise, the new ingester instances will not be considered by the queriers. Warning: This may increase resource usage. +1. Before starting the migration, you should ensure that the querier uses all ingesters during the migration. This means that shuffle sharding should be disabled and sharding by all labels should be enabled. + It is sufficient to set these settings on the querier using `querier.extraArgs`. Set `distributor.sharding-strategy` to `default` and `distributor.shard-by-all-labels` to `"true"` there. + Warning: This may increase resource usage of the queriers. 1. Set `ingester.zoneAwareReplication.enabled=true`, `ingester.zoneAwareReplication.migration.enabled=true`, `ingester.zoneAwareReplication.zones` to the desired zones but with `replicas=0`. Set `rollout_operator.enabled=true`. Upgrade the chart. ```yaml @@ -74,7 +76,7 @@ Make sure to set the following settings before starting the migration: 1. Remove all values below `ingester.zoneAwareReplication.migration`. This will delete the old stateful set. -1. If you have previously disabled shuffle-sharding, wait `-querier.shuffle-sharding-ingesters-lookback-period` before removing `querier.extraArgs`. +1. If you have set any querier arguments in the first step, wait `-querier.shuffle-sharding-ingesters-lookback-period` before removing `querier.extraArgs`. ## Faster rollouts diff --git a/templates/querier/querier-dep.yaml b/templates/querier/querier-dep.yaml index 1f2a31ea..18dfb2e4 100644 --- a/templates/querier/querier-dep.yaml +++ b/templates/querier/querier-dep.yaml @@ -57,6 +57,9 @@ spec: (or (not .Values.ingester.zoneAwareReplication.migration.enabled) .Values.ingester.zoneAwareReplication.migration.readPath) }} - "-distributor.zone-awareness-enabled" + {{- if .Values.ingester.zoneAwareReplication.migration.enabled }} + - "-distributor.excluded-zones=default" + {{- end }} {{- end }} {{- if .Values.query_scheduler.enabled }} - "-querier.scheduler-address={{ template "cortex.querySchedulerFullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:{{ .Values.config.server.grpc_listen_port }}"