diff --git a/.agents/architecture.md b/.agents/architecture.md index cd87c316..6b947096 100644 --- a/.agents/architecture.md +++ b/.agents/architecture.md @@ -3,7 +3,7 @@ ## Core Systems - Terraform in `infra/` provisions the DigitalOcean VPC, Kubernetes cluster, managed Postgres, Redis, Spaces, and optional registry before seeding namespaces and registry credentials (infra/main.tf:80-195). - Argo CD Applications (`echo-*`, `echo-monitoring-*`) track this repository’s `main` branch with automated prune/self-heal and namespace creation (argo/echo-dev.yaml:1-23, argo/echo-monitoring-prod.yaml:1-23). -- The `helm/echo` chart deploys the API server, worker tiers (worker, workerCpu, workerScheduler), Directus, and Neo4j along with shared env configuration and ingress/rollout settings (helm/echo/values.yaml:1-166). +- The `helm/echo` chart deploys the API server, worker tiers (worker, workerCpu, workerScheduler) and Directus along with shared env configuration and ingress/rollout settings (helm/echo/values.yaml:1-166). - `helm/monitoring` delivers Prometheus, Grafana, Loki, promtail, node-exporter, blackbox checks, and alertmanager with storage and ingress defaults (helm/monitoring/values.yaml:1-112). - Sealed secrets house sensitive config for each namespace, edited locally via `secret-manager.sh` before sealing (secret-manager.sh:104-189, secrets/sealed-backend-secrets-dev.yaml:1-18). - `ai-infra` bootstraps a GCS-backed Terraform state bucket and Vertex AI endpoint plus service-account IAM for Gemini usage (ai-infra/state/main.tf:1-31, ai-infra/vertex/main.tf:1-20, ai-infra/README.md:5-30). diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 00000000..051ab561 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,148 @@ +name: validate + +# Why this exists: Argo CD syncs `main` straight to the clusters with +# prune + selfHeal. Anything that merges here is deployed. A chart that does not +# render, or a manifest the API server rejects, becomes a production problem with +# no gate in between. +# +# Precedent: a Helm/Python `{{ }}` brace collision in cronjob-warning-digest.yaml +# silently blocked the whole monitoring chart from rendering for weeks. A plain +# `helm template` would have caught it on the pull request. +# +# The render matrix below MUST mirror the valueFiles in argo/*.yaml. If you add +# an Argo Application or change its valueFiles, add it here in the same commit. + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + helm: + name: helm render (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # mirrors argo/echo-prod.yaml + - name: echo-prod + chart: helm/echo + values: "-f helm/echo/values-prod.yaml -f helm/echo/values-extended-env.yaml" + # mirrors argo/echo-dev.yaml + - name: echo-dev + chart: helm/echo + values: "-f helm/echo/values.yaml -f helm/echo/values-extended-env.yaml -f helm/echo/values-echo-next.yaml" + # mirrors argo/echo-testing.yaml + - name: echo-testing + chart: helm/echo + values: "-f helm/echo/values-testing.yaml -f helm/echo/values-extended-env.yaml" + # mirrors argo/echo-monitoring-prod.yaml + - name: echo-monitoring-prod + chart: helm/monitoring + values: "-f helm/monitoring/values-prod.yaml" + # mirrors argo/echo-monitoring-dev.yaml + - name: echo-monitoring-dev + chart: helm/monitoring + values: "-f helm/monitoring/values.yaml" + steps: + - uses: actions/checkout@v4 + + - uses: azure/setup-helm@v4 + with: + version: v3.16.3 + + - name: helm lint + run: helm lint ${{ matrix.chart }} ${{ matrix.values }} + + - name: helm template + run: | + mkdir -p rendered + helm template ${{ matrix.name }} ${{ matrix.chart }} ${{ matrix.values }} \ + > rendered/${{ matrix.name }}.yaml + # An empty or near-empty render means the chart silently produced nothing. + lines=$(wc -l < rendered/${{ matrix.name }}.yaml) + echo "rendered $lines lines" + test "$lines" -gt 50 + + - name: install kubeconform + run: | + curl -sSL -o /tmp/kubeconform.tar.gz \ + https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz + tar -xzf /tmp/kubeconform.tar.gz -C /tmp kubeconform + sudo mv /tmp/kubeconform /usr/local/bin/ + + - name: kubeconform + # Kubernetes version is pinned to the DOKS version in infra/main.tf. + # Bump both together. + run: | + kubeconform \ + -kubernetes-version 1.32.5 \ + -strict \ + -ignore-missing-schemas \ + -summary \ + rendered/${{ matrix.name }}.yaml + + argo-manifests: + name: argo application manifests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: install pyyaml + run: python3 -m pip install --quiet pyyaml + + - name: parse every Argo Application + run: | + python3 - <<'PY' + import glob, sys, yaml + bad = 0 + for path in sorted(glob.glob("argo/*.yaml")): + try: + docs = [d for d in yaml.safe_load_all(open(path)) if d] + except yaml.YAMLError as e: + print(f"FAIL {path}: {e}"); bad = 1; continue + for d in docs: + if d.get("kind") != "Application": + continue + spec = d.get("spec", {}) + src = spec.get("source", {}) + rev = src.get("targetRevision") + # A feature branch pinned here is exactly the drift that left + # echo-monitoring-prod eight months behind main. + if rev != "main": + print(f"FAIL {path}: targetRevision is {rev!r}, expected 'main'") + bad = 1 + if not spec.get("destination", {}).get("namespace"): + print(f"FAIL {path}: no destination.namespace") + bad = 1 + print(f"ok {path} -> {src.get('path')} @ {rev}") + sys.exit(bad) + PY + + terraform: + name: terraform ${{ matrix.dir }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dir: [infra, ai-infra/state, ai-infra/vertex] + steps: + - uses: actions/checkout@v4 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.9.8 + terraform_wrapper: false + + - name: terraform fmt + run: terraform -chdir=${{ matrix.dir }} fmt -check -recursive + + - name: terraform init (no backend) + run: terraform -chdir=${{ matrix.dir }} init -backend=false -input=false + + - name: terraform validate + run: terraform -chdir=${{ matrix.dir }} validate diff --git a/AGENTS.md b/AGENTS.md index 9368403e..0da32038 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,15 @@ # AGENTS.md +> **This file is a generated snapshot and it goes stale.** It has been wrong +> before: it described Neo4j as a deployed workload after it stopped being used, +> and it says "no CI workflows tracked in repo" while `dembrane/echo`'s GitHub +> Actions commit image tags into this repo several times a day. +> +> **For deployments, the comment block at the top of `infra/main.tf` is the +> source of truth.** That is the runbook actually in use (workspace selection, +> which tfvars, the `doctl` kubeconfig lines). Prefer code and config over any +> prose in this file, and fix this file when you find it wrong. + ## 1) Snapshot - Project: Dembrane ECHO GitOps • Repo type: mono (Terraform + Helm + Argo CD GitOps) (README.md:40, infra/main.tf:80) - Entrypoints: Terraform stacks (`infra/`, `ai-infra/`), Argo CD apps (`echo-*`, `echo-monitoring-*`), Helm charts (`helm/echo`, `helm/monitoring`) (infra/main.tf:80-195, ai-infra/README.md:16-24, argo/echo-dev.yaml:1-23, helm/monitoring/Chart.yaml:1-6) @@ -11,12 +21,12 @@ ## 2) Tech & Tooling - Runtimes & package managers: Terraform ≥1.0, kubectl, Helm 3, kubeseal, doctl per prerequisites; Python 3 + `requests` for Loki tooling (README.md:84-103, scripts/LOKI_LOG_QUERY.md:5-17). -- Core libraries by role: DigitalOcean Terraform resources provision VPC/K8s/DB/cache/object storage (infra/main.tf:80-145); Argo CD Applications drive GitOps sync (argo/echo-dev.yaml:1-23); `helm/echo` manages API, Directus, worker tiers, and Neo4j (helm/echo/values.yaml:42-166); `helm/monitoring` delivers Prometheus/Grafana/Loki stack (helm/monitoring/values.yaml:1-61); `ai-infra` provisions Vertex AI endpoints and IAM (ai-infra/vertex/main.tf:1-20). +- Core libraries by role: DigitalOcean Terraform resources provision VPC/K8s/DB/cache/object storage (infra/main.tf:80-145); Argo CD Applications drive GitOps sync (argo/echo-dev.yaml:1-23); `helm/echo` manages API, Directus and worker tiers (helm/echo/values.yaml:42-166); `helm/monitoring` delivers Prometheus/Grafana/Loki stack (helm/monitoring/values.yaml:1-61); `ai-infra` provisions Vertex AI endpoints and IAM (ai-infra/vertex/main.tf:1-20). - Scripts you’ll actually use: `secret-manager.sh` for base64 edits, batch updates, and compares (secret-manager.sh:4-193); `scripts/query_logs.py` wraps Loki queries with chunking/pagination (scripts/LOKI_LOG_QUERY.md:29-132); `scripts/k6/sendChunks.js` replays participant uploads via k6 (scripts/k6/README.md:11-35). - Code style (lint/format/type): Terraform providers are version-locked by `.terraform.lock.hcl`; run CLI formatters (`terraform fmt`, `helm lint`) locally as needed (infra/.terraform.lock.hcl:1-33). ## 3) Architecture (mental model) -- Modules/services & responsibilities: Terraform builds DigitalOcean infra then seeds namespaces/secrets; Helm deploys application workloads (API, workers, Directus, Neo4j) and monitoring stack (infra/main.tf:80-195, helm/echo/values.yaml:42-166, helm/monitoring/values.yaml:1-112). +- Modules/services & responsibilities: Terraform builds DigitalOcean infra then seeds namespaces/secrets; Helm deploys application workloads (API, workers, Directus) and monitoring stack (infra/main.tf:80-195, helm/echo/values.yaml:42-166, helm/monitoring/values.yaml:1-112). - Data & external surfaces: Postgres, Redis, and Spaces are managed services; ingress exposes `directus`/`api` hostnames with TLS and monitoring endpoints with optional auth (infra/main.tf:101-145, helm/echo/values.yaml:143-159, helm/monitoring/values.yaml:1-60). - Notable patterns: Argo CD auto-prune/self-heal enforces drift control; HPAs and priority classes tune scaling for core workloads (argo/echo-dev.yaml:18-23, helm/echo/templates/hpa-api-server.yaml:1-32, helm/echo/templates/priorityclass-echo-critical.yaml:1-11). - Diagram → see `.agents/architecture.md`. diff --git a/NEED_HELP.md b/NEED_HELP.md index e037b2f6..5cbb42ca 100644 --- a/NEED_HELP.md +++ b/NEED_HELP.md @@ -1,5 +1,27 @@ @Dembrane, @spashii needs your help! +## done (2026-08-13) +- ~~CI validations~~ → `.github/workflows/validate.yml`: `helm lint` and + `helm template` for all five Argo render combinations, `kubeconform` on the + output, `terraform fmt`/`validate`, and a check that every Argo Application + tracks `main`. It caught a duplicate-annotation bug in the monitoring ingress + on its first run. +- ~~Neo4j~~ deleted (deployment, service, PVC, the `echo-critical` PriorityClass + that existed only for it, and its committed password). Zero references in + `server/`, `agent/` or `frontend/` confirmed before removal. +- ~~Grafana `adminPassword: "admin"` in values~~ deleted. It was dead config; the + live value comes from the SealedSecret. + +## still open, highest value first +- **protect `main`** (see the section below; still unprotected as of 2026-08-13 + while Argo syncs prod from it with prune + selfHeal) +- **one Argo, app-of-apps** — Application CRs are still applied by hand, which is + the drift bug from `docs/triage-2026-05-14.md`. Note `docs/migration-plan.md` + decision 6 ("don't adopt app-of-apps") is void: the repo is staying and the + target is now OVHcloud, not GCP Cloud Run. +- `imagePullPolicy: Always` on every deployment while image tags are immutable + git SHAs. Puts the registry in the pod startup path for no benefit. + ## infra - add Azure LLMs - add Runpod Servvice (or use Az Serverless / Google Cloud Run) diff --git a/helm/echo/templates/deployment-neo4j.yaml b/helm/echo/templates/deployment-neo4j.yaml deleted file mode 100644 index c2e9a155..00000000 --- a/helm/echo/templates/deployment-neo4j.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: echo-neo4j - labels: - app: echo - component: neo4j -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app: echo - component: neo4j - template: - metadata: - labels: - app: echo - component: neo4j - spec: - priorityClassName: echo-critical - containers: - - name: neo4j - image: "{{ .Values.neo4j.image.repository }}:{{ .Values.neo4j.image.tag | default "5.18.0-community" }}" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 7474 - name: http - - containerPort: 7687 - name: bolt - env: - - name: NEO4J_AUTH - value: "neo4j/{{ .Values.neo4j.password }}" - - name: NEO4J_ACCEPT_LICENSE_AGREEMENT - value: "yes" - - name: NEO4J_server_memory_pagecache_size - value: "{{ .Values.neo4j.config.pagecacheSize | default "512M" }}" - - name: NEO4J_server_memory_heap_initial__size - value: "{{ default (default "512M" .Values.neo4j.config.heapSize) .Values.neo4j.config.heapInitialSize }}" - - name: NEO4J_server_memory_heap_max__size - value: "{{ default (default "512M" .Values.neo4j.config.heapSize) .Values.neo4j.config.heapMaxSize }}" - - name: NEO4J_server_bolt_listen__address - value: "0.0.0.0:7687" - - name: NEO4J_server_config_strict__validation_enabled - value: "false" - volumeMounts: - - name: neo4j-data - mountPath: /data - resources: - requests: - cpu: "{{ .Values.neo4j.resources.requests.cpu | default "500m" }}" - memory: "{{ .Values.neo4j.resources.requests.memory | default "1Gi" }}" - limits: - cpu: "{{ .Values.neo4j.resources.limits.cpu | default "1000m" }}" - memory: "{{ .Values.neo4j.resources.limits.memory | default "2Gi" }}" - readinessProbe: - tcpSocket: - port: 7687 - initialDelaySeconds: 45 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 6 - livenessProbe: - tcpSocket: - port: 7687 - initialDelaySeconds: 90 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 6 - volumes: - - name: neo4j-data - persistentVolumeClaim: - claimName: neo4j-data \ No newline at end of file diff --git a/helm/echo/templates/priorityclass-echo-critical.yaml b/helm/echo/templates/priorityclass-echo-critical.yaml deleted file mode 100644 index 9ed2b3f1..00000000 --- a/helm/echo/templates/priorityclass-echo-critical.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: echo-critical - labels: - app: echo - component: neo4j -value: 1000000000 -globalDefault: false -preemptionPolicy: PreemptLowerPriority -description: "PriorityClass for echo Neo4j to ensure scheduling of larger resources" - - diff --git a/helm/echo/templates/pvc-neo4j.yaml b/helm/echo/templates/pvc-neo4j.yaml deleted file mode 100644 index ea29fe4d..00000000 --- a/helm/echo/templates/pvc-neo4j.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: neo4j-data - labels: - app: echo - component: neo4j -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.neo4j.storage.size | default "10Gi" }} - storageClassName: {{ .Values.storage.storageClassName | default "do-block-storage" }} \ No newline at end of file diff --git a/helm/echo/templates/service-neo4j.yaml b/helm/echo/templates/service-neo4j.yaml deleted file mode 100644 index c3d8e4f6..00000000 --- a/helm/echo/templates/service-neo4j.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: echo-neo4j - labels: - app: echo - component: neo4j -spec: - ports: - - port: 7474 - targetPort: 7474 - name: http - - port: 7687 - targetPort: 7687 - name: bolt - selector: - app: echo - component: neo4j \ No newline at end of file diff --git a/helm/echo/values-prod.yaml b/helm/echo/values-prod.yaml index 0fa74c3d..a2f4946f 100644 --- a/helm/echo/values-prod.yaml +++ b/helm/echo/values-prod.yaml @@ -126,25 +126,6 @@ workerScheduler: cpu: "800m" memory: "1Gi" -neo4j: - image: - repository: "neo4j" - tag: "5.18.0-community" - password: "admin@dembrane" - storage: - size: "40Gi" - config: - pagecacheSize: "2G" - heapInitialSize: "2G" - heapMaxSize: "4G" - resources: - requests: - cpu: "2" - memory: "2Gi" - limits: - cpu: "4" - memory: "8Gi" - ingress: enabled: true className: "nginx" diff --git a/helm/echo/values-testing.yaml b/helm/echo/values-testing.yaml index 9ad80067..538cc6fd 100644 --- a/helm/echo/values-testing.yaml +++ b/helm/echo/values-testing.yaml @@ -134,24 +134,6 @@ workerScheduler: cpu: "600m" memory: "1Gi" -neo4j: - image: - repository: "neo4j" - tag: "5.18.0-community" - password: "admin@dembrane" - storage: - size: "5Gi" # Smaller than dev - config: - pagecacheSize: "256M" - heapSize: "256M" - resources: - requests: - cpu: "300m" - memory: "512Mi" - limits: - cpu: "800m" - memory: "2Gi" - ingress: enabled: true className: "nginx" diff --git a/helm/echo/values.yaml b/helm/echo/values.yaml index 5cde9f3f..b50f1597 100644 --- a/helm/echo/values.yaml +++ b/helm/echo/values.yaml @@ -140,24 +140,6 @@ workerScheduler: cpu: "800m" memory: "1Gi" -neo4j: - image: - repository: "neo4j" - tag: "5.18.0-community" - password: "admin@dembrane" - storage: - size: "10Gi" - config: - pagecacheSize: "512M" - heapSize: "512M" - resources: - requests: - cpu: "500m" - memory: "1Gi" - limits: - cpu: "1" - memory: "4Gi" - ingress: enabled: true className: "nginx" diff --git a/helm/monitoring/templates/configmap-grafana-dashboards.yaml b/helm/monitoring/templates/configmap-grafana-dashboards.yaml index 301a2f43..7bf4689e 100644 --- a/helm/monitoring/templates/configmap-grafana-dashboards.yaml +++ b/helm/monitoring/templates/configmap-grafana-dashboards.yaml @@ -483,7 +483,7 @@ data: ]} }, {"type": "timeseries", "title": "Pod restarts by service (1h)", "id": 8, "gridPos": {"h": 8, "w": 8, "x": 8, "y": 9}, - "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\"}[1h]), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler|neo4j).*\" ) )", "legendFormat": "{{ "{{" }}svc{{ "}}" }}"}], + "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\"}[1h]), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler).*\" ) )", "legendFormat": "{{ "{{" }}svc{{ "}}" }}"}], "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []} }, {"type": "row", "title": "Capacity", "id": 9, "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 17}}, @@ -504,11 +504,11 @@ data: {"type": "row", "title": "Resource hotspots", "id": 12, "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 26}}, {"type": "timeseries", "title": "CPU usage by service (cores)", "id": 13, "gridPos": {"h": 8, "w": 12, "x": 0, "y": 27}, "fieldConfig": {"defaults": {"unit": "cores"}}, - "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( sum by (pod) ( rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", image!=\"\", container!=\"POD\", container!=\"\"}[5m]) ), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler|neo4j).*\" ) )"}] + "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( sum by (pod) ( rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", image!=\"\", container!=\"POD\", container!=\"\"}[5m]) ), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler).*\" ) )"}] }, {"type": "timeseries", "title": "Memory usage by service (bytes)", "id": 14, "gridPos": {"h": 8, "w": 12, "x": 12, "y": 27}, "fieldConfig": {"defaults": {"unit": "bytes"}}, - "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( sum by (pod) ( container_memory_working_set_bytes{namespace=\"$namespace\", image!=\"\", container!=\"POD\", container!=\"\"} ), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler|neo4j).*\" ) )"}] + "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( sum by (pod) ( container_memory_working_set_bytes{namespace=\"$namespace\", image!=\"\", container!=\"POD\", container!=\"\"} ), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler).*\" ) )"}] } , {"type": "row", "title": "Error Logs", "id": 200, "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 35}}, diff --git a/helm/monitoring/templates/configmap-prometheus.yaml b/helm/monitoring/templates/configmap-prometheus.yaml index f89e1a70..b01c5ff1 100644 --- a/helm/monitoring/templates/configmap-prometheus.yaml +++ b/helm/monitoring/templates/configmap-prometheus.yaml @@ -348,8 +348,8 @@ data: - alert: DeploymentAvailabilityShortfall expr: | ( - sum by (deployment) (kube_deployment_spec_replicas{namespace="echo-prod", deployment=~"echo-(api|directus|worker|worker-cpu|worker-scheduler|neo4j).*"}) - - sum by (deployment) (kube_deployment_status_replicas_available{namespace="echo-prod", deployment=~"echo-(api|directus|worker|worker-cpu|worker-scheduler|neo4j).*"}) + sum by (deployment) (kube_deployment_spec_replicas{namespace="echo-prod", deployment=~"echo-(api|directus|worker|worker-cpu|worker-scheduler).*"}) + - sum by (deployment) (kube_deployment_status_replicas_available{namespace="echo-prod", deployment=~"echo-(api|directus|worker|worker-cpu|worker-scheduler).*"}) ) > 0 for: 10m labels: diff --git a/helm/monitoring/templates/ingress-monitoring.yaml b/helm/monitoring/templates/ingress-monitoring.yaml index e80c7b9c..c6b26a75 100644 --- a/helm/monitoring/templates/ingress-monitoring.yaml +++ b/helm/monitoring/templates/ingress-monitoring.yaml @@ -5,16 +5,25 @@ metadata: name: monitoring-ingress namespace: monitoring annotations: - kubernetes.io/ingress.class: {{ .Values.ingress.className }} - cert-manager.io/cluster-issuer: {{ .Values.clusterIssuerName }} - nginx.ingress.kubernetes.io/ssl-redirect: "true" + {{- /* + Built as one dict so a key can never be emitted twice. Before this, the + hardcoded defaults below and the `ingress.annotations` range both wrote + cert-manager.io/cluster-issuer and nginx.ingress.kubernetes.io/ssl-redirect, + producing a duplicate-key Ingress. Values win over the defaults here. + */}} + {{- $ann := dict + "kubernetes.io/ingress.class" .Values.ingress.className + "cert-manager.io/cluster-issuer" .Values.clusterIssuerName + "nginx.ingress.kubernetes.io/ssl-redirect" "true" }} {{- if .Values.ingress.basicAuth.enabled }} - # Basic auth protection - nginx.ingress.kubernetes.io/auth-type: basic - nginx.ingress.kubernetes.io/auth-secret: monitoring-basic-auth - nginx.ingress.kubernetes.io/auth-realm: "Authentication Required" + {{- $_ := set $ann "nginx.ingress.kubernetes.io/auth-type" "basic" }} + {{- $_ := set $ann "nginx.ingress.kubernetes.io/auth-secret" "monitoring-basic-auth" }} + {{- $_ := set $ann "nginx.ingress.kubernetes.io/auth-realm" "Authentication Required" }} {{- end }} {{- range $key, $value := .Values.ingress.annotations }} + {{- $_ := set $ann $key $value }} + {{- end }} + {{- range $key, $value := $ann }} {{ $key }}: {{ $value | quote }} {{- end }} spec: diff --git a/helm/monitoring/values-prod.yaml b/helm/monitoring/values-prod.yaml index e6ef9592..27c79377 100644 --- a/helm/monitoring/values-prod.yaml +++ b/helm/monitoring/values-prod.yaml @@ -29,8 +29,7 @@ prometheus: # Grafana settings grafana: - # This will be overridden by the SealedSecret in production - adminPassword: "admin" + # Admin password comes from secrets/sealed-monitoring-secrets-prod.yaml resources: requests: cpu: "200m" diff --git a/helm/monitoring/values.yaml b/helm/monitoring/values.yaml index a613fb8e..7038e2e2 100644 --- a/helm/monitoring/values.yaml +++ b/helm/monitoring/values.yaml @@ -28,7 +28,8 @@ prometheus: # Grafana settings grafana: - adminPassword: "admin" # Override this with values-prod.yaml or --set + # Admin password comes from the monitoring-secrets SealedSecret + # (GF_SECURITY_ADMIN_PASSWORD in deployment-grafana.yaml). Never set it here. resources: requests: cpu: "100m"