From 94af710485ef3638a508fbfb60e56c6bde86daa0 Mon Sep 17 00:00:00 2001 From: spashii Date: Thu, 13 Aug 2026 16:19:38 +0200 Subject: [PATCH] infra: add CI validations, delete Neo4j, drop dead Grafana password Argo syncs main straight to the clusters with prune and selfHeal, so anything merged here deploys. Until now nothing checked that the charts even render. CI (.github/workflows/validate.yml) - helm lint + helm template for all five combinations in argo/*.yaml, so the matrix mirrors what Argo actually renders. Add an Application, add a row. - kubeconform against the rendered output, pinned to the DOKS 1.32.5 version in infra/main.tf. - terraform fmt -check and validate for infra, ai-infra/state, ai-infra/vertex. - every Argo Application must track main. A feature branch pinned in an Application is the exact drift in docs/triage-2026-05-14.md, where echo-monitoring-prod sat eight months behind. It found a real bug immediately: the monitoring ingress emitted cert-manager.io/cluster-issuer and nginx.ingress.kubernetes.io/ssl-redirect twice, once hardcoded and once from ingress.annotations. Annotations are now built as one dict, so values override defaults and no key repeats. Neo4j Deleted the deployment, service and PVC, plus the echo-critical PriorityClass that existed only for it and is referenced by nothing. Confirmed zero references across server/, agent/ and frontend/ before removing. Also removes a plaintext neo4j password that was committed in all three values files. Frees 2.7Gi memory and a 40Gi PVC per cluster. Grafana Removed adminPassword from both monitoring values files. No template read it; the live value comes from the monitoring-secrets SealedSecret. It rendered to nothing and only read like a live credential in a public repo. Docs AGENTS.md and .agents/architecture.md still described Neo4j as deployed. Corrected, and AGENTS.md now says what it is: a generated snapshot that goes stale, with the comment block in infra/main.tf as the deployment source of truth. --- .agents/architecture.md | 2 +- .github/workflows/validate.yml | 148 ++++++++++++++++++ AGENTS.md | 14 +- NEED_HELP.md | 22 +++ helm/echo/templates/deployment-neo4j.yaml | 74 --------- .../priorityclass-echo-critical.yaml | 13 -- helm/echo/templates/pvc-neo4j.yaml | 14 -- helm/echo/templates/service-neo4j.yaml | 18 --- helm/echo/values-prod.yaml | 19 --- helm/echo/values-testing.yaml | 18 --- helm/echo/values.yaml | 18 --- .../configmap-grafana-dashboards.yaml | 6 +- .../templates/configmap-prometheus.yaml | 4 +- .../templates/ingress-monitoring.yaml | 23 ++- helm/monitoring/values-prod.yaml | 3 +- helm/monitoring/values.yaml | 3 +- 16 files changed, 207 insertions(+), 192 deletions(-) create mode 100644 .github/workflows/validate.yml delete mode 100644 helm/echo/templates/deployment-neo4j.yaml delete mode 100644 helm/echo/templates/priorityclass-echo-critical.yaml delete mode 100644 helm/echo/templates/pvc-neo4j.yaml delete mode 100644 helm/echo/templates/service-neo4j.yaml diff --git a/.agents/architecture.md b/.agents/architecture.md index cd87c316..6b947096 100644 --- a/.agents/architecture.md +++ b/.agents/architecture.md @@ -3,7 +3,7 @@ ## Core Systems - Terraform in `infra/` provisions the DigitalOcean VPC, Kubernetes cluster, managed Postgres, Redis, Spaces, and optional registry before seeding namespaces and registry credentials (infra/main.tf:80-195). - Argo CD Applications (`echo-*`, `echo-monitoring-*`) track this repository’s `main` branch with automated prune/self-heal and namespace creation (argo/echo-dev.yaml:1-23, argo/echo-monitoring-prod.yaml:1-23). -- The `helm/echo` chart deploys the API server, worker tiers (worker, workerCpu, workerScheduler), Directus, and Neo4j along with shared env configuration and ingress/rollout settings (helm/echo/values.yaml:1-166). +- The `helm/echo` chart deploys the API server, worker tiers (worker, workerCpu, workerScheduler) and Directus along with shared env configuration and ingress/rollout settings (helm/echo/values.yaml:1-166). - `helm/monitoring` delivers Prometheus, Grafana, Loki, promtail, node-exporter, blackbox checks, and alertmanager with storage and ingress defaults (helm/monitoring/values.yaml:1-112). - Sealed secrets house sensitive config for each namespace, edited locally via `secret-manager.sh` before sealing (secret-manager.sh:104-189, secrets/sealed-backend-secrets-dev.yaml:1-18). - `ai-infra` bootstraps a GCS-backed Terraform state bucket and Vertex AI endpoint plus service-account IAM for Gemini usage (ai-infra/state/main.tf:1-31, ai-infra/vertex/main.tf:1-20, ai-infra/README.md:5-30). diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 00000000..051ab561 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,148 @@ +name: validate + +# Why this exists: Argo CD syncs `main` straight to the clusters with +# prune + selfHeal. Anything that merges here is deployed. A chart that does not +# render, or a manifest the API server rejects, becomes a production problem with +# no gate in between. +# +# Precedent: a Helm/Python `{{ }}` brace collision in cronjob-warning-digest.yaml +# silently blocked the whole monitoring chart from rendering for weeks. A plain +# `helm template` would have caught it on the pull request. +# +# The render matrix below MUST mirror the valueFiles in argo/*.yaml. If you add +# an Argo Application or change its valueFiles, add it here in the same commit. + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + helm: + name: helm render (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # mirrors argo/echo-prod.yaml + - name: echo-prod + chart: helm/echo + values: "-f helm/echo/values-prod.yaml -f helm/echo/values-extended-env.yaml" + # mirrors argo/echo-dev.yaml + - name: echo-dev + chart: helm/echo + values: "-f helm/echo/values.yaml -f helm/echo/values-extended-env.yaml -f helm/echo/values-echo-next.yaml" + # mirrors argo/echo-testing.yaml + - name: echo-testing + chart: helm/echo + values: "-f helm/echo/values-testing.yaml -f helm/echo/values-extended-env.yaml" + # mirrors argo/echo-monitoring-prod.yaml + - name: echo-monitoring-prod + chart: helm/monitoring + values: "-f helm/monitoring/values-prod.yaml" + # mirrors argo/echo-monitoring-dev.yaml + - name: echo-monitoring-dev + chart: helm/monitoring + values: "-f helm/monitoring/values.yaml" + steps: + - uses: actions/checkout@v4 + + - uses: azure/setup-helm@v4 + with: + version: v3.16.3 + + - name: helm lint + run: helm lint ${{ matrix.chart }} ${{ matrix.values }} + + - name: helm template + run: | + mkdir -p rendered + helm template ${{ matrix.name }} ${{ matrix.chart }} ${{ matrix.values }} \ + > rendered/${{ matrix.name }}.yaml + # An empty or near-empty render means the chart silently produced nothing. + lines=$(wc -l < rendered/${{ matrix.name }}.yaml) + echo "rendered $lines lines" + test "$lines" -gt 50 + + - name: install kubeconform + run: | + curl -sSL -o /tmp/kubeconform.tar.gz \ + https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz + tar -xzf /tmp/kubeconform.tar.gz -C /tmp kubeconform + sudo mv /tmp/kubeconform /usr/local/bin/ + + - name: kubeconform + # Kubernetes version is pinned to the DOKS version in infra/main.tf. + # Bump both together. + run: | + kubeconform \ + -kubernetes-version 1.32.5 \ + -strict \ + -ignore-missing-schemas \ + -summary \ + rendered/${{ matrix.name }}.yaml + + argo-manifests: + name: argo application manifests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: install pyyaml + run: python3 -m pip install --quiet pyyaml + + - name: parse every Argo Application + run: | + python3 - <<'PY' + import glob, sys, yaml + bad = 0 + for path in sorted(glob.glob("argo/*.yaml")): + try: + docs = [d for d in yaml.safe_load_all(open(path)) if d] + except yaml.YAMLError as e: + print(f"FAIL {path}: {e}"); bad = 1; continue + for d in docs: + if d.get("kind") != "Application": + continue + spec = d.get("spec", {}) + src = spec.get("source", {}) + rev = src.get("targetRevision") + # A feature branch pinned here is exactly the drift that left + # echo-monitoring-prod eight months behind main. + if rev != "main": + print(f"FAIL {path}: targetRevision is {rev!r}, expected 'main'") + bad = 1 + if not spec.get("destination", {}).get("namespace"): + print(f"FAIL {path}: no destination.namespace") + bad = 1 + print(f"ok {path} -> {src.get('path')} @ {rev}") + sys.exit(bad) + PY + + terraform: + name: terraform ${{ matrix.dir }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dir: [infra, ai-infra/state, ai-infra/vertex] + steps: + - uses: actions/checkout@v4 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.9.8 + terraform_wrapper: false + + - name: terraform fmt + run: terraform -chdir=${{ matrix.dir }} fmt -check -recursive + + - name: terraform init (no backend) + run: terraform -chdir=${{ matrix.dir }} init -backend=false -input=false + + - name: terraform validate + run: terraform -chdir=${{ matrix.dir }} validate diff --git a/AGENTS.md b/AGENTS.md index 9368403e..0da32038 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,15 @@ # AGENTS.md +> **This file is a generated snapshot and it goes stale.** It has been wrong +> before: it described Neo4j as a deployed workload after it stopped being used, +> and it says "no CI workflows tracked in repo" while `dembrane/echo`'s GitHub +> Actions commit image tags into this repo several times a day. +> +> **For deployments, the comment block at the top of `infra/main.tf` is the +> source of truth.** That is the runbook actually in use (workspace selection, +> which tfvars, the `doctl` kubeconfig lines). Prefer code and config over any +> prose in this file, and fix this file when you find it wrong. + ## 1) Snapshot - Project: Dembrane ECHO GitOps • Repo type: mono (Terraform + Helm + Argo CD GitOps) (README.md:40, infra/main.tf:80) - Entrypoints: Terraform stacks (`infra/`, `ai-infra/`), Argo CD apps (`echo-*`, `echo-monitoring-*`), Helm charts (`helm/echo`, `helm/monitoring`) (infra/main.tf:80-195, ai-infra/README.md:16-24, argo/echo-dev.yaml:1-23, helm/monitoring/Chart.yaml:1-6) @@ -11,12 +21,12 @@ ## 2) Tech & Tooling - Runtimes & package managers: Terraform ≥1.0, kubectl, Helm 3, kubeseal, doctl per prerequisites; Python 3 + `requests` for Loki tooling (README.md:84-103, scripts/LOKI_LOG_QUERY.md:5-17). -- Core libraries by role: DigitalOcean Terraform resources provision VPC/K8s/DB/cache/object storage (infra/main.tf:80-145); Argo CD Applications drive GitOps sync (argo/echo-dev.yaml:1-23); `helm/echo` manages API, Directus, worker tiers, and Neo4j (helm/echo/values.yaml:42-166); `helm/monitoring` delivers Prometheus/Grafana/Loki stack (helm/monitoring/values.yaml:1-61); `ai-infra` provisions Vertex AI endpoints and IAM (ai-infra/vertex/main.tf:1-20). +- Core libraries by role: DigitalOcean Terraform resources provision VPC/K8s/DB/cache/object storage (infra/main.tf:80-145); Argo CD Applications drive GitOps sync (argo/echo-dev.yaml:1-23); `helm/echo` manages API, Directus and worker tiers (helm/echo/values.yaml:42-166); `helm/monitoring` delivers Prometheus/Grafana/Loki stack (helm/monitoring/values.yaml:1-61); `ai-infra` provisions Vertex AI endpoints and IAM (ai-infra/vertex/main.tf:1-20). - Scripts you’ll actually use: `secret-manager.sh` for base64 edits, batch updates, and compares (secret-manager.sh:4-193); `scripts/query_logs.py` wraps Loki queries with chunking/pagination (scripts/LOKI_LOG_QUERY.md:29-132); `scripts/k6/sendChunks.js` replays participant uploads via k6 (scripts/k6/README.md:11-35). - Code style (lint/format/type): Terraform providers are version-locked by `.terraform.lock.hcl`; run CLI formatters (`terraform fmt`, `helm lint`) locally as needed (infra/.terraform.lock.hcl:1-33). ## 3) Architecture (mental model) -- Modules/services & responsibilities: Terraform builds DigitalOcean infra then seeds namespaces/secrets; Helm deploys application workloads (API, workers, Directus, Neo4j) and monitoring stack (infra/main.tf:80-195, helm/echo/values.yaml:42-166, helm/monitoring/values.yaml:1-112). +- Modules/services & responsibilities: Terraform builds DigitalOcean infra then seeds namespaces/secrets; Helm deploys application workloads (API, workers, Directus) and monitoring stack (infra/main.tf:80-195, helm/echo/values.yaml:42-166, helm/monitoring/values.yaml:1-112). - Data & external surfaces: Postgres, Redis, and Spaces are managed services; ingress exposes `directus`/`api` hostnames with TLS and monitoring endpoints with optional auth (infra/main.tf:101-145, helm/echo/values.yaml:143-159, helm/monitoring/values.yaml:1-60). - Notable patterns: Argo CD auto-prune/self-heal enforces drift control; HPAs and priority classes tune scaling for core workloads (argo/echo-dev.yaml:18-23, helm/echo/templates/hpa-api-server.yaml:1-32, helm/echo/templates/priorityclass-echo-critical.yaml:1-11). - Diagram → see `.agents/architecture.md`. diff --git a/NEED_HELP.md b/NEED_HELP.md index e037b2f6..5cbb42ca 100644 --- a/NEED_HELP.md +++ b/NEED_HELP.md @@ -1,5 +1,27 @@ @Dembrane, @spashii needs your help! +## done (2026-08-13) +- ~~CI validations~~ → `.github/workflows/validate.yml`: `helm lint` and + `helm template` for all five Argo render combinations, `kubeconform` on the + output, `terraform fmt`/`validate`, and a check that every Argo Application + tracks `main`. It caught a duplicate-annotation bug in the monitoring ingress + on its first run. +- ~~Neo4j~~ deleted (deployment, service, PVC, the `echo-critical` PriorityClass + that existed only for it, and its committed password). Zero references in + `server/`, `agent/` or `frontend/` confirmed before removal. +- ~~Grafana `adminPassword: "admin"` in values~~ deleted. It was dead config; the + live value comes from the SealedSecret. + +## still open, highest value first +- **protect `main`** (see the section below; still unprotected as of 2026-08-13 + while Argo syncs prod from it with prune + selfHeal) +- **one Argo, app-of-apps** — Application CRs are still applied by hand, which is + the drift bug from `docs/triage-2026-05-14.md`. Note `docs/migration-plan.md` + decision 6 ("don't adopt app-of-apps") is void: the repo is staying and the + target is now OVHcloud, not GCP Cloud Run. +- `imagePullPolicy: Always` on every deployment while image tags are immutable + git SHAs. Puts the registry in the pod startup path for no benefit. + ## infra - add Azure LLMs - add Runpod Servvice (or use Az Serverless / Google Cloud Run) diff --git a/helm/echo/templates/deployment-neo4j.yaml b/helm/echo/templates/deployment-neo4j.yaml deleted file mode 100644 index c2e9a155..00000000 --- a/helm/echo/templates/deployment-neo4j.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: echo-neo4j - labels: - app: echo - component: neo4j -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app: echo - component: neo4j - template: - metadata: - labels: - app: echo - component: neo4j - spec: - priorityClassName: echo-critical - containers: - - name: neo4j - image: "{{ .Values.neo4j.image.repository }}:{{ .Values.neo4j.image.tag | default "5.18.0-community" }}" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 7474 - name: http - - containerPort: 7687 - name: bolt - env: - - name: NEO4J_AUTH - value: "neo4j/{{ .Values.neo4j.password }}" - - name: NEO4J_ACCEPT_LICENSE_AGREEMENT - value: "yes" - - name: NEO4J_server_memory_pagecache_size - value: "{{ .Values.neo4j.config.pagecacheSize | default "512M" }}" - - name: NEO4J_server_memory_heap_initial__size - value: "{{ default (default "512M" .Values.neo4j.config.heapSize) .Values.neo4j.config.heapInitialSize }}" - - name: NEO4J_server_memory_heap_max__size - value: "{{ default (default "512M" .Values.neo4j.config.heapSize) .Values.neo4j.config.heapMaxSize }}" - - name: NEO4J_server_bolt_listen__address - value: "0.0.0.0:7687" - - name: NEO4J_server_config_strict__validation_enabled - value: "false" - volumeMounts: - - name: neo4j-data - mountPath: /data - resources: - requests: - cpu: "{{ .Values.neo4j.resources.requests.cpu | default "500m" }}" - memory: "{{ .Values.neo4j.resources.requests.memory | default "1Gi" }}" - limits: - cpu: "{{ .Values.neo4j.resources.limits.cpu | default "1000m" }}" - memory: "{{ .Values.neo4j.resources.limits.memory | default "2Gi" }}" - readinessProbe: - tcpSocket: - port: 7687 - initialDelaySeconds: 45 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 6 - livenessProbe: - tcpSocket: - port: 7687 - initialDelaySeconds: 90 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 6 - volumes: - - name: neo4j-data - persistentVolumeClaim: - claimName: neo4j-data \ No newline at end of file diff --git a/helm/echo/templates/priorityclass-echo-critical.yaml b/helm/echo/templates/priorityclass-echo-critical.yaml deleted file mode 100644 index 9ed2b3f1..00000000 --- a/helm/echo/templates/priorityclass-echo-critical.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: echo-critical - labels: - app: echo - component: neo4j -value: 1000000000 -globalDefault: false -preemptionPolicy: PreemptLowerPriority -description: "PriorityClass for echo Neo4j to ensure scheduling of larger resources" - - diff --git a/helm/echo/templates/pvc-neo4j.yaml b/helm/echo/templates/pvc-neo4j.yaml deleted file mode 100644 index ea29fe4d..00000000 --- a/helm/echo/templates/pvc-neo4j.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: neo4j-data - labels: - app: echo - component: neo4j -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.neo4j.storage.size | default "10Gi" }} - storageClassName: {{ .Values.storage.storageClassName | default "do-block-storage" }} \ No newline at end of file diff --git a/helm/echo/templates/service-neo4j.yaml b/helm/echo/templates/service-neo4j.yaml deleted file mode 100644 index c3d8e4f6..00000000 --- a/helm/echo/templates/service-neo4j.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: echo-neo4j - labels: - app: echo - component: neo4j -spec: - ports: - - port: 7474 - targetPort: 7474 - name: http - - port: 7687 - targetPort: 7687 - name: bolt - selector: - app: echo - component: neo4j \ No newline at end of file diff --git a/helm/echo/values-prod.yaml b/helm/echo/values-prod.yaml index 0fa74c3d..a2f4946f 100644 --- a/helm/echo/values-prod.yaml +++ b/helm/echo/values-prod.yaml @@ -126,25 +126,6 @@ workerScheduler: cpu: "800m" memory: "1Gi" -neo4j: - image: - repository: "neo4j" - tag: "5.18.0-community" - password: "admin@dembrane" - storage: - size: "40Gi" - config: - pagecacheSize: "2G" - heapInitialSize: "2G" - heapMaxSize: "4G" - resources: - requests: - cpu: "2" - memory: "2Gi" - limits: - cpu: "4" - memory: "8Gi" - ingress: enabled: true className: "nginx" diff --git a/helm/echo/values-testing.yaml b/helm/echo/values-testing.yaml index 9ad80067..538cc6fd 100644 --- a/helm/echo/values-testing.yaml +++ b/helm/echo/values-testing.yaml @@ -134,24 +134,6 @@ workerScheduler: cpu: "600m" memory: "1Gi" -neo4j: - image: - repository: "neo4j" - tag: "5.18.0-community" - password: "admin@dembrane" - storage: - size: "5Gi" # Smaller than dev - config: - pagecacheSize: "256M" - heapSize: "256M" - resources: - requests: - cpu: "300m" - memory: "512Mi" - limits: - cpu: "800m" - memory: "2Gi" - ingress: enabled: true className: "nginx" diff --git a/helm/echo/values.yaml b/helm/echo/values.yaml index 5cde9f3f..b50f1597 100644 --- a/helm/echo/values.yaml +++ b/helm/echo/values.yaml @@ -140,24 +140,6 @@ workerScheduler: cpu: "800m" memory: "1Gi" -neo4j: - image: - repository: "neo4j" - tag: "5.18.0-community" - password: "admin@dembrane" - storage: - size: "10Gi" - config: - pagecacheSize: "512M" - heapSize: "512M" - resources: - requests: - cpu: "500m" - memory: "1Gi" - limits: - cpu: "1" - memory: "4Gi" - ingress: enabled: true className: "nginx" diff --git a/helm/monitoring/templates/configmap-grafana-dashboards.yaml b/helm/monitoring/templates/configmap-grafana-dashboards.yaml index 301a2f43..7bf4689e 100644 --- a/helm/monitoring/templates/configmap-grafana-dashboards.yaml +++ b/helm/monitoring/templates/configmap-grafana-dashboards.yaml @@ -483,7 +483,7 @@ data: ]} }, {"type": "timeseries", "title": "Pod restarts by service (1h)", "id": 8, "gridPos": {"h": 8, "w": 8, "x": 8, "y": 9}, - "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\"}[1h]), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler|neo4j).*\" ) )", "legendFormat": "{{ "{{" }}svc{{ "}}" }}"}], + "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\"}[1h]), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler).*\" ) )", "legendFormat": "{{ "{{" }}svc{{ "}}" }}"}], "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []} }, {"type": "row", "title": "Capacity", "id": 9, "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 17}}, @@ -504,11 +504,11 @@ data: {"type": "row", "title": "Resource hotspots", "id": 12, "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 26}}, {"type": "timeseries", "title": "CPU usage by service (cores)", "id": 13, "gridPos": {"h": 8, "w": 12, "x": 0, "y": 27}, "fieldConfig": {"defaults": {"unit": "cores"}}, - "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( sum by (pod) ( rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", image!=\"\", container!=\"POD\", container!=\"\"}[5m]) ), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler|neo4j).*\" ) )"}] + "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( sum by (pod) ( rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", image!=\"\", container!=\"POD\", container!=\"\"}[5m]) ), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler).*\" ) )"}] }, {"type": "timeseries", "title": "Memory usage by service (bytes)", "id": 14, "gridPos": {"h": 8, "w": 12, "x": 12, "y": 27}, "fieldConfig": {"defaults": {"unit": "bytes"}}, - "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( sum by (pod) ( container_memory_working_set_bytes{namespace=\"$namespace\", image!=\"\", container!=\"POD\", container!=\"\"} ), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler|neo4j).*\" ) )"}] + "targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (svc) ( label_replace( sum by (pod) ( container_memory_working_set_bytes{namespace=\"$namespace\", image!=\"\", container!=\"POD\", container!=\"\"} ), \"svc\", \"$1\", \"pod\", \"echo-(api|directus|worker|worker-cpu|worker-scheduler).*\" ) )"}] } , {"type": "row", "title": "Error Logs", "id": 200, "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 35}}, diff --git a/helm/monitoring/templates/configmap-prometheus.yaml b/helm/monitoring/templates/configmap-prometheus.yaml index f89e1a70..b01c5ff1 100644 --- a/helm/monitoring/templates/configmap-prometheus.yaml +++ b/helm/monitoring/templates/configmap-prometheus.yaml @@ -348,8 +348,8 @@ data: - alert: DeploymentAvailabilityShortfall expr: | ( - sum by (deployment) (kube_deployment_spec_replicas{namespace="echo-prod", deployment=~"echo-(api|directus|worker|worker-cpu|worker-scheduler|neo4j).*"}) - - sum by (deployment) (kube_deployment_status_replicas_available{namespace="echo-prod", deployment=~"echo-(api|directus|worker|worker-cpu|worker-scheduler|neo4j).*"}) + sum by (deployment) (kube_deployment_spec_replicas{namespace="echo-prod", deployment=~"echo-(api|directus|worker|worker-cpu|worker-scheduler).*"}) + - sum by (deployment) (kube_deployment_status_replicas_available{namespace="echo-prod", deployment=~"echo-(api|directus|worker|worker-cpu|worker-scheduler).*"}) ) > 0 for: 10m labels: diff --git a/helm/monitoring/templates/ingress-monitoring.yaml b/helm/monitoring/templates/ingress-monitoring.yaml index e80c7b9c..c6b26a75 100644 --- a/helm/monitoring/templates/ingress-monitoring.yaml +++ b/helm/monitoring/templates/ingress-monitoring.yaml @@ -5,16 +5,25 @@ metadata: name: monitoring-ingress namespace: monitoring annotations: - kubernetes.io/ingress.class: {{ .Values.ingress.className }} - cert-manager.io/cluster-issuer: {{ .Values.clusterIssuerName }} - nginx.ingress.kubernetes.io/ssl-redirect: "true" + {{- /* + Built as one dict so a key can never be emitted twice. Before this, the + hardcoded defaults below and the `ingress.annotations` range both wrote + cert-manager.io/cluster-issuer and nginx.ingress.kubernetes.io/ssl-redirect, + producing a duplicate-key Ingress. Values win over the defaults here. + */}} + {{- $ann := dict + "kubernetes.io/ingress.class" .Values.ingress.className + "cert-manager.io/cluster-issuer" .Values.clusterIssuerName + "nginx.ingress.kubernetes.io/ssl-redirect" "true" }} {{- if .Values.ingress.basicAuth.enabled }} - # Basic auth protection - nginx.ingress.kubernetes.io/auth-type: basic - nginx.ingress.kubernetes.io/auth-secret: monitoring-basic-auth - nginx.ingress.kubernetes.io/auth-realm: "Authentication Required" + {{- $_ := set $ann "nginx.ingress.kubernetes.io/auth-type" "basic" }} + {{- $_ := set $ann "nginx.ingress.kubernetes.io/auth-secret" "monitoring-basic-auth" }} + {{- $_ := set $ann "nginx.ingress.kubernetes.io/auth-realm" "Authentication Required" }} {{- end }} {{- range $key, $value := .Values.ingress.annotations }} + {{- $_ := set $ann $key $value }} + {{- end }} + {{- range $key, $value := $ann }} {{ $key }}: {{ $value | quote }} {{- end }} spec: diff --git a/helm/monitoring/values-prod.yaml b/helm/monitoring/values-prod.yaml index e6ef9592..27c79377 100644 --- a/helm/monitoring/values-prod.yaml +++ b/helm/monitoring/values-prod.yaml @@ -29,8 +29,7 @@ prometheus: # Grafana settings grafana: - # This will be overridden by the SealedSecret in production - adminPassword: "admin" + # Admin password comes from secrets/sealed-monitoring-secrets-prod.yaml resources: requests: cpu: "200m" diff --git a/helm/monitoring/values.yaml b/helm/monitoring/values.yaml index a613fb8e..7038e2e2 100644 --- a/helm/monitoring/values.yaml +++ b/helm/monitoring/values.yaml @@ -28,7 +28,8 @@ prometheus: # Grafana settings grafana: - adminPassword: "admin" # Override this with values-prod.yaml or --set + # Admin password comes from the monitoring-secrets SealedSecret + # (GF_SECURITY_ADMIN_PASSWORD in deployment-grafana.yaml). Never set it here. resources: requests: cpu: "100m"