diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..8ff468308 --- /dev/null +++ b/.env.example @@ -0,0 +1,10 @@ +# Local development values. ./deploy.sh dev copies this file to .env, which compose reads. +# Production never sees this file: its deploy writes .env from the repository secrets. +SECRET_KEY=CHANGE_ME_min_32_chars_long_!!!! +MONGO_USER=root +MONGO_PASSWORD=rootpassword +REDIS_PASSWORD=redispassword +MONGO_MONITOR_PASSWORD=monitorpassword +ADMIN_USER_PASSWORD=admin123 +DEFAULT_USER_PASSWORD=user123 +GRAFANA_ADMIN_PASSWORD=admin123 diff --git a/.github/actions/e2e-boot/action.yml b/.github/actions/e2e-boot/action.yml index 65de445c4..1a84cfb83 100644 --- a/.github/actions/e2e-boot/action.yml +++ b/.github/actions/e2e-boot/action.yml @@ -8,10 +8,6 @@ inputs: github-token: description: 'GitHub token for GHCR authentication' required: true - install-kueue: - description: 'Pre-pull the Kueue controller image, for jobs that install Kueue (default: true)' - required: false - default: 'true' runs: using: 'composite' @@ -28,11 +24,12 @@ runs: env: IMAGE_TAG: ${{ inputs.image-tag }} run: | + cp .env.example .env nohup bash -c ' IMAGE_TAG='"$IMAGE_TAG"' docker compose pull --quiet 2>&1 echo "--- pull done, starting infra ---" docker compose up -d --no-build \ - mongo redis shared-ca kafka 2>&1 + mongo redis kafka 2>&1 echo $? > /tmp/infra-pull.exit ' > /tmp/infra-pull.log 2>&1 & echo $! > /tmp/infra-pull.pid @@ -46,18 +43,9 @@ runs: -v /etc/rancher/k3s:/etc/rancher/k3s -e K3S_KUBECONFIG_MODE=644 \ "$K3S_IMAGE" server --disable=traefik --disable-network-policy --tls-san host.docker.internal - - name: Pre-pull executor images and Kueue manifest (background) + - name: Pre-pull executor images (background) shell: bash run: | - KUEUE_VERSION="${KUEUE_VERSION:-v0.20.1}" - - nohup bash -c ' - curl --proto =https --tlsv1.2 -fsSL -o /tmp/kueue-manifests.yaml \ - "https://github.com/kubernetes-sigs/kueue/releases/download/'"$KUEUE_VERSION"'/manifests.yaml" - echo $? > /tmp/kueue-download.exit - ' > /tmp/kueue-download.log 2>&1 & - echo $! > /tmp/kueue-download.pid - # No docker exec before k3s writes its kubeconfig: one during its root cgroup evacuation kills the kubelet nohup bash -c ' timeout 120 bash -c "until [ -s /etc/rancher/k3s/k3s.yaml ] && docker exec k3s crictl info >/dev/null 2>&1; do sleep 2; done" @@ -65,16 +53,3 @@ runs: echo $? > /tmp/crictl-python.exit ' > /tmp/crictl-python.log 2>&1 & echo $! > /tmp/crictl-python.pid - - # registry.k8s.io can take minutes for this image, longer than e2e-ready's wait for the controller - - name: Pre-pull the Kueue controller image (background) - if: inputs.install-kueue == 'true' - shell: bash - run: | - KUEUE_VERSION="${KUEUE_VERSION:-v0.20.1}" - nohup bash -c ' - timeout 120 bash -c "until [ -s /etc/rancher/k3s/k3s.yaml ] && docker exec k3s crictl info >/dev/null 2>&1; do sleep 2; done" - docker exec k3s crictl pull registry.k8s.io/kueue/kueue:'"$KUEUE_VERSION"' - echo $? > /tmp/crictl-kueue.exit - ' > /tmp/crictl-kueue.log 2>&1 & - echo $! > /tmp/crictl-kueue.pid diff --git a/.github/actions/e2e-ready/action.yml b/.github/actions/e2e-ready/action.yml index 6a12cb21a..5e7c5628c 100644 --- a/.github/actions/e2e-ready/action.yml +++ b/.github/actions/e2e-ready/action.yml @@ -9,10 +9,6 @@ inputs: description: 'Also wait for frontend health check (default: false)' required: false default: 'false' - install-kueue: - description: 'Install Kueue CRDs for executor pods (default: true)' - required: false - default: 'true' runs: using: 'composite' @@ -27,72 +23,11 @@ runs: export KUBECONFIG=/home/runner/.kube/config # kubectl wait fails at once while the node isn't registered yet, so retry until it is timeout 120 bash -c 'until kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null 2>&1; do sleep 2; done' - kubectl create namespace integr8scode --dry-run=client -o yaml | kubectl apply -f - - sed -E 's#https://(127\.0\.0\.1|0\.0\.0\.0):6443#https://host.docker.internal:6443#g' \ - /home/runner/.kube/config > backend/kubeconfig.yaml - chmod 644 backend/kubeconfig.yaml - - name: Install Kueue and wait for executor image pulls - if: inputs.install-kueue == 'true' + - name: Wait for the executor image pull shell: bash run: | set -e - - KUEUE_VERSION="${KUEUE_VERSION:-v0.20.1}" - KUEUE_MANIFEST_SHA256="${KUEUE_MANIFEST_SHA256:-c48514295365aae0c7dc578c70330befdd0015fec57cb4a06ba872bbef8750dd}" - KUEUE_MANIFEST="/tmp/kueue-manifests.yaml" - - timeout 120 tail --pid="$(cat /tmp/kueue-download.pid)" -f /dev/null 2>/dev/null || true - cat /tmp/kueue-download.log 2>/dev/null || true - [ "$(cat /tmp/kueue-download.exit)" = "0" ] - - echo "${KUEUE_MANIFEST_SHA256} ${KUEUE_MANIFEST}" | sha256sum -c - - timeout 600 tail --pid="$(cat /tmp/crictl-kueue.pid)" -f /dev/null 2>/dev/null || true - cat /tmp/crictl-kueue.log 2>/dev/null || true - [ "$(cat /tmp/crictl-kueue.exit)" = "0" ] - - kubectl apply --server-side -f "$KUEUE_MANIFEST" - rm -f "$KUEUE_MANIFEST" - # Kueue's cert bootstrap answers /readyz with 404 until the kubelet mounts its new secret, which can take minutes - if ! kubectl wait --for=condition=Available --timeout=300s deployment/kueue-controller-manager -n kueue-system; then - kubectl -n kueue-system get pods -o wide - kubectl -n kueue-system describe pods - kubectl -n kueue-system get events --sort-by=.lastTimestamp - kubectl -n kueue-system logs deployment/kueue-controller-manager --all-containers --tail=300 - exit 1 - fi - - kubectl apply --server-side -f - <<'EOF' - apiVersion: kueue.x-k8s.io/v1beta2 - kind: ResourceFlavor - metadata: - name: default-flavor - --- - apiVersion: kueue.x-k8s.io/v1beta2 - kind: ClusterQueue - metadata: - name: executor-queue - spec: - namespaceSelector: {} - resourceGroups: - - coveredResources: ["cpu", "memory"] - flavors: - - name: default-flavor - resources: - - name: cpu - nominalQuota: "32" - - name: memory - nominalQuota: "4Gi" - --- - apiVersion: kueue.x-k8s.io/v1beta2 - kind: LocalQueue - metadata: - name: executor-queue - namespace: integr8scode - spec: - clusterQueue: executor-queue - EOF - timeout 120 tail --pid="$(cat /tmp/crictl-python.pid)" -f /dev/null 2>/dev/null || true cat /tmp/crictl-python.log 2>/dev/null || true [ "$(cat /tmp/crictl-python.exit)" = "0" ] diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2fa392eb9..1bed0a9a4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,12 +29,11 @@ updates: - dependency-name: "typescript" versions: [">=7.0.0"] - # Docker base images, one entry so one group covers all three Dockerfiles + # Docker base images, one entry so one group covers both Dockerfiles - package-ecosystem: "docker" directories: - "/backend" - "/frontend" - - "/cert-generator" schedule: interval: "weekly" commit-message: diff --git a/.github/labeler.yml b/.github/labeler.yml index b52635ccf..bbbce0f68 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -76,7 +76,7 @@ - docker-compose.yaml - deploy.sh - "**/Dockerfile*" - - cert-generator/** + - .env.example - .github/workflows/docker.yml - .github/workflows/release-deploy.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f15fcde94..ef547997b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,8 +20,6 @@ concurrency: env: REGISTRY: ghcr.io K3S_IMAGE: rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d - KUEUE_VERSION: v0.20.1 - KUEUE_MANIFEST_SHA256: c48514295365aae0c7dc578c70330befdd0015fec57cb4a06ba872bbef8750dd jobs: changes: @@ -52,7 +50,7 @@ jobs: - 'frontend/**' infra: - *workflow - - 'cert-generator/**' + - '.env.example' - 'docker-compose.yaml' - 'deploy.sh' - '.github/actions/**' @@ -432,7 +430,7 @@ jobs: id: base-cache with: path: /tmp/base-image.tar.zst - key: base-image-${{ runner.os }}-${{ hashFiles('backend/Dockerfile.base', 'backend/pyproject.toml', 'backend/uv.lock') }} + key: base-image-${{ runner.os }}-${{ hashFiles('backend/Dockerfile', 'backend/pyproject.toml', 'backend/uv.lock') }} - name: Load base image from cache if: steps.base-cache.outputs.cache-hit == 'true' @@ -443,7 +441,8 @@ jobs: uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: ./backend - file: ./backend/Dockerfile.base + file: ./backend/Dockerfile + target: base load: true tags: integr8scode-base:latest cache-from: type=gha,scope=backend-base @@ -453,24 +452,13 @@ jobs: if: steps.base-cache.outputs.cache-hit != 'true' run: docker save integr8scode-base:latest | zstd -T0 -3 > /tmp/base-image.tar.zst - # ── Backend (depends on local base image) ─────────────── + # ── Backend (the cached base image stands in for the Dockerfile's base stage) ─────────────── - name: Build backend image run: | docker build -t integr8scode-backend:latest \ --build-context base=docker-image://integr8scode-base:latest \ -f ./backend/Dockerfile ./backend - # ── Utility images (GHA-cached, independent of base) ──────────── - - name: Build cert-generator image - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 - with: - context: ./cert-generator - file: ./cert-generator/Dockerfile - load: true - tags: integr8scode-cert-generator:latest - cache-from: type=gha,scope=cert-generator - cache-to: type=gha,mode=max,scope=cert-generator - # ── Frontend (nginx + SSL) ───────────────────────────────────── - name: Build frontend image uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 @@ -490,14 +478,12 @@ jobs: IMG: ${{ env.REGISTRY }}/${{ steps.tags.outputs.image-prefix }} run: | # Tag all images for GHCR - docker tag integr8scode-base:latest "$IMG/base:$TAG" docker tag integr8scode-backend:latest "$IMG/backend:$TAG" - docker tag integr8scode-cert-generator:latest "$IMG/cert-generator:$TAG" docker tag integr8scode-frontend:latest "$IMG/frontend:$TAG" - # Push all 4 images in parallel, tracking each PID + # Push both images in parallel, tracking each PID declare -A PIDS - for name in base backend cert-generator frontend; do + for name in backend frontend; do docker push "$IMG/$name:$TAG" & PIDS[$name]=$! done @@ -606,7 +592,6 @@ jobs: with: image-tag: ${{ needs.build-images.outputs.sha-tag }} github-token: ${{ secrets.GITHUB_TOKEN }} - install-kueue: 'false' # Phase 2: Node + Playwright setup (overlaps with k3s boot + image pull) - name: Setup Node.js @@ -642,7 +627,6 @@ jobs: with: image-tag: ${{ needs.build-images.outputs.sha-tag }} wait-for-frontend: 'true' - install-kueue: 'false' - name: Run Playwright tests timeout-minutes: 10 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ba8a95c16..4bcc2d53c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -32,10 +32,8 @@ jobs: fail-fast: false matrix: image: - - base - backend - frontend - - cert-generator steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -108,10 +106,8 @@ jobs: echo "Promoting tag: $TAG → latest" echo "" - crane copy "$REGISTRY/$PREFIX/base:$TAG" "$REGISTRY/$PREFIX/base:latest" crane copy "$REGISTRY/$PREFIX/backend:$TAG" "$REGISTRY/$PREFIX/backend:latest" crane copy "$REGISTRY/$PREFIX/frontend:$TAG" "$REGISTRY/$PREFIX/frontend:latest" - crane copy "$REGISTRY/$PREFIX/cert-generator:$TAG" "$REGISTRY/$PREFIX/cert-generator:latest" summary: name: Summary @@ -138,9 +134,8 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "| Image | Pull Command |" >> $GITHUB_STEP_SUMMARY echo "|-------|--------------|" >> $GITHUB_STEP_SUMMARY - echo "| Base | \`docker pull $REGISTRY/$PREFIX/base:latest\` |" >> $GITHUB_STEP_SUMMARY echo "| Backend | \`docker pull $REGISTRY/$PREFIX/backend:latest\` |" >> $GITHUB_STEP_SUMMARY echo "| Frontend | \`docker pull $REGISTRY/$PREFIX/frontend:latest\` |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "### Security Scans" >> $GITHUB_STEP_SUMMARY - echo "All 4 images scanned with Trivy (CRITICAL + HIGH, unfixed ignored)." >> $GITHUB_STEP_SUMMARY + echo "Both images scanned with Trivy (CRITICAL + HIGH, unfixed ignored)." >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/release-deploy.yml b/.github/workflows/release-deploy.yml index 36a358172..ffbff9bac 100644 --- a/.github/workflows/release-deploy.yml +++ b/.github/workflows/release-deploy.yml @@ -95,7 +95,7 @@ jobs: PREFIX="${GITHUB_REPOSITORY_OWNER,,}/integr8scode" VERSION="${{ steps.version.outputs.version }}" TAG="sha-${{ steps.sha.outputs.short-sha }}" - for img in base backend frontend cert-generator; do + for img in backend frontend; do crane copy "$REGISTRY/$PREFIX/$img:$TAG" "$REGISTRY/$PREFIX/$img:$VERSION" done @@ -159,14 +159,6 @@ jobs: command_timeout: 10m script: | set -e - # compose falls back to public defaults for these, so a missing secret must stop the deploy - : "${SECRET_KEY:?the JWT_SECRET_KEY secret is empty}" - : "${MONGO_USER:?the MONGO_USER secret is empty}" - : "${MONGO_PASSWORD:?the MONGO_PASSWORD secret is empty}" - : "${MONGO_MONITOR_PASSWORD:?the MONGO_MONITOR_PASSWORD secret is empty}" - : "${REDIS_PASSWORD:?the REDIS_PASSWORD secret is empty}" - : "${ADMIN_USER_PASSWORD:?the ADMIN_USER_PASSWORD secret is empty}" - : "${DEFAULT_USER_PASSWORD:?the DEFAULT_USER_PASSWORD secret is empty}" cd /opt/Integr8sCode # Images the running stack doesn't use; old tags can be pulled from GHCR again @@ -176,23 +168,28 @@ jobs: git fetch --tags --force origin git checkout --detach "$IMAGE_TAG" - export IMAGE_TAG="$IMAGE_TAG" - export COMPOSE_PROFILES=observability - export SECRET_KEY="$SECRET_KEY" - export MONGO_USER="$MONGO_USER" - export MONGO_PASSWORD="$MONGO_PASSWORD" - export MONGO_MONITOR_PASSWORD="$MONGO_MONITOR_PASSWORD" - export REDIS_PASSWORD="$REDIS_PASSWORD" - export ADMIN_USER_PASSWORD="$ADMIN_USER_PASSWORD" - export DEFAULT_USER_PASSWORD="$DEFAULT_USER_PASSWORD" - export GRAFANA_ROOT_URL="https://grafana.integr8scode.cc/" - export GRAFANA_ADMIN_USER="$GRAFANA_ADMIN_USER" - export GRAFANA_ADMIN_PASSWORD="$GRAFANA_ADMIN_PASSWORD" - export MAILJET_API_KEY="$MAILJET_API_KEY" - export MAILJET_SECRET_KEY="$MAILJET_SECRET_KEY" - export MAILJET_FROM_ADDRESS="$MAILJET_FROM_ADDRESS" - export GF_SMTP_HOST="$MAILJET_HOST" - export GRAFANA_ALERT_RECIPIENTS="$GRAFANA_ALERT_RECIPIENTS" + # compose reads .env for every command, so the backup cron and commands run by hand get the same values; it refuses to start on an empty secret + umask 077 + cat > .env < ({ - Run through npm scripts: `npm run test:e2e:deps` and `npm run test:e2e:browsers` once, then `npm run test:e2e` against the compose stack on :5001 (`npm run dev` can't reach the API from the host) - `frontend/e2e/fixtures.ts` registers one account per worker through the API and signs in the seeded `admin` (password from `ADMIN_USER_PASSWORD`, default `admin123`). Each test gets a fresh context: `page` (guest), `userPage` or `adminPage`; `api` is a client signed in as the worker's user and `execution` starts an execution through it - Specs are flat `test()` calls with no `describe` wrappers or helpers defined inside functions. Assert with retrying `expect`, never `waitForTimeout` or a polling loop; hold a request with `page.route(url, holdRequest)` to see a loading state. Labels come from `fieldTitle` and option lists from the generated enums and schemas -- Users are seeded by the `user-seed` compose service (`scripts/seed_users.py`) on every `./deploy.sh dev`; rerun it with `docker compose run --rm user-seed`. For manual checks log in as `user` / `user123` or `admin` / `admin123` +- The backend container seeds the users (`scripts/seed_users.py`) before gunicorn starts; rerun it with `docker compose exec backend python -m scripts.seed_users`. For manual checks log in as `user` / `user123` or `admin` / `admin123` ## CI Gates (All Must Pass) diff --git a/README.md b/README.md index 4737cadbf..27b845be2 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ ### Prerequisites - Docker and Docker Compose -- Kubernetes cluster (k3s, Docker Desktop K8s, or minikube) with `kubectl` configured +- k3s, installed with the flags in [Getting started](https://hardmax71.github.io/Integr8sCode/getting-started/) ### Quick start @@ -69,7 +69,7 @@ cd Integr8sCode ./deploy.sh dev ``` -Secrets (MongoDB, Redis, JWT) fall back to dev defaults, so local development needs no config. For production, set `SECRET_KEY`, `MONGO_USER`, `MONGO_PASSWORD` and `REDIS_PASSWORD` as env vars, plus `MONGO_MONITOR_PASSWORD` with `--observability`. +`./deploy.sh dev` copies `.env.example` to `.env` on its first run, so local development needs no config. Compose has no defaults for secrets: anywhere else, write a `.env` with your own values, and compose refuses to start while one is missing. ### Verify @@ -89,7 +89,7 @@ curl -k https://localhost/api/v1/health/live `./deploy.sh dev --observability` adds Grafana, Jaeger, Victoria Metrics, Loki and the OTel Collector. `--debug` adds all of those plus Kafdrop. -Self-signed TLS certs are generated automatically, so accept the browser warning. +nginx and the backend each make a self-signed certificate when they first start, so accept the browser warning. nginx verifies the backend's certificate on every proxied request. A certificate mounted at `/etc/nginx/certs` in the frontend container is used as is. ### Run tests diff --git a/backend/.trivyignore b/backend/.trivyignore index bcd77ed34..0640113b6 100644 --- a/backend/.trivyignore +++ b/backend/.trivyignore @@ -1,6 +1 @@ # CVE IDs Trivy should ignore, one per line, each with a reason comment above it. - -# cert-generator: kubectl v1.37.1, the newest release, is built with Go 1.26.8; it only talks to the local k3s API during setup -CVE-2026-78667 -CVE-2026-78669 -CVE-2026-97031 diff --git a/backend/Dockerfile b/backend/Dockerfile index bb01914df..eebcd04b4 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,21 +1,84 @@ -# Backend API service +# Backend image, used by the API and the worker +# The builder stage keeps gcc and the dev headers out of the final image + +FROM python:3.14-slim AS builder + +WORKDIR /app + +# Install build-time dependencies (gcc + dev headers for C extensions) +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + gcc \ + liblzma-dev \ + libsnappy-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install uv (using Docker Hub mirror - ghcr.io has rate limiting issues) +COPY --from=astral/uv:0.13.0 /uv /uvx /bin/ + +# Pre-compile bytecode for faster startup; copy mode avoids symlink issues with cache mounts +ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy + +# Copy dependency files +COPY pyproject.toml uv.lock ./ + +# Install Python dependencies with BuildKit cache mount for faster rebuilds +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --locked --no-dev --no-install-project --no-build + +# Python, runtime libs and the locked dependencies; CI builds and caches this stage on its own +FROM python:3.14-slim AS base + +WORKDIR /app + +# Runtime libs only (no -dev headers, no gcc); uv installs the app, so pip goes too, since its vendored packages trip the scan +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ + curl \ + liblzma5 \ + libsnappy1v5 \ + && rm -rf /var/lib/apt/lists/* \ + && python -m pip uninstall --yes --root-user-action=ignore pip + +# Copy uv from builder (avoids second Docker Hub pull, guarantees version consistency) +COPY --from=builder /bin/uv /bin/uvx /bin/ + +# Copy pre-built virtual environment from builder stage +COPY --from=builder /app/.venv /app/.venv + +# Copy dependency files (needed for uv to recognize the project) +COPY pyproject.toml uv.lock ./ + +# Set paths: PYTHONPATH for imports, PATH for venv binaries (no uv run needed at runtime) +ENV PYTHONPATH=/app +ENV PATH="/app/.venv/bin:$PATH" +# OpenTelemetry instrumentations emit the stable HTTP semantic conventions (http.server.request.duration) +ENV OTEL_SEMCONV_STABILITY_OPT_IN=http + FROM base # Copy application files and configuration COPY ./app /app/app COPY ./scripts /app/scripts COPY config.toml /app/config.toml -COPY openssl.cnf /app/openssl.cnf -# Ensure the certs directory exists -RUN mkdir -p /app/certs +# Mount points for the key (backend only) and the certificate (nginx verifies the backend against it) +RUN mkdir -p /app/certs/key /app/certs/cert # Create entrypoint script inline (BuildKit heredoc) -# Note: cert wait is handled by docker-compose depends_on (cert-generator: service_completed_successfully) +# The certificate is made once and kept in its volume, since nginx loads it only when nginx starts COPY <<'EOF' /entrypoint.sh #!/bin/bash set -e +if [ ! -s /app/certs/key/server.key ] || [ ! -s /app/certs/cert/server.crt ]; then + openssl req -x509 -newkey rsa:2048 -nodes -days 3650 -quiet -subj "/CN=backend" \ + -addext "subjectAltName=DNS:backend,DNS:localhost,IP:127.0.0.1" \ + -keyout /app/certs/key/server.key -out /app/certs/cert/server.crt +fi + +python -m scripts.seed_users + echo "Starting application..." exec gunicorn 'app.main:create_app()' \ @@ -30,8 +93,8 @@ exec gunicorn 'app.main:create_app()' \ --log-level info \ --access-logfile - \ --error-logfile - \ - --keyfile /app/certs/server.key \ - --certfile /app/certs/server.crt + --keyfile /app/certs/key/server.key \ + --certfile /app/certs/cert/server.crt EOF RUN chmod +x /entrypoint.sh diff --git a/backend/Dockerfile.base b/backend/Dockerfile.base deleted file mode 100644 index db7ed0736..000000000 --- a/backend/Dockerfile.base +++ /dev/null @@ -1,57 +0,0 @@ -# Shared base image for all backend services -# Contains: Python, system deps, uv, and all Python dependencies -# Multi-stage build: gcc + dev headers only in builder, not in final image - -FROM python:3.14-slim AS builder - -WORKDIR /app - -# Install build-time dependencies (gcc + dev headers for C extensions) -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - gcc \ - liblzma-dev \ - libsnappy-dev \ - && rm -rf /var/lib/apt/lists/* - -# Install uv (using Docker Hub mirror - ghcr.io has rate limiting issues) -COPY --from=astral/uv:0.13.0 /uv /uvx /bin/ - -# Pre-compile bytecode for faster startup; copy mode avoids symlink issues with cache mounts -ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy - -# Copy dependency files -COPY pyproject.toml uv.lock ./ - -# Install Python dependencies with BuildKit cache mount for faster rebuilds -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-dev --no-install-project --no-build - -FROM python:3.14-slim - -WORKDIR /app - -# Runtime libs only (no -dev headers, no gcc); uv installs the app, so pip goes too, since its vendored packages trip the scan -RUN apt-get update && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends \ - curl \ - liblzma5 \ - libsnappy1v5 \ - && rm -rf /var/lib/apt/lists/* \ - && python -m pip uninstall --yes --root-user-action=ignore pip - -# Copy uv from builder (avoids second Docker Hub pull, guarantees version consistency) -COPY --from=builder /bin/uv /bin/uvx /bin/ - -# Copy pre-built virtual environment from builder stage -COPY --from=builder /app/.venv /app/.venv - -# Copy dependency files (needed for uv to recognize the project) -COPY pyproject.toml uv.lock ./ - -# Set paths: PYTHONPATH for imports, PATH for venv binaries (no uv run needed at runtime) -ENV PYTHONPATH=/app -ENV PATH="/app/.venv/bin:$PATH" -ENV KUBECONFIG=/app/kubeconfig.yaml -# OpenTelemetry instrumentations emit the stable HTTP semantic conventions (http.server.request.duration) -ENV OTEL_SEMCONV_STABILITY_OPT_IN=http diff --git a/backend/app/core/container.py b/backend/app/core/container.py index 2f5ff6b41..fe6d20508 100644 --- a/backend/app/core/container.py +++ b/backend/app/core/container.py @@ -57,7 +57,6 @@ def create_app_container(settings: Settings) -> AsyncContainer: AdminServicesProvider(), EventReplayProvider(), BusinessServicesProvider(), - KubernetesProvider(), FastapiProvider(), context={Settings: settings}, ) diff --git a/backend/app/core/providers.py b/backend/app/core/providers.py index 5b5554c6c..778338aa6 100644 --- a/backend/app/core/providers.py +++ b/backend/app/core/providers.py @@ -228,8 +228,12 @@ async def get_api_client( self, settings: Settings, logger: structlog.stdlib.BoundLogger ) -> AsyncIterator[k8s_client.ApiClient]: """Provide Kubernetes ApiClient with config loading and cleanup.""" - await k8s_config.load_kube_config(config_file=settings.KUBERNETES_CONFIG_PATH) - api_client = k8s_client.ApiClient() + configuration = k8s_client.Configuration() + await k8s_config.load_kube_config( + config_file=settings.KUBERNETES_CONFIG_PATH, client_configuration=configuration + ) + configuration.host = settings.KUBERNETES_API_SERVER + api_client = k8s_client.ApiClient(configuration) logger.info("Kubernetes API client ready", host=api_client.configuration.host) try: yield api_client diff --git a/backend/app/services/k8s_worker/pod_builder.py b/backend/app/services/k8s_worker/pod_builder.py index 27634c998..ba7429436 100644 --- a/backend/app/services/k8s_worker/pod_builder.py +++ b/backend/app/services/k8s_worker/pod_builder.py @@ -162,7 +162,6 @@ def _build_pod_metadata( ) -> k8s_client.V1ObjectMeta: """Build pod metadata with saga tracking""" labels = {"app": "integr8s", "component": "executor", "execution-id": execution_id, "language": language} - labels["kueue.x-k8s.io/queue-name"] = "executor-queue" labels["user-id"] = user_id[:63] # K8s label value limit diff --git a/backend/app/services/k8s_worker/worker.py b/backend/app/services/k8s_worker/worker.py index b85a1e8a7..11794e488 100644 --- a/backend/app/services/k8s_worker/worker.py +++ b/backend/app/services/k8s_worker/worker.py @@ -249,15 +249,17 @@ async def _publish_pod_creation_failed(self, command: CreatePodCommandEvent, err await self.producer.produce(event_to_produce=event, key=command.execution_id) async def ensure_namespace_security(self) -> None: - """Apply security controls to the executor namespace at startup. + """Create the executor namespace and apply its security controls at startup. - Creates: + Applies: + - The namespace with Pod Security Admission labels (Restricted profile) - Default-deny NetworkPolicy for executor pods (blocks lateral movement and exfiltration) - - Pod Security Admission labels (Restricted profile) + - ResourceQuota capping what all executor pods together may request """ namespace = self._settings.K8S_NAMESPACE + await self._apply_namespace(namespace) await self._ensure_executor_network_policy(namespace) - await self._apply_psa_labels(namespace) + await self._ensure_resource_quota(namespace) async def _ensure_executor_network_policy(self, namespace: str) -> None: """Create or update default-deny NetworkPolicy for executor pods.""" @@ -289,17 +291,57 @@ async def _ensure_executor_network_policy(self, namespace: str) -> None: ) self.logger.info("NetworkPolicy applied", policy_name=policy_name, namespace=namespace) - async def _apply_psa_labels(self, namespace: str) -> None: - """Apply Pod Security Admission labels to the executor namespace.""" + async def _apply_namespace(self, namespace: str) -> None: + """Create or update the executor namespace with its Pod Security Admission labels.""" psa_labels = { "pod-security.kubernetes.io/enforce": "restricted", "pod-security.kubernetes.io/enforce-version": "latest", "pod-security.kubernetes.io/warn": "restricted", "pod-security.kubernetes.io/audit": "restricted", } + body = k8s_client.V1Namespace( + api_version="v1", + kind="Namespace", + metadata=k8s_client.V1ObjectMeta(name=namespace, labels=psa_labels), + ) + + await self.v1.patch_namespace( # type: ignore[call-arg] + name=namespace, + body=body, + field_manager="integr8s", + force=True, + _content_type="application/apply-patch+yaml", + ) + self.logger.info("Namespace applied with Pod Security Admission labels", namespace=namespace) + + async def _ensure_resource_quota(self, namespace: str) -> None: + """Create or update the quota on what all pods in the executor namespace may request together.""" + quota_name = "executor-quota" + quota = k8s_client.V1ResourceQuota( + api_version="v1", + kind="ResourceQuota", + metadata=k8s_client.V1ObjectMeta( + name=quota_name, + namespace=namespace, + labels={"app": "integr8s", "component": "security"}, + ), + spec=k8s_client.V1ResourceQuotaSpec( + hard={ + "requests.cpu": self._settings.K8S_NAMESPACE_CPU_QUOTA, + "requests.memory": self._settings.K8S_NAMESPACE_MEMORY_QUOTA, + } + ), + ) - await self.v1.patch_namespace(name=namespace, body={"metadata": {"labels": psa_labels}}) - self.logger.info("Pod Security Admission labels applied", namespace=namespace) + await self.v1.patch_namespaced_resource_quota( # type: ignore[call-arg] + name=quota_name, + namespace=namespace, + body=quota, + field_manager="integr8s", + force=True, + _content_type="application/apply-patch+yaml", + ) + self.logger.info("ResourceQuota applied", quota_name=quota_name, namespace=namespace) async def ensure_image_pre_puller_daemonset(self) -> None: """Ensure the runtime image pre-puller DaemonSet exists.""" diff --git a/backend/app/services/pod_monitor/monitor.py b/backend/app/services/pod_monitor/monitor.py index 6d301d11b..dfc8adb07 100644 --- a/backend/app/services/pod_monitor/monitor.py +++ b/backend/app/services/pod_monitor/monitor.py @@ -223,7 +223,7 @@ def _track_pod_phase(self, event_type: WatchEventType, pod: k8s_client.V1Pod, ph async def _delete_pod(self, pod: k8s_client.V1Pod) -> None: """Delete a pod after its data has been fully extracted. - Frees the Kueue quota slot so gated executor pods can be admitted. + Frees its share of the namespace's ResourceQuota. The ConfigMap is garbage-collected automatically via ownerReference. """ pod_name = pod.metadata.name diff --git a/backend/app/settings.py b/backend/app/settings.py index 02dc495da..bc6eb2015 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -78,18 +78,22 @@ def __init__( MONGO_PASSWORD: str = "" MONGODB_URL: str = "" KUBERNETES_CONFIG_PATH: str = "~/.kube/config" - KUBERNETES_CA_CERTIFICATE_PATH: str | None = None + # k3s writes 127.0.0.1 into its kubeconfig; a container reaches the same API through the host + KUBERNETES_API_SERVER: str = "https://host.docker.internal:6443" RATE_LIMITS: str = "100/minute" RATE_LIMIT_ENABLED: bool = True # Set to False to disable rate limiting entirely - SSL_KEYFILE: str = "/app/certs/server.key" - SSL_CERTFILE: str = "/app/certs/server.crt" + SSL_KEYFILE: str = "/app/certs/key/server.key" + SSL_CERTFILE: str = "/app/certs/cert/server.crt" SERVER_HOST: str = "localhost" SERVER_PORT: int = 443 # Kubernetes namespace for execution pods K8S_NAMESPACE: str = "integr8scode" + # ResourceQuota on what all pods in that namespace may request together + K8S_NAMESPACE_CPU_QUOTA: str = "32" + K8S_NAMESPACE_MEMORY_QUOTA: str = "4Gi" # Settings for Kubernetes resource limits and requests K8S_POD_CPU_LIMIT: str = "1000m" diff --git a/backend/config.test.toml b/backend/config.test.toml index e1e70f4ab..c5f4a247b 100644 --- a/backend/config.test.toml +++ b/backend/config.test.toml @@ -7,8 +7,7 @@ PROJECT_NAME = "integr8scode" ALGORITHM = "HS256" ACCESS_TOKEN_EXPIRE_MINUTES = 1440 -KUBERNETES_CONFIG_PATH = "/app/kubeconfig.yaml" -KUBERNETES_CA_CERTIFICATE_PATH = "/app/certs/k8s-ca.pem" +KUBERNETES_CONFIG_PATH = "/etc/rancher/k3s/k3s.yaml" K8S_POD_CPU_LIMIT = "1000m" K8S_POD_MEMORY_LIMIT = "128Mi" K8S_POD_CPU_REQUEST = "200m" diff --git a/backend/config.toml b/backend/config.toml index 1ca84a4c2..cd6604d1b 100644 --- a/backend/config.toml +++ b/backend/config.toml @@ -14,8 +14,7 @@ MONGO_DB = "integr8scode" # --8<-- [end:core] # --8<-- [start:kubernetes] -KUBERNETES_CONFIG_PATH = "/app/kubeconfig.yaml" -KUBERNETES_CA_CERTIFICATE_PATH = "/app/certs/k8s-ca.pem" +KUBERNETES_CONFIG_PATH = "/etc/rancher/k3s/k3s.yaml" K8S_POD_CPU_LIMIT = "1000m" K8S_POD_MEMORY_LIMIT = "128Mi" K8S_POD_CPU_REQUEST = "200m" diff --git a/backend/openssl.cnf b/backend/openssl.cnf deleted file mode 100644 index fc54c921b..000000000 --- a/backend/openssl.cnf +++ /dev/null @@ -1,21 +0,0 @@ -[ req ] -default_bits = 4096 -prompt = no -default_md = sha256 -req_extensions = req_ext -distinguished_name = dn - -[ dn ] -C = US -ST = Test -L = Test -O = Test -CN = localhost - -[ req_ext ] -subjectAltName = @alt_names - -[ alt_names ] -DNS.1 = localhost -IP.1 = 127.0.0.1 -IP.2 = ::1 diff --git a/backend/scripts/seed_users.py b/backend/scripts/seed_users.py index 9475a9367..7af1c7de2 100755 --- a/backend/scripts/seed_users.py +++ b/backend/scripts/seed_users.py @@ -5,15 +5,17 @@ Creates: 1. Default user (role=user) for testing/demo 2. Admin user (role=admin, is_superuser=True) for administration + 3. The collector's monitoring user, with a role that can only read server and index stats Also drops duplicate system settings documents, which an old get-or-insert race could create and which would stop the unique settings_id index from building, and rewrites the stored theme -"auto" to its current name "system". Compose starts the API and the workers only after this -script succeeds. +"auto" to its current name "system". The backend container runs this script before gunicorn +starts, and the worker starts once the backend is healthy. -Uses main Settings for MongoDB connection. Password env vars are script-specific: - DEFAULT_USER_PASSWORD: Default user password (default: user123) - ADMIN_USER_PASSWORD: Admin user password (default: admin123) +Uses main Settings for MongoDB connection. Password env vars are script-specific and required: + DEFAULT_USER_PASSWORD: Default user password + ADMIN_USER_PASSWORD: Admin user password + MONGO_MONITOR_USER, MONGO_MONITOR_PASSWORD: the collector's mongodb receiver login """ import asyncio @@ -97,11 +99,25 @@ async def rename_auto_theme(db: AsyncDatabase[dict[str, Any]]) -> None: print(f"Renamed the auto theme to system in {result.modified_count} {collection.name} documents") +async def sync_monitor_user(client: AsyncMongoClient[dict[str, Any]], db_name: str, user: str, password: str) -> None: + """Create or update the collector's mongodb receiver login; root fails $indexStats on admin.system.*.""" + admin = client.admin + privileges = [ + {"resource": {"cluster": True}, "actions": ["serverStatus", "top"]}, + {"resource": {"db": db_name, "collection": ""}, "actions": ["dbStats", "indexStats", "listCollections"]}, + ] + role_exists = (await admin.command("rolesInfo", "otelMonitor"))["roles"] + await admin.command("updateRole" if role_exists else "createRole", "otelMonitor", privileges=privileges, roles=[]) + user_exists = (await admin.command("usersInfo", user))["users"] + await admin.command("updateUser" if user_exists else "createUser", user, pwd=password, roles=["otelMonitor"]) + print(f"Synced monitoring user '{user}'") + + async def seed_users(settings: Settings) -> None: """Seed default users using provided settings for MongoDB connection.""" pwd_hasher = PasswordHash((BcryptHasher(rounds=settings.BCRYPT_ROUNDS),)) - default_password = os.environ.get("DEFAULT_USER_PASSWORD", "user123") - admin_password = os.environ.get("ADMIN_USER_PASSWORD", "admin123") + default_password = os.environ["DEFAULT_USER_PASSWORD"] + admin_password = os.environ["ADMIN_USER_PASSWORD"] client: AsyncMongoClient[dict[str, Any]] = AsyncMongoClient(settings.MONGODB_URL) db = client.get_default_database(default=settings.MONGO_DB) @@ -109,6 +125,7 @@ async def seed_users(settings: Settings) -> None: await drop_duplicate_system_settings(db) await rename_auto_theme(db) + await sync_monitor_user(client, db.name, os.environ["MONGO_MONITOR_USER"], os.environ["MONGO_MONITOR_PASSWORD"]) # Default user await upsert_user( @@ -132,12 +149,7 @@ async def seed_users(settings: Settings) -> None: is_superuser=True, ) - print("\n" + "=" * 50) - print("SEEDED USERS:") - print("=" * 50) - print("Default: user / user123 (or DEFAULT_USER_PASSWORD)") - print("Admin: admin / admin123 (or ADMIN_USER_PASSWORD)") - print("=" * 50) + print("Seeded users 'user' and 'admin', passwords from DEFAULT_USER_PASSWORD and ADMIN_USER_PASSWORD") await client.close() diff --git a/backend/tests/unit/core/test_kubernetes_provider.py b/backend/tests/unit/core/test_kubernetes_provider.py new file mode 100644 index 000000000..77ae2aba9 --- /dev/null +++ b/backend/tests/unit/core/test_kubernetes_provider.py @@ -0,0 +1,54 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import structlog +from dishka import Provider, Scope, make_async_container +from kubernetes_asyncio import client as k8s_client + +from app.core.providers import KubernetesProvider +from app.settings import Settings + +pytestmark = pytest.mark.unit + +K3S_KUBECONFIG = """\ +apiVersion: v1 +kind: Config +clusters: +- name: default + cluster: + server: https://127.0.0.1:6443 + insecure-skip-tls-verify: true +users: +- name: default + user: + token: k3s-token +contexts: +- name: default + context: + cluster: default + user: default +current-context: default +""" + + +async def test_api_client_reaches_k3s_through_the_configured_server(test_settings: Settings, tmp_path: Path) -> None: + kubeconfig = tmp_path / "k3s.yaml" + kubeconfig.write_text(K3S_KUBECONFIG) + settings = test_settings.model_copy( + update={"KUBERNETES_CONFIG_PATH": str(kubeconfig), "KUBERNETES_API_SERVER": "https://k3s.example:6443"} + ) + context = Provider(scope=Scope.APP) + context.from_context(provides=Settings, scope=Scope.APP) + context.from_context(provides=structlog.stdlib.BoundLogger, scope=Scope.APP) + container = make_async_container( + KubernetesProvider(), + context, + context={Settings: settings, structlog.stdlib.BoundLogger: MagicMock(spec=structlog.stdlib.BoundLogger)}, + ) + + api_client = await container.get(k8s_client.ApiClient) + + assert api_client.configuration.host == "https://k3s.example:6443" + assert api_client.configuration.api_key == {"BearerToken": "Bearer k3s-token"} + await container.close() diff --git a/backend/tests/unit/services/test_k8s_worker_namespace.py b/backend/tests/unit/services/test_k8s_worker_namespace.py new file mode 100644 index 000000000..a0e45c5ea --- /dev/null +++ b/backend/tests/unit/services/test_k8s_worker_namespace.py @@ -0,0 +1,49 @@ +from unittest.mock import AsyncMock, MagicMock + +import pytest +import structlog +from kubernetes_asyncio import client as k8s_client +from opentelemetry import trace + +from app.events.core import UnifiedProducer +from app.services.k8s_worker import KubernetesWorker +from app.settings import Settings + +pytestmark = pytest.mark.unit + +SERVER_SIDE_APPLY = {"field_manager": "integr8s", "force": True, "_content_type": "application/apply-patch+yaml"} + + +async def test_namespace_security_applies_namespace_policy_and_quota( + test_settings: Settings, tracer: trace.Tracer +) -> None: + settings = test_settings.model_copy(update={"K8S_NAMESPACE_CPU_QUOTA": "8", "K8S_NAMESPACE_MEMORY_QUOTA": "2Gi"}) + worker = KubernetesWorker( + api_client=MagicMock(spec=k8s_client.ApiClient), + producer=AsyncMock(spec=UnifiedProducer), + settings=settings, + logger=MagicMock(spec=structlog.stdlib.BoundLogger), + tracer=tracer, + ) + core = MagicMock(patch_namespace=AsyncMock(), patch_namespaced_resource_quota=AsyncMock()) + networking = MagicMock(patch_namespaced_network_policy=AsyncMock()) + worker.v1 = core + worker.networking_v1 = networking + + await worker.ensure_namespace_security() + + namespace_call = core.patch_namespace.await_args + assert namespace_call is not None + assert namespace_call.kwargs["name"] == settings.K8S_NAMESPACE + assert namespace_call.kwargs["body"].metadata.labels["pod-security.kubernetes.io/enforce"] == "restricted" + assert {k: namespace_call.kwargs[k] for k in SERVER_SIDE_APPLY} == SERVER_SIDE_APPLY + + policy_call = networking.patch_namespaced_network_policy.await_args + assert policy_call is not None + assert policy_call.kwargs["body"].spec.policy_types == ["Ingress", "Egress"] + + quota_call = core.patch_namespaced_resource_quota.await_args + assert quota_call is not None + assert quota_call.kwargs["namespace"] == settings.K8S_NAMESPACE + assert quota_call.kwargs["body"].spec.hard == {"requests.cpu": "8", "requests.memory": "2Gi"} + assert {k: quota_call.kwargs[k] for k in SERVER_SIDE_APPLY} == SERVER_SIDE_APPLY diff --git a/cert-generator/Dockerfile b/cert-generator/Dockerfile deleted file mode 100644 index 1057a8a2b..000000000 --- a/cert-generator/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM alpine:3.24 - -# Pin versions for reproducible builds -# kubectl: Use supported version (N-2 policy: 1.37, 1.36, 1.35 as of Oct 2026) -ARG KUBECTL_VERSION=v1.37.1 - -# Install required packages and tools for all architectures -RUN apk upgrade --no-cache && \ - apk add --no-cache ca-certificates curl dos2unix iproute2 netcat-openbsd openssl && \ - update-ca-certificates && \ - # Detect architecture and install appropriate binaries - ARCH=$(uname -m); \ - case "$ARCH" in \ - x86_64) KUBECTL_ARCH=amd64 ;; \ - aarch64) KUBECTL_ARCH=arm64 ;; \ - armv7l) KUBECTL_ARCH=arm ;; \ - *) echo "Unsupported architecture: $ARCH" && exit 1 ;; \ - esac && \ - # Install kubectl (pinned version for reproducibility) - curl --proto =https --tlsv1.2 -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${KUBECTL_ARCH}/kubectl" -o /usr/local/bin/kubectl && \ - chmod +x /usr/local/bin/kubectl - -# Create shared directory for root CA -RUN mkdir -p /shared_ca - -# Copy and prepare setup scripts -COPY setup-k8s.sh /setup-k8s.sh -COPY setup-certs.sh /setup-certs.sh -RUN dos2unix /setup-k8s.sh /setup-certs.sh && chmod +x /setup-k8s.sh /setup-certs.sh - -# Run both setup scripts (k8s setup failure must not block cert generation) -ENTRYPOINT ["/bin/sh", "-c", "/setup-k8s.sh; /setup-certs.sh"] diff --git a/cert-generator/setup-certs.sh b/cert-generator/setup-certs.sh deleted file mode 100644 index 5f4b8d954..000000000 --- a/cert-generator/setup-certs.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -set -e - -echo "Generating certificates..." - -BACKEND_CERT_DIR=${BACKEND_CERT_DIR:-/backend-certs} -FRONTEND_CERT_DIR=${FRONTEND_CERT_DIR:-/frontend-certs} - -mkdir -p "$BACKEND_CERT_DIR" "$FRONTEND_CERT_DIR" -echo "BACKEND_CERT_DIR=${BACKEND_CERT_DIR}" -echo "FRONTEND_CERT_DIR=${FRONTEND_CERT_DIR}" - -# Generate Self-Signed Certificate -openssl req -x509 -newkey rsa:2048 -nodes \ - -keyout "$BACKEND_CERT_DIR/server.key" \ - -out "$BACKEND_CERT_DIR/server.crt" \ - -subj "/CN=backend" \ - -addext "subjectAltName = DNS:backend,DNS:localhost,IP:127.0.0.1,IP:::1" \ - -days 365 -echo "Created cert: $BACKEND_CERT_DIR/server.crt" -echo "Created key: $BACKEND_CERT_DIR/server.key" -echo "Certificate fingerprint (SHA256): $(openssl x509 -in "$BACKEND_CERT_DIR/server.crt" -noout -sha256 -fingerprint | sed 's/^.*=//')" - -# Copy for frontend -cp "$BACKEND_CERT_DIR/server.crt" "$FRONTEND_CERT_DIR/server.crt" -cp "$BACKEND_CERT_DIR/server.key" "$FRONTEND_CERT_DIR/server.key" -echo "Copied cert/key to frontend directory" - -# Copy to shared CA directory -if [ -n "$SHARED_CA_DIR" ]; then - cp "$BACKEND_CERT_DIR/server.crt" "$SHARED_CA_DIR/ca.pem" - echo "Certificate copied to shared CA directory" -fi - -echo "✅ Certificates generated" diff --git a/cert-generator/setup-k8s.sh b/cert-generator/setup-k8s.sh deleted file mode 100644 index d931f1e1d..000000000 --- a/cert-generator/setup-k8s.sh +++ /dev/null @@ -1,311 +0,0 @@ -#!/bin/sh -set -e - -echo "Setting up Kubernetes resources..." - -# Auto-configure kubectl for k3s if needed -# k3s stores its kubeconfig at /etc/rancher/k3s/k3s.yaml -# When running in bridge network, we need to use the routable host IP instead of 127.0.0.1 -configure_kubectl() { - # If kubectl already works, nothing to do - if kubectl version --request-timeout=2s >/dev/null 2>&1; then - return 0 - fi - # Try k3s kubeconfig with routable IP (for bridge network containers) - if [ -r /etc/rancher/k3s/k3s.yaml ]; then - # Get the k3s node-ip from config (routable from containers) - K3S_HOST_IP="" - if [ -r /etc/rancher/k3s/config.yaml ]; then - K3S_HOST_IP=$(grep -E '^node-ip:' /etc/rancher/k3s/config.yaml 2>/dev/null | sed -E 's/^node-ip:[[:space:]]*"?([^"[:space:]]+)"?.*/\1/' | head -1) - fi - # If no node-ip found, try to detect host from container (for CI/Docker environments) - if [ -z "$K3S_HOST_IP" ] || [ "$K3S_HOST_IP" = "127.0.0.1" ]; then - # Prefer host.docker.internal (works with TLS cert SANs, requires extra_hosts in compose) - if getent hosts host.docker.internal >/dev/null 2>&1; then - K3S_HOST_IP="host.docker.internal" - fi - fi - if [ -z "$K3S_HOST_IP" ] || [ "$K3S_HOST_IP" = "127.0.0.1" ]; then - # Fallback: Docker gateway (may need insecure TLS if IP not in cert SANs) - K3S_HOST_IP=$(ip route 2>/dev/null | grep default | awk '{print $3}' | head -1) - fi - if [ -n "$K3S_HOST_IP" ] && [ "$K3S_HOST_IP" != "127.0.0.1" ]; then - # Create modified kubeconfig with routable IP/hostname - # Handle both 127.0.0.1 and 0.0.0.0 (k3s may use either depending on config) - mkdir -p /tmp/kube - sed -E "s#https://(127\.0\.0\.1|0\.0\.0\.0):#https://${K3S_HOST_IP}:#g" /etc/rancher/k3s/k3s.yaml > /tmp/kube/config - export KUBECONFIG=/tmp/kube/config - echo "Using k3s kubeconfig with routable IP: $K3S_HOST_IP" - else - export KUBECONFIG=/etc/rancher/k3s/k3s.yaml - echo "Using k3s kubeconfig: $KUBECONFIG" - fi - return 0 - fi - return 1 -} - -configure_kubectl || true - -# In CI mode, skip k8s setup if connection fails -if [ -n "$CI" ]; then - echo "Running in CI mode" - if ! kubectl version --request-timeout=5s >/dev/null 2>&1; then - echo "WARNING: Cannot connect to Kubernetes in CI - skipping k8s setup" - echo "Creating dummy kubeconfig for CI..." - mkdir -p /backend - cat > /backend/kubeconfig.yaml </dev/null 2>&1; then - echo "ERROR: Cannot connect to Kubernetes cluster" - exit 1 -fi -echo "Connected to Kubernetes" - -# Create namespace -kubectl create namespace integr8scode --dry-run=client -o yaml | kubectl apply -f - - -# Install Kueue (scheduling-gate based quota management) -KUEUE_VERSION="${KUEUE_VERSION:-v0.20.1}" -KUEUE_MANIFEST_SHA256="${KUEUE_MANIFEST_SHA256:-c48514295365aae0c7dc578c70330befdd0015fec57cb4a06ba872bbef8750dd}" -echo "Installing Kueue ${KUEUE_VERSION}..." -KUEUE_MANIFEST="$(mktemp /tmp/kueue-manifests.XXXXXXXXXX)" -curl --proto =https --tlsv1.2 -fsSL -o "$KUEUE_MANIFEST" "https://github.com/kubernetes-sigs/kueue/releases/download/${KUEUE_VERSION}/manifests.yaml" -echo "${KUEUE_MANIFEST_SHA256} ${KUEUE_MANIFEST}" | sha256sum -c - -kubectl apply --server-side -f "$KUEUE_MANIFEST" -rm -f "$KUEUE_MANIFEST" -# Kueue's cert bootstrap answers /readyz with 404 until the kubelet mounts its new secret, which can take minutes -kubectl wait --for=condition=Available --timeout=300s \ - deployment/kueue-controller-manager -n kueue-system - -# Kueue resources: ResourceFlavor + ClusterQueue + LocalQueue -kubectl apply --server-side -f - <<'KUEUE_EOF' -apiVersion: kueue.x-k8s.io/v1beta2 -kind: ResourceFlavor -metadata: - name: default-flavor ---- -apiVersion: kueue.x-k8s.io/v1beta2 -kind: ClusterQueue -metadata: - name: executor-queue -spec: - namespaceSelector: {} - resourceGroups: - - coveredResources: ["cpu", "memory"] - flavors: - - name: default-flavor - resources: - - name: cpu - nominalQuota: "32" - - name: memory - nominalQuota: "4Gi" ---- -apiVersion: kueue.x-k8s.io/v1beta2 -kind: LocalQueue -metadata: - name: executor-queue - namespace: integr8scode -spec: - clusterQueue: executor-queue -KUEUE_EOF -echo "Kueue installed and configured" - -# Create ServiceAccount -kubectl apply -f - </dev/null; then - echo "ERROR: the token controller did not fill secret integr8scode-sa-token" - exit 1 -fi -TOKEN_B64=$(kubectl get secret integr8scode-sa-token -n default -o jsonpath='{.data.token}') -TOKEN=$(printf %s "$TOKEN_B64" | base64 -d) -TOKEN_LEN=$(printf %s "$TOKEN" | wc -c | awk '{print $1}') -TOKEN_HEAD=$(printf %s "$TOKEN" | cut -c1-10) -echo "ServiceAccount token acquired (len=${TOKEN_LEN}, head=${TOKEN_HEAD}...)" - -# Determine the host IP that containers can reach -# Priority: 1) k3s node-ip config, 2) server URL from kubeconfig, 3) fallback to host.docker.internal -get_container_host_ip() { - # Try k3s config node-ip (most reliable for k3s setups) - if [ -f /etc/rancher/k3s/config.yaml ]; then - K3S_NODE_IP=$(grep -E '^node-ip:' /etc/rancher/k3s/config.yaml 2>/dev/null | sed -E 's/^node-ip:[[:space:]]*"?([^"[:space:]]+)"?.*/\1/' | head -1) - if [ -n "$K3S_NODE_IP" ] && [ "$K3S_NODE_IP" != "127.0.0.1" ]; then - echo "$K3S_NODE_IP" - return - fi - fi - # Try extracting from kubeconfig server URL (if not localhost) - if [ -n "$SERVER_HOST" ] && [ "$SERVER_HOST" != "127.0.0.1" ] && [ "$SERVER_HOST" != "localhost" ]; then - echo "$SERVER_HOST" - return - fi - # Fallback to host.docker.internal (works on Docker Desktop, may need extra_hosts on Linux) - echo "host.docker.internal" -} - -CONTAINER_HOST_IP=$(get_container_host_ip) -CONTAINER_SERVER="https://${CONTAINER_HOST_IP}:${K8S_PORT}" -echo "Detected container-accessible host IP: ${CONTAINER_HOST_IP}" - -echo "Writing kubeconfig for containers:" -echo " cluster: ${CLUSTER_NAME}" -echo " server (container view): ${CONTAINER_SERVER}" -echo " tls-server-name (cert host): ${SERVER_HOST}" - -cat > /backend/kubeconfig.yaml </dev/null || exit 1"] + test: ["CMD-SHELL", "curl -f -s --cacert /app/certs/cert/server.crt https://localhost:443/api/v1/health/live >/dev/null || exit 1"] interval: 2s timeout: 3s retries: 30 @@ -161,13 +112,11 @@ services: context: ./frontend dockerfile: Dockerfile depends_on: - cert-generator: - condition: service_completed_successfully + # nginx loads the backend's certificate when it starts, so the backend has to have made it backend: - condition: service_started + condition: service_healthy volumes: - - ./frontend/certs:/etc/nginx/certs:ro - - shared_ca:/shared_ca:ro + - backend_tls_cert:/etc/nginx/backend:ro ports: - "5001:5001" networks: @@ -233,7 +182,7 @@ services: - GF_LOG_LEVEL=warn - GF_SERVER_ROOT_URL=${GRAFANA_ROOT_URL:-http://localhost:3000/} - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin123} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:?see .env.example} - GF_SMTP_HOST=${GF_SMTP_HOST:-} - GF_SMTP_USER=${MAILJET_API_KEY:-} - GF_SMTP_PASSWORD=${MAILJET_SECRET_KEY:-} @@ -328,27 +277,6 @@ services: networks: - app-network - # Seed default users (runs once after mongo is ready) - user-seed: - logging: *logging - image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} - container_name: user-seed - depends_on: - mongo: - condition: service_healthy - environment: - <<: *backend-secrets - DEFAULT_USER_PASSWORD: ${DEFAULT_USER_PASSWORD:-user123} - ADMIN_USER_PASSWORD: ${ADMIN_USER_PASSWORD:-admin123} - volumes: - - ./backend/app:/app/app:ro - - ./backend/scripts:/app/scripts:ro - - ./backend/config.toml:/app/config.toml:ro - command: ["python", "-m", "scripts.seed_users"] - networks: - - app-network - restart: "no" # Run once and exit - # Kafka consumers, the pod watch and the scheduled jobs worker: logging: *logging @@ -357,17 +285,17 @@ services: mem_limit: 256m command: ["python", "-m", "app.worker"] depends_on: - user-seed: - condition: service_completed_successfully + # healthy means the backend has run scripts.seed_users, data fixes included + backend: + condition: service_healthy kafka: condition: service_healthy - mongo: - condition: service_started environment: <<: *backend-secrets volumes: - ./backend/app:/app/app:ro - - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro + # k3s's own kubeconfig; the folder, so a missing file never turns into a directory + - /etc/rancher/k3s:/etc/rancher/k3s:ro - ./backend/config.toml:/app/config.toml:ro - ./backend/config.worker.toml:/app/config.worker.toml:ro networks: @@ -456,8 +384,9 @@ services: - /proc:/hostfs/proc:ro - /sys:/hostfs/sys:ro environment: + # The backend creates this user on start; until then the mongodb receiver's scrapes fail and retry - MONGO_MONITOR_USER=${MONGO_MONITOR_USER:-otel-monitor} - - MONGO_MONITOR_PASSWORD=${MONGO_MONITOR_PASSWORD:-monitorpassword} + - MONGO_MONITOR_PASSWORD=${MONGO_MONITOR_PASSWORD:?see .env.example} ports: - "127.0.0.1:4317:4317" # OTLP gRPC - "127.0.0.1:4318:4318" # OTLP HTTP @@ -473,56 +402,16 @@ services: condition: service_started kafka: condition: service_healthy - mongo-monitor-user: - condition: service_completed_successfully restart: unless-stopped - # Least-privilege MongoDB user for the collector's mongodb receiver (root fails $indexStats on admin.system.*) - mongo-monitor-user: - logging: *logging - image: mongo:9.0.2 - container_name: mongo-monitor-user - profiles: ["observability"] - depends_on: - mongo: - condition: service_healthy - environment: - MONGO_USER: ${MONGO_USER:-root} - MONGO_PASSWORD: ${MONGO_PASSWORD:-rootpassword} - MONGO_MONITOR_USER: ${MONGO_MONITOR_USER:-otel-monitor} - MONGO_MONITOR_PASSWORD: ${MONGO_MONITOR_PASSWORD:-monitorpassword} - entrypoint: - - mongosh - - --quiet - - --nodb - - --eval - - | - const admin = connect("mongodb://mongo:27017/admin"); - admin.auth(process.env.MONGO_USER, process.env.MONGO_PASSWORD); - const role = { - privileges: [ - { resource: { cluster: true }, actions: ["serverStatus", "top"] }, - { resource: { db: "integr8scode", collection: "" }, actions: ["dbStats", "indexStats", "listCollections"] }, - ], - roles: [], - }; - if (admin.getRole("otelMonitor")) admin.updateRole("otelMonitor", role); - else admin.createRole({ role: "otelMonitor", ...role }); - const user = process.env.MONGO_MONITOR_USER; - const spec = { pwd: process.env.MONGO_MONITOR_PASSWORD, roles: ["otelMonitor"] }; - if (admin.getUser(user)) admin.updateUser(user, spec); - else admin.createUser({ user, ...spec }); - networks: - - app-network - restart: "no" - # --8<-- [start:dev_volumes] volumes: mongo_data: redis_data: + backend_tls_key: + backend_tls_cert: grafana_data: victoria_metrics_data: - shared_ca: loki_data: kafka_data: kafka_logs: diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 50b193b08..177493fe6 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -16,10 +16,10 @@ The app's Content Security Policy is a `` tag SvelteKit writes into the pa User code runs as a non-root user with a read-only root filesystem, no capabilities, `RuntimeDefault` seccomp, user namespaces (`hostUsers: false`) and no service account token. Each pod has an `activeDeadlineSeconds` set from the execution timeout, raised to a per-language minimum where one is defined. `K8S_POD_RUNTIME_CLASS_NAME` can put pods under gVisor or another sandboxed runtime. -The k8s worker owns the namespace controls and applies them at startup: the `executor-deny-all` NetworkPolicy, which blocks all executor ingress and egress on a cluster that enforces NetworkPolicies, and the `restricted` Pod Security Admission labels. Kueue, which setup-k8s.sh installs, caps the total CPU and memory that executor pods request; pods over quota wait behind a scheduling gate instead of failing. See [Pod and namespace security](security/policies.md). +The k8s worker owns the namespace controls and applies them at startup: the `executor-deny-all` NetworkPolicy, which blocks all executor ingress and egress on a cluster that enforces NetworkPolicies, and the `restricted` Pod Security Admission labels. It also creates the namespace itself and applies the `executor-quota` ResourceQuota, which caps the total CPU and memory that executor pods request; a pod over the quota is refused. See [Pod and namespace security](security/policies.md). Passwords are hashed with `pwdlib`'s `BcryptHasher`, with `BCRYPT_ROUNDS` rounds. Sessions are a JWT in an httpOnly cookie plus a CSRF double-submit token, and repeated failed logins lock the account. See [Authentication](architecture/authentication.md). -Redis requires a password and MongoDB requires authentication. MongoDB, Redis, Kafka and the observability services only listen on `127.0.0.1` on the host; the backend (443) and frontend (5001) are the public ports, both TLS with self-signed certificates from the cert-generator container. Third-party images in compose are pinned to fixed versions. +Redis requires a password and MongoDB requires authentication. MongoDB, Redis, Kafka and the observability services only listen on `127.0.0.1` on the host; the frontend (5001) is the public port. nginx serves it over TLS with the certificate mounted at `/etc/nginx/certs`, or with a self-signed one it makes when it starts. The backend listens on `127.0.0.1:443` over TLS with a self-signed certificate it makes on first start, and nginx verifies that certificate on every proxied request. Third-party images in compose are pinned to fixed versions. The secrets (`SECRET_KEY`, `MONGO_USER`, `MONGO_PASSWORD`, `REDIS_PASSWORD`, and the Grafana, Mailjet and MongoDB monitor credentials) come from environment variables. Compose has development defaults for all of them, so set real values in production. The release workflow does that from repository secrets. diff --git a/docs/architecture/frontend-build.md b/docs/architecture/frontend-build.md index 7ad0a445a..3f6d3f145 100644 --- a/docs/architecture/frontend-build.md +++ b/docs/architecture/frontend-build.md @@ -38,8 +38,8 @@ Imports use package.json subpath imports instead of the `$lib` alias that Svelte `frontend/Dockerfile` has two stages. `node:26-alpine` runs `npm ci` and `npm run build`. `nginx:1.31-alpine` copies `build/` to `/usr/share/nginx/html` and puts `nginx.conf.template` in `/etc/nginx/templates/`, so the image's entrypoint fills in `BACKEND_URL`, `GRAFANA_URL` and `GRAFANA_PUBLIC_URL` with envsubst at startup. -nginx listens on 5001 with TLS, using the certificates cert-generator writes to `frontend/certs`. It proxies `/api/` to `BACKEND_URL` (`https://backend:443` in compose) and gives `/api/v1/events/` SSE settings: no buffering and 24 hour timeouts. `/grafana` is a 302 redirect to `GRAFANA_PUBLIC_URL`. Static assets get a one year `expires`, which is safe since their names carry content hashes, and HTML gets `expires -1` so browsers revalidate it. Any path that isn't a file gets `200.html`, so deep links and reloads work. [Nginx configuration](../operations/nginx-configuration.md) walks through the whole file. +nginx listens on 5001 with TLS. Its certificate is the one mounted at `/etc/nginx/certs`, or, when nothing is mounted, a self-signed one that a script in the image's `/docker-entrypoint.d/` makes before nginx starts. It proxies `/api/` to `BACKEND_URL` (`https://backend:443` in compose), verifying the backend against the certificate the backend keeps in the `backend_tls_cert` volume, and gives `/api/v1/events/` SSE settings: no buffering and 24 hour timeouts. `/grafana` is a 302 redirect to `GRAFANA_PUBLIC_URL`. Static assets get a one year `expires`, which is safe since their names carry content hashes, and HTML gets `expires -1` so browsers revalidate it. Any path that isn't a file gets `200.html`, so deep links and reloads work. [Nginx configuration](../operations/nginx-configuration.md) walks through the whole file. `./deploy.sh dev` starts the compose stack, and the `frontend` service there is the same nginx image on `https://localhost:5001`. It has no source mount, so a frontend change needs `./deploy.sh dev --build` to show up. -`npm run dev` runs the Vite dev server on port 5001 with hot module replacement. It uses HTTPS when `./certs/server.key` and `./certs/server.crt` exist (or `SSL_KEY_FILE` and `SSL_CERT_FILE`), and proxies `/api` to `https://backend:443` without verifying its certificate. The `backend` hostname only resolves inside compose, so API calls only work from a container on `app-network`, with the compose `frontend` service stopped to free the port. +`npm run dev` runs the Vite dev server on port 5001 with hot module replacement. It uses HTTPS when `./certs/server.key` and `./certs/server.crt` exist (or `SSL_KEY_FILE` and `SSL_CERT_FILE`), and proxies `/api` to `https://backend:443` with the certificate checked, so it needs `NODE_EXTRA_CA_CERTS` pointing at the backend's `server.crt`. The `backend` hostname only resolves inside compose, so API calls only work from a container on `app-network`, with the compose `frontend` service stopped to free the port. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index f36978a05..94b4b2619 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -30,11 +30,11 @@ Integr8sCode runs user scripts in Python, Node.js, Ruby, Bash or Go, each in its ![System diagram](/assets/images/system_diagram.svg) -The browser loads the SPA from the frontend container, an nginx on port 5001 that also proxies `/api` to the backend over HTTPS. The backend runs FastAPI under Gunicorn with Uvicorn workers and serves both REST and SSE. Kafka runs in KRaft mode and creates topics on first use, so there is no topic setup step. Events travel as JSON that FastStream serializes from the Pydantic models. +The browser loads the SPA from the frontend container, an nginx that serves HTTPS on port 5001 and proxies `/api` to the backend over HTTPS, checking the backend's own certificate. The backend runs FastAPI under Gunicorn with Uvicorn workers and serves both REST and SSE. Kafka runs in KRaft mode and creates topics on first use, so there is no topic setup step. Events travel as JSON that FastStream serializes from the Pydantic models. The worker is one more container from the same backend image, running `python -m app.worker`. Inside it, the k8s worker creates pods through the Kubernetes API, the pod monitor watches them, the result processor writes results to MongoDB, and the saga orchestrator schedules executions with a Redis-backed queue and tracks sagas in MongoDB. A DLQ job retries failed events every 10 seconds. -The observability profile adds an OpenTelemetry Collector that ships traces to Jaeger, metrics to VictoriaMetrics and logs to Loki, with Grafana on top of all three. Kafdrop is under the debug profile. The cert-generator creates TLS certificates for the frontend and backend from a shared CA, and it also prepares the cluster: it creates the `integr8scode` namespace, installs Kueue, sets up a service account with RBAC and writes the kubeconfig the backend containers use. +The observability profile adds an OpenTelemetry Collector that ships traces to Jaeger, metrics to VictoriaMetrics and logs to Loki, with Grafana on top of all three. Kafdrop is under the debug profile. No container runs once and exits. nginx and the backend each make their own TLS certificate when they first start, and the backend seeds the users before it serves. The worker reads k3s's own kubeconfig and creates the `integr8scode` namespace with its NetworkPolicy and ResourceQuota. ## Backend wiring @@ -112,7 +112,7 @@ sequenceDiagram The rate limit middleware has already run by the time the route sees the request. If the client sent an `Idempotency-Key` header it used before, the [idempotency](idempotency.md) layer returns the earlier execution instead of creating a new one, or 409 while that first request is still in flight. Otherwise the execution service checks the language and version against the [runtime registry](runtime-registry.md) and rejects an unknown pair with 422. Then it stores the execution in MongoDB, publishes `execution_requested` and answers with the execution ID and its status. -The saga orchestrator puts the request into the [execution queue](execution-queue.md) and, once a slot is free, runs the saga steps, the last of which publishes `create_pod_command`. The k8s worker creates a ConfigMap with the script and a hardened pod, then publishes `pod_created`, or `execution_failed` if the Kubernetes API refused. The pod monitor watches the pod and publishes `pod_scheduled`, `pod_running` and finally one of `execution_completed`, `execution_failed` or `execution_timeout`. Then it deletes the pod, which frees the Kueue quota, and Kubernetes garbage-collects the ConfigMap through its owner reference. +The saga orchestrator puts the request into the [execution queue](execution-queue.md) and, once a slot is free, runs the saga steps, the last of which publishes `create_pod_command`. The k8s worker creates a ConfigMap with the script and a hardened pod, then publishes `pod_created`, or `execution_failed` if the Kubernetes API refused. The pod monitor watches the pod and publishes `pod_scheduled`, `pod_running` and finally one of `execution_completed`, `execution_failed` or `execution_timeout`. Then it deletes the pod, which frees its share of the ResourceQuota, and Kubernetes garbage-collects the ConfigMap through its owner reference. The result processor writes the final state and output to MongoDB and publishes `result_stored`, or `result_failed` if it couldn't. The saga orchestrator also listens for the terminal events: it marks the saga done, releases the queue slot and schedules the next execution. diff --git a/docs/architecture/services-overview.md b/docs/architecture/services-overview.md index c36d70288..9653a4bff 100644 --- a/docs/architecture/services-overview.md +++ b/docs/architecture/services-overview.md @@ -42,7 +42,7 @@ The saga orchestrator puts each request into the execution queue and runs the sa The k8s worker creates a ConfigMap holding the script and the entrypoint, then the pod, then sets the pod as the ConfigMap's owner so Kubernetes deletes both together. The pod runs as user 1000 with a read-only root filesystem, all capabilities dropped, seccomp `RuntimeDefault`, no service account token, no service links and no host namespaces. The worker refuses to start if `K8S_NAMESPACE` is `default`. -The pod monitor lists and then watches pods labelled `app=integr8s,component=executor`. Each watch window lasts up to 30 seconds, and an APScheduler job checks every 5 seconds and opens the next one once the last has ended. It maps pod phases to events, reads the exit code, resource usage and output from the termination message and logs, and deletes the pod once it has published a terminal execution event. Deleting the pod frees the Kueue quota for the next one. +The pod monitor lists and then watches pods labelled `app=integr8s,component=executor`. Each watch window lasts up to 30 seconds, and an APScheduler job checks every 5 seconds and opens the next one once the last has ended. It maps pod phases to events, reads the exit code, resource usage and output from the termination message and logs, and deletes the pod once it has published a terminal execution event. Deleting the pod frees its share of the ResourceQuota for the next one. The result processor writes the final status, output and resource usage to MongoDB and records execution metrics. It doesn't touch Kubernetes. diff --git a/docs/assets/images/system_diagram.svg b/docs/assets/images/system_diagram.svg index b694284a1..6d07b9f05 100644 --- a/docs/assets/images/system_diagram.svg +++ b/docs/assets/images/system_diagram.svg @@ -1,4 +1,4 @@ -
HTTPS 443
(SPA + Assets)
Browser SPA
HTTPS /api/v1/*
(REST/SSE endpoints)
frontend
(Nginx + Svelte)
SSE /api/v1/events/*
(EventSource)
CRUD:
- executions
- settings
- events
backend
(FastAPI + SSE)
share CA
bundle
shared-ca
TLS cert
cert-generator
Cert Generator
Observability
Prom
remote_write
(metrics)
otlphtpp
otel-collectorJaeger
OTLP gRPC
(traces)
query
dashboards
(PromQL)
Grafana
kafdrop
Kubernetes API
consume commands and events
publish pod, execution and result events
worker
saga orchestrator
k8s worker
pod monitor
result processor
DLQ retries
create/delete,
watch/list pods
kubeconfig
cluster bootstrap
Redis
Mongo
Rate limiting keys
+ SSE pub/sub
consume for
API handlers
web UI:
topics/consumers
produce
domain events
and replays
results, sagas,
DLQ rows
execution queue,
idempotency keys
Project
Scope
log queries
Loki
Text is not SVG - cannot display \ No newline at end of file +
HTTPS 5001
(SPA + Assets)
Browser SPA
HTTPS /api/v1/*
(REST/SSE endpoints)
frontend
(Nginx + Svelte)
SSE /api/v1/events/*
(EventSource)
CRUD:
- executions
- settings
- events
backend
(FastAPI + SSE)
Observability
Prom
remote_write
(metrics)
otlphtpp
otel-collectorJaeger
OTLP gRPC
(traces)
query
dashboards
(PromQL)
Grafana
kafdrop
Kubernetes API
consume commands and events
publish pod, execution and result events
worker
saga orchestrator
k8s worker
pod monitor
result processor
DLQ retries
create/delete,
watch/list pods
Redis
Mongo
Rate limiting keys
+ SSE pub/sub
consume for
API handlers
web UI:
topics/consumers
produce
domain events
and replays
results, sagas,
DLQ rows
execution queue,
idempotency keys
Project
Scope
log queries
Loki
Text is not SVG - cannot display \ No newline at end of file diff --git a/docs/components/workers/index.md b/docs/components/workers/index.md index b98d66baf..41a917882 100644 --- a/docs/components/workers/index.md +++ b/docs/components/workers/index.md @@ -39,7 +39,7 @@ cd backend uv run python -m app.worker ``` -The worker reads the same `config.toml` as the container, so it connects to the compose hostnames `mongo`, `redis` and `kafka`. Map those to `127.0.0.1` in `/etc/hosts`, since compose publishes their ports on localhost. It also reads the kubeconfig from `KUBERNETES_CONFIG_PATH`, which is `/app/kubeconfig.yaml`, a path that exists only inside the container. Set it in `config.worker.toml` to a kubeconfig the host can use, such as `~/.kube/config`. +The worker reads the same `config.toml` as the container, so it connects to the compose hostnames `mongo`, `redis` and `kafka`. Map those to `127.0.0.1` in `/etc/hosts`, since compose publishes their ports on localhost. It also reads the kubeconfig from `KUBERNETES_CONFIG_PATH` (`/etc/rancher/k3s/k3s.yaml`) and sends requests to `KUBERNETES_API_SERVER` (`https://host.docker.internal:6443`). On the host, set the second to `https://127.0.0.1:6443` in `config.worker.toml`. ## Scaling diff --git a/docs/components/workers/k8s_worker.md b/docs/components/workers/k8s_worker.md index 0ee8b45bc..603670b1f 100644 --- a/docs/components/workers/k8s_worker.md +++ b/docs/components/workers/k8s_worker.md @@ -44,7 +44,7 @@ Before it consumes anything, the worker applies two things to the executor names It also applies the `runtime-image-pre-puller` DaemonSet, which pulls every image in the runtime registry on each node. The worker doesn't wait for those pulls, so an execution that arrives before they finish still waits for its image. -Executor pods also carry the label `kueue.x-k8s.io/queue-name: executor-queue`, which points at the Kueue LocalQueue that `cert-generator/setup-k8s.sh` creates in the `integr8scode` namespace. That script sets up no other namespace, so a different `K8S_NAMESPACE` needs its own LocalQueue. +Every executor pod counts against the `executor-quota` ResourceQuota that the worker applies to its namespace on start (`K8S_NAMESPACE_CPU_QUOTA` and `K8S_NAMESPACE_MEMORY_QUOTA`, 32 CPU and 4Gi of requests). A pod over the quota is refused with `403 Forbidden`, and the worker publishes `pod_creation_failed`. ## Key files diff --git a/docs/getting-started.md b/docs/getting-started.md index f221a1fe5..f024e5006 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,6 +1,6 @@ # Getting started -You need Docker with Compose and a Kubernetes cluster that kubectl on your machine can reach. Docker alone isn't enough: every script runs in a Kubernetes pod, and there's no Docker fallback. The project runs on k3s, locally and in CI. +You need Docker with Compose and k3s on the same machine. Docker alone isn't enough: every script runs in a Kubernetes pod, and there's no Docker fallback. The project runs on k3s, locally and in CI. ## Set up k3s @@ -12,7 +12,7 @@ curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik --bind-addres CI also passes `--disable-network-policy` to dodge the iptables race under [Troubleshooting](#troubleshooting). Leave it out here: without the policy controller, k3s doesn't enforce the NetworkPolicy that cuts executor pods off the network. -On startup the `cert-generator` container looks for the cluster through `~/.kube` and `/etc/rancher/k3s`. When it finds one, it creates the `integr8scode` namespace, installs Kueue, sets up a service account and writes `backend/kubeconfig.yaml` for the workers. When it can't connect, it logs "Cannot connect to Kubernetes cluster" and skips all of that. The rest of the stack still starts and you can log in, but the worker has no kubeconfig and keeps restarting, so scripts stay queued forever. +That's all the cluster needs. The worker reads k3s's own kubeconfig from `/etc/rancher/k3s/k3s.yaml`, talks to `https://host.docker.internal:6443` (the name the `--tls-san` flag puts on k3s's certificate) and on start creates the `integr8scode` namespace with its Pod Security labels, NetworkPolicy and ResourceQuota. ## Start the stack @@ -22,12 +22,12 @@ cd Integr8sCode ./deploy.sh dev ``` -The first start pulls or builds the images and takes a few minutes. Secrets (JWT key, MongoDB and Redis passwords) come from environment variables with dev defaults, so there's nothing to copy or fill in. Add `--wait` to block until every service is running, and healthy where it has a healthcheck. +The first start pulls or builds the images and takes a few minutes. On its first run `./deploy.sh dev` copies `.env.example` to `.env`, which holds the local secrets (JWT key, MongoDB and Redis passwords, the two logins), so there's nothing to fill in. Add `--wait` to block until every service is running, and healthy where it has a healthcheck. Check the backend: ```bash -curl -k https://localhost/api/v1/health/live +curl -k https://localhost:5001/api/v1/health/live ``` ```json @@ -36,7 +36,7 @@ curl -k https://localhost/api/v1/health/live ## Log in and run a script -Open [https://localhost:5001](https://localhost:5001) and accept the self-signed certificate. Log in with username `user` and password `user123`. For the admin pages (events, executions, sagas, users, system settings) use `admin` / `admin123`. A one-shot `user-seed` container creates both accounts, or resets their passwords, on every start. Set `DEFAULT_USER_PASSWORD` and `ADMIN_USER_PASSWORD` before deploying anywhere that isn't your laptop. +Open [https://localhost:5001](https://localhost:5001) and accept the self-signed certificate. Log in with username `user` and password `user123`. For the admin pages (events, executions, sagas, users, system settings) use `admin` / `admin123`. The backend creates both accounts, or resets their passwords, every time it starts. Anywhere that isn't your laptop, put your own values in `.env`. After login you land in the editor, which starts with a small Python script. Click **Run Script** and the editor shows the status while the pod runs, then the output once it finishes. The language picker offers what the runtime registry lists: Python 3.14, 3.13, 3.12 and 3.11, Node 26, 24 and 22, Ruby 4.0, 3.4 and 3.3, Go 1.27 and 1.26, and Bash 5.3. @@ -66,7 +66,7 @@ Kafdrop, Grafana and Jaeger aren't part of the default stack. `./deploy.sh dev - ## Troubleshooting -If scripts stay queued, check `docker compose logs cert-generator` for the connection error above, then `docker compose logs worker`. A k3s that stopped after a VPN or IP change is covered in [Deployment](operations/deployment.md). +If scripts stay queued, check `docker compose logs worker`, which logs the Kubernetes error when it can't reach k3s. A k3s that stopped after a VPN or IP change is covered in [Deployment](operations/deployment.md). If every service times out talking to mongo or kafka right after k3s started or restarted, k3s's policy sync probably wiped Docker's iptables rules ([kube-router#1248](https://github.com/cloudnativelabs/kube-router/issues/1248)). `docker compose down` followed by `./deploy.sh dev` recreates the network and its rules. diff --git a/docs/operations/cicd.md b/docs/operations/cicd.md index c988da5be..9ce0661d7 100644 --- a/docs/operations/cicd.md +++ b/docs/operations/cicd.md @@ -5,13 +5,13 @@ CI is six GitHub Actions workflows, two composite actions and a Dependabot confi ```mermaid graph LR subgraph ST["CI"] - UB["backend unit + Grafana contract"] --> Build["build and push 4 images"] + UB["backend unit + Grafana contract"] --> Build["build and push 2 images"] UF["frontend unit"] --> Build Build --> EB["backend E2E"] Build --> EF["frontend E2E, 2 shards"] end subgraph DSP["Docker Scan & Promote"] - Scan["Trivy, 4 images"] --> Promote["crane: sha tag to latest"] + Scan["Trivy, 2 images"] --> Promote["crane: sha tag to latest"] end subgraph RD["Release & Deploy"] Release["CalVer tag + GitHub release"] --> Deploy["SSH deploy"] @@ -49,7 +49,7 @@ CI is the only workflow a PR has to pass, and it runs only what the PR can affec |-----------------------------------------------------------------|----------------------------------------------------------------------| | `backend/` | ruff, mypy, grimp, bandit, backend unit tests | | `frontend/` | frontend lint and type check, frontend unit tests | -| `backend/`, `frontend/`, cert-generator, compose, deploy.sh or the composite actions | the images and frontend E2E; backend E2E unless only `frontend/` changed | +| `backend/`, `frontend/`, `.env.example`, compose, deploy.sh or the composite actions | the images and frontend E2E; backend E2E unless only `frontend/` changed | | `backend/`, the spec, the generated client or its generator config | Spec and client match the code | | `docs/`, mkdocs.yml, or `backend/` and `frontend/`, since the docs include snippets of them | Docs Build | @@ -63,14 +63,12 @@ Stacked PRs come from `gh stack`. CI runs for every PR in a stack, each against The backend unit job installs with `uv sync --frozen --group test --no-dev --no-install-project --no-build` and runs `pytest tests/unit` with coverage. The pytest config adds `-n auto --dist=loadgroup`, so tests run in parallel with xdist and tests that share a group stay on one worker. The same job then runs the [Grafana contract tests](../testing/contract-testing.md) with `pytest tests/contract -m grafana_contract`. The frontend unit job runs `npm ci --ignore-scripts` and `npm run test:coverage`. Coverage goes to Codecov with the flags `backend-unit`, `frontend-unit` and later `backend-e2e`. -Once both pass, or are skipped because only one side changed, one job builds the four images and pushes them to GHCR as `sha-<7 chars>`: +Once both pass, or are skipped because only one side changed, one job builds the two images and pushes them to GHCR as `sha-<7 chars>`: -| Image | Dockerfile | Build | -|------------------|-----------------------------|---------------------------------------------------------------| -| `base` | `backend/Dockerfile.base` | cached as a zstd tarball keyed on `Dockerfile.base`, pyproject.toml and uv.lock; GHA layer cache on a miss | -| `backend` | `backend/Dockerfile` | plain `docker build` on top of the local base image | -| `cert-generator` | `cert-generator/Dockerfile` | GHA layer cache | -| `frontend` | `frontend/Dockerfile` | GHA layer cache | +| Image | Dockerfile | Build | +|------------|-----------------------|---------------------------------------------------------------| +| `backend` | `backend/Dockerfile` | the `base` stage (Python, system packages, locked dependencies) is built on its own and cached as a zstd tarball keyed on the Dockerfile, pyproject.toml and uv.lock, with the GHA layer cache on a miss; then `docker build --build-context base=docker-image://...` builds only the app layers on top | +| `frontend` | `frontend/Dockerfile` | GHA layer cache | The pushes run in parallel and each failure is reported on its own. Fork PRs can't push to GHCR, so for them the push and both E2E jobs are skipped. Both E2E jobs check `!cancelled()` and the build's result in their own `if`. Without a status function, GitHub skips every job below a skipped one in the `needs` chain, so a backend-only PR, which skips the frontend unit job, would skip both E2E jobs. Workers reuse the backend image, so there are no worker images. @@ -78,19 +76,19 @@ The pushes run in parallel and each failure is reported on its own. Fork PRs can Each E2E job gets its own runner with k3s and the full compose stack, pulling the images it just built. Two composite actions set that up, and the frontend job installs Node and Playwright between them so that work overlaps with k3s booting. -`e2e-boot` logs in to GHCR, starts `docker compose pull` and then `mongo redis shared-ca kafka` in the background, and starts k3s in a privileged container named `k3s`, from the `rancher/k3s` image, with its network policy controller off. No E2E test checks policy enforcement, and the worker still creates the NetworkPolicy object. The container publishes 6443 and mounts the runner's `/etc/rancher/k3s`, so the kubeconfig lands where the cert-generator service reads it, the same as with a host install. k3s runs in a container because its pods then get their network interfaces inside it. With k3s on the runner itself, every executor pod changed the runner's network, and Chromium aborted Playwright navigations in flight with `net::ERR_NETWORK_CHANGED`. Also in the background, it downloads the Kueue manifest and, once k3s has written its kubeconfig, pre-pulls `python:3.14-slim`, the default runtime and the only one the tests execute, into k3s with `docker exec k3s crictl`, so the first executor pods don't wait on an image pull. Nothing execs into the container before that kubeconfig exists: at startup k3s moves every process out of the container's root cgroup and then enables the cgroup controllers there, and a `docker exec` landing in between made that fail with "device or resource busy", so the kubelet exited and the node never came up. Jobs that install Kueue also pre-pull `registry.k8s.io/kueue/kueue` that way, and `e2e-ready` waits for that pull before it waits for the controller, since the image alone once took over five minutes from registry.k8s.io. +`e2e-boot` logs in to GHCR, copies `.env.example` to `.env`, starts `docker compose pull` and then `mongo redis kafka` in the background, and starts k3s in a privileged container named `k3s`, from the `rancher/k3s` image, with its network policy controller off. No E2E test checks policy enforcement, and the worker still creates the NetworkPolicy object. The container publishes 6443 and mounts the runner's `/etc/rancher/k3s`, so the kubeconfig lands where the worker reads it, the same as with a host install. k3s runs in a container because its pods then get their network interfaces inside it. With k3s on the runner itself, every executor pod changed the runner's network, and Chromium aborted Playwright navigations in flight with `net::ERR_NETWORK_CHANGED`. Also in the background, once k3s has written its kubeconfig, it pre-pulls `python:3.14-slim`, the default runtime and the only one the tests execute, into k3s with `docker exec k3s crictl`, so the first executor pods don't wait on an image pull. Nothing execs into the container before that kubeconfig exists: at startup k3s moves every process out of the container's root cgroup and then enables the cgroup controllers there, and a `docker exec` landing in between made that fail with "device or resource busy", so the kubelet exited and the node never came up. -`e2e-ready` copies k3s's kubeconfig for the runner's kubectl, waits for the node, writes `backend/kubeconfig.yaml` pointing at `host.docker.internal:6443` and creates the `integr8scode` namespace. Unless `install-kueue` is `false`, it applies the Kueue manifest, waits for the controller and creates the `ResourceFlavor`, `ClusterQueue` and `LocalQueue` (`kueue.x-k8s.io/v1beta2`), then waits for the image pre-pulls. Kueue starts with a certificate bootstrap that answers `/readyz` with 404 until the kubelet has mounted its new certificate secret, and it checks for the file with an exponential backoff, so on a slow run the controller takes over two minutes. If it isn't Available within 300 seconds, the step prints the `kueue-system` pods, their descriptions, events and the controller's last 300 log lines before it fails. It copies config.test.toml over config.toml, waits for the background infra start, runs `docker compose up -d --no-build` and polls `/api/v1/health/live` (and the frontend, when asked). It then waits until the five Kafka consumer groups report `Stable` in `kafka-consumer-groups --describe --state`. On a fresh Kafka most of them start at the newest message, so a request sent while they were still joining would never be read, and an execution test would time out. +`e2e-ready` copies k3s's kubeconfig for the runner's kubectl, waits for the node and then for the image pre-pull. The worker creates the `integr8scode` namespace, its NetworkPolicy and its ResourceQuota itself when it starts. It copies config.test.toml over config.toml, waits for the background infra start, runs `docker compose up -d --no-build` and polls `/api/v1/health/live` (and the frontend, when asked). It then waits until the five Kafka consumer groups report `Stable` in `kafka-consumer-groups --describe --state`. On a fresh Kafka most of them start at the newest message, so a request sent while they were still joining would never be read, and an execution test would time out. -The backend E2E job runs `pytest tests/e2e` inside the backend container after `uv sync --group test --no-dev --frozen --no-install-project --no-build`. The frontend E2E job runs as two shards. It caches Playwright browsers by package-lock.json, always runs `npm run test:e2e:deps`, runs `npm run test:e2e:browsers` on a cache miss, skips Kueue, and runs `npm run test:e2e -- --shard=N/2`. On failure both jobs upload compose logs, the k3s container's log and `kubectl describe nodes`; the backend one adds `kubectl get events`. +The backend E2E job runs `pytest tests/e2e` inside the backend container after `uv sync --group test --no-dev --frozen --no-install-project --no-build`. The frontend E2E job runs as two shards. It caches Playwright browsers by package-lock.json, always runs `npm run test:e2e:deps`, runs `npm run test:e2e:browsers` on a cache miss, and runs `npm run test:e2e -- --shard=N/2`. On failure both jobs upload compose logs, the k3s container's log and `kubectl describe nodes`; the backend one adds `kubectl get events`. ## Docker Scan & Promote -The jobs run only when CI succeeded on `main`, or on a manual run. A matrix scans `base`, `backend`, `frontend` and `cert-generator` with Trivy v0.75.0 (`CRITICAL,HIGH`, unfixed ignored, `backend/.trivyignore`) and uploads SARIF to the Security tab. A finding fails the scan job, and `promote` needs every scan to pass, so an image with a fixable `CRITICAL` or `HIGH` vulnerability never becomes `latest`. `limit-severities-for-sarif` keeps the SARIF to those two levels; without it the action drops the severity filter for SARIF and the gate would trip on everything. The promote job then copies each `sha-` tag to `latest` with `crane copy`, a registry-side manifest copy with no rebuild. +The jobs run only when CI succeeded on `main`, or on a manual run. A matrix scans `backend` and `frontend` with Trivy v0.75.0 (`CRITICAL,HIGH`, unfixed ignored, `backend/.trivyignore`) and uploads SARIF to the Security tab. A finding fails the scan job, and `promote` needs every scan to pass, so an image with a fixable `CRITICAL` or `HIGH` vulnerability never becomes `latest`. `limit-severities-for-sarif` keeps the SARIF to those two levels; without it the action drops the severity filter for SARIF and the gate would trip on everything. The promote job then copies each `sha-` tag to `latest` with `crane copy`, a registry-side manifest copy with no rebuild. ## Release & Deploy -The release job first finds the commit `latest` was built from, by matching the digest of `backend:latest` against the `sha-` tags of the last 100 commits on `main`. That commit can be behind `main`'s head when a later merge is still in CI or failed it. If the commit already has a CalVer tag, nothing new passed since the last release: a scheduled run stops there, and a manual run deploys that release again. Otherwise the job computes a CalVer version `YYYY.M.PATCH`, where `PATCH` is one more than the highest tag of the current month, starting at 0. It copies the commit's `sha-` images to that tag for all four images with crane, pushes an annotated git tag on that commit and creates a GitHub release with generated notes. +The release job first finds the commit `latest` was built from, by matching the digest of `backend:latest` against the `sha-` tags of the last 100 commits on `main`. That commit can be behind `main`'s head when a later merge is still in CI or failed it. If the commit already has a CalVer tag, nothing new passed since the last release: a scheduled run stops there, and a manual run deploys that release again. Otherwise the job computes a CalVer version `YYYY.M.PATCH`, where `PATCH` is one more than the highest tag of the current month, starting at 0. It copies the commit's `sha-` images to that tag for both images with crane, pushes an annotated git tag on that commit and creates a GitHub release with generated notes. The deploy job runs `appleboy/ssh-action` against the production server. It's skipped on a manual run with `skip_deploy`, and on a scheduled run with nothing new. What it runs on the server is in [Deployment](deployment.md#production). @@ -103,7 +101,7 @@ The deploy job runs `appleboy/ssh-action` against the production server. It's sk | `MONGO_USER`, `MONGO_PASSWORD` | Release & Deploy | MongoDB root credentials | | `MONGO_MONITOR_PASSWORD` | Release & Deploy | password of the collector's MongoDB user | | `REDIS_PASSWORD` | Release & Deploy | Redis password | -| `ADMIN_USER_PASSWORD`, `DEFAULT_USER_PASSWORD` | Release & Deploy | passwords the seed job sets for `admin` and `user` | +| `ADMIN_USER_PASSWORD`, `DEFAULT_USER_PASSWORD` | Release & Deploy | passwords the backend sets for `admin` and `user` on start | | `GRAFANA_ADMIN_USER`, `GRAFANA_ADMIN_PASSWORD` | Release & Deploy | Grafana admin login | | `MAILJET_HOST`, `MAILJET_API_KEY`, `MAILJET_SECRET_KEY`, `MAILJET_FROM_ADDRESS` | Release & Deploy | Grafana alert mail over SMTP | | `GRAFANA_ALERT_RECIPIENTS` | Release & Deploy | who gets alert mail | @@ -118,7 +116,7 @@ Every action, the composite ones included, is pinned to a full commit SHA with t Tools are pinned exactly: mkdocs 1.6.1 with material 9.7.7, mermaid2 1.2.3 and swagger-ui-tag 0.8.1; bandit 1.9.4; Trivy v0.75.0. Python installs use `uv sync --frozen` with `--no-build`, so only wheels from the lockfile get installed, and `--no-install-project`. `uvx` calls also pass `--no-build`. npm installs use `npm ci --ignore-scripts`, in CI and in the frontend image. -k3s is the `rancher/k3s:v1.37.1-k3s1` image, pinned by digest in `K3S_IMAGE` in ci.yml. The Kueue manifest (`v0.20.1`) is checked against its SHA256 before it's applied. All these downloads use `curl --proto =https --tlsv1.2`. +k3s is the `rancher/k3s:v1.37.1-k3s1` image, pinned by digest in `K3S_IMAGE` in ci.yml. `latest` is only ever set by the promote job, so after an automatic promote it points at images that passed the tests and the Trivy scan. A manual promote still needs the scan but can pick a SHA whose E2E jobs failed, since the build job pushes the `sha-` tags before E2E runs. Any build stays pullable by its `sha-` tag. diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index 6e5211c2b..0c511a887 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -1,8 +1,8 @@ # Deployment -Everything runs from one docker-compose.yaml: the backend, the frontend, the worker, MongoDB, Redis, Kafka and the optional observability stack. The worker reuses the backend image with a different `command:`, so there is one application image to build. Kubernetes only runs the executor pods; the backend and the worker reach it through a kubeconfig that the cert-generator container writes. +Everything runs from one docker-compose.yaml: the backend, the frontend, the worker, MongoDB, Redis, Kafka and the optional observability stack. The worker reuses the backend image with a different `command:`, so there is one application image to build. Kubernetes only runs the executor pods. The worker reads k3s's own kubeconfig from `/etc/rancher/k3s`, reaches the API at `host.docker.internal:6443` and on start creates the `integr8scode` namespace with its Pod Security labels, NetworkPolicy and ResourceQuota. -`backend/Dockerfile.base` is a two-stage build on `python:3.14-slim`. The builder stage runs `uv sync --locked --no-dev --no-install-project --no-build`, and the final stage copies only the virtualenv plus a few runtime libraries. `backend/Dockerfile` adds the application code on top and starts gunicorn with uvicorn workers on port 443. The frontend image builds the Svelte app with `node:26-alpine` and serves it from `nginx:1.31-alpine`, see [Nginx configuration](nginx-configuration.md). +`backend/Dockerfile` builds on `python:3.14-slim` in three stages. The builder stage runs `uv sync --locked --no-dev --no-install-project --no-build`, the `base` stage copies only the virtualenv plus a few runtime libraries, and the last stage adds the application code. On first start the container makes a self-signed certificate and keeps it in the `backend_tls_key` and `backend_tls_cert` volumes, then it seeds the default users and starts gunicorn with uvicorn workers on port 443 over TLS. nginx mounts the certificate volume and verifies the backend against it. The frontend image builds the Svelte app with `node:26-alpine` and serves it from `nginx:1.31-alpine`, see [Nginx configuration](nginx-configuration.md). ## deploy.sh @@ -22,7 +22,7 @@ Everything runs from one docker-compose.yaml: the backend, the frontend, the wor ./deploy.sh dev --debug # all of the above plus Kafdrop ``` -The core stack is MongoDB, Redis, Kafka (KRaft, no ZooKeeper), the backend, the frontend and the worker. A few one-shot containers run first and exit: `shared-ca` prepares the shared CA volume, `cert-generator` writes TLS certs and `backend/kubeconfig.yaml`, and `user-seed` creates the default accounts. Kafka creates topics on first use (`KAFKA_AUTO_CREATE_TOPICS_ENABLE`). Topic names are the event types themselves, such as `execution_requested`, with no prefix. +The core stack is MongoDB, Redis, Kafka (KRaft, no ZooKeeper), the backend, the frontend and the worker. No container runs once and exits. Secrets come from `.env`, which `./deploy.sh dev` copies from `.env.example` on its first run; compose has no defaults for them and refuses to start while one is empty. Kafka creates topics on first use (`KAFKA_AUTO_CREATE_TOPICS_ENABLE`). Topic names are the event types themselves, such as `execution_requested`, with no prefix. | Service | URL | Needs | |--------------------|-------------------------|-------------------------------| @@ -39,11 +39,7 @@ The core stack is MongoDB, Redis, Kafka (KRaft, no ZooKeeper), the backend, the Inside the compose network Kafka is `kafka:29092`. From the host use `localhost:9092`, since that's the listener advertised for host clients. -The seed job creates `user` / `user123` and `admin` / `admin123`, and every run sets the passwords of accounts that already exist again. Override the passwords with env vars: - -```bash -DEFAULT_USER_PASSWORD=mypass ADMIN_USER_PASSWORD=myadmin ./deploy.sh dev -``` +Every time the backend starts, it creates `user` / `user123` and `admin` / `admin123` with `scripts/seed_users.py`, and sets the passwords of accounts that already exist again. The passwords are `DEFAULT_USER_PASSWORD` and `ADMIN_USER_PASSWORD` in `.env`. There's no hot reload. Gunicorn runs without `--reload` and the frontend is an nginx production build. The backend and the worker bind-mount `backend/app`, so a Python change needs `docker compose restart backend worker`. A frontend change needs `docker compose up -d --build frontend`. @@ -62,7 +58,7 @@ Data lives in named volumes: | MongoDB | `mongosh` ping, with the root credentials read from the container env | | Redis | `redis-cli -a $REDIS_PASSWORD ping` | | Kafka | `kafka-broker-api-versions --bootstrap-server localhost:9092` | -| Backend | `curl -k https://localhost:443/api/v1/health/live` | +| Backend | `curl --cacert /app/certs/cert/server.crt https://localhost:443/api/v1/health/live` | | Frontend | `curl -k https://localhost:5001` | | Grafana | `curl localhost:3000/api/health` | | Victoria Metrics | `wget http://127.0.0.1:8428/-/healthy` | @@ -91,7 +87,7 @@ Kafdrop runs with `-Xmx512M` and no limit. All services above except Jaeger have ## Observability profile -The `observability` profile adds a one-shot `mongo-monitor-user` container. It connects as root and creates (or updates) an `otelMonitor` role with `serverStatus`, `top`, `dbStats`, `indexStats` and `listCollections`, plus a user with only that role. The collector's mongodb receiver logs in as that user, since root fails `$indexStats` on `admin.system.*`. Set the credentials with `MONGO_MONITOR_USER` (default `otel-monitor`) and `MONGO_MONITOR_PASSWORD` (default `monitorpassword`). +The backend's start creates (or updates) an `otelMonitor` role with `serverStatus`, `top`, `dbStats`, `indexStats` and `listCollections`, plus a user with only that role. The collector's mongodb receiver logs in as that user, since root fails `$indexStats` on `admin.system.*`. The credentials are `MONGO_MONITOR_USER` (default `otel-monitor`) and `MONGO_MONITOR_PASSWORD` in `.env`. Until the backend has run, the receiver's scrapes fail and retry. Grafana starts through a small entrypoint. It resets the admin password from `GF_SECURITY_ADMIN_PASSWORD`, starts Grafana without the alert rules, waits until the dashboard provider has created the `integr8scode` folder, then copies the rules back and calls the alerting reload API. Loaded first, the rules would create their own folder with a random UID. See [Grafana integration](grafana-integration.md). @@ -125,7 +121,7 @@ docker compose exec mongo sh -c 'mongosh -u "$MONGO_INITDB_ROOT_USERNAME" -p "$M |-----------------------|------------------------------------|-------------------------------------------------------------| | Unknown topic errors | Kafka not ready yet | `docker compose logs kafka` | | MongoDB auth errors | Volume created with other credentials | `MONGO_USER`/`MONGO_PASSWORD` must match what the volume was initialized with | -| Worker can't reach k8s | Stale kubeconfig after the cluster was reinstalled or its API address changed | `docker compose restart cert-generator`, then restart the worker | +| Worker can't reach k8s | k3s was reinstalled and wrote a new kubeconfig | `docker compose restart worker` | List topics from inside the broker: @@ -145,20 +141,19 @@ sudo systemctl stop k3s sudo rm -rf /var/lib/rancher/k3s/server/db/ /var/lib/rancher/k3s/server/tls/ sudo systemctl start k3s -# regenerate the kubeconfig and reconnect the worker -docker compose restart cert-generator +# reconnect the worker with the new kubeconfig docker compose restart worker ``` -The cluster comes back empty, so Kueue and the `integr8scode` namespace are recreated by cert-generator on that restart. With NordVPN, `nordvpn set lan-discovery enabled` keeps the host reachable from containers. See [k3s#277](https://github.com/k3s-io/k3s/issues/277) and the [k3s certificate docs](https://docs.k3s.io/cli/certificate). +The cluster comes back empty, and the worker recreates the `integr8scode` namespace, its NetworkPolicy and ResourceQuota when it restarts. With NordVPN, `nordvpn set lan-discovery enabled` keeps the host reachable from containers. See [k3s#277](https://github.com/k3s-io/k3s/issues/277) and the [k3s certificate docs](https://docs.k3s.io/cli/certificate). ### k3s certificates expired -k3s issues its certificates for a year and renews them only when it starts. On a server that runs longer than that without a restart, kubectl and the worker fail with `x509: certificate has expired`, and `sudo systemctl restart k3s` fixes it. If k3s then hangs in `activating` and its journal repeats `dynamiclistener [::]:6443: failed to update cert with connection local address: EOF`, the cached serving certificate in /var/lib/rancher/k3s/server/tls/dynamic-cert.json is empty, which happens when the disk fills up. Move that file aside and start k3s again; it writes a new one. +k3s issues its certificates for a year and renews them only when it starts. On a server that runs longer than that without a restart, kubectl and the worker fail with `x509: certificate has expired`. `sudo systemctl restart k3s` renews them, then `docker compose restart worker`, since the worker reads k3s's kubeconfig, client certificate included, only when it starts. If k3s then hangs in `activating` and its journal repeats `dynamiclistener [::]:6443: failed to update cert with connection local address: EOF`, the cached serving certificate in /var/lib/rancher/k3s/server/tls/dynamic-cert.json is empty, which happens when the disk fills up. Move that file aside and start k3s again; it writes a new one. ## Pre-built images -CI builds `base`, `backend`, `frontend` and `cert-generator` and pushes them to GHCR as `sha-<7 chars>`. On `main`, after the Trivy scan job, they're also tagged `latest`, and each release adds a CalVer tag such as `2026.10.0`. To run them instead of building: +CI builds `backend` and `frontend` and pushes them to GHCR as `sha-<7 chars>`. On `main`, after the Trivy scan job, they're also tagged `latest`, and each release adds a CalVer tag such as `2026.10.0`. To run them instead of building: ```bash IMAGE_TAG=sha-abc1234 docker compose up -d --no-build @@ -166,7 +161,7 @@ IMAGE_TAG=sha-abc1234 docker compose up -d --no-build ## Production -Production gets one release a day. Every merge to `main` goes through CI and Docker Scan & Promote, which moves `latest`; at 23:00 Europe/Berlin [Release & Deploy](cicd.md#release-deploy) releases whatever `latest` is and deploys it, and skips the night when nothing new passed. The deploy job SSHes into the server, removes images no container uses, checks out the release's git tag in `/opt/Integr8sCode` (so the compose file and config match the images) and exports the secrets from the repo (`SECRET_KEY`, `MONGO_USER`, `MONGO_PASSWORD`, `MONGO_MONITOR_PASSWORD`, `REDIS_PASSWORD`, `ADMIN_USER_PASSWORD`, `DEFAULT_USER_PASSWORD`, the Grafana and Mailjet values) together with `COMPOSE_PROFILES=observability` and `GRAFANA_ROOT_URL=https://grafana.integr8scode.cc/`. Then it runs: +Production gets one release a day. Every merge to `main` goes through CI and Docker Scan & Promote, which moves `latest`; at 23:00 Europe/Berlin [Release & Deploy](cicd.md#release-deploy) releases whatever `latest` is and deploys it, and skips the night when nothing new passed. The deploy job SSHes into the server, removes images no container uses, checks out the release's git tag in `/opt/Integr8sCode` (so the compose file and config match the images) and writes `/opt/Integr8sCode/.env` (mode 600) from the repo's secrets (`SECRET_KEY`, `MONGO_USER`, `MONGO_PASSWORD`, `MONGO_MONITOR_PASSWORD`, `REDIS_PASSWORD`, `ADMIN_USER_PASSWORD`, `DEFAULT_USER_PASSWORD`, the Grafana and Mailjet values) together with `IMAGE_TAG`, `COMPOSE_PROFILES=observability` and `GRAFANA_ROOT_URL=https://grafana.integr8scode.cc/`. Compose reads that file for every command, so the backup cron and commands run by hand on the server use the same values, and a secret that came through empty stops compose before anything starts. The values sit in single quotes, so a secret can't contain one. Then it runs: ```bash docker compose pull @@ -174,7 +169,7 @@ docker compose up -d --remove-orphans --no-build --wait --wait-timeout 180 docker image prune -af ``` -Pruning before the pull and after the deploy keeps only the images the running stack uses. To roll back by hand, check out an older CalVer tag, set `IMAGE_TAG` to it and run the same `pull` and `up`; the old images come back from GHCR. Export the same variables the deploy job does first. Without them compose recreates the containers with the dev defaults. +Pruning before the pull and after the deploy keeps only the images the running stack uses. To roll back by hand, check out an older CalVer tag, set `IMAGE_TAG` in `.env` to it and run the same `pull` and `up`; the old images come back from GHCR. The images on GHCR are public, so the server pulls them without logging in. To set up the server and the repository secrets, see [Required secrets](cicd.md#required-secrets). diff --git a/docs/operations/metrics-reference.md b/docs/operations/metrics-reference.md index d3c451e30..f08d03f92 100644 --- a/docs/operations/metrics-reference.md +++ b/docs/operations/metrics-reference.md @@ -250,4 +250,4 @@ sum by (auth_reason) (rate(token_validation_failures_total[5m])) sum(rate(mongodb_cache_operations_total{type="hit"}[5m])) / sum(rate(mongodb_cache_operations_total[5m])) ``` -`sum(pods_by_phase{pod_phase="Pending"})` shows executor pods waiting for a node or a Kueue slot. +`sum(pods_by_phase{pod_phase="Pending"})` shows executor pods waiting for a node or an image pull. diff --git a/docs/operations/nginx-configuration.md b/docs/operations/nginx-configuration.md index 344cbe353..74e635335 100644 --- a/docs/operations/nginx-configuration.md +++ b/docs/operations/nginx-configuration.md @@ -18,7 +18,7 @@ flowchart LR --8<-- "frontend/nginx.conf.template:server_block" ``` -Both server blocks listen on `5001 ssl` with the cert and key that cert-generator writes into `frontend/certs`. Compose publishes the port as `5001:5001`. +Both server blocks listen on `5001 ssl` with `/etc/nginx/certs/server.crt` and `server.key`. Mount your own pair there and nginx uses it; otherwise a script the image puts in `/docker-entrypoint.d/` makes a self-signed pair before nginx starts and keeps it for the container's lifetime. Compose publishes the port as `5001:5001`. ### Security headers @@ -58,7 +58,7 @@ Text types only. Images other than SVG and icons are already compressed. The backend's address goes through a variable and Docker's resolver, like Grafana's below, so nginx looks `backend` up again every 30 seconds. With a plain `proxy_pass https://backend:443`, nginx resolves it once at startup. A backend that comes back with a new IP, for example after a Docker restart, then gets 502 until the frontend restarts too. -`proxy_ssl_verify off` is fine here because the backend sits on the internal compose network with a self-signed cert. Don't copy it to a proxy that leaves the host. +`proxy_ssl_verify on` checks the backend's certificate on every request. The backend makes that certificate when it first starts and keeps it in the `backend_tls_cert` volume, nginx mounts the volume read-only, and `proxy_ssl_trusted_certificate` trusts exactly that file, so a different server answering as `backend` gets a 502. The frontend waits for the backend's healthcheck, since nginx reads the file only when it starts. The SSE location under `/api/v1/events/` speaks HTTP/1.1 with an empty `Connection` header, turns off buffering and caching, and raises both timeouts to 24 hours so long-lived streams aren't cut after the default 60 seconds. `proxy_buffering off` alone is enough to pass each event through as it arrives. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 350ee69c1..c8516ca2c 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -18,7 +18,7 @@ In the tables, the value is what applies with the shipped `config.toml`. The sou ## Secrets -For each secret, `Settings` takes the environment variable if it's set and non-empty, then the TOML value, then a dev default. Compose passes all four to every backend service through the `x-backend-secrets` anchor. +For each secret, `Settings` takes the environment variable if it's set and non-empty, then the TOML value, then a dev default. Compose passes all four to every backend service through the `x-backend-secrets` anchor and has no defaults of its own: the values come from `.env` (`./deploy.sh dev` copies `.env.example`, the production deploy writes it from the repository secrets), and compose refuses to start while one is empty. | Variable | Purpose | Dev default | |----------|---------|-------------| @@ -50,9 +50,10 @@ For each secret, `Settings` takes the environment variable if it's set and non-e | Key | Value | Source | What it does | |-----|-------|--------|--------------| -| `KUBERNETES_CONFIG_PATH` | `/app/kubeconfig.yaml` | config.toml | Kubeconfig for the API, k8s worker and pod monitor | -| `KUBERNETES_CA_CERTIFICATE_PATH` | `/app/certs/k8s-ca.pem` | config.toml | Unused | +| `KUBERNETES_CONFIG_PATH` | `/etc/rancher/k3s/k3s.yaml` | config.toml | k3s's own kubeconfig, read by the k8s worker and pod monitor | +| `KUBERNETES_API_SERVER` | `https://host.docker.internal:6443` | default | API address the client uses instead of the kubeconfig's `127.0.0.1` | | `K8S_NAMESPACE` | `integr8scode` | config.toml | Namespace for executor pods; `default` is refused | +| `K8S_NAMESPACE_CPU_QUOTA`, `K8S_NAMESPACE_MEMORY_QUOTA` | `32`, `4Gi` | default | ResourceQuota on what all pods in that namespace request together | | `K8S_POD_CPU_LIMIT` | `1000m` | config.toml | Seeds the `cpu_limit` system setting | | `K8S_POD_MEMORY_LIMIT` | `128Mi` | config.toml | Seeds the `memory_limit` system setting; also the base for the memory utilization metric | | `K8S_POD_EXECUTION_TIMEOUT` | `5` | config.toml | Seeds the `max_timeout_seconds` system setting | @@ -157,7 +158,7 @@ In the container, the backend runs gunicorn from the Dockerfile's entrypoint, wh | `WEB_THREADS` | `4` | config.toml | Unused by the app; gunicorn reads the environment variable instead | | `SERVER_HOST` | `0.0.0.0` | config.toml | Bind address under `python -m app.main` | | `SERVER_PORT` | `443` | default | Port under `python -m app.main` | -| `SSL_KEYFILE`, `SSL_CERTFILE` | `/app/certs/server.key`, `/app/certs/server.crt` | default | TLS files under `python -m app.main` | +| `SSL_KEYFILE`, `SSL_CERTFILE` | `/app/certs/key/server.key`, `/app/certs/cert/server.crt` | default | TLS files under `python -m app.main`; the container's entrypoint makes them on first start | | `BCRYPT_ROUNDS` | `12` | config.toml | bcrypt cost for passwords | | `REDIS_MAX_CONNECTIONS` | `200` | config.toml | Redis connection pool size | @@ -199,10 +200,10 @@ A few variables are read by containers other than the backend's `Settings`: | Variable | Default | Read by | |----------|---------|---------| | `WEB_CONCURRENCY`, `WEB_THREADS`, `WEB_TIMEOUT` | `2`, `4`, `60` | gunicorn in the backend container | -| `DEFAULT_USER_PASSWORD`, `ADMIN_USER_PASSWORD` | `user123`, `admin123` | `user-seed`, which creates the `user` and `admin` accounts | -| `MONGO_MONITOR_USER`, `MONGO_MONITOR_PASSWORD` | `otel-monitor`, `monitorpassword` | `mongo-monitor-user` creates this least-privilege MongoDB user, and the OTel collector's mongodb receiver logs in with it | +| `DEFAULT_USER_PASSWORD`, `ADMIN_USER_PASSWORD` | none; `.env.example` has `user123`, `admin123` | `scripts.seed_users` in the backend container, which creates the `user` and `admin` accounts before gunicorn starts | +| `MONGO_MONITOR_USER`, `MONGO_MONITOR_PASSWORD` | `otel-monitor`; none for the password, `.env.example` has `monitorpassword` | `scripts.seed_users` creates this least-privilege MongoDB user, and the OTel collector's mongodb receiver logs in with it | | `GRAFANA_ROOT_URL` | `http://localhost:3000/` | Grafana's root URL, and the frontend's `GRAFANA_PUBLIC_URL`, where `/grafana` redirects | -| `GRAFANA_ADMIN_USER`, `GRAFANA_ADMIN_PASSWORD` | `admin`, `admin123` | Grafana | +| `GRAFANA_ADMIN_USER`, `GRAFANA_ADMIN_PASSWORD` | `admin`; none for the password, `.env.example` has `admin123` | Grafana | | `GF_SMTP_HOST`, `MAILJET_API_KEY`, `MAILJET_SECRET_KEY` | empty | Grafana, as the SMTP server and login for alert mail. The release deploy fills the host from the `MAILJET_HOST` secret | | `MAILJET_FROM_ADDRESS`, `GRAFANA_ALERT_RECIPIENTS` | `noreply@integr8scode.cc` | Grafana, as the alert mail sender and recipients | | `IMAGE_TAG` | `latest` | Image tag for every service built from this repo | diff --git a/docs/security/policies.md b/docs/security/policies.md index 6b6054c6b..8ac8eae9f 100644 --- a/docs/security/policies.md +++ b/docs/security/policies.md @@ -1,6 +1,6 @@ # Pod and namespace security -User code runs in executor pods in the `integr8scode` namespace. The pod builder locks down each pod, the k8s worker locks down the namespace at startup, and Kueue caps how much CPU and memory the pods can request in total. +User code runs in executor pods in the `integr8scode` namespace. The pod builder locks down each pod, the k8s worker locks down the namespace at startup, and a ResourceQuota caps how much CPU and memory the pods can request in total. ## Executor pods @@ -22,9 +22,9 @@ The timeout and the CPU and memory limits are the admin settings `max_timeout_se ## Namespace -The k8s worker calls `ensure_namespace_security()` when it starts. The first step is a server-side apply and the second a plain namespace patch, and both are safe to repeat. +The k8s worker calls `ensure_namespace_security()` when it starts. It server-side applies three objects, so each step is safe to repeat: the namespace with its Pod Security labels, the NetworkPolicy and the ResourceQuota. -The first creates or updates the `executor-deny-all` NetworkPolicy. It selects pods labelled `component: executor`, which every executor pod has, and denies all ingress and egress. The worker is the only owner of this policy; setup-k8s.sh doesn't create it. +The first creates or updates the `executor-deny-all` NetworkPolicy. It selects pods labelled `component: executor`, which every executor pod has, and denies all ingress and egress. The worker is the only owner of this policy. ```yaml apiVersion: networking.k8s.io/v1 @@ -45,19 +45,13 @@ The second labels the namespace with the `restricted` Pod Security Standard for The worker also applies a `runtime-image-pre-puller` DaemonSet that pulls every runtime image onto each node, so executor pods don't wait on a pull. Its containers also follow the restricted profile. -The worker reaches the cluster with the `integr8scode-sa` service account that setup-k8s.sh creates. setup-k8s.sh also writes the kubeconfig with that account's token. A Role in the namespace lets it manage pods (with logs), ConfigMaps, services, DaemonSets and NetworkPolicies, and a ClusterRole lets it get, list and patch namespaces for the labels. +The worker reaches the cluster with k3s's own kubeconfig, mounted read-only from `/etc/rancher/k3s` into the worker container only, so it holds the cluster's admin rights. A service account limited to the namespace would not hold it back: the worker sets the namespace's Pod Security labels itself, and an account allowed to change those labels can lift the namespace out of `restricted`. The API container mounts no kubeconfig. -## Quota with Kueue +## Quota -Kueue caps the total CPU and memory that executor pods request, not what they use: a default pod requests 200m CPU but may use up to its 1000m limit. Neither the worker nor a ResourceQuota enforces this. The quota sits on a ClusterQueue that would admit a LocalQueue from any namespace, and only `integr8scode` has one. setup-k8s.sh installs Kueue v0.20.1 (manifest checked by SHA256) and creates the queues: +The `executor-quota` ResourceQuota caps the total CPU and memory that pods in the namespace request, not what they use: a default pod requests 200m CPU but may use up to its 1000m limit. The worker applies it with `requests.cpu` from `K8S_NAMESPACE_CPU_QUOTA` (32) and `requests.memory` from `K8S_NAMESPACE_MEMORY_QUOTA` (4Gi). The API server refuses a pod that would go over it with `403 Forbidden`, and the worker publishes `pod_creation_failed` for its execution. -| Resource | Name | Purpose | -|----------------|------------------|------------------------------------------------| -| ResourceFlavor | `default-flavor` | the cluster's default resources | -| ClusterQueue | `executor-queue` | quota of 32 CPU and 4Gi memory | -| LocalQueue | `executor-queue` | queue in `integr8scode`, bound to the ClusterQueue | - -Executor pods carry `kueue.x-k8s.io/queue-name: executor-queue`. When the quota is used up, Kueue holds new pods behind a scheduling gate instead of rejecting them the way a ResourceQuota would, and releases them as quota frees up. `activeDeadlineSeconds` only starts once a pod is running on a node, so time spent gated doesn't eat into the execution timeout. How many executions run at once is still decided by the saga orchestrator's queue; Kueue is the backstop for total resource requests. +How many executions run at once is decided by the saga orchestrator's queue (`max_concurrent_executions`, 10 by default). Every pod requests 200m CPU and 128Mi, Go's included, since Go raises only its limits, so 10 executions request 2 CPU and 1.25Gi. The memory quota fills at 32 pods, which makes it a backstop that refuses pods only when the admin settings allow more than that at once. ## Network policy enforcement diff --git a/docs/security/supply-chain.md b/docs/security/supply-chain.md index 67ce4933b..0a6560336 100644 --- a/docs/security/supply-chain.md +++ b/docs/security/supply-chain.md @@ -10,13 +10,13 @@ Findings show up in the repository's Security tab under code scanning, as `backe ## Image and code scans -Docker Scan & Promote runs Trivy v0.75.0 on the `base`, `backend`, `frontend` and `cert-generator` images of every commit that passed CI on `main`. It reports `CRITICAL` and `HIGH` findings that have a fix, skips anything listed in `backend/.trivyignore`, and uploads SARIF per image as `trivy-`. A finding fails the job and blocks the promote to `latest`. The images keep the count at zero by upgrading their OS packages at build time, and the backend drops pip, whose vendored packages were the only Python findings. The ignore file holds three Go CVEs in kubectl v1.37.1, the newest release, which cert-generator only uses to talk to the local k3s API. See [CI/CD](../operations/cicd.md#docker-scan-promote). +Docker Scan & Promote runs Trivy v0.75.0 on the `backend` and `frontend` images of every commit that passed CI on `main`. It reports `CRITICAL` and `HIGH` findings that have a fix, skips anything listed in `backend/.trivyignore`, and uploads SARIF per image as `trivy-`. A finding fails the job and blocks the promote to `latest`. The images keep the count at zero by upgrading their OS packages at build time, and the backend drops pip, whose vendored packages were the only Python findings. The ignore file lists no CVE today. See [CI/CD](../operations/cicd.md#docker-scan-promote). CI's Security Scanning job runs `uvx --no-build --python 3.14 bandit==1.9.4 -r backend/ -x backend/tests/ -ll` on pushes to `main` and on PRs that change `backend/`. `-ll` reports medium severity and up, and any finding fails the workflow. ## Pinning -Every GitHub Action, including those used inside the composite actions, is pinned to a full commit SHA with the version in a comment. The Trivy action is pinned to the commit of v0.36.0, since its older tags were deleted upstream. Tools are pinned to exact versions (bandit, Trivy, mkdocs and its plugins), Python installs are `uv sync --frozen --no-build` from the lockfile, and npm installs are `npm ci --ignore-scripts`. The k3s install script and the Kueue manifest are fetched over HTTPS only and checked against a SHA256 before use. +Every GitHub Action, including those used inside the composite actions, is pinned to a full commit SHA with the version in a comment. The Trivy action is pinned to the commit of v0.36.0, since its older tags were deleted upstream. Tools are pinned to exact versions (bandit, Trivy, mkdocs and its plugins), Python installs are `uv sync --frozen --no-build` from the lockfile, and npm installs are `npm ci --ignore-scripts`. The k3s image is pinned by digest. Dependabot opens one grouped PR per ecosystem every week, for uv, npm, the Dockerfiles, the compose images and the GitHub Actions, and updates SHA and version comment together. Details are in [CI/CD](../operations/cicd.md#dependabot). diff --git a/frontend/Dockerfile b/frontend/Dockerfile index cc74599d2..0d433b0dd 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -13,8 +13,8 @@ RUN npm run build # Production stage FROM nginx:1.31-alpine -# Upgrade the base image's packages to pick up fixes newer than the image, and add curl for the healthcheck -RUN apk upgrade --no-cache && apk add --no-cache curl +# Upgrade the base image's packages to pick up fixes newer than the image, and add curl for the healthcheck and openssl for the certificate +RUN apk upgrade --no-cache && apk add --no-cache curl openssl # Copy built static files COPY --from=builder /app/build /usr/share/nginx/html @@ -26,10 +26,22 @@ COPY nginx.conf.template /etc/nginx/templates/default.conf.template ENV GRAFANA_PUBLIC_URL=http://localhost:3000/ # Create writable directories for nginx (required for read-only root filesystem) -# and certs directory for TLS certificates mounted at runtime +# and certs directory for TLS certificates RUN mkdir -p /var/cache/nginx /var/run /etc/nginx/certs && \ chown -R nginx:nginx /var/cache/nginx /var/run /etc/nginx/certs +# nginx's entrypoint runs this before it starts; a certificate mounted at /etc/nginx/certs is kept as is +COPY --chmod=755 <<'EOF' /docker-entrypoint.d/05-self-signed-cert.sh +#!/bin/sh +set -e +if [ -s /etc/nginx/certs/server.crt ]; then + exit 0 +fi +openssl req -x509 -newkey rsa:2048 -nodes -days 3650 -quiet -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1" \ + -keyout /etc/nginx/certs/server.key -out /etc/nginx/certs/server.crt +EOF + EXPOSE 5001 CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/nginx.conf.template b/frontend/nginx.conf.template index 1a6d3efe8..5f0d86082 100644 --- a/frontend/nginx.conf.template +++ b/frontend/nginx.conf.template @@ -46,7 +46,9 @@ server { location /api/ { proxy_pass $backend_upstream; - proxy_ssl_verify off; + # The backend's own certificate, from the volume it writes it to; the SSE location below inherits this + proxy_ssl_verify on; + proxy_ssl_trusted_certificate /etc/nginx/backend/server.crt; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -61,7 +63,6 @@ server { # SSE specific settings for /api/v1/events/ location ~ ^/api/v1/events/ { proxy_pass $backend_upstream; - proxy_ssl_verify off; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 90a8324d8..905caa782 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -15,7 +15,8 @@ function devServer(): ServerOptions { strictPort: true, https: tls ? { key: fs.readFileSync(keyFile), cert: fs.readFileSync(certFile) } : undefined, proxy: { - '/api': { target: 'https://backend:443', secure: false }, + // Node trusts the backend's self-signed certificate once NODE_EXTRA_CA_CERTS points at it + '/api': { target: 'https://backend:443', secure: true }, }, }; }