From d32d0a7e83fdf0c64e513fdc2cfa69d6854064c8 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 24 Jul 2026 21:42:20 -0400 Subject: [PATCH] fix: prevent sandbox virtio-fs fd exhaustion --- .github/workflows/ci.yml | 14 + README.md | 13 +- api/Dockerfile | 15 +- docker-compose.local-dev.yml | 7 + docker-compose.scalable.yml | 2 + docker-compose.yaml | 6 +- docker/Dockerfile.worker-sandbox | 97 ++++++- helm/codeapi/Chart.yaml | 2 +- helm/codeapi/README.md | 50 +++- helm/codeapi/templates/package-init-job.yaml | 2 +- helm/codeapi/templates/pvc.yaml | 2 +- .../templates/worker-sandbox-deployment.yaml | 18 +- helm/codeapi/values-local.yaml | 3 + helm/codeapi/values.yaml | 18 +- tests/block_root_package_delivery.sh | 274 ++++++++++++++++++ 15 files changed, 493 insertions(+), 30 deletions(-) create mode 100755 tests/block_root_package_delivery.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d73538a..bd43f21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,20 @@ concurrency: cancel-in-progress: true jobs: + deployment-config-tests: + name: Deployment Config Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Block-root package delivery + run: tests/block_root_package_delivery.sh + + - name: Validate sandbox Dockerfiles + run: | + docker buildx build --check -f api/Dockerfile . + docker buildx build --check -f docker/Dockerfile.worker-sandbox . + api-unit-tests: name: API Unit Tests runs-on: ubuntu-latest diff --git a/README.md b/README.md index 01c26bf..718db6a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ Code Interpreter (internally `codeapi`, the prefix used by its env vars, images, - **Worker Sandbox** - Executes code in NsJail (or libkrun microVM) sandboxes with resource limits - **File Server** - Manages file uploads/downloads via S3 (IRSA authentication) - **Tool Call Server** - Handles programmatic tool calls from within sandbox sessions -- **Package Init** - One-time job that pre-installs language runtimes (Python, Node, Bun) onto a shared PVC +- **Package Delivery** - Bakes Python, Node, and Bun into the default microVM + block-root image; a package-init PVC mode remains available for direct NsJail + development ## Architecture @@ -56,6 +58,15 @@ public issue (see [CONTRIBUTING](CONTRIBUTING.md)). docker-compose up --build ``` +The default KVM Compose path builds `sandbox-runner-baked`: the guest root and +`/pkgs` tree live in a read-only ext4 block image instead of a long-lived +virtio-fs mount. The first image build takes longer because it compiles the +language runtimes, but package-heavy workloads do not accumulate host file +descriptors in the launcher. + +Setting `KVM_ENABLED=false` still selects the directory-root target and the +host package mount automatically for direct NsJail development. + Local Docker Compose files set `CODEAPI_INTERNAL_SERVICE_TOKEN` to a shared development value by default. Production deployments must override it with a strong secret; when it is unset, file object routes and Tool Call Server diff --git a/api/Dockerfile b/api/Dockerfile index ff4e848..b8aedbf 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -170,10 +170,12 @@ RUN cargo build --release # # Split into: # - sandbox-runner-base : everything except the guest rootfs copy -# - sandbox-runner : default target; copies rootfs without packages -# (packages come from a runtime PVC mount) +# - sandbox-runner : legacy/direct target; copies rootfs without +# packages (packages come from a runtime PVC mount) # - sandbox-runner-baked: copies rootfs AND bakes /pkgs from package-builder # (no PVC needed; runner can spread across nodes) +# - sandbox-runner-{true,false}: Compose aliases selected by KVM_ENABLED +# - sandbox-runner-default: final alias for the safe baked KVM image # ============================================================================ FROM fedora:43 AS sandbox-runner-base @@ -232,3 +234,12 @@ COPY --from=sandbox-build / /sandbox-rootfs/ RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg RUN mkdir -p /host-packages + +# Compose derives these aliases from KVM_ENABLED so existing no-KVM overrides +# keep selecting the directory-root image automatically. +FROM sandbox-runner-baked AS sandbox-runner-true +FROM sandbox-runner AS sandbox-runner-false + +# Keep the safe KVM image as the Dockerfile default. Direct NsJail and legacy +# virtio-fs deployments can still select --target sandbox-runner explicitly. +FROM sandbox-runner-true AS sandbox-runner-default diff --git a/docker-compose.local-dev.yml b/docker-compose.local-dev.yml index 5edf343..bf58b59 100644 --- a/docker-compose.local-dev.yml +++ b/docker-compose.local-dev.yml @@ -12,6 +12,7 @@ services: build: context: . dockerfile: api/Dockerfile + target: sandbox-runner-${KVM_ENABLED:-true} container_name: sandbox privileged: false devices: @@ -22,6 +23,7 @@ services: - ${SANDBOX_PACKAGES_PATH:-./data/pkgs}:/host-packages:ro environment: - KVM_ENABLED=${KVM_ENABLED:-true} + - LAUNCHER_PACKAGES_HOST=/disabled-host-packages - LAUNCHER_VCPUS=2 - LAUNCHER_RAM_MIB=2048 - LAUNCHER_LOG_LEVEL=3 @@ -40,6 +42,11 @@ services: - CODEAPI_INTERNAL_SERVICE_TOKEN=${CODEAPI_INTERNAL_SERVICE_TOKEN:-localdev-internal-service-token} - SANDBOX_ALLOWED_LOCAL_NETWORK_PORT=3033 - SANDBOX_FORWARD_TARGET=tool_call_server:3033 + healthcheck: + test: ["CMD", "/usr/local/bin/sandbox-runner-healthcheck.sh"] + interval: 10s + timeout: 5s + retries: 6 depends_on: - tool_call_server diff --git a/docker-compose.scalable.yml b/docker-compose.scalable.yml index 4b74114..49c8914 100644 --- a/docker-compose.scalable.yml +++ b/docker-compose.scalable.yml @@ -169,6 +169,7 @@ services: build: context: . dockerfile: docker/Dockerfile.worker-sandbox + target: worker-sandbox-${KVM_ENABLED:-true} privileged: false devices: - ${KVM_DEVICE_PATH:-/dev/kvm}:/dev/kvm @@ -207,6 +208,7 @@ services: - SANDBOX_DISABLE_NETWORKING=true - SANDBOX_ALLOWED_LOCAL_NETWORK_PORT=3033 - SANDBOX_FORWARD_TARGET=tool_call_server:3033 + - LAUNCHER_PACKAGES_HOST=/disabled-host-packages volumes: - ${SANDBOX_PACKAGES_PATH:-./data/pkgs}:/host-packages:ro depends_on: diff --git a/docker-compose.yaml b/docker-compose.yaml index 3b6560b..ac98d91 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -124,6 +124,7 @@ services: build: context: . dockerfile: api/Dockerfile + target: sandbox-runner-${KVM_ENABLED:-true} container_name: sandbox-runner restart: always privileged: false @@ -135,6 +136,9 @@ services: - ${SANDBOX_PACKAGES_PATH:-./data/pkgs}:/host-packages:ro environment: - KVM_ENABLED=${KVM_ENABLED:-true} + # The direct path uses /host-packages itself. The KVM path must not + # hand that directory to libkrun as a long-lived virtio-fs share. + - LAUNCHER_PACKAGES_HOST=/disabled-host-packages - CODEAPI_HARDENED_SANDBOX_MODE=${CODEAPI_HARDENED_SANDBOX_MODE:-true} - EGRESS_GATEWAY_URL=http://egress_gateway:3190 - SANDBOX_EXECUTE_BODY_LIMIT=${SANDBOX_EXECUTE_BODY_LIMIT:-50mb} @@ -161,7 +165,7 @@ services: egress_gateway: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "curl -fsS http://localhost:2000/api/v2/runtimes >/dev/null || exit 1"] + test: ["CMD", "/usr/local/bin/sandbox-runner-healthcheck.sh"] interval: 5s timeout: 3s retries: 60 diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 5e0c97d..cd3edd3 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -5,7 +5,7 @@ # # Architecture within this container: # ┌──────────────────────────────────────────────────────────────────────────┐ -# │ Worker-Sandbox Container (Fedora 41, libkrun) │ +# │ Worker-Sandbox Container (Fedora 43, libkrun) │ # │ │ # │ ┌─────────────────┐ ┌──────────────────────────────────────────┐│ # │ │ Worker Process │ ──────▶│ libkrun MicroVM (own kernel) ││ @@ -45,6 +45,36 @@ COPY api/src/spec-guard.c /tmp/spec-guard.c RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && chmod 0111 /usr/local/bin/spec-guard +# ============================================================================ +# Stage 1b: Build language runtime packages for the baked KVM root disk +# +# Mirrors docker/Dockerfile.package-init, but populates /pkgs at image-build +# time so the combined worker can boot without a long-lived /host-packages +# virtio-fs mount. +# ============================================================================ +FROM buildpack-deps:bookworm AS package-builder + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHON_VERSION=3.14.4 +ENV BUN_VERSION=1.3.14 + +COPY docker/apt-install.sh /usr/local/bin/apt-install +RUN chmod +x /usr/local/bin/apt-install + +RUN apt-install \ + curl unzip wget ca-certificates \ + build-essential libssl-dev libffi-dev libsqlite3-dev \ + zlib1g-dev libbz2-dev libreadline-dev libncurses5-dev \ + tk-dev xz-utils libcurl4-openssl-dev libfontconfig1-dev \ + libudunits2-dev libpng-dev libxml2-dev libcairo2-dev \ + libfreetype6-dev + +RUN mkdir -p /pkgs + +COPY docker/package-init.sh /package-init.sh +COPY javascript-packages.txt /javascript-packages.txt +RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh + # ============================================================================ # Stage 2: Build worker # ============================================================================ @@ -153,9 +183,35 @@ COPY launcher/src/ ./src/ RUN cargo build --release # ============================================================================ -# Stage 6: Final combined image (Fedora for libkrun runtime) +# Stage 5b: Build the read-only ext4 guest root disk +# ============================================================================ +FROM debian:bookworm AS worker-sandbox-rootfs-image + +RUN apt-get update && apt-get install -y --no-install-recommends \ + e2fsprogs \ + coreutils \ + gawk \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=sandbox-rootfs / /sandbox-rootfs/ +COPY --from=package-builder /pkgs /sandbox-rootfs/pkgs +COPY docker/build-rootfs-image.sh /usr/local/bin/build-rootfs-image.sh + +RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg +RUN chmod +x /usr/local/bin/build-rootfs-image.sh \ + && /usr/local/bin/build-rootfs-image.sh /sandbox-rootfs /sandbox-rootfs.img + +# ============================================================================ +# Stage 6: Shared combined worker host (Fedora for libkrun runtime) +# +# Final targets: +# - worker-sandbox-baked: KVM default; packages live in a read-only ext4 +# block root, so no package virtio-fs mount is needed +# - worker-sandbox-legacy: directory root plus /host-packages for legacy KVM +# deployments and KVM_ENABLED=false direct mode +# - worker-sandbox-{true,false}: Compose aliases selected by KVM_ENABLED # ============================================================================ -FROM fedora:43 +FROM fedora:43 AS worker-sandbox-base RUN dnf install -y --setopt=install_weak_deps=False \ libkrun \ @@ -176,13 +232,6 @@ ENV PATH="/root/.bun/bin:${PATH}" # --- Launcher (runs on host, boots microVM) --- COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/local/bin/launcher -# --- Sandbox rootfs (becomes the microVM guest filesystem) --- -COPY --from=sandbox-rootfs / /sandbox-rootfs/ - -RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg - -RUN mkdir -p /host-packages - # --- Launcher entrypoint (DNS resolution + socat relay before VM boot) --- COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh COPY docker/start-direct-sandbox.sh /usr/local/bin/start-direct-sandbox.sh @@ -207,3 +256,31 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ CMD curl -f http://localhost:3113/health || exit 1 CMD ["/supervisor.sh"] + +# Legacy directory-root target. Keep this for deployments that intentionally +# mount packages at /host-packages, and for the KVM_ENABLED=false direct path. +FROM worker-sandbox-base AS worker-sandbox-legacy + +COPY --from=sandbox-rootfs / /sandbox-rootfs/ + +RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg \ + && mkdir -p /host-packages + +# KVM production default. The package tree is part of the read-only block root, +# avoiding host file-descriptor retention from package lookups over virtio-fs. +FROM worker-sandbox-base AS worker-sandbox-baked + +COPY --from=worker-sandbox-rootfs-image /sandbox-rootfs.img /sandbox-rootfs.img + +ENV LAUNCHER_ROOT_DISK=/sandbox-rootfs.img \ + LAUNCHER_ROOT_DISK_READ_ONLY=true \ + LAUNCHER_ROOT_DEVICE=/dev/vda \ + LAUNCHER_ROOT_FSTYPE=ext4 \ + LAUNCHER_ROOT_OPTIONS=ro + +# Compose derives these aliases from KVM_ENABLED so the direct path keeps +# selecting the directory-root image without duplicating the baked rootfs. +FROM worker-sandbox-baked AS worker-sandbox-true +FROM worker-sandbox-legacy AS worker-sandbox-false + +FROM worker-sandbox-true AS worker-sandbox-default diff --git a/helm/codeapi/Chart.yaml b/helm/codeapi/Chart.yaml index 12fceb4..9737e90 100644 --- a/helm/codeapi/Chart.yaml +++ b/helm/codeapi/Chart.yaml @@ -3,7 +3,7 @@ apiVersion: v2 name: codeapi description: A Helm chart for Code Interpreter API - scalable code execution service type: application -version: 0.2.0 # Chart version (bump this when you change the chart) +version: 0.3.0 # Chart version (bump this when you change the chart) appVersion: "2.0.0" # App version (bump this when you change the app) # Keywords for searching diff --git a/helm/codeapi/README.md b/helm/codeapi/README.md index a60f89c..9198b1a 100644 --- a/helm/codeapi/README.md +++ b/helm/codeapi/README.md @@ -78,6 +78,30 @@ For development only, `LOCAL_MODE=true` bypasses authentication — see **TLS to Redis.** Set `REDIS_TLS=true` via `extraEnv` on each component when your Redis (e.g. a managed cache) requires TLS. +**Package delivery.** KVM deployments default to +`workerSandbox.packages.source=image`. Build and publish the baked runner target +under the existing `workerSandbox.sandboxImage` repository and tag: + +```bash +docker build \ + --target sandbox-runner-baked \ + -t codeapi-sandbox-runner:latest \ + -f api/Dockerfile . +``` + +That image boots the guest and its `/pkgs` tree from a read-only ext4 block +image. It intentionally does not expose `/host-packages` through virtio-fs, +which prevents package imports from pinning host-side launcher file +descriptors. Set `workerSandbox.packages.source=pvc` only for compatibility with +runtime package rebuilds; Kubernetes will then use the FD-aware liveness probe +to recycle the runner before descriptor exhaustion. + +When upgrading from chart 0.2.x, rebuild the configured `sandboxImage` from the +baked target before enabling the 0.3.x default. To keep an existing +directory-root image during migration, set +`workerSandbox.packages.source=pvc`; the established image repository/tag +override remains unchanged in either mode. + ## Quick Start (Local Development) ### 1. Start Minikube @@ -93,7 +117,7 @@ eval $(minikube docker-env) # Build all images, from the codeapi root docker build -t codeapi-api:latest -f service/Dockerfile.api . docker build -t codeapi-worker:latest -f service/Dockerfile.worker . -docker build -t codeapi-sandbox-runner:latest -f api/Dockerfile . +docker build --target sandbox-runner -t codeapi-sandbox-runner:latest -f api/Dockerfile . docker build -t codeapi-file-server:latest -f service/Dockerfile --target production . docker build -t codeapi-tool-call-server:latest -f service/Dockerfile.tool-call-server --target production . docker build -t codeapi-package-init:latest -f docker/Dockerfile.package-init . @@ -113,9 +137,13 @@ helm dependency update helm install codeapi . -f values-local.yaml ``` -### 4. Language Packages (Automatic) +### 4. Language Packages (Local PVC Mode) -The chart includes a **package-init Job** that runs as a Helm `pre-install` hook. It automatically compiles Python, downloads Node/Bun, installs offline package sets, and registers Bash into the packages PVC before the worker pods start. +`values-local.yaml` disables KVM and selects the legacy PVC package source. In +that mode the chart includes a **package-init Job** that runs as a Helm +`pre-install` hook. It compiles Python, downloads Node/Bun, installs offline +package sets, and registers Bash into the packages PVC before the worker pods +start. This happens automatically on `helm install`. To force a rebuild: @@ -168,7 +196,7 @@ curl http://localhost:3112/v1/health # Start minikube minikube start -# Deploy (package-init job runs automatically) +# Deploy local direct mode (package-init job runs automatically) helm install codeapi ./helm/codeapi -f ./helm/codeapi/values-local.yaml # Port forward @@ -204,7 +232,7 @@ helm upgrade codeapi ./helm/codeapi -f ./helm/codeapi/values-local.yaml \ # Rebuild images (must be in minikube docker env) eval $(minikube docker-env) docker build -t codeapi-worker:latest -f service/Dockerfile.worker . -docker build -t codeapi-sandbox-runner:latest -f api/Dockerfile . +docker build --target sandbox-runner -t codeapi-sandbox-runner:latest -f api/Dockerfile . # Restart deployments to pick up new images kubectl rollout restart deployment/codeapi-service-worker @@ -285,9 +313,9 @@ kubectl logs deployment/codeapi-service-worker --tail=5 | +--------------+ +--------------+ +--------------+ | | | | +---------------------------------------------------------------+ | -| | PersistentVolume (Packages) | | -| | /pkgs - Python, Node, Bun runtimes | | -| | ReadOnlyMany - shared across all sandbox-runner pods | | +| | Package delivery | | +| | KVM default: /pkgs in each baked ext4 block-root image | | +| | Direct mode: shared package PVC populated by package-init | | | +---------------------------------------------------------------+ | | | | Total sandbox capacity: 3 pods x 8 concurrent jobs = 24 jobs | @@ -315,7 +343,7 @@ kubectl describe pod ### "runtime is unknown" error ```bash -# Language packages PVC is empty. Check if the init job ran: +# In source=pvc mode, check whether the package-init job populated the PVC: kubectl get jobs -l app.kubernetes.io/component=package-init kubectl logs job/codeapi-package-init @@ -326,6 +354,10 @@ helm upgrade codeapi . --set workerSandbox.packages.initJob.forceRebuild=true kubectl rollout restart deployment/codeapi-sandbox-runner ``` +In the default `source=image` mode, rebuild and publish +`workerSandbox.sandboxImage` from the `sandbox-runner-baked` target instead; no +package-init Job or packages PVC is rendered. + ### Connection refused on port 3112 ```bash # Make sure port-forward is running diff --git a/helm/codeapi/templates/package-init-job.yaml b/helm/codeapi/templates/package-init-job.yaml index d7fbfe2..8d92750 100644 --- a/helm/codeapi/templates/package-init-job.yaml +++ b/helm/codeapi/templates/package-init-job.yaml @@ -3,7 +3,7 @@ Package Init Job Runs as a Helm pre-install/pre-upgrade hook to populate the packages PVC with Python, Node, Bun, and Bash runtimes for the NsJail sandbox. */}} -{{- if and .Values.workerSandbox.enabled .Values.workerSandbox.packages.initJob.enabled }} +{{- if and .Values.workerSandbox.enabled (eq (.Values.workerSandbox.packages.source | default "image") "pvc") .Values.workerSandbox.packages.initJob.enabled }} {{- $packageInitNodeSelector := .Values.workerSandbox.sandboxRunner.nodeSelector }} {{- $packageInitTolerations := .Values.workerSandbox.sandboxRunner.tolerations }} {{- if .Values.workerSandbox.sandboxRunner.inheritSharedScheduling }} diff --git a/helm/codeapi/templates/pvc.yaml b/helm/codeapi/templates/pvc.yaml index d54c34b..883395c 100644 --- a/helm/codeapi/templates/pvc.yaml +++ b/helm/codeapi/templates/pvc.yaml @@ -12,7 +12,7 @@ If using the initJob to populate packages: - The init job writes packages to the PVC - Worker pods then mount the PVC (read-only at container level) */}} -{{- if and .Values.workerSandbox.enabled .Values.workerSandbox.packages.persistence.enabled (not .Values.workerSandbox.packages.persistence.existingClaim) }} +{{- if and .Values.workerSandbox.enabled (eq (.Values.workerSandbox.packages.source | default "image") "pvc") .Values.workerSandbox.packages.persistence.enabled (not .Values.workerSandbox.packages.persistence.existingClaim) }} apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml index ff23008..8a82a61 100644 --- a/helm/codeapi/templates/worker-sandbox-deployment.yaml +++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml @@ -19,6 +19,14 @@ Split runtime deployments: {{- if not .Values.executionManifest.publicKey }} {{- fail "workerSandbox.enabled requires executionManifest.publicKey so sandbox-runner can verify signed execute manifests without a signing secret. The chart ships no default keypair; derive the public key from your Ed25519 private key (see helm/codeapi/README.md 'Execution manifest signing keys') and set it via --set or a values file (base64 DER or PEM with escaped newlines). For minikube local dev use -f values-local.yaml" }} {{- end }} +{{- $packagesSource := .Values.workerSandbox.packages.source | default "image" }} +{{- if and (ne $packagesSource "image") (ne $packagesSource "pvc") }} +{{- fail "workerSandbox.packages.source must be image or pvc" }} +{{- end }} +{{- $packagesFromPvc := eq $packagesSource "pvc" }} +{{- if and (not $packagesFromPvc) (not .Values.workerSandbox.kvmEnabled) }} +{{- fail "workerSandbox.packages.source=image requires workerSandbox.kvmEnabled=true because the baked runner boots from a libkrun block root image" }} +{{- end }} {{- range .Values.workerSandbox.sandboxExtraEnv }} {{- $name := .name | default "" }} {{- if and $name (or (regexMatch "(?i)(SECRET|TOKEN|PASSWORD|KEY)" $name) (hasPrefix "REDIS_" $name) (hasPrefix "AWS_" $name) (hasPrefix "S3_" $name) (hasPrefix "MINIO_" $name) (hasPrefix "CODEAPI_" $name) (eq $name "FILE_SERVER_URL") (eq $name "TOOL_CALL_SERVER_URL")) }} @@ -249,7 +257,7 @@ spec: - name: LAUNCHER_FILTER_VSOCK_ENOTCONN value: {{ .Values.workerSandbox.launcher.filterVsockEnotconn | quote }} - name: SANDBOX_RUNNER_FD_LIVENESS_LIMIT - value: {{ .Values.workerSandbox.sandboxRunner.fdLivenessLimit | default 40000 | quote }} + value: {{ .Values.workerSandbox.sandboxRunner.fdLivenessLimit | quote }} - name: EGRESS_GATEWAY_URL value: "http://{{ include "codeapi.fullname" . }}-egress-gateway:{{ .Values.egressGateway.service.port }}" - name: SANDBOX_LOG_LEVEL @@ -297,14 +305,18 @@ spec: {{- with .Values.workerSandbox.sandboxExtraEnv }} {{- toYaml . | nindent 12 }} {{- end }} + {{- if or $packagesFromPvc $useKvmHostPath }} volumeMounts: + {{- if $packagesFromPvc }} - name: packages mountPath: /host-packages readOnly: true + {{- end }} {{- if $useKvmHostPath }} - name: dev-kvm mountPath: /dev/kvm {{- end }} + {{- end }} {{- if .Values.workerSandbox.kvmEnabled }} securityContext: privileged: false @@ -369,7 +381,9 @@ spec: {{- $_ := set $sandboxResources "limits" $limits }} {{- end }} {{- toYaml $sandboxResources | nindent 12 }} + {{- if or $packagesFromPvc $useKvmHostPath }} volumes: + {{- if $packagesFromPvc }} - name: packages {{- if .Values.workerSandbox.packages.hostPath }} hostPath: @@ -384,12 +398,14 @@ spec: {{- else }} emptyDir: {} {{- end }} + {{- end }} {{- if $useKvmHostPath }} - name: dev-kvm hostPath: path: /dev/kvm type: CharDevice {{- end }} + {{- end }} {{- with $sandboxRunnerNodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/values-local.yaml b/helm/codeapi/values-local.yaml index 652e23d..ea56632 100644 --- a/helm/codeapi/values-local.yaml +++ b/helm/codeapi/values-local.yaml @@ -74,6 +74,9 @@ workerSandbox: # Use PVC for packages (fast local storage, no 9p overhead) # Populate manually: see cheat sheet pvc-populator section packages: + # The baked block-root image requires KVM; local direct NsJail mode keeps + # the legacy PVC package path. + source: pvc hostPath: "" # Don't use hostPath (9p is slow) persistence: enabled: true diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index ce16c34..c926e76 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -120,6 +120,9 @@ workerSandbox: pullPolicy: IfNotPresent sandboxImage: + # With packages.source=image, build the api/Dockerfile default (or + # sandbox-runner-baked). With source=pvc/direct mode, build + # --target sandbox-runner under the same configured repository/tag. repository: codeapi-sandbox-runner tag: latest pullPolicy: IfNotPresent @@ -241,9 +244,18 @@ workerSandbox: jobUidCount: null workspaceReaperMaxAgeSeconds: 3600 - # Language runtime packages (mount from host or PVC) + # Language runtime package delivery packages: - # Use a PVC for packages + # image (default): boot a baked /pkgs tree from a read-only ext4 block root. + # This avoids long-lived virtio-fs inode handles exhausting the launcher FD + # limit. Requires kvmEnabled=true and sandboxImage built from the + # sandbox-runner-baked target (the api/Dockerfile default). + # + # pvc: legacy compatibility mode. Mount a host/PVC package tree through + # virtio-fs and recycle runners with the FD liveness guard before exhaustion. + source: image + + # Used only when source=pvc. persistence: enabled: true size: 10Gi @@ -252,7 +264,7 @@ workerSandbox: # If you have pre-built packages, specify the existing claim existingClaim: "" - # Init job to populate packages on first install + # Used only when source=pvc. Populates packages on first install. # Runs as a Helm pre-install/pre-upgrade hook # Compiles Python from source and downloads Node/Bun binaries initJob: diff --git a/tests/block_root_package_delivery.sh b/tests/block_root_package_delivery.sh new file mode 100755 index 0000000..c549533 --- /dev/null +++ b/tests/block_root_package_delivery.sh @@ -0,0 +1,274 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Regression coverage for issue #15: KVM deployment defaults must boot the +# baked package tree from a block root and must not reintroduce /host-packages +# as a long-lived virtio-fs mount. + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +for command in docker helm jq; do + if ! command -v "$command" >/dev/null 2>&1; then + echo "missing required command: $command" >&2 + exit 1 + fi +done + +assert_contains() { + local file="$1" + local pattern="$2" + local message="$3" + if ! grep -Eq "$pattern" "$file"; then + echo "$message" >&2 + exit 1 + fi +} + +assert_not_contains() { + local file="$1" + local pattern="$2" + local message="$3" + if grep -Eq "$pattern" "$file"; then + echo "$message" >&2 + exit 1 + fi +} + +assert_compose_mode() { + local compose_file="$1" + local service="$2" + local kvm_enabled="$3" + local expected_target="$4" + local output="$5" + + KVM_ENABLED="$kvm_enabled" \ + docker compose -f "$compose_file" config --format json > "$output" + + local target + target="$(jq -r --arg service "$service" '.services[$service].build.target // ""' "$output")" + if [[ "$target" != "$expected_target" ]]; then + echo "$compose_file: $service target is '$target', expected '$expected_target'" >&2 + exit 1 + fi + + if ! jq -e --arg service "$service" --arg expected "$kvm_enabled" \ + '.services[$service].environment.KVM_ENABLED == $expected' \ + "$output" >/dev/null; then + echo "$compose_file: $service did not preserve KVM_ENABLED=$kvm_enabled" >&2 + exit 1 + fi + + if [[ "$kvm_enabled" == "true" ]]; then + local packages_host + packages_host="$(jq -r --arg service "$service" \ + '.services[$service].environment.LAUNCHER_PACKAGES_HOST // ""' "$output")" + if [[ "$packages_host" != "/disabled-host-packages" ]]; then + echo "$compose_file: KVM mode could expose /host-packages to libkrun" >&2 + exit 1 + fi + elif ! jq -e --arg service "$service" \ + 'any((.services[$service].volumes // [])[]; .target == "/host-packages")' \ + "$output" >/dev/null; then + echo "$compose_file: direct mode lost its /host-packages compatibility mount" >&2 + exit 1 + fi +} + +assert_compose_mode \ + "$ROOT/docker-compose.yaml" \ + sandbox-runner \ + true \ + sandbox-runner-true \ + "$TMP_DIR/compose.json" +assert_compose_mode \ + "$ROOT/docker-compose.local-dev.yml" \ + sandbox \ + true \ + sandbox-runner-true \ + "$TMP_DIR/compose-local.json" +assert_compose_mode \ + "$ROOT/docker-compose.scalable.yml" \ + worker-sandbox \ + true \ + worker-sandbox-true \ + "$TMP_DIR/compose-scalable.json" + +assert_compose_mode \ + "$ROOT/docker-compose.yaml" \ + sandbox-runner \ + false \ + sandbox-runner-false \ + "$TMP_DIR/compose-direct.json" +assert_compose_mode \ + "$ROOT/docker-compose.local-dev.yml" \ + sandbox \ + false \ + sandbox-runner-false \ + "$TMP_DIR/compose-local-direct.json" +assert_compose_mode \ + "$ROOT/docker-compose.scalable.yml" \ + worker-sandbox \ + false \ + worker-sandbox-false \ + "$TMP_DIR/compose-scalable-direct.json" + +if [[ "$(awk '/^FROM / { stage=$NF } END { print stage }' "$ROOT/api/Dockerfile")" != "sandbox-runner-default" ]]; then + echo "api/Dockerfile must default to the baked block-root runner" >&2 + exit 1 +fi +if [[ "$(awk '/^FROM / { stage=$NF } END { print stage }' "$ROOT/docker/Dockerfile.worker-sandbox")" != "worker-sandbox-default" ]]; then + echo "docker/Dockerfile.worker-sandbox must default to the baked block-root runner" >&2 + exit 1 +fi + +assert_contains \ + "$ROOT/api/Dockerfile" \ + '^FROM sandbox-runner-baked AS sandbox-runner-true$' \ + "sandbox-runner-true must inherit the baked target" +assert_contains \ + "$ROOT/api/Dockerfile" \ + '^FROM sandbox-runner AS sandbox-runner-false$' \ + "sandbox-runner-false must inherit the direct/PVC target" +assert_contains \ + "$ROOT/docker/Dockerfile.worker-sandbox" \ + '^FROM worker-sandbox-baked AS worker-sandbox-true$' \ + "worker-sandbox-true must inherit the baked target" +assert_contains \ + "$ROOT/docker/Dockerfile.worker-sandbox" \ + '^FROM worker-sandbox-legacy AS worker-sandbox-false$' \ + "worker-sandbox-false must inherit the direct target" + +awk ' + /^FROM / { + if (inside) exit + inside = ($NF == "sandbox-runner-baked") + } + inside { print } +' "$ROOT/api/Dockerfile" > "$TMP_DIR/api-baked-stage" +awk ' + /^FROM / { + if (inside) exit + inside = ($NF == "worker-sandbox-baked") + } + inside { print } +' "$ROOT/docker/Dockerfile.worker-sandbox" > "$TMP_DIR/worker-baked-stage" + +for baked_stage in "$TMP_DIR/api-baked-stage" "$TMP_DIR/worker-baked-stage"; do + assert_contains \ + "$baked_stage" \ + '^ENV LAUNCHER_ROOT_DISK=/sandbox-rootfs.img' \ + "$baked_stage must configure the block root" + assert_not_contains \ + "$baked_stage" \ + '/host-packages' \ + "$baked_stage must not create the package virtio-fs path" +done + +# The chart dependencies are unrelated to package delivery. Copy the chart to +# a temporary directory without its dependency block so this test remains +# hermetic and does not download Redis or MinIO. +mkdir "$TMP_DIR/chart" +cp "$ROOT/helm/codeapi/values.yaml" "$TMP_DIR/chart/values.yaml" +cp -R "$ROOT/helm/codeapi/templates" "$TMP_DIR/chart/templates" +awk '/^dependencies:/{exit} {print}' \ + "$ROOT/helm/codeapi/Chart.yaml" > "$TMP_DIR/chart/Chart.yaml" + +helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + > "$TMP_DIR/helm-image.yaml" + +assert_contains \ + "$TMP_DIR/helm-image.yaml" \ + 'image: "codeapi-sandbox-runner:latest"' \ + "default Helm render must preserve sandboxImage" +assert_not_contains \ + "$TMP_DIR/helm-image.yaml" \ + 'mountPath: /host-packages' \ + "default Helm render must not mount /host-packages" +assert_not_contains \ + "$TMP_DIR/helm-image.yaml" \ + '^kind: PersistentVolumeClaim$' \ + "default Helm render must not create the packages PVC" +assert_not_contains \ + "$TMP_DIR/helm-image.yaml" \ + 'app.kubernetes.io/component: package-init' \ + "default Helm render must not create package-init" + +helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.sandboxImage.repository=registry.example/sandbox \ + --set workerSandbox.sandboxImage.tag=custom \ + > "$TMP_DIR/helm-custom-image.yaml" + +assert_contains \ + "$TMP_DIR/helm-custom-image.yaml" \ + 'image: "registry.example/sandbox:custom"' \ + "image package mode must preserve the established sandboxImage override" + +helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.packages.source=pvc \ + > "$TMP_DIR/helm-pvc.yaml" + +assert_contains \ + "$TMP_DIR/helm-pvc.yaml" \ + 'image: "codeapi-sandbox-runner:latest"' \ + "PVC compatibility mode must select sandboxImage" +assert_contains \ + "$TMP_DIR/helm-pvc.yaml" \ + 'mountPath: /host-packages' \ + "PVC compatibility mode must mount /host-packages" +assert_contains \ + "$TMP_DIR/helm-pvc.yaml" \ + '^kind: PersistentVolumeClaim$' \ + "PVC compatibility mode must create the packages PVC" +assert_contains \ + "$TMP_DIR/helm-pvc.yaml" \ + 'app.kubernetes.io/component: package-init' \ + "PVC compatibility mode must create package-init" + +helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.sandboxRunner.fdLivenessLimit=0 \ + > "$TMP_DIR/helm-no-fd-liveness.yaml" + +assert_contains \ + "$TMP_DIR/helm-no-fd-liveness.yaml" \ + 'value: "0"' \ + "fdLivenessLimit=0 must reach the healthcheck instead of reverting to 40000" + +if helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.kvmEnabled=false \ + > "$TMP_DIR/invalid.yaml" 2> "$TMP_DIR/invalid.log"; then + echo "image package source must fail without KVM" >&2 + exit 1 +fi + +assert_contains \ + "$TMP_DIR/invalid.log" \ + 'packages.source=image requires workerSandbox.kvmEnabled=true' \ + "KVM validation failed for an unexpected reason" + +if helm template codeapi "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.packages.source=invalid \ + > "$TMP_DIR/invalid-source.yaml" 2> "$TMP_DIR/invalid-source.log"; then + echo "unknown package source must fail" >&2 + exit 1 +fi + +assert_contains \ + "$TMP_DIR/invalid-source.log" \ + 'workerSandbox.packages.source must be image or pvc' \ + "package source validation failed for an unexpected reason" + +echo "block-root package delivery checks passed"