fix: prevent sandbox virtio-fs fd exhaustion

.github/workflows/ci.yml

@@ -22,6 +22,20 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  deployment-config-tests:
+    name: Deployment Config Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Block-root package delivery
+        run: tests/block_root_package_delivery.sh
+
+      - name: Validate sandbox Dockerfiles
+        run: |
+          docker buildx build --check -f api/Dockerfile .
+          docker buildx build --check -f docker/Dockerfile.worker-sandbox .
+
   api-unit-tests:
     name: API Unit Tests
     runs-on: ubuntu-latest

README.md

@@ -12,7 +12,9 @@ Code Interpreter (internally `codeapi`, the prefix used by its env vars, images,
 - **Worker Sandbox** - Executes code in NsJail (or libkrun microVM) sandboxes with resource limits
 - **File Server** - Manages file uploads/downloads via S3 (IRSA authentication)
 - **Tool Call Server** - Handles programmatic tool calls from within sandbox sessions
-- **Package Init** - One-time job that pre-installs language runtimes (Python, Node, Bun) onto a shared PVC
+- **Package Delivery** - Bakes Python, Node, and Bun into the default microVM
+  block-root image; a package-init PVC mode remains available for direct NsJail
+  development
 
 ## Architecture
 
@@ -56,6 +58,15 @@ public issue (see [CONTRIBUTING](CONTRIBUTING.md)).
 docker-compose up --build
 ```
 
+The default KVM Compose path builds `sandbox-runner-baked`: the guest root and
+`/pkgs` tree live in a read-only ext4 block image instead of a long-lived
+virtio-fs mount. The first image build takes longer because it compiles the
+language runtimes, but package-heavy workloads do not accumulate host file
+descriptors in the launcher.
+
+Setting `KVM_ENABLED=false` still selects the directory-root target and the
+host package mount automatically for direct NsJail development.
+
 Local Docker Compose files set `CODEAPI_INTERNAL_SERVICE_TOKEN` to a shared
 development value by default. Production deployments must override it with a
 strong secret; when it is unset, file object routes and Tool Call Server

api/Dockerfile

@@ -170,10 +170,12 @@ RUN cargo build --release
 #
 # Split into:
 #   - sandbox-runner-base : everything except the guest rootfs copy
-#   - sandbox-runner      : default target; copies rootfs without packages
-#                           (packages come from a runtime PVC mount)
+#   - sandbox-runner      : legacy/direct target; copies rootfs without
+#                           packages (packages come from a runtime PVC mount)
 #   - sandbox-runner-baked: copies rootfs AND bakes /pkgs from package-builder
 #                           (no PVC needed; runner can spread across nodes)
+#   - sandbox-runner-{true,false}: Compose aliases selected by KVM_ENABLED
+#   - sandbox-runner-default: final alias for the safe baked KVM image
 # ============================================================================
 FROM fedora:43 AS sandbox-runner-base
 
@@ -232,3 +234,12 @@ COPY --from=sandbox-build / /sandbox-rootfs/
 RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg
 
 RUN mkdir -p /host-packages
+
+# Compose derives these aliases from KVM_ENABLED so existing no-KVM overrides
+# keep selecting the directory-root image automatically.
+FROM sandbox-runner-baked AS sandbox-runner-true
+FROM sandbox-runner AS sandbox-runner-false
+
+# Keep the safe KVM image as the Dockerfile default. Direct NsJail and legacy
+# virtio-fs deployments can still select --target sandbox-runner explicitly.
+FROM sandbox-runner-true AS sandbox-runner-default

docker-compose.local-dev.yml

@@ -12,6 +12,7 @@ services:
     build:
       context: .
       dockerfile: api/Dockerfile
+      target: sandbox-runner-${KVM_ENABLED:-true}
     container_name: sandbox
     privileged: false
     devices:
@@ -22,6 +23,7 @@ services:
       - ${SANDBOX_PACKAGES_PATH:-./data/pkgs}:/host-packages:ro
     environment:
       - KVM_ENABLED=${KVM_ENABLED:-true}
+      - LAUNCHER_PACKAGES_HOST=/disabled-host-packages
       - LAUNCHER_VCPUS=2
       - LAUNCHER_RAM_MIB=2048
       - LAUNCHER_LOG_LEVEL=3
@@ -40,6 +42,11 @@ services:
       - CODEAPI_INTERNAL_SERVICE_TOKEN=${CODEAPI_INTERNAL_SERVICE_TOKEN:-localdev-internal-service-token}
       - SANDBOX_ALLOWED_LOCAL_NETWORK_PORT=3033
       - SANDBOX_FORWARD_TARGET=tool_call_server:3033
+    healthcheck:
+      test: ["CMD", "/usr/local/bin/sandbox-runner-healthcheck.sh"]
+      interval: 10s
+      timeout: 5s
+      retries: 6
     depends_on:
       - tool_call_server
 

docker-compose.scalable.yml

@@ -169,6 +169,7 @@ services:
     build:
       context: .
       dockerfile: docker/Dockerfile.worker-sandbox
+      target: worker-sandbox-${KVM_ENABLED:-true}
     privileged: false
     devices:
       - ${KVM_DEVICE_PATH:-/dev/kvm}:/dev/kvm
@@ -207,6 +208,7 @@ services:
       - SANDBOX_DISABLE_NETWORKING=true
       - SANDBOX_ALLOWED_LOCAL_NETWORK_PORT=3033
       - SANDBOX_FORWARD_TARGET=tool_call_server:3033
+      - LAUNCHER_PACKAGES_HOST=/disabled-host-packages
     volumes:
       - ${SANDBOX_PACKAGES_PATH:-./data/pkgs}:/host-packages:ro
     depends_on:

docker-compose.yaml

@@ -124,6 +124,7 @@ services:
         build:
           context: .
           dockerfile: api/Dockerfile
+          target: sandbox-runner-${KVM_ENABLED:-true}
         container_name: sandbox-runner
         restart: always
         privileged: false
@@ -135,6 +136,9 @@ services:
             - ${SANDBOX_PACKAGES_PATH:-./data/pkgs}:/host-packages:ro
         environment:
             - KVM_ENABLED=${KVM_ENABLED:-true}
+            # The direct path uses /host-packages itself. The KVM path must not
+            # hand that directory to libkrun as a long-lived virtio-fs share.
+            - LAUNCHER_PACKAGES_HOST=/disabled-host-packages
             - CODEAPI_HARDENED_SANDBOX_MODE=${CODEAPI_HARDENED_SANDBOX_MODE:-true}
             - EGRESS_GATEWAY_URL=http://egress_gateway:3190
             - SANDBOX_EXECUTE_BODY_LIMIT=${SANDBOX_EXECUTE_BODY_LIMIT:-50mb}
@@ -161,7 +165,7 @@ services:
             egress_gateway:
                 condition: service_healthy
         healthcheck:
-            test: ["CMD-SHELL", "curl -fsS http://localhost:2000/api/v2/runtimes >/dev/null || exit 1"]
+            test: ["CMD", "/usr/local/bin/sandbox-runner-healthcheck.sh"]
             interval: 5s
             timeout: 3s
             retries: 60

docker/Dockerfile.worker-sandbox

@@ -5,7 +5,7 @@
 #
 # Architecture within this container:
 # ┌──────────────────────────────────────────────────────────────────────────┐
-# │  Worker-Sandbox Container (Fedora 41, libkrun)                          │
+# │  Worker-Sandbox Container (Fedora 43, libkrun)                          │
 # │                                                                         │
 # │  ┌─────────────────┐        ┌──────────────────────────────────────────┐│
 # │  │  Worker Process │ ──────▶│  libkrun MicroVM (own kernel)            ││
@@ -45,6 +45,36 @@ COPY api/src/spec-guard.c /tmp/spec-guard.c
 RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \
     && chmod 0111 /usr/local/bin/spec-guard
 
+# ============================================================================
+# Stage 1b: Build language runtime packages for the baked KVM root disk
+#
+# Mirrors docker/Dockerfile.package-init, but populates /pkgs at image-build
+# time so the combined worker can boot without a long-lived /host-packages
+# virtio-fs mount.
+# ============================================================================
+FROM buildpack-deps:bookworm AS package-builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHON_VERSION=3.14.4
+ENV BUN_VERSION=1.3.14
+
+COPY docker/apt-install.sh /usr/local/bin/apt-install
+RUN chmod +x /usr/local/bin/apt-install
+
+RUN apt-install \
+    curl unzip wget ca-certificates \
+    build-essential libssl-dev libffi-dev libsqlite3-dev \
+    zlib1g-dev libbz2-dev libreadline-dev libncurses5-dev \
+    tk-dev xz-utils libcurl4-openssl-dev libfontconfig1-dev \
+    libudunits2-dev libpng-dev libxml2-dev libcairo2-dev \
+    libfreetype6-dev
+
+RUN mkdir -p /pkgs
+
+COPY docker/package-init.sh /package-init.sh
+COPY javascript-packages.txt /javascript-packages.txt
+RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh
+
 # ============================================================================
 # Stage 2: Build worker
 # ============================================================================
@@ -153,9 +183,35 @@ COPY launcher/src/ ./src/
 RUN cargo build --release
 
 # ============================================================================
-# Stage 6: Final combined image (Fedora for libkrun runtime)
+# Stage 5b: Build the read-only ext4 guest root disk
+# ============================================================================
+FROM debian:bookworm AS worker-sandbox-rootfs-image
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    e2fsprogs \
+    coreutils \
+    gawk \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=sandbox-rootfs / /sandbox-rootfs/
+COPY --from=package-builder /pkgs /sandbox-rootfs/pkgs
+COPY docker/build-rootfs-image.sh /usr/local/bin/build-rootfs-image.sh
+
+RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg
+RUN chmod +x /usr/local/bin/build-rootfs-image.sh \
+    && /usr/local/bin/build-rootfs-image.sh /sandbox-rootfs /sandbox-rootfs.img
+
+# ============================================================================
+# Stage 6: Shared combined worker host (Fedora for libkrun runtime)
+#
+# Final targets:
+#   - worker-sandbox-baked: KVM default; packages live in a read-only ext4
+#                           block root, so no package virtio-fs mount is needed
+#   - worker-sandbox-legacy: directory root plus /host-packages for legacy KVM
+#                            deployments and KVM_ENABLED=false direct mode
+#   - worker-sandbox-{true,false}: Compose aliases selected by KVM_ENABLED
 # ============================================================================
-FROM fedora:43
+FROM fedora:43 AS worker-sandbox-base
 
 RUN dnf install -y --setopt=install_weak_deps=False \
     libkrun \
@@ -176,13 +232,6 @@ ENV PATH="/root/.bun/bin:${PATH}"
 # --- Launcher (runs on host, boots microVM) ---
 COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/local/bin/launcher
 
-# --- Sandbox rootfs (becomes the microVM guest filesystem) ---
-COPY --from=sandbox-rootfs / /sandbox-rootfs/
-
-RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg
-
-RUN mkdir -p /host-packages
-
 # --- Launcher entrypoint (DNS resolution + socat relay before VM boot) ---
 COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh
 COPY docker/start-direct-sandbox.sh /usr/local/bin/start-direct-sandbox.sh
@@ -207,3 +256,31 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
   CMD curl -f http://localhost:3113/health || exit 1
 
 CMD ["/supervisor.sh"]
+
+# Legacy directory-root target. Keep this for deployments that intentionally
+# mount packages at /host-packages, and for the KVM_ENABLED=false direct path.
+FROM worker-sandbox-base AS worker-sandbox-legacy
+
+COPY --from=sandbox-rootfs / /sandbox-rootfs/
+
+RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg \
+    && mkdir -p /host-packages
+
+# KVM production default. The package tree is part of the read-only block root,
+# avoiding host file-descriptor retention from package lookups over virtio-fs.
+FROM worker-sandbox-base AS worker-sandbox-baked
+
+COPY --from=worker-sandbox-rootfs-image /sandbox-rootfs.img /sandbox-rootfs.img
+
+ENV LAUNCHER_ROOT_DISK=/sandbox-rootfs.img \
+    LAUNCHER_ROOT_DISK_READ_ONLY=true \
+    LAUNCHER_ROOT_DEVICE=/dev/vda \
+    LAUNCHER_ROOT_FSTYPE=ext4 \
+    LAUNCHER_ROOT_OPTIONS=ro
+
+# Compose derives these aliases from KVM_ENABLED so the direct path keeps
+# selecting the directory-root image without duplicating the baked rootfs.
+FROM worker-sandbox-baked AS worker-sandbox-true
+FROM worker-sandbox-legacy AS worker-sandbox-false
+
+FROM worker-sandbox-true AS worker-sandbox-default

helm/codeapi/Chart.yaml

@@ -3,7 +3,7 @@ apiVersion: v2
 name: codeapi
 description: A Helm chart for Code Interpreter API - scalable code execution service
 type: application
-version: 0.2.0       # Chart version (bump this when you change the chart)
+version: 0.3.0       # Chart version (bump this when you change the chart)
 appVersion: "2.0.0"  # App version (bump this when you change the app)
 
 # Keywords for searching

helm/codeapi/README.md

@@ -78,6 +78,30 @@ For development only, `LOCAL_MODE=true` bypasses authentication — see
 **TLS to Redis.** Set `REDIS_TLS=true` via `extraEnv` on each component when
 your Redis (e.g. a managed cache) requires TLS.
 
+**Package delivery.** KVM deployments default to
+`workerSandbox.packages.source=image`. Build and publish the baked runner target
+under the existing `workerSandbox.sandboxImage` repository and tag:
+
+```bash
+docker build \
+  --target sandbox-runner-baked \
+  -t codeapi-sandbox-runner:latest \
+  -f api/Dockerfile .
+```
+
+That image boots the guest and its `/pkgs` tree from a read-only ext4 block
+image. It intentionally does not expose `/host-packages` through virtio-fs,
+which prevents package imports from pinning host-side launcher file
+descriptors. Set `workerSandbox.packages.source=pvc` only for compatibility with
+runtime package rebuilds; Kubernetes will then use the FD-aware liveness probe
+to recycle the runner before descriptor exhaustion.
+
+When upgrading from chart 0.2.x, rebuild the configured `sandboxImage` from the
+baked target before enabling the 0.3.x default. To keep an existing
+directory-root image during migration, set
+`workerSandbox.packages.source=pvc`; the established image repository/tag
+override remains unchanged in either mode.
+
 ## Quick Start (Local Development)
 
 ### 1. Start Minikube
@@ -93,7 +117,7 @@ eval $(minikube docker-env)
 # Build all images, from the codeapi root
 docker build -t codeapi-api:latest -f service/Dockerfile.api .
 docker build -t codeapi-worker:latest -f service/Dockerfile.worker .
-docker build -t codeapi-sandbox-runner:latest -f api/Dockerfile .
+docker build --target sandbox-runner -t codeapi-sandbox-runner:latest -f api/Dockerfile .
 docker build -t codeapi-file-server:latest -f service/Dockerfile --target production .
 docker build -t codeapi-tool-call-server:latest -f service/Dockerfile.tool-call-server --target production .
 docker build -t codeapi-package-init:latest -f docker/Dockerfile.package-init .
@@ -113,9 +137,13 @@ helm dependency update
 helm install codeapi . -f values-local.yaml
 ```
 
-### 4. Language Packages (Automatic)
+### 4. Language Packages (Local PVC Mode)
 
-The chart includes a **package-init Job** that runs as a Helm `pre-install` hook. It automatically compiles Python, downloads Node/Bun, installs offline package sets, and registers Bash into the packages PVC before the worker pods start.
+`values-local.yaml` disables KVM and selects the legacy PVC package source. In
+that mode the chart includes a **package-init Job** that runs as a Helm
+`pre-install` hook. It compiles Python, downloads Node/Bun, installs offline
+package sets, and registers Bash into the packages PVC before the worker pods
+start.
 
 This happens automatically on `helm install`. To force a rebuild:
 
@@ -168,7 +196,7 @@ curl http://localhost:3112/v1/health
 # Start minikube
 minikube start
 
-# Deploy (package-init job runs automatically)
+# Deploy local direct mode (package-init job runs automatically)
 helm install codeapi ./helm/codeapi -f ./helm/codeapi/values-local.yaml
 
 # Port forward
@@ -204,7 +232,7 @@ helm upgrade codeapi ./helm/codeapi -f ./helm/codeapi/values-local.yaml \
 # Rebuild images (must be in minikube docker env)
 eval $(minikube docker-env)
 docker build -t codeapi-worker:latest -f service/Dockerfile.worker .
-docker build -t codeapi-sandbox-runner:latest -f api/Dockerfile .
+docker build --target sandbox-runner -t codeapi-sandbox-runner:latest -f api/Dockerfile .
 
 # Restart deployments to pick up new images
 kubectl rollout restart deployment/codeapi-service-worker
@@ -285,9 +313,9 @@ kubectl logs deployment/codeapi-service-worker --tail=5
 |  +--------------+  +--------------+  +--------------+                |
 |                                                                      |
 |  +---------------------------------------------------------------+   |
-|  |              PersistentVolume (Packages)                       |   |
-|  |  /pkgs - Python, Node, Bun runtimes                            |   |
-|  |  ReadOnlyMany - shared across all sandbox-runner pods          |   |
+|  |                 Package delivery                               |   |
+|  |  KVM default: /pkgs in each baked ext4 block-root image        |   |
+|  |  Direct mode: shared package PVC populated by package-init     |   |
 |  +---------------------------------------------------------------+   |
 |                                                                      |
 |  Total sandbox capacity: 3 pods x 8 concurrent jobs = 24 jobs        |
@@ -315,7 +343,7 @@ kubectl describe pod <pod-name>
 
 ### "runtime is unknown" error
 ```bash
-# Language packages PVC is empty. Check if the init job ran:
+# In source=pvc mode, check whether the package-init job populated the PVC:
 kubectl get jobs -l app.kubernetes.io/component=package-init
 kubectl logs job/codeapi-package-init
 
@@ -326,6 +354,10 @@ helm upgrade codeapi . --set workerSandbox.packages.initJob.forceRebuild=true
 kubectl rollout restart deployment/codeapi-sandbox-runner
 ```
 
+In the default `source=image` mode, rebuild and publish
+`workerSandbox.sandboxImage` from the `sandbox-runner-baked` target instead; no
+package-init Job or packages PVC is rendered.
+
 ### Connection refused on port 3112
 ```bash
 # Make sure port-forward is running