IsuraManchanayake · Oct 15, 2026 · Oct 15, 2026 · Oct 15, 2026 · Oct 15, 2026 · Oct 15, 2026
diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml
@@ -0,0 +1,326 @@
+name: Nightly runtimes
+
+# Each night the newest upstream release of each runtime is installed untested
+# and the adapter's tests run on it; one that passes is added to the recipe by
+# a PR, with the digests its project lists, and one that fails opens an issue.
+# `workflow_dispatch` tests a named release. A schedule fires only from the
+# default branch, and a release the recipe already lists is a no-op.
+
+on:
+  schedule:
+    - cron: "23 3 * * *"
+  workflow_dispatch:
+    inputs:
+      llamacpp:
+        description: "A llama.cpp build to test instead of the newest (bNNNN)"
+        required: false
+        default: ""
+      mlx:
+        description: "An MLX version to test instead of the newest (X.Y.Z)"
+        required: false
+        default: ""
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+
+concurrency:
+  group: nightly-runtimes
+  cancel-in-progress: false
+
+env:
+  # A small GGUF the llama.cpp adapter's model tests run on; without one they
+  # skip, and a pass would say little.
+  TEST_GGUF_URL: https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_K_M.gguf
+
+jobs:
+  pick:
+    name: What is new upstream
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    outputs:
+      llamacpp: ${{ steps.llamacpp.outputs.version }}
+      mlx: ${{ steps.mlx.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: llama.cpp
+        id: llamacpp
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          REQUESTED: ${{ inputs.llamacpp }}
+        run: python3 scripts/nightly_runtimes.py llamacpp pick --requested "$REQUESTED" | tee -a "$GITHUB_OUTPUT"
+      - name: MLX
+        id: mlx
+        env:
+          REQUESTED: ${{ inputs.mlx }}
+        run: python3 scripts/nightly_runtimes.py mlx pick --requested "$REQUESTED" | tee -a "$GITHUB_OUTPUT"
+
+  llamacpp:
+    name: llama.cpp ${{ needs.pick.outputs.llamacpp }} on ${{ matrix.os }}
+    needs: pick
+    if: needs.pick.outputs.llamacpp != ''
+    strategy:
+      fail-fast: false
+      matrix:
+        # The macOS leg runs on a self-hosted Apple silicon runner. The hosted
+        # macos-14 runner's virtualized GPU takes llama.cpp's fallback Metal
+        # kernels, which fail the adapter's numeric tests whatever the build,
+        # and its memory leaves the pool shared where the tests expect a
+        # buffer per lane.
+        include:
+          - os: macos
+            runner: [self-hosted, macOS, ARM64]
+          - os: ubuntu-24.04
+            runner: ubuntu-24.04
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
+    env:
+      BUILD: ${{ needs.pick.outputs.llamacpp }}
+    steps:
+      - uses: actions/checkout@v4
+      # A self-hosted runner keeps the toolchain it has, where rustup puts it;
+      # its PATH does not say so unless its owner's shell did.
+      - uses: dtolnay/rust-toolchain@stable
+        if: runner.environment == 'github-hosted'
+      - if: runner.environment == 'self-hosted'
+        run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+      # Never on a self-hosted runner: its save step deletes every file in
+      # ~/.cargo/bin that cargo did not install during the job, rustup
+      # included. A self-hosted runner keeps its target directory anyway.
+      - uses: Swatinem/rust-cache@v2
+        if: runner.environment == 'github-hosted'
+
+      - name: Build the worker and plan the untested install
+        id: plan
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          cargo build -p superfluid-adapter-llamacpp -p superfluid-tokenizer-llamacpp
+          plan=$(target/debug/superfluid-worker-llamacpp plan --version "$BUILD" --untested \
+            | python3 -c 'import sys, json; print(json.dumps(json.load(sys.stdin)["plans"][0]))')
+          {
+            echo "plan<<PLAN_EOF"
+            echo "$plan"
+            echo "PLAN_EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Install it
+        env:
+          PLAN: ${{ steps.plan.outputs.plan }}
+        run: target/debug/superfluid-worker-llamacpp install --into "$RUNNER_TEMP/llamacpp" --plan "$PLAN"
+
+      # A hosted runner starts empty and gets the model from the cache; a
+      # self-hosted one keeps the directory between jobs, and would spend
+      # minutes uploading it.
+      - uses: actions/cache@v4
+        if: runner.environment == 'github-hosted'
+        with:
+          path: ~/.cache/superfluid-nightly/models
+          key: nightly-gguf-${{ hashFiles('.github/workflows/nightly-runtimes.yml') }}
+      - name: A model for the tests
+        run: |
+          mkdir -p ~/.cache/superfluid-nightly/models
+          model=~/.cache/superfluid-nightly/models/$(basename "$TEST_GGUF_URL")
+          [ -s "$model" ] || curl -fsSL --retry 3 -o "$model" "$TEST_GGUF_URL"
+          echo "SUPERFLUID_TEST_GGUF=$model" >> "$GITHUB_ENV"
+
+      - name: The adapter's tests on it
+        run: |
+          set -o pipefail
+          export SUPERFLUID_LLAMA_LIB="$RUNNER_TEMP/llamacpp/lib"
+          cargo test -p superfluid-adapter-llamacpp -- --nocapture 2>&1 \
+            | tee "$RUNNER_TEMP/llamacpp-tests.log" \
+            | python3 scripts/skip_report.py "llama.cpp adapter"
+
+      - if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: llamacpp-${{ needs.pick.outputs.llamacpp }}-${{ matrix.os }}
+          path: ${{ runner.temp }}/llamacpp-tests.log
+          if-no-files-found: ignore
+
+  record-llamacpp:
+    name: Record llama.cpp ${{ needs.pick.outputs.llamacpp }}
+    needs: [pick, llamacpp]
+    if: always() && needs.pick.outputs.llamacpp != ''
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    env:
+      BUILD: ${{ needs.pick.outputs.llamacpp }}
+      GH_TOKEN: ${{ github.token }}
+      RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      BASE: ${{ github.ref_name }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Passed, so add it to the recipe in a PR
+        if: needs.llamacpp.result == 'success'
+        run: |
+          set -euo pipefail
+          branch="nightly/llamacpp-$BUILD"
+          open=$(gh pr list --head "$branch" --state open --json number --jq 'length') || { echo "::error::cannot list this repository's pull requests (may Actions open them? see Settings > Actions > General)"; exit 1; }
+          if [ "$open" != 0 ]; then
+            echo "a PR from $branch is already open"
+            exit 0
+          fi
+          git switch -c "$branch"
+          GITHUB_TOKEN="$GH_TOKEN" python3 scripts/nightly_runtimes.py llamacpp append "$BUILD"
+          git -c user.name="github-actions[bot]" -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \
+            commit -am "llamacpp: tested with $BUILD"
+          git push -f origin "$branch"
+          p1="The nightly job installed llama.cpp $BUILD untested and ran the adapter's tests on it on Apple silicon (Metal) and ubuntu-24.04 (CPU): $RUN"
+          p2="This adds the build to the adapter's recipe as a tested build, with the size and SHA-256 GitHub lists for each of its assets, so the next release installs it on its own. The upstream release is https://github.com/ggml-org/llama.cpp/releases/tag/$BUILD."
+          body="$p1"$'\n\n'"$p2"
+          gh pr create --base "$BASE" --head "$branch" --title "llamacpp: tested with $BUILD" --body "$body"
+
+      - name: Failed, so open an issue
+        if: needs.llamacpp.result != 'success'
+        run: |
+          set -euo pipefail
+          title="llama.cpp $BUILD breaks the adapter"
+          # One open issue per runtime: a later build that fails too is a comment on it, since
+          # the newest build is tested every night and the adapter may take days to catch up.
+          existing=$(gh issue list --search "\"breaks the adapter\" llama.cpp in:title" --state open --json number,title --jq '.[0] | "\(.number) \(.title)"') || { echo "::error::cannot list this repository's issues (are issues enabled, and may Actions write them?)"; exit 1; }
+          if [ -n "$existing" ]; then
+            number=${existing%% *}
+            case "$existing" in *"$title"*) echo "an issue for $BUILD is already open: #$number"; exit 0;; esac
+            gh issue comment "$number" --body "$BUILD fails too: $RUN"
+            echo "noted $BUILD on the open issue #$number"
+            exit 0
+          fi
+          p1="The nightly job installed llama.cpp $BUILD untested and the adapter's tests did not pass on it: $RUN"
+          p2="The test logs are the run's artifacts. Until the adapter is fixed, or the build is found to be at fault, the recipe keeps its last tested build. The upstream release is https://github.com/ggml-org/llama.cpp/releases/tag/$BUILD."
+          body="$p1"$'\n\n'"$p2"
+          gh issue create --title "$title" --body "$body"
+
+  mlx:
+    name: MLX ${{ needs.pick.outputs.mlx }}
+    needs: pick
+    if: needs.pick.outputs.mlx != ''
+    runs-on: macos-14
+    timeout-minutes: 60
+    env:
+      VERSION: ${{ needs.pick.outputs.mlx }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions/cache@v4
+        with:
+          path: ~/.cache/superfluid-nightly/adapter-python
+          key: adapter-python-${{ hashFiles('crates/superfluid-adapters/superfluid-adapter-mlx/recipe.json') }}
+
+      - name: Build the worker as a release ships it
+        run: SUPERFLUID_ADAPTER_CACHE="$HOME/.cache/superfluid-nightly/adapter-python" bash scripts/build_runtime_adapters.sh "$RUNNER_TEMP/adapters"
+
+      - name: Install MLX untested
+        run: |
+          set -euo pipefail
+          worker="$RUNNER_TEMP/adapters/superfluid-worker-mlx"
+          plan=$("$worker" plan --version "$VERSION" --untested \
+            | python3 -c 'import sys, json; print(json.dumps(json.load(sys.stdin)["plans"][0]))')
+          "$worker" install --into "$RUNNER_TEMP/mlx" --plan "$plan"
+          # The tests run in env/, where mlx went, and are built against the bundled
+          # python/, the 3.12 the wheels are for, whatever Python the runner has.
+          env="$RUNNER_TEMP/mlx/env"
+          "$env/bin/python" -c 'import mlx.core, mlx_lm; print("mlx", mlx.core.__version__, "in", __import__("sys").prefix)' \
+            || { echo "::error::$env is not an environment with mlx and mlx-lm"; exit 1; }
+          echo "SUPERFLUID_MLX_VENV=$env" >> "$GITHUB_ENV"
+          echo "PYO3_PYTHON=$RUNNER_TEMP/mlx/python/bin/python3" >> "$GITHUB_ENV"
+
+      - name: Install it again with a report, so the recipe records what the tests run on
+        # A real install, not a dry run, so the report is the environment the
+        # tests then run on and not a resolution that could differ from it.
+        run: |
+          set -euo pipefail
+          py="$SUPERFLUID_MLX_VENV/bin/python"
+          [ -x "$py" ] || py="$SUPERFLUID_MLX_VENV/bin/python3"
+          "$py" -m pip --version >/dev/null 2>&1 || "$py" -m ensurepip --upgrade >/dev/null
+          "$py" -m pip install --ignore-installed --only-binary=:all: \
+            --report "$RUNNER_TEMP/mlx-report.json" "mlx==$VERSION" mlx-lm >/dev/null
+          "$py" -c 'import mlx.core, mlx_lm' || { echo "::error::the reinstalled environment does not import mlx and mlx-lm"; exit 1; }
+          python3 -c 'import json, sys; print(len(json.load(open(sys.argv[1]))["install"]), "distributions installed and reported")' "$RUNNER_TEMP/mlx-report.json"
+
+      - name: The adapter's tests on it
+        # The test executable links @rpath/libpython3.12.dylib like the worker, but has no
+        # rpath into the install, so the install's lib directory is named for dyld.
+        env:
+          DYLD_LIBRARY_PATH: ${{ runner.temp }}/mlx/python/lib
+          DYLD_FALLBACK_LIBRARY_PATH: ${{ runner.temp }}/mlx/python/lib
+        run: |
+          set -o pipefail
+          cargo test -p superfluid-adapter-mlx -- --nocapture 2>&1 \
+            | tee "$RUNNER_TEMP/mlx-tests.log" \
+            | python3 scripts/skip_report.py "MLX adapter"
+
+      - if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: mlx-${{ needs.pick.outputs.mlx }}
+          path: |
+            ${{ runner.temp }}/mlx-tests.log
+            ${{ runner.temp }}/mlx-report.json
+          if-no-files-found: ignore
+
+  record-mlx:
+    name: Record MLX ${{ needs.pick.outputs.mlx }}
+    needs: [pick, mlx]
+    if: always() && needs.pick.outputs.mlx != ''
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    env:
+      VERSION: ${{ needs.pick.outputs.mlx }}
+      RESULT: ${{ needs.mlx.result }}
+      GH_TOKEN: ${{ github.token }}
+      RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    steps:
+      - uses: actions/checkout@v4
+      - if: needs.mlx.result == 'success'
+        uses: actions/download-artifact@v4
+        with:
+          name: mlx-${{ needs.pick.outputs.mlx }}
+          path: ${{ runner.temp }}/mlx
+
+      - name: Passed, so add it to the recipe in a PR
+        if: needs.mlx.result == 'success'
+        env:
+          BASE: ${{ github.ref_name }}
+        run: |
+          set -euo pipefail
+          branch="nightly/mlx-$VERSION"
+          open=$(gh pr list --head "$branch" --state open --json number --jq 'length') || { echo "::error::cannot list this repository's pull requests (may Actions open them? see Settings > Actions > General)"; exit 1; }
+          if [ "$open" != 0 ]; then
+            echo "a PR from $branch is already open"
+            exit 0
+          fi
+          git switch -c "$branch"
+          python3 scripts/nightly_runtimes.py mlx append "$VERSION" --report "$RUNNER_TEMP/mlx/mlx-report.json"
+          git -c user.name="github-actions[bot]" -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \
+            commit -am "mlx: tested with $VERSION"
+          git push -f origin "$branch"
+          p1="The nightly job installed MLX $VERSION untested on macos-14 and ran the adapter's tests on it: $RUN"
+          p2="This adds the version to the adapter's recipe as a tested one: the environment pip resolved for it on the runner, every wheel with the size and SHA-256 PyPI lists for it, so the next release installs it on its own. The Python the adapter ships is unchanged."
+          body="$p1"$'\n\n'"$p2"
+          gh pr create --base "$BASE" --head "$branch" --title "mlx: tested with $VERSION" --body "$body"
+
+      - name: Failed, so open an issue
+        if: needs.mlx.result != 'success'
+        # Among the failures: an MLX that needs a newer Python than the
+        # adapter ships, which is a change to the worker's build, not a pin.
+        run: |
+          set -euo pipefail
+          title="MLX $VERSION breaks the adapter"
+          # One open issue per runtime: a later version that fails too is a comment on it.
+          existing=$(gh issue list --search "\"breaks the adapter\" MLX in:title" --state open --json number,title --jq '.[0] | "\(.number) \(.title)"') || { echo "::error::cannot list this repository's issues (are issues enabled, and may Actions write them?)"; exit 1; }
+          if [ -n "$existing" ]; then
+            number=${existing%% *}
+            case "$existing" in *"$title"*) echo "an issue for MLX $VERSION is already open: #$number"; exit 0;; esac
+            gh issue comment "$number" --body "MLX $VERSION fails too: $RUN"
+            echo "noted MLX $VERSION on the open issue #$number"
+            exit 0
+          fi
+          p1="The nightly job installed MLX $VERSION untested on macos-14 and the adapter's tests did not pass on it: $RUN"
+          p2="The test log is the run's artifact. Until the adapter is fixed, or the version is found to be at fault, the recipe keeps its last tested version. A version that needs a newer Python than the adapter ships fails here too: that is a change to the worker's build, not a pin."
+          body="$p1"$'\n\n'"$p2"
+          gh issue create --title "$title" --body "$body"
diff --git a/crates/superfluid-adapters/superfluid-adapter-kit/src/install.rs b/crates/superfluid-adapters/superfluid-adapter-kit/src/install.rs
@@ -375,11 +375,16 @@ pub fn fetch_json(url: &str) -> Result<Value, String> {
     if !url.starts_with("https://") {
         return Err(format!("{url}: only https:// URLs are fetched"));
     }
-    let out = Command::new("curl")
-        .args(["-fsSL", "--proto", "=https", "--tlsv1.2", "--retry", "2", "-H", "Accept: application/json"])
-        .arg(url)
-        .output()
-        .map_err(|e| format!("curl: {e}"))?;
+    let mut curl = Command::new("curl");
+    curl.args(["-fsSL", "--proto", "=https", "--tlsv1.2", "--retry", "2", "-H", "Accept: application/json"]);
+    // GitHub's API allows sixty unauthenticated requests an hour per address, which hosted
+    // runners share; a token in the environment lifts that, for the API alone.
+    if url.starts_with("https://api.github.com/") {
+        if let Some(token) = std::env::var_os("GITHUB_TOKEN").filter(|t| !t.is_empty()) {
+            curl.arg("-H").arg(format!("Authorization: Bearer {}", token.to_string_lossy()));
+        }
+    }
+    let out = curl.arg(url).output().map_err(|e| format!("curl: {e}"))?;
     if !out.status.success() {
         return Err(format!("fetching {url} failed: {}", String::from_utf8_lossy(&out.stderr).trim()));
     }

diff --git a/docs/models/runtimes.md b/docs/models/runtimes.md
@@ -124,6 +124,10 @@ Requests a model cannot serve are refused with a 400:
 | `unsupported_runtime` | a load the runtime cannot serve as asked |
 | `unsupported_parameter` | a request parameter the path does not support |
 
+## How an upstream release becomes a tested one
+
+Each adapter installs on its own only the upstream versions it was built and tested against, listed in its `recipe.json`; another installs with `--untested` and is still checked at load. A nightly job (`.github/workflows/nightly-runtimes.yml`) keeps those lists current: it installs the newest upstream release untested, runs the adapter's tests on it, and records the result. A release that passes arrives as a PR adding it to the recipe (`scripts/nightly_runtimes.py <runtime> append`): for llama.cpp with the size and SHA-256 GitHub lists for each of its assets, for MLX with the environment pip resolved for it on the runner, every wheel with the size and SHA-256 PyPI lists for it. One that fails opens an issue with the run attached; an MLX that needs a newer Python than the adapter ships fails that way, since it changes the worker's build rather than a pin. `workflow_dispatch` tests a named release instead of the newest.
+
 ## Per-runtime limits
 
 ### llamacpp