From 4e5cee5a8f8d2df6647c29f96519a322c91ade4b Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Thu, 15 Oct 2026 13:39:05 +1100 Subject: [PATCH 01/12] ci: a nightly job tests each runtime's newest upstream release and records the result Each adapter installs on its own only the upstream versions it was built and tested against, so those lists go stale as llama.cpp and MLX release. The job picks the newest upstream release the recipe does not list (scripts/nightly_runtimes.py), installs it untested, and runs the adapter's tests on it: llama.cpp on macos-14 and ubuntu-24.04 with a small GGUF for the model tests, MLX on macos-14 with the worker built as a release ships it. A llama.cpp build that passes arrives as a PR adding it to the recipe with the size and SHA-256 GitHub lists for each of its assets, the same asset names the previous tested build carries; one that fails opens an issue with the run attached. An MLX version is pinned by hand, wheel by wheel, so its result is an issue either way. workflow_dispatch tests a named release instead of the newest. --- .github/workflows/nightly-runtimes.yml | 249 +++++++++++++++++++++++++ docs/models/runtimes.md | 4 + scripts/nightly_runtimes.py | 146 +++++++++++++++ 3 files changed, 399 insertions(+) create mode 100644 .github/workflows/nightly-runtimes.yml create mode 100755 scripts/nightly_runtimes.py diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml new file mode 100644 index 0000000..6622da0 --- /dev/null +++ b/.github/workflows/nightly-runtimes.yml @@ -0,0 +1,249 @@ +name: Nightly runtimes + +# Each night, the newest upstream release of each runtime this superfluid +# drives is installed untested and the adapter's tests run on it. A llama.cpp +# build that passes is added to the adapter's recipe by a PR, with the size +# and SHA-256 GitHub lists for each of its assets, so the next release +# installs it on its own; a build that fails opens an issue with the run +# attached. An MLX version is pinned by hand, wheel by wheel, so its result +# is an issue either way. `workflow_dispatch` tests a named release instead +# of the newest. +# +# A schedule fires only from the default branch. The job is a no-op for a +# runtime whose newest release the recipe already lists. + +on: + schedule: + - cron: "23 3 * * *" + workflow_dispatch: + inputs: + llamacpp: + description: "A llama.cpp build to test instead of the newest (bNNNN)" + required: false + default: "" + mlx: + description: "An MLX version to test instead of the newest (X.Y.Z)" + required: false + default: "" + +permissions: + contents: write + pull-requests: write + issues: write + +concurrency: + group: nightly-runtimes + cancel-in-progress: false + +env: + # A small GGUF the llama.cpp adapter's model tests run on; without one they + # skip, and a pass would say little. + TEST_GGUF_URL: https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_K_M.gguf + +jobs: + pick: + name: What is new upstream + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + llamacpp: ${{ steps.llamacpp.outputs.version }} + mlx: ${{ steps.mlx.outputs.version }} + steps: + - uses: actions/checkout@v4 + - name: llama.cpp + id: llamacpp + env: + GITHUB_TOKEN: ${{ github.token }} + REQUESTED: ${{ inputs.llamacpp }} + run: python3 scripts/nightly_runtimes.py llamacpp pick --requested "$REQUESTED" | tee -a "$GITHUB_OUTPUT" + - name: MLX + id: mlx + env: + REQUESTED: ${{ inputs.mlx }} + run: python3 scripts/nightly_runtimes.py mlx pick --requested "$REQUESTED" | tee -a "$GITHUB_OUTPUT" + + llamacpp: + name: llama.cpp ${{ needs.pick.outputs.llamacpp }} on ${{ matrix.os }} + needs: pick + if: needs.pick.outputs.llamacpp != '' + strategy: + fail-fast: false + matrix: + os: [macos-14, ubuntu-24.04] + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + env: + BUILD: ${{ needs.pick.outputs.llamacpp }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - name: Build the worker and plan the untested install + id: plan + run: | + cargo build -p superfluid-adapter-llamacpp -p superfluid-tokenizer-llamacpp + plan=$(target/debug/superfluid-worker-llamacpp plan --version "$BUILD" --untested \ + | python3 -c 'import sys, json; print(json.dumps(json.load(sys.stdin)["plans"][0]))') + { + echo "plan<> "$GITHUB_OUTPUT" + + - name: Install it + env: + PLAN: ${{ steps.plan.outputs.plan }} + run: target/debug/superfluid-worker-llamacpp install --into "$RUNNER_TEMP/llamacpp" --plan "$PLAN" + + - uses: actions/cache@v4 + with: + path: ~/.cache/superfluid-nightly/models + key: nightly-gguf-${{ hashFiles('.github/workflows/nightly-runtimes.yml') }} + - name: A model for the tests + run: | + mkdir -p ~/.cache/superfluid-nightly/models + model=~/.cache/superfluid-nightly/models/$(basename "$TEST_GGUF_URL") + [ -s "$model" ] || curl -fsSL --retry 3 -o "$model" "$TEST_GGUF_URL" + echo "SUPERFLUID_TEST_GGUF=$model" >> "$GITHUB_ENV" + + - name: The adapter's tests on it + run: | + set -o pipefail + export SUPERFLUID_LLAMA_LIB="$RUNNER_TEMP/llamacpp/lib" + cargo test -p superfluid-adapter-llamacpp -- --nocapture 2>&1 \ + | tee "$RUNNER_TEMP/llamacpp-tests.log" \ + | python3 scripts/skip_report.py "llama.cpp adapter" + + - if: always() + uses: actions/upload-artifact@v4 + with: + name: llamacpp-${{ needs.pick.outputs.llamacpp }}-${{ matrix.os }} + path: ${{ runner.temp }}/llamacpp-tests.log + if-no-files-found: ignore + + record-llamacpp: + name: Record llama.cpp ${{ needs.pick.outputs.llamacpp }} + needs: [pick, llamacpp] + if: always() && needs.pick.outputs.llamacpp != '' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + BUILD: ${{ needs.pick.outputs.llamacpp }} + GH_TOKEN: ${{ github.token }} + RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + BASE: ${{ github.ref_name }} + steps: + - uses: actions/checkout@v4 + + - name: Passed, so add it to the recipe in a PR + if: needs.llamacpp.result == 'success' + run: | + set -euo pipefail + branch="nightly/llamacpp-$BUILD" + if [ "$(gh pr list --head "$branch" --state open --json number --jq 'length')" != 0 ]; then + echo "a PR from $branch is already open" + exit 0 + fi + git switch -c "$branch" + GITHUB_TOKEN="$GH_TOKEN" python3 scripts/nightly_runtimes.py llamacpp append "$BUILD" + git -c user.name="github-actions[bot]" -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -am "llamacpp: tested with $BUILD" + git push -f origin "$branch" + p1="The nightly job installed llama.cpp $BUILD untested and ran the adapter's tests on it on macos-14 (Metal) and ubuntu-24.04 (CPU): $RUN" + p2="This adds the build to the adapter's recipe as a tested build, with the size and SHA-256 GitHub lists for each of its assets, so the next release installs it on its own. The upstream release is https://github.com/ggml-org/llama.cpp/releases/tag/$BUILD." + body="$p1"$'\n\n'"$p2" + gh pr create --base "$BASE" --head "$branch" --title "llamacpp: tested with $BUILD" --body "$body" + + - name: Failed, so open an issue + if: needs.llamacpp.result != 'success' + run: | + set -euo pipefail + title="llama.cpp $BUILD breaks the adapter" + if [ "$(gh issue list --search "in:title \"$title\"" --state open --json number --jq 'length')" != 0 ]; then + echo "an issue for $BUILD is already open" + exit 0 + fi + p1="The nightly job installed llama.cpp $BUILD untested and the adapter's tests did not pass on it: $RUN" + p2="The test logs are the run's artifacts. Until the adapter is fixed, or the build is found to be at fault, the recipe keeps its last tested build. The upstream release is https://github.com/ggml-org/llama.cpp/releases/tag/$BUILD." + body="$p1"$'\n\n'"$p2" + gh issue create --title "$title" --body "$body" + + mlx: + name: MLX ${{ needs.pick.outputs.mlx }} + needs: pick + if: needs.pick.outputs.mlx != '' + runs-on: macos-14 + timeout-minutes: 60 + env: + VERSION: ${{ needs.pick.outputs.mlx }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: actions/cache@v4 + with: + path: ~/.cache/superfluid-nightly/adapter-python + key: adapter-python-${{ hashFiles('crates/superfluid-adapters/superfluid-adapter-mlx/recipe.json') }} + + - name: Build the worker as a release ships it + run: SUPERFLUID_ADAPTER_CACHE="$HOME/.cache/superfluid-nightly/adapter-python" bash scripts/build_runtime_adapters.sh "$RUNNER_TEMP/adapters" + + - name: Install MLX untested + run: | + set -euo pipefail + worker="$RUNNER_TEMP/adapters/superfluid-worker-mlx" + plan=$("$worker" plan --version "$VERSION" --untested \ + | python3 -c 'import sys, json; print(json.dumps(json.load(sys.stdin)["plans"][0]))') + "$worker" install --into "$RUNNER_TEMP/mlx" --plan "$plan" + site=$(find "$RUNNER_TEMP/mlx" -type d -name site-packages | head -1) + [ -n "$site" ] || { echo "::error::the install holds no site-packages"; exit 1; } + echo "SUPERFLUID_MLX_VENV=$(dirname "$(dirname "$(dirname "$site")")")" >> "$GITHUB_ENV" + + - name: The adapter's tests on it + run: | + set -o pipefail + cargo test -p superfluid-adapter-mlx -- --nocapture 2>&1 \ + | tee "$RUNNER_TEMP/mlx-tests.log" \ + | python3 scripts/skip_report.py "MLX adapter" + + - if: always() + uses: actions/upload-artifact@v4 + with: + name: mlx-${{ needs.pick.outputs.mlx }} + path: ${{ runner.temp }}/mlx-tests.log + if-no-files-found: ignore + + record-mlx: + name: Record MLX ${{ needs.pick.outputs.mlx }} + needs: [pick, mlx] + if: always() && needs.pick.outputs.mlx != '' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + VERSION: ${{ needs.pick.outputs.mlx }} + RESULT: ${{ needs.mlx.result }} + GH_TOKEN: ${{ github.token }} + RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - uses: actions/checkout@v4 + - name: Open an issue with the result + # An MLX version is pinned by hand (every wheel with its SHA-256, see + # the recipe's `about`), so a pass asks for that PR rather than making it. + run: | + set -euo pipefail + if [ "$RESULT" = success ]; then + title="MLX $VERSION passes the adapter's tests: pin it" + p1="The nightly job installed MLX $VERSION untested on macos-14 and the adapter's tests passed on it: $RUN" + p2="To install it on its own, the next release needs the version pinned in the MLX adapter's recipe, wheel by wheel with the SHA-256 PyPI lists for each (see the recipe's about)." + else + title="MLX $VERSION breaks the adapter" + p1="The nightly job installed MLX $VERSION untested on macos-14 and the adapter's tests did not pass on it: $RUN" + p2="The test log is the run's artifact. Until the adapter is fixed, or the version is found to be at fault, the recipe keeps its last tested version." + fi + body="$p1"$'\n\n'"$p2" + if [ "$(gh issue list --search "in:title \"$title\"" --state open --json number --jq 'length')" != 0 ]; then + echo "an issue for MLX $VERSION is already open" + exit 0 + fi + gh issue create --title "$title" --body "$body" diff --git a/docs/models/runtimes.md b/docs/models/runtimes.md index fcaf5c5..68a3691 100644 --- a/docs/models/runtimes.md +++ b/docs/models/runtimes.md @@ -124,6 +124,10 @@ Requests a model cannot serve are refused with a 400: | `unsupported_runtime` | a load the runtime cannot serve as asked | | `unsupported_parameter` | a request parameter the path does not support | +## How an upstream release becomes a tested one + +Each adapter installs on its own only the upstream versions it was built and tested against, listed in its `recipe.json`; another installs with `--untested` and is still checked at load. A nightly job (`.github/workflows/nightly-runtimes.yml`) keeps those lists current: it installs the newest upstream release untested, runs the adapter's tests on it, and records the result. A llama.cpp build that passes arrives as a PR adding it to the recipe with the size and SHA-256 GitHub lists for each of its assets (`scripts/nightly_runtimes.py llamacpp append `); one that fails opens an issue with the run attached. An MLX version is pinned by hand, wheel by wheel, so its result opens an issue either way. `workflow_dispatch` tests a named release instead of the newest. + ## Per-runtime limits ### llamacpp diff --git a/scripts/nightly_runtimes.py b/scripts/nightly_runtimes.py new file mode 100755 index 0000000..bdeec9e --- /dev/null +++ b/scripts/nightly_runtimes.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""The nightly runtimes job's two decisions: which upstream release to test, +and how to record one that passed. + + nightly_runtimes.py llamacpp pick [--requested bNNNN] + nightly_runtimes.py llamacpp append bNNNN + nightly_runtimes.py mlx pick [--requested X.Y.Z] + +`pick` prints `version=` for the newest upstream release the adapter's +recipe does not list yet, or `version=` when there is nothing new (or when +`--requested` names a release already listed); the workflow reads it from +$GITHUB_OUTPUT. `append` adds a llama.cpp build to the adapter's recipe as a +tested build, newest first: the same asset names the previous tested build +carries, with the size and SHA-256 GitHub lists for each, so a human reviews +a PR rather than typing digests. An MLX version is pinned by hand, wheel by +wheel, so `mlx` has no `append`; the job opens an issue with the result. + +Only the standard library; GITHUB_TOKEN is used when set, for the API rate +limit. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +RECIPES = { + "llamacpp": ROOT / "crates/superfluid-adapters/superfluid-adapter-llamacpp/recipe.json", + "mlx": ROOT / "crates/superfluid-adapters/superfluid-adapter-mlx/recipe.json", +} +LLAMACPP_RELEASES = "https://api.github.com/repos/ggml-org/llama.cpp/releases?per_page=30" +LLAMACPP_RELEASE = "https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/{version}" +MLX_PYPI = "https://pypi.org/pypi/mlx/json" +BUILD = re.compile(r"^b(\d+)$") + + +def fetch_json(url: str) -> object: + req = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json", "User-Agent": "superfluid-nightly"}) + token = os.environ.get("GITHUB_TOKEN") + if token and "api.github.com" in url: + req.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen(req, timeout=60) as resp: + return json.load(resp) + + +def recipe(runtime: str) -> dict: + return json.loads(RECIPES[runtime].read_text(encoding="utf-8")) + + +def tested_versions(runtime: str) -> list[str]: + return [t["version"] for t in recipe(runtime).get("tested", [])] + + +def build_number(tag: str) -> int: + m = BUILD.match(tag) + return int(m.group(1)) if m else -1 + + +def newest_llamacpp_build() -> dict: + """The newest `b` release that carries the macOS arm64 archive, which + every tested build has; llama.cpp also tags semver releases, which are + not builds of the library.""" + releases = fetch_json(LLAMACPP_RELEASES) + builds = [ + r + for r in releases + if build_number(r["tag_name"]) >= 0 + and not r.get("draft") + and any(a["name"] == f"llama-{r['tag_name']}-bin-macos-arm64.tar.gz" for a in r.get("assets", [])) + ] + if not builds: + raise SystemExit("no llama.cpp build with a macOS arm64 archive in the newest 30 releases") + return max(builds, key=lambda r: build_number(r["tag_name"])) + + +def pick(runtime: str, requested: str | None) -> str: + listed = tested_versions(runtime) + if requested: + return "" if requested in listed else requested + if runtime == "llamacpp": + newest = newest_llamacpp_build()["tag_name"] + have = max((build_number(v) for v in listed), default=-1) + return newest if build_number(newest) > have else "" + info = fetch_json(MLX_PYPI)["info"]["version"] + return "" if info in listed else info + + +def append_llamacpp(version: str) -> None: + path = RECIPES["llamacpp"] + r = recipe("llamacpp") + tested = r.setdefault("tested", []) + if any(t["version"] == version for t in tested): + print(f"{version} is already a tested build", file=sys.stderr) + return + if not tested: + raise SystemExit("the recipe lists no tested build to take the asset names from") + previous = tested[0] + release = fetch_json(LLAMACPP_RELEASE.format(version=version)) + by_name = {a["name"]: a for a in release.get("assets", [])} + assets, missing = [], [] + for old in previous["assets"]: + name = old["name"].replace(previous["version"], version) + a = by_name.get(name) + digest = (a or {}).get("digest") or "" + if not a or not digest.startswith("sha256:"): + missing.append(name) + continue + assets.append({"name": name, "size": a["size"], "sha256": digest[len("sha256:") :]}) + if missing: + # The archives the tests ran on must be there; a CUDA runtime archive + # dropped upstream is reported and left out. + required = [n for n in missing if n.endswith(("bin-macos-arm64.tar.gz", "bin-ubuntu-x64.tar.gz"))] + if required: + raise SystemExit(f"{version} lacks, or lists no SHA-256 for, {', '.join(required)}") + print(f"note: {version} lacks {', '.join(missing)}; left out", file=sys.stderr) + entry = {"version": version, "published": release["published_at"][:10], "assets": assets} + tested.insert(0, entry) + path.write_text(json.dumps(r, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"recorded {version} ({release['published_at'][:10]}) with {len(assets)} assets in {path.relative_to(ROOT)}") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("runtime", choices=["llamacpp", "mlx"]) + sub = ap.add_subparsers(dest="action", required=True) + p = sub.add_parser("pick", help="the newest upstream release the recipe does not list, as version=") + p.add_argument("--requested", default="", help="test this release instead of the newest") + a = sub.add_parser("append", help="record a llama.cpp build that passed as a tested build") + a.add_argument("version") + args = ap.parse_args() + if args.action == "pick": + print(f"version={pick(args.runtime, args.requested.strip() or None)}") + return 0 + if args.runtime != "llamacpp": + raise SystemExit("an MLX version is pinned by hand, wheel by wheel; see the recipe's `about`") + append_llamacpp(args.version) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From f3fd56894997f602f8e2a581dce300df520bc13d Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Thu, 15 Oct 2026 17:00:16 +1100 Subject: [PATCH 02/12] ci: the nightly MLX job tests the install's environment with its own Python recipe::install leaves two site-packages: the bundled interpreter's under python/ and the environment's under env/, where mlx and mlx-lm went. The job took the first find found, which can be the bare interpreter's, and built its test executables against whatever Python PyO3 picked on the runner. Now SUPERFLUID_MLX_VENV names env/, checked by importing mlx and mlx-lm from it, and PYO3_PYTHON names the install's Python 3.12. --- .github/workflows/nightly-runtimes.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml index 6622da0..17fba8b 100644 --- a/.github/workflows/nightly-runtimes.yml +++ b/.github/workflows/nightly-runtimes.yml @@ -196,9 +196,15 @@ jobs: plan=$("$worker" plan --version "$VERSION" --untested \ | python3 -c 'import sys, json; print(json.dumps(json.load(sys.stdin)["plans"][0]))') "$worker" install --into "$RUNNER_TEMP/mlx" --plan "$plan" - site=$(find "$RUNNER_TEMP/mlx" -type d -name site-packages | head -1) - [ -n "$site" ] || { echo "::error::the install holds no site-packages"; exit 1; } - echo "SUPERFLUID_MLX_VENV=$(dirname "$(dirname "$(dirname "$site")")")" >> "$GITHUB_ENV" + # The install holds the bundled interpreter under python/ and, beside it, the + # environment under env/ where mlx and mlx-lm went: the tests get the latter, + # and their test executables are built against the former, the Python 3.12 the + # wheels are for, whatever Python the runner has. + env="$RUNNER_TEMP/mlx/env" + "$env/bin/python" -c 'import mlx.core, mlx_lm; print("mlx", mlx.core.__version__, "in", __import__("sys").prefix)' \ + || { echo "::error::$env is not an environment with mlx and mlx-lm"; exit 1; } + echo "SUPERFLUID_MLX_VENV=$env" >> "$GITHUB_ENV" + echo "PYO3_PYTHON=$RUNNER_TEMP/mlx/python/bin/python3" >> "$GITHUB_ENV" - name: The adapter's tests on it run: | From ebf5029950bdc998e8ca01db95ea7879b1b6cfa1 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Thu, 15 Oct 2026 14:10:41 +1100 Subject: [PATCH 03/12] ci: the nightly job records a passing MLX version as a PR An MLX version that passes the adapter's tests arrives the same way a llama.cpp build does: as a PR adding it to the adapter's recipe, built from pip's report of the environment it resolved on the runner, every wheel with the size and SHA-256 PyPI lists for it. The issue stays for a version that fails, which includes one needing a newer Python than the adapter ships. --- .github/workflows/nightly-runtimes.yml | 67 ++++++++++++++++----- docs/models/runtimes.md | 2 +- scripts/nightly_runtimes.py | 83 ++++++++++++++++++++++---- 3 files changed, 125 insertions(+), 27 deletions(-) diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml index 17fba8b..a9e645e 100644 --- a/.github/workflows/nightly-runtimes.yml +++ b/.github/workflows/nightly-runtimes.yml @@ -5,9 +5,10 @@ name: Nightly runtimes # build that passes is added to the adapter's recipe by a PR, with the size # and SHA-256 GitHub lists for each of its assets, so the next release # installs it on its own; a build that fails opens an issue with the run -# attached. An MLX version is pinned by hand, wheel by wheel, so its result -# is an issue either way. `workflow_dispatch` tests a named release instead -# of the newest. +# attached. An MLX version that passes arrives the same way, its entry built +# from pip's report of the environment it resolved on the runner, every +# wheel with the size and SHA-256 PyPI lists for it. `workflow_dispatch` +# tests a named release instead of the newest. # # A schedule fires only from the default branch. The job is a no-op for a # runtime whose newest release the recipe already lists. @@ -206,6 +207,18 @@ jobs: echo "SUPERFLUID_MLX_VENV=$env" >> "$GITHUB_ENV" echo "PYO3_PYTHON=$RUNNER_TEMP/mlx/python/bin/python3" >> "$GITHUB_ENV" + - name: What pip resolved for it, for the recipe + # The same interpreter the install used, so the report names the + # wheels for Python 3.12 on macOS arm64, which the recipe pins. + run: | + set -euo pipefail + py="$SUPERFLUID_MLX_VENV/bin/python" + [ -x "$py" ] || py="$SUPERFLUID_MLX_VENV/bin/python3" + "$py" -m pip --version >/dev/null 2>&1 || "$py" -m ensurepip --upgrade >/dev/null + "$py" -m pip install --dry-run --ignore-installed --only-binary=:all: \ + --report "$RUNNER_TEMP/mlx-report.json" "mlx==$VERSION" mlx-lm >/dev/null + python3 -c 'import json, sys; print(len(json.load(open(sys.argv[1]))["install"]), "distributions resolved")' "$RUNNER_TEMP/mlx-report.json" + - name: The adapter's tests on it run: | set -o pipefail @@ -217,7 +230,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: mlx-${{ needs.pick.outputs.mlx }} - path: ${{ runner.temp }}/mlx-tests.log + path: | + ${{ runner.temp }}/mlx-tests.log + ${{ runner.temp }}/mlx-report.json if-no-files-found: ignore record-mlx: @@ -233,23 +248,45 @@ jobs: RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} steps: - uses: actions/checkout@v4 - - name: Open an issue with the result - # An MLX version is pinned by hand (every wheel with its SHA-256, see - # the recipe's `about`), so a pass asks for that PR rather than making it. + - if: needs.mlx.result == 'success' + uses: actions/download-artifact@v4 + with: + name: mlx-${{ needs.pick.outputs.mlx }} + path: ${{ runner.temp }}/mlx + + - name: Passed, so add it to the recipe in a PR + if: needs.mlx.result == 'success' + env: + BASE: ${{ github.ref_name }} run: | set -euo pipefail - if [ "$RESULT" = success ]; then - title="MLX $VERSION passes the adapter's tests: pin it" - p1="The nightly job installed MLX $VERSION untested on macos-14 and the adapter's tests passed on it: $RUN" - p2="To install it on its own, the next release needs the version pinned in the MLX adapter's recipe, wheel by wheel with the SHA-256 PyPI lists for each (see the recipe's about)." - else - title="MLX $VERSION breaks the adapter" - p1="The nightly job installed MLX $VERSION untested on macos-14 and the adapter's tests did not pass on it: $RUN" - p2="The test log is the run's artifact. Until the adapter is fixed, or the version is found to be at fault, the recipe keeps its last tested version." + branch="nightly/mlx-$VERSION" + if [ "$(gh pr list --head "$branch" --state open --json number --jq 'length')" != 0 ]; then + echo "a PR from $branch is already open" + exit 0 fi + git switch -c "$branch" + python3 scripts/nightly_runtimes.py mlx append "$VERSION" --report "$RUNNER_TEMP/mlx/mlx-report.json" + git -c user.name="github-actions[bot]" -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -am "mlx: tested with $VERSION" + git push -f origin "$branch" + p1="The nightly job installed MLX $VERSION untested on macos-14 and ran the adapter's tests on it: $RUN" + p2="This adds the version to the adapter's recipe as a tested one: the environment pip resolved for it on the runner, every wheel with the size and SHA-256 PyPI lists for it, so the next release installs it on its own. The Python the adapter ships is unchanged." body="$p1"$'\n\n'"$p2" + gh pr create --base "$BASE" --head "$branch" --title "mlx: tested with $VERSION" --body "$body" + + - name: Failed, so open an issue + if: needs.mlx.result != 'success' + # Among the failures: an MLX that needs a newer Python than the + # adapter ships, which is a change to the worker's build, not a pin. + run: | + set -euo pipefail + title="MLX $VERSION breaks the adapter" if [ "$(gh issue list --search "in:title \"$title\"" --state open --json number --jq 'length')" != 0 ]; then echo "an issue for MLX $VERSION is already open" exit 0 fi + p1="The nightly job installed MLX $VERSION untested on macos-14 and the adapter's tests did not pass on it: $RUN" + p2="The test log is the run's artifact. Until the adapter is fixed, or the version is found to be at fault, the recipe keeps its last tested version. A version that needs a newer Python than the adapter ships fails here too: that is a change to the worker's build, not a pin." + body="$p1"$'\n\n'"$p2" gh issue create --title "$title" --body "$body" diff --git a/docs/models/runtimes.md b/docs/models/runtimes.md index 68a3691..d8ad01c 100644 --- a/docs/models/runtimes.md +++ b/docs/models/runtimes.md @@ -126,7 +126,7 @@ Requests a model cannot serve are refused with a 400: ## How an upstream release becomes a tested one -Each adapter installs on its own only the upstream versions it was built and tested against, listed in its `recipe.json`; another installs with `--untested` and is still checked at load. A nightly job (`.github/workflows/nightly-runtimes.yml`) keeps those lists current: it installs the newest upstream release untested, runs the adapter's tests on it, and records the result. A llama.cpp build that passes arrives as a PR adding it to the recipe with the size and SHA-256 GitHub lists for each of its assets (`scripts/nightly_runtimes.py llamacpp append `); one that fails opens an issue with the run attached. An MLX version is pinned by hand, wheel by wheel, so its result opens an issue either way. `workflow_dispatch` tests a named release instead of the newest. +Each adapter installs on its own only the upstream versions it was built and tested against, listed in its `recipe.json`; another installs with `--untested` and is still checked at load. A nightly job (`.github/workflows/nightly-runtimes.yml`) keeps those lists current: it installs the newest upstream release untested, runs the adapter's tests on it, and records the result. A release that passes arrives as a PR adding it to the recipe (`scripts/nightly_runtimes.py append`): for llama.cpp with the size and SHA-256 GitHub lists for each of its assets, for MLX with the environment pip resolved for it on the runner, every wheel with the size and SHA-256 PyPI lists for it. One that fails opens an issue with the run attached; an MLX that needs a newer Python than the adapter ships fails that way, since it changes the worker's build rather than a pin. `workflow_dispatch` tests a named release instead of the newest. ## Per-runtime limits diff --git a/scripts/nightly_runtimes.py b/scripts/nightly_runtimes.py index bdeec9e..fb963d9 100755 --- a/scripts/nightly_runtimes.py +++ b/scripts/nightly_runtimes.py @@ -5,15 +5,19 @@ nightly_runtimes.py llamacpp pick [--requested bNNNN] nightly_runtimes.py llamacpp append bNNNN nightly_runtimes.py mlx pick [--requested X.Y.Z] + nightly_runtimes.py mlx append X.Y.Z --report `pick` prints `version=` for the newest upstream release the adapter's recipe does not list yet, or `version=` when there is nothing new (or when `--requested` names a release already listed); the workflow reads it from -$GITHUB_OUTPUT. `append` adds a llama.cpp build to the adapter's recipe as a -tested build, newest first: the same asset names the previous tested build -carries, with the size and SHA-256 GitHub lists for each, so a human reviews -a PR rather than typing digests. An MLX version is pinned by hand, wheel by -wheel, so `mlx` has no `append`; the job opens an issue with the result. +$GITHUB_OUTPUT. `append` adds a release to the adapter's recipe as a tested +one, newest first, so a human reviews a PR rather than typing digests. For +llama.cpp that is the same asset names the previous tested build carries, +with the size and SHA-256 GitHub lists for each. For MLX it is the whole +environment pip resolved for the version on the runner, read from pip's +install report (`pip install --dry-run --report`): every distribution's +wheel, with the size and SHA-256 PyPI lists for it, which must agree with +the digest pip reported. Only the standard library; GITHUB_TOKEN is used when set, for the API rate limit. @@ -25,6 +29,7 @@ import os import re import sys +import urllib.error import urllib.request from pathlib import Path @@ -36,6 +41,7 @@ LLAMACPP_RELEASES = "https://api.github.com/repos/ggml-org/llama.cpp/releases?per_page=30" LLAMACPP_RELEASE = "https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/{version}" MLX_PYPI = "https://pypi.org/pypi/mlx/json" +PYPI_RELEASE = "https://pypi.org/pypi/{name}/{version}/json" BUILD = re.compile(r"^b(\d+)$") @@ -44,8 +50,11 @@ def fetch_json(url: str) -> object: token = os.environ.get("GITHUB_TOKEN") if token and "api.github.com" in url: req.add_header("Authorization", f"Bearer {token}") - with urllib.request.urlopen(req, timeout=60) as resp: - return json.load(resp) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + return json.load(resp) + except urllib.error.HTTPError as e: + raise SystemExit(f"{url}: HTTP {e.code}") from None def recipe(runtime: str) -> dict: @@ -124,21 +133,73 @@ def append_llamacpp(version: str) -> None: print(f"recorded {version} ({release['published_at'][:10]}) with {len(assets)} assets in {path.relative_to(ROOT)}") +def canonical(name: str) -> str: + """A project name as PyPI's URLs spell it; the recipe keeps the spelling + the distribution gives itself (`PyYAML`, `Jinja2`) and orders by this.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def pypi_wheel(name: str, version: str, file: str) -> dict: + """The size and SHA-256 PyPI lists for one wheel of a release.""" + listing = fetch_json(PYPI_RELEASE.format(name=name, version=version)) + for u in listing.get("urls", []): + if u["filename"] == file: + return {"size": u["size"], "sha256": u["digests"]["sha256"]} + raise SystemExit(f"PyPI lists no {file} for {name} {version}") + + +def append_mlx(version: str, report_path: Path) -> None: + path = RECIPES["mlx"] + r = recipe("mlx") + tested = r.setdefault("tested", []) + if any(t["version"] == version for t in tested): + print(f"{version} is already a tested version", file=sys.stderr) + return + report = json.loads(report_path.read_text(encoding="utf-8")) + wheels, mlx_lm = [], None + for item in report.get("install", []): + meta = item["metadata"] + name, ver = meta["name"], meta["version"] + url = item["download_info"]["url"] + file = url.rsplit("/", 1)[-1] + if not file.endswith(".whl"): + raise SystemExit(f"{name} {ver} resolved to {file}, not a wheel; the recipe pins wheels only") + if canonical(name) == "mlx" and ver != version: + raise SystemExit(f"the report resolved mlx {ver}, not {version}") + if canonical(name) == "mlx-lm": + mlx_lm = ver + pinned = pypi_wheel(canonical(name), ver, file) + reported = item["download_info"].get("archive_info", {}).get("hashes", {}).get("sha256") + if reported and reported != pinned["sha256"]: + raise SystemExit(f"{file}: the SHA-256 pip reported differs from the one PyPI lists") + wheels.append({"name": name, "version": ver, "file": file, "size": pinned["size"], "sha256": pinned["sha256"]}) + if not any(canonical(w["name"]) == "mlx" for w in wheels) or mlx_lm is None: + raise SystemExit("the report holds no mlx or no mlx-lm wheel") + wheels.sort(key=lambda w: canonical(w["name"])) + tested.insert(0, {"version": version, "mlx_lm": mlx_lm, "wheels": wheels}) + path.write_text(json.dumps(r, indent=1, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"recorded mlx {version} (mlx-lm {mlx_lm}) with {len(wheels)} wheels in {path.relative_to(ROOT)}") + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("runtime", choices=["llamacpp", "mlx"]) sub = ap.add_subparsers(dest="action", required=True) p = sub.add_parser("pick", help="the newest upstream release the recipe does not list, as version=") p.add_argument("--requested", default="", help="test this release instead of the newest") - a = sub.add_parser("append", help="record a llama.cpp build that passed as a tested build") + a = sub.add_parser("append", help="record a release that passed as a tested one") a.add_argument("version") + a.add_argument("--report", help="mlx: pip's install report (--dry-run --report) for the version on the runner") args = ap.parse_args() if args.action == "pick": print(f"version={pick(args.runtime, args.requested.strip() or None)}") return 0 - if args.runtime != "llamacpp": - raise SystemExit("an MLX version is pinned by hand, wheel by wheel; see the recipe's `about`") - append_llamacpp(args.version) + if args.runtime == "llamacpp": + append_llamacpp(args.version) + return 0 + if not args.report: + raise SystemExit("mlx append needs --report, pip's install report for the version") + append_mlx(args.version, Path(args.report)) return 0 From 355d18cf2d1dd3005c8e29b61b9e6d53b9985878 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Thu, 15 Oct 2026 17:00:39 +1100 Subject: [PATCH 04/12] ci: the MLX report describes the environment the tests ran on A dry-run resolution made after the install could name other versions or wheels than the environment installed moments before, and the recipe would pin those as tested. The report now comes from a real install into the test environment, which the tests then run on. --- .github/workflows/nightly-runtimes.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml index a9e645e..e16bb66 100644 --- a/.github/workflows/nightly-runtimes.yml +++ b/.github/workflows/nightly-runtimes.yml @@ -207,17 +207,20 @@ jobs: echo "SUPERFLUID_MLX_VENV=$env" >> "$GITHUB_ENV" echo "PYO3_PYTHON=$RUNNER_TEMP/mlx/python/bin/python3" >> "$GITHUB_ENV" - - name: What pip resolved for it, for the recipe - # The same interpreter the install used, so the report names the - # wheels for Python 3.12 on macOS arm64, which the recipe pins. + - name: Install it again with a report, so the recipe records what the tests run on + # Not a dry run: pip installs the environment the tests then use and + # reports exactly that, wheel by wheel. A resolution made separately + # could differ from the install if a package were released or yanked + # in between, and the recipe would then pin something untested. run: | set -euo pipefail py="$SUPERFLUID_MLX_VENV/bin/python" [ -x "$py" ] || py="$SUPERFLUID_MLX_VENV/bin/python3" "$py" -m pip --version >/dev/null 2>&1 || "$py" -m ensurepip --upgrade >/dev/null - "$py" -m pip install --dry-run --ignore-installed --only-binary=:all: \ + "$py" -m pip install --ignore-installed --only-binary=:all: \ --report "$RUNNER_TEMP/mlx-report.json" "mlx==$VERSION" mlx-lm >/dev/null - python3 -c 'import json, sys; print(len(json.load(open(sys.argv[1]))["install"]), "distributions resolved")' "$RUNNER_TEMP/mlx-report.json" + "$py" -c 'import mlx.core, mlx_lm' || { echo "::error::the reinstalled environment does not import mlx and mlx-lm"; exit 1; } + python3 -c 'import json, sys; print(len(json.load(open(sys.argv[1]))["install"]), "distributions installed and reported")' "$RUNNER_TEMP/mlx-report.json" - name: The adapter's tests on it run: | From 5a60fde95f19510a798acce57485edd5cdd1a37d Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Thu, 15 Oct 2026 18:45:29 +1100 Subject: [PATCH 05/12] comments: say why, as the tree does --- .github/workflows/nightly-runtimes.yml | 29 ++++++++------------------ 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml index e16bb66..9800dbd 100644 --- a/.github/workflows/nightly-runtimes.yml +++ b/.github/workflows/nightly-runtimes.yml @@ -1,17 +1,10 @@ name: Nightly runtimes -# Each night, the newest upstream release of each runtime this superfluid -# drives is installed untested and the adapter's tests run on it. A llama.cpp -# build that passes is added to the adapter's recipe by a PR, with the size -# and SHA-256 GitHub lists for each of its assets, so the next release -# installs it on its own; a build that fails opens an issue with the run -# attached. An MLX version that passes arrives the same way, its entry built -# from pip's report of the environment it resolved on the runner, every -# wheel with the size and SHA-256 PyPI lists for it. `workflow_dispatch` -# tests a named release instead of the newest. -# -# A schedule fires only from the default branch. The job is a no-op for a -# runtime whose newest release the recipe already lists. +# Each night the newest upstream release of each runtime is installed untested +# and the adapter's tests run on it; one that passes is added to the recipe by +# a PR, with the digests its project lists, and one that fails opens an issue. +# `workflow_dispatch` tests a named release. A schedule fires only from the +# default branch, and a release the recipe already lists is a no-op. on: schedule: @@ -197,10 +190,8 @@ jobs: plan=$("$worker" plan --version "$VERSION" --untested \ | python3 -c 'import sys, json; print(json.dumps(json.load(sys.stdin)["plans"][0]))') "$worker" install --into "$RUNNER_TEMP/mlx" --plan "$plan" - # The install holds the bundled interpreter under python/ and, beside it, the - # environment under env/ where mlx and mlx-lm went: the tests get the latter, - # and their test executables are built against the former, the Python 3.12 the - # wheels are for, whatever Python the runner has. + # The tests run in env/, where mlx went, and are built against the bundled + # python/, the 3.12 the wheels are for, whatever Python the runner has. env="$RUNNER_TEMP/mlx/env" "$env/bin/python" -c 'import mlx.core, mlx_lm; print("mlx", mlx.core.__version__, "in", __import__("sys").prefix)' \ || { echo "::error::$env is not an environment with mlx and mlx-lm"; exit 1; } @@ -208,10 +199,8 @@ jobs: echo "PYO3_PYTHON=$RUNNER_TEMP/mlx/python/bin/python3" >> "$GITHUB_ENV" - name: Install it again with a report, so the recipe records what the tests run on - # Not a dry run: pip installs the environment the tests then use and - # reports exactly that, wheel by wheel. A resolution made separately - # could differ from the install if a package were released or yanked - # in between, and the recipe would then pin something untested. + # A real install, not a dry run, so the report is the environment the + # tests then run on and not a resolution that could differ from it. run: | set -euo pipefail py="$SUPERFLUID_MLX_VENV/bin/python" From dc70e77da8917371be205dff23d622bb5d878914 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Thu, 15 Oct 2026 20:53:08 +1100 Subject: [PATCH 06/12] ci: what three dry runs of the nightly on a fork taught - A listing that fails (issues disabled, no permission) was read as an open issue and the run recorded nothing: every listing check now stops with a clear error. - The MLX test executable, built against the install's Python, links @rpath/libpython with no rpath into the install: the install's lib directory is named for dyld. - Hosted runners share GitHub's unauthenticated API rate limit, and the worker's plan of an untested build hit it: the kit sends GITHUB_TOKEN for api.github.com when set, and the plan step has the job's token. - A runtime has one open issue; a later build that fails too is noted on it instead of opening one a day. Seen on the fork: https://github.com/IsuraManchanayake/superfluid/actions/runs/38715098729 --- .github/workflows/nightly-runtimes.yml | 32 +++++++++++++++---- .../superfluid-adapter-kit/src/install.rs | 15 ++++++--- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml index 9800dbd..f003637 100644 --- a/.github/workflows/nightly-runtimes.yml +++ b/.github/workflows/nightly-runtimes.yml @@ -75,6 +75,8 @@ jobs: - name: Build the worker and plan the untested install id: plan + env: + GITHUB_TOKEN: ${{ github.token }} run: | cargo build -p superfluid-adapter-llamacpp -p superfluid-tokenizer-llamacpp plan=$(target/debug/superfluid-worker-llamacpp plan --version "$BUILD" --untested \ @@ -135,7 +137,8 @@ jobs: run: | set -euo pipefail branch="nightly/llamacpp-$BUILD" - if [ "$(gh pr list --head "$branch" --state open --json number --jq 'length')" != 0 ]; then + open=$(gh pr list --head "$branch" --state open --json number --jq 'length') || { echo "::error::cannot list this repository's pull requests (may Actions open them? see Settings > Actions > General)"; exit 1; } + if [ "$open" != 0 ]; then echo "a PR from $branch is already open" exit 0 fi @@ -154,8 +157,14 @@ jobs: run: | set -euo pipefail title="llama.cpp $BUILD breaks the adapter" - if [ "$(gh issue list --search "in:title \"$title\"" --state open --json number --jq 'length')" != 0 ]; then - echo "an issue for $BUILD is already open" + # One open issue per runtime: a later build that fails too is a comment on it, since + # the newest build is tested every night and the adapter may take days to catch up. + existing=$(gh issue list --search "\"breaks the adapter\" llama.cpp in:title" --state open --json number,title --jq '.[0] | "\(.number) \(.title)"') || { echo "::error::cannot list this repository's issues (are issues enabled, and may Actions write them?)"; exit 1; } + if [ -n "$existing" ]; then + number=${existing%% *} + case "$existing" in *"$title"*) echo "an issue for $BUILD is already open: #$number"; exit 0;; esac + gh issue comment "$number" --body "$BUILD fails too: $RUN" + echo "noted $BUILD on the open issue #$number" exit 0 fi p1="The nightly job installed llama.cpp $BUILD untested and the adapter's tests did not pass on it: $RUN" @@ -212,6 +221,11 @@ jobs: python3 -c 'import json, sys; print(len(json.load(open(sys.argv[1]))["install"]), "distributions installed and reported")' "$RUNNER_TEMP/mlx-report.json" - name: The adapter's tests on it + # The test executable links @rpath/libpython3.12.dylib like the worker, but has no + # rpath into the install, so the install's lib directory is named for dyld. + env: + DYLD_LIBRARY_PATH: ${{ runner.temp }}/mlx/python/lib + DYLD_FALLBACK_LIBRARY_PATH: ${{ runner.temp }}/mlx/python/lib run: | set -o pipefail cargo test -p superfluid-adapter-mlx -- --nocapture 2>&1 \ @@ -253,7 +267,8 @@ jobs: run: | set -euo pipefail branch="nightly/mlx-$VERSION" - if [ "$(gh pr list --head "$branch" --state open --json number --jq 'length')" != 0 ]; then + open=$(gh pr list --head "$branch" --state open --json number --jq 'length') || { echo "::error::cannot list this repository's pull requests (may Actions open them? see Settings > Actions > General)"; exit 1; } + if [ "$open" != 0 ]; then echo "a PR from $branch is already open" exit 0 fi @@ -274,8 +289,13 @@ jobs: run: | set -euo pipefail title="MLX $VERSION breaks the adapter" - if [ "$(gh issue list --search "in:title \"$title\"" --state open --json number --jq 'length')" != 0 ]; then - echo "an issue for MLX $VERSION is already open" + # One open issue per runtime: a later version that fails too is a comment on it. + existing=$(gh issue list --search "\"breaks the adapter\" MLX in:title" --state open --json number,title --jq '.[0] | "\(.number) \(.title)"') || { echo "::error::cannot list this repository's issues (are issues enabled, and may Actions write them?)"; exit 1; } + if [ -n "$existing" ]; then + number=${existing%% *} + case "$existing" in *"$title"*) echo "an issue for MLX $VERSION is already open: #$number"; exit 0;; esac + gh issue comment "$number" --body "MLX $VERSION fails too: $RUN" + echo "noted MLX $VERSION on the open issue #$number" exit 0 fi p1="The nightly job installed MLX $VERSION untested on macos-14 and the adapter's tests did not pass on it: $RUN" diff --git a/crates/superfluid-adapters/superfluid-adapter-kit/src/install.rs b/crates/superfluid-adapters/superfluid-adapter-kit/src/install.rs index 7974ecf..e1687f4 100644 --- a/crates/superfluid-adapters/superfluid-adapter-kit/src/install.rs +++ b/crates/superfluid-adapters/superfluid-adapter-kit/src/install.rs @@ -375,11 +375,16 @@ pub fn fetch_json(url: &str) -> Result { if !url.starts_with("https://") { return Err(format!("{url}: only https:// URLs are fetched")); } - let out = Command::new("curl") - .args(["-fsSL", "--proto", "=https", "--tlsv1.2", "--retry", "2", "-H", "Accept: application/json"]) - .arg(url) - .output() - .map_err(|e| format!("curl: {e}"))?; + let mut curl = Command::new("curl"); + curl.args(["-fsSL", "--proto", "=https", "--tlsv1.2", "--retry", "2", "-H", "Accept: application/json"]); + // GitHub's API allows sixty unauthenticated requests an hour per address, which hosted + // runners share; a token in the environment lifts that, for the API alone. + if url.starts_with("https://api.github.com/") { + if let Some(token) = std::env::var_os("GITHUB_TOKEN").filter(|t| !t.is_empty()) { + curl.arg("-H").arg(format!("Authorization: Bearer {}", token.to_string_lossy())); + } + } + let out = curl.arg(url).output().map_err(|e| format!("curl: {e}"))?; if !out.status.success() { return Err(format!("fetching {url} failed: {}", String::from_utf8_lossy(&out.stderr).trim())); } From 8c0cbe2fe420acb246c5858d77b641e2285723ff Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Thu, 15 Oct 2026 21:17:46 +1100 Subject: [PATCH 07/12] ci: the MLX recipe entry pins the wheel built for the oldest macOS the adapter runs on pip on a runner or machine newer than macOS 14 resolves the mlx and mlx-metal wheels built for its own macOS, which an older Mac cannot install. The recording script now pins, for a wheel with a macOS tag, the same wheel built for the oldest macOS at or above 14 that PyPI offers, with the size and digest PyPI lists for it. --- scripts/nightly_runtimes.py | 41 ++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/scripts/nightly_runtimes.py b/scripts/nightly_runtimes.py index fb963d9..679f812 100755 --- a/scripts/nightly_runtimes.py +++ b/scripts/nightly_runtimes.py @@ -139,12 +139,37 @@ def canonical(name: str) -> str: return re.sub(r"[-_.]+", "-", name).lower() -def pypi_wheel(name: str, version: str, file: str) -> dict: - """The size and SHA-256 PyPI lists for one wheel of a release.""" +MACOS_TAG = re.compile(r"-macosx_(\d+)_(\d+)_arm64\.whl$") +# The oldest macOS the adapter runs on: a wheel built for it installs there and +# on every later one, where a wheel built for the runner's own macOS would not. +MACOS_FLOOR = (14, 0) + + +def floored(file: str, files: list[str]) -> str: + """The wheel to pin for `file`: the same wheel built for the oldest macOS + at or above the floor that PyPI offers, since pip on a newer runner or + machine resolves the newest, which an older Mac cannot install.""" + m = MACOS_TAG.search(file) + if not m: + return file + stem = file[: m.start()] + offered = [] + for f in files: + fm = MACOS_TAG.search(f) + if fm and f[: fm.start()] == stem: + offered.append(((int(fm.group(1)), int(fm.group(2))), f)) + at_or_above = sorted(v for v in offered if v[0] >= MACOS_FLOOR) + return at_or_above[0][1] if at_or_above else file + + +def pypi_wheel(name: str, version: str, file: str) -> tuple[str, dict]: + """The wheel pinned for `file` (see `floored`) and the size and SHA-256 PyPI lists for it.""" listing = fetch_json(PYPI_RELEASE.format(name=name, version=version)) - for u in listing.get("urls", []): + urls = listing.get("urls", []) + file = floored(file, [u["filename"] for u in urls]) + for u in urls: if u["filename"] == file: - return {"size": u["size"], "sha256": u["digests"]["sha256"]} + return file, {"size": u["size"], "sha256": u["digests"]["sha256"]} raise SystemExit(f"PyPI lists no {file} for {name} {version}") @@ -168,11 +193,13 @@ def append_mlx(version: str, report_path: Path) -> None: raise SystemExit(f"the report resolved mlx {ver}, not {version}") if canonical(name) == "mlx-lm": mlx_lm = ver - pinned = pypi_wheel(canonical(name), ver, file) + pinned_file, pinned = pypi_wheel(canonical(name), ver, file) reported = item["download_info"].get("archive_info", {}).get("hashes", {}).get("sha256") - if reported and reported != pinned["sha256"]: + if pinned_file == file and reported and reported != pinned["sha256"]: raise SystemExit(f"{file}: the SHA-256 pip reported differs from the one PyPI lists") - wheels.append({"name": name, "version": ver, "file": file, "size": pinned["size"], "sha256": pinned["sha256"]}) + if pinned_file != file: + print(f"note: {file} pinned as {pinned_file}, the same wheel built for macOS {MACOS_FLOOR[0]}", file=sys.stderr) + wheels.append({"name": name, "version": ver, "file": pinned_file, "size": pinned["size"], "sha256": pinned["sha256"]}) if not any(canonical(w["name"]) == "mlx" for w in wheels) or mlx_lm is None: raise SystemExit("the report holds no mlx or no mlx-lm wheel") wheels.sort(key=lambda w: canonical(w["name"])) From 1472dd8e1045e3e5e8b0c27320b05303d222467c Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 16 Oct 2026 00:38:45 +1100 Subject: [PATCH 08/12] ci: the nightly's macOS leg runs on a self-hosted Apple silicon runner The hosted macos-14 runner's GPU is virtualized: llama.cpp takes its fallback Metal kernels there, a 40-token prompt costs seconds, a padded batch answers a zero logit row, and a batched round differs from solo decode past the tests' bound, whatever the build; and with 2 GB free the adapter keeps one shared pool where the tests expect a buffer per lane. The leg now asks for a self-hosted Apple silicon runner and keeps the toolchain that runner has. --- .github/workflows/nightly-runtimes.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml index f003637..9fcf23e 100644 --- a/.github/workflows/nightly-runtimes.yml +++ b/.github/workflows/nightly-runtimes.yml @@ -63,14 +63,25 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-14, ubuntu-24.04] - runs-on: ${{ matrix.os }} + # The macOS leg runs on a self-hosted Apple silicon runner. The hosted + # macos-14 runner's virtualized GPU takes llama.cpp's fallback Metal + # kernels, which fail the adapter's numeric tests whatever the build, + # and its memory leaves the pool shared where the tests expect a + # buffer per lane. + include: + - os: macos + runner: [self-hosted, macOS, ARM64] + - os: ubuntu-24.04 + runner: ubuntu-24.04 + runs-on: ${{ matrix.runner }} timeout-minutes: 60 env: BUILD: ${{ needs.pick.outputs.llamacpp }} steps: - uses: actions/checkout@v4 + # A self-hosted runner keeps the toolchain it has. - uses: dtolnay/rust-toolchain@stable + if: runner.environment == 'github-hosted' - uses: Swatinem/rust-cache@v2 - name: Build the worker and plan the untested install From 38d1f51bde3eec2d278391ac89eeb5646d443103 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 16 Oct 2026 00:56:51 +1100 Subject: [PATCH 09/12] ci: a self-hosted runner finds cargo where rustup puts it The runner's PATH is its owner's shell's at start, which need not name ~/.cargo/bin; the first run on a Mac died in fifteen seconds on 'cargo: command not found'. --- .github/workflows/nightly-runtimes.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml index 9fcf23e..08b5d29 100644 --- a/.github/workflows/nightly-runtimes.yml +++ b/.github/workflows/nightly-runtimes.yml @@ -79,9 +79,12 @@ jobs: BUILD: ${{ needs.pick.outputs.llamacpp }} steps: - uses: actions/checkout@v4 - # A self-hosted runner keeps the toolchain it has. + # A self-hosted runner keeps the toolchain it has, where rustup puts it; + # its PATH does not say so unless its owner's shell did. - uses: dtolnay/rust-toolchain@stable if: runner.environment == 'github-hosted' + - if: runner.environment == 'self-hosted' + run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - uses: Swatinem/rust-cache@v2 - name: Build the worker and plan the untested install From dcdeacf2a80c4df13091d0153f871d1a9607c977 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 16 Oct 2026 01:06:40 +1100 Subject: [PATCH 10/12] ci: no rust-cache on a self-hosted runner, whose save step deletes rustup Swatinem/rust-cache's save step removes every regular file in ~/.cargo/bin that cargo did not install during the job: on a hosted runner that is nothing of value, on a developer's Mac it was rustup itself. A self-hosted runner keeps its work directory between jobs and needs no cache. --- .github/workflows/nightly-runtimes.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml index 08b5d29..29ed162 100644 --- a/.github/workflows/nightly-runtimes.yml +++ b/.github/workflows/nightly-runtimes.yml @@ -85,7 +85,11 @@ jobs: if: runner.environment == 'github-hosted' - if: runner.environment == 'self-hosted' run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + # Never on a self-hosted runner: its save step deletes every file in + # ~/.cargo/bin that cargo did not install during the job, rustup + # included. A self-hosted runner keeps its target directory anyway. - uses: Swatinem/rust-cache@v2 + if: runner.environment == 'github-hosted' - name: Build the worker and plan the untested install id: plan From 55751323572f7a74391b9d5135b857ebe32b8748 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 16 Oct 2026 01:13:55 +1100 Subject: [PATCH 11/12] ci: the model cache is for hosted runners; a self-hosted one keeps the file Its key is the workflow file's hash, so every edit to the workflow missed the cache and the post step uploaded the model from the runner: on a laptop, minutes at a megabyte a second. --- .github/workflows/nightly-runtimes.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml index 29ed162..928ffe9 100644 --- a/.github/workflows/nightly-runtimes.yml +++ b/.github/workflows/nightly-runtimes.yml @@ -110,7 +110,11 @@ jobs: PLAN: ${{ steps.plan.outputs.plan }} run: target/debug/superfluid-worker-llamacpp install --into "$RUNNER_TEMP/llamacpp" --plan "$PLAN" + # A hosted runner starts empty and gets the model from the cache; a + # self-hosted one keeps the directory between jobs, and would spend + # minutes uploading it. - uses: actions/cache@v4 + if: runner.environment == 'github-hosted' with: path: ~/.cache/superfluid-nightly/models key: nightly-gguf-${{ hashFiles('.github/workflows/nightly-runtimes.yml') }} From 1cdbbcea98127779ac01cb47163329b65f7e72ce Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 16 Oct 2026 01:35:01 +1100 Subject: [PATCH 12/12] ci: the recipe PR says the macOS tests ran on Apple silicon, not macos-14 --- .github/workflows/nightly-runtimes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-runtimes.yml b/.github/workflows/nightly-runtimes.yml index 928ffe9..2161e68 100644 --- a/.github/workflows/nightly-runtimes.yml +++ b/.github/workflows/nightly-runtimes.yml @@ -169,7 +169,7 @@ jobs: git -c user.name="github-actions[bot]" -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ commit -am "llamacpp: tested with $BUILD" git push -f origin "$branch" - p1="The nightly job installed llama.cpp $BUILD untested and ran the adapter's tests on it on macos-14 (Metal) and ubuntu-24.04 (CPU): $RUN" + p1="The nightly job installed llama.cpp $BUILD untested and ran the adapter's tests on it on Apple silicon (Metal) and ubuntu-24.04 (CPU): $RUN" p2="This adds the build to the adapter's recipe as a tested build, with the size and SHA-256 GitHub lists for each of its assets, so the next release installs it on its own. The upstream release is https://github.com/ggml-org/llama.cpp/releases/tag/$BUILD." body="$p1"$'\n\n'"$p2" gh pr create --base "$BASE" --head "$branch" --title "llamacpp: tested with $BUILD" --body "$body"