diff --git a/.github/workflows/publish-proof-eval-image.yml b/.github/workflows/publish-proof-eval-image.yml new file mode 100644 index 000000000..5672ff2aa --- /dev/null +++ b/.github/workflows/publish-proof-eval-image.yml @@ -0,0 +1,180 @@ +name: publish-proof-eval-image + +# Publishes ghcr.io/cortexlm/proof-eval and prints the pushed sha256 digest. +# That digest is what the control plane pins in config/proof-pin.toml as +# eval_image_digest; a tag is never the pin. Do not invent a sha256. +# +# Two images, two Dockerfiles: +# +# : scoring — CUDA base + torch. Pin this, and only after +# this job has pulled THAT digest and run the harvest-PATH +# check + selftest against it. +# :-contract slim contract only. Fast, cannot score, not the pin. + +on: + push: + branches: ["main", "cursor/**"] + paths: + - "eval/**" + - ".github/workflows/publish-proof-eval-image.yml" + workflow_dispatch: + +permissions: + contents: read + packages: write + +concurrency: + group: publish-proof-eval-${{ github.ref }} + cancel-in-progress: false + +jobs: + contract: + runs-on: ubuntu-latest + outputs: + digest: ${{ steps.push.outputs.digest }} + steps: + - uses: actions/checkout@v4 + - name: resolve the image name + run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/proof-eval" >> "${GITHUB_ENV}" + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: build and push the contract image + id: push + uses: docker/build-push-action@v6 + with: + context: . + file: eval/Dockerfile + push: true + provenance: false + platforms: linux/amd64 + tags: | + ${{ env.IMAGE }}:${{ github.sha }}-contract + build-args: | + WITH_RUNTIME=0 + PROOF_GIT_SHA=${{ github.sha }} + cache-from: type=gha,scope=proof-eval-contract + cache-to: type=gha,mode=max,scope=proof-eval-contract + - name: report + run: | + { + echo "### proof-eval (contract only)" + echo + echo '```' + echo "image = \"${IMAGE}\"" + echo "digest = \"${{ steps.push.outputs.digest }}\"" + echo '```' + echo + echo "Contract layer only: slim, no model runtime, so it refuses to score." + echo "Do not pin this digest." + } >> "${GITHUB_STEP_SUMMARY}" + + runtime: + runs-on: ubuntu-latest + timeout-minutes: 120 + outputs: + digest: ${{ steps.push.outputs.digest }} + steps: + - uses: actions/checkout@v4 + - name: resolve the image name + run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/proof-eval" >> "${GITHUB_ENV}" + - name: free disk for the CUDA base and runtime wheels + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /opt/hostedtoolcache /usr/local/share/boost \ + /opt/hostedtoolcache/CodeQL /usr/local/lib/node_modules \ + /usr/share/swift /opt/az || true + df -h / + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: build and push the scoring image + id: push + uses: docker/build-push-action@v6 + with: + context: . + file: eval/Dockerfile.scoring + push: true + provenance: false + platforms: linux/amd64 + tags: | + ${{ env.IMAGE }}:${{ github.sha }} + build-args: | + WITH_RUNTIME=1 + PROOF_GIT_SHA=${{ github.sha }} + - name: pull the published digest and prove it can score + env: + DIGEST: ${{ steps.push.outputs.digest }} + run: | + set -eu + case "${DIGEST}" in + sha256:*) ;; + *) echo "no sha256 digest was published"; exit 1 ;; + esac + ref="${IMAGE}@${DIGEST}" + docker pull "${ref}" + docker run --rm --entrypoint /bin/sh "${ref}" -c \ + 'test -f /usr/bin/proof-eval && test -x /usr/bin/proof-eval && env -i PATH=/usr/bin:/bin /usr/bin/proof-eval --help' + docker run --rm --entrypoint /bin/sh "${ref}" -c \ + 'test ! -L /usr/bin/proof-eval' + docker run --rm --entrypoint /bin/sh "${ref}" -c \ + 'env -i PATH=/usr/bin:/bin /usr/bin/proof-eval score --help' + docker run --rm --entrypoint /opt/proof-eval-venv/bin/python "${ref}" \ + -c 'import torch, transformers' + docker run --rm --entrypoint /bin/sh "${ref}" -c \ + 'test -f /opt/proof-eval/baked_proxies.json && grep -q "Qwen/Qwen3.8-0.6B" /opt/proof-eval/baked_proxies.json' + docker run --rm --entrypoint /bin/sh "${ref}" -c \ + 'env -i PATH=/usr/bin:/bin HOME=/root PROOF_SELFTEST_REQUIRE_RUNTIME=1 /usr/bin/proof-eval selftest' + - name: publicize the GHCR package + env: + GH_TOKEN: ${{ github.token }} + OWNER: ${{ github.repository_owner }} + run: | + set -euo pipefail + enc=$(python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=""))' "proof-eval") + code=$(curl -sS -o /tmp/vis.json -w "%{http_code}" \ + -X PUT \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/${OWNER}/packages/container/${enc}/visibility" \ + -d '{"visibility":"public"}' || true) + echo "pkg=proof-eval http=${code} body=$(head -c 200 /tmp/vis.json || true)" + - name: report the digest to pin + env: + DIGEST: ${{ steps.push.outputs.digest }} + run: | + set -eu + case "${DIGEST}" in + sha256:*) ;; + *) echo "no sha256 digest was published"; exit 1 ;; + esac + { + echo "### proof-eval (scoring image, CUDA base)" + echo + echo "Pulled this digest after push, ran the harvest-PATH check," + echo "proved \`import torch, transformers\`, baked proxy Qwen/Qwen3.8-0.6B," + echo "and ran \`proof-eval selftest\` (fabric 12.5 Gbit/s / no IB-NVLink-NCCL)." + echo "Paste into the control plane's \`config/proof-pin.toml\`:" + echo + echo '```toml' + echo "eval_image = \"${IMAGE}\"" + echo "eval_image_digest = \"${DIGEST}\"" + echo "proxy_model = \"Qwen/Qwen3.8-0.6B\"" + echo "proxy_models = [\"Qwen/Qwen3.8-0.6B\"]" + echo '```' + } >> "${GITHUB_STEP_SUMMARY}" + echo "eval_image_digest = \"${DIGEST}\"" + echo "${DIGEST}" > /tmp/proof-eval.digest + - name: upload digest + uses: actions/upload-artifact@v4 + with: + name: proof-eval-digest + path: /tmp/proof-eval.digest + if-no-files-found: error diff --git a/AGENTS.md b/AGENTS.md index 8197d068d..2c9715b1b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ Short contract for agents and operators. Prefer linking over restating runbooks. -**Product:** Cortex ([`CortexLM/cortex`](https://github.com/CortexLM/cortex)) — Bittensor subnet control plane. **Two live challenges:** `bounty` (2000 bps) and `proof` (8000 bps). Proof-weighted 20%/80% lock regardless of eval digest. Proof's `eval_image_digest` is empty (submits 503); do not invent a sha256. Sum is 10000. `relearn`, `relearn-image`, `relearn-agent`, `relearn-mm`, `design`, and `prism` are **off** — no trust-root row, so they have no emission and no leaf may verify. Relearn* code stays behind the `relearn` / `mm` compose profiles. Proof scores operator-published research topics (dynamic `topic_id`, digest-pinned RLM judge, `wta` or `discovery` payout). Naming split (Cortex vs leftover `base` / `BASE_*`): [`docs/NAMING.md`](docs/NAMING.md). +**Product:** Cortex ([`CortexLM/cortex`](https://github.com/CortexLM/cortex)) — Bittensor subnet control plane. **Two live challenges:** `bounty` (2000 bps) and `proof` (8000 bps). Proof-weighted 20%/80% lock regardless of eval digest. Proof eval digest is pinned (`ghcr.io/cortexlm/proof-eval@sha256:78b614a1…`, proxy `Qwen/Qwen3.8-0.6B`); live submits still 503 until harvest is wired, a baseline is sealed, and ≥1 topic is open. Empty digest stays fail-closed (do not invent a sha256). Sum is 10000. `relearn`, `relearn-image`, `relearn-agent`, `relearn-mm`, `design`, and `prism` are **off** — no trust-root row, so they have no emission and no leaf may verify. Relearn* code stays behind the `relearn` / `mm` compose profiles. Proof scores operator-published research topics (dynamic `topic_id`, digest-pinned RLM judge, `wta` or `discovery` payout). Naming split (Cortex vs leftover `base` / `BASE_*`): [`docs/NAMING.md`](docs/NAMING.md). PRs require a [Greptile](https://greptile.com) review (`.greptile/`). If the bot is silent, comment `@greptileai review`. diff --git a/config/proof-pin.toml b/config/proof-pin.toml index a462201f4..45d161900 100644 --- a/config/proof-pin.toml +++ b/config/proof-pin.toml @@ -5,6 +5,12 @@ # # Deploy = bump eval_image_digest after proof-eval CI is green. Empty digest # is the pre-launch state: live submits answer 503. Do not invent a sha256. +# proxy_model must be an id the image actually bakes (see proxy_models). +# +# Scoring image published by .github/workflows/publish-proof-eval-image.yml +# run 33892650063 (commit 51f937c7818f0eb1e3ed1412972de98b6994952b). +# Pulled that digest after push; harvest-PATH + selftest + baked proxy +# Qwen/Qwen3.8-0.6B + 12.5 Gbit/s fabric enforcement proved on those bytes. # # Topic documents are signed by the `proof` row key in config/challenges.toml # (sr25519, domain `base-proof-topic-v1`). `topic_pubkey` must match that row. @@ -24,14 +30,13 @@ challenge_id = "proof" scoring_version = 1 base_model_family = "Qwen/Qwen3.8" -# Empty until the eval image exists. A topic naming a proxy the image does -# not bake is a publish 400. -proxy_model = "" -proxy_models = [] +# Baked into ghcr.io/cortexlm/proof-eval@sha256:78b614a1… (see baked_proxies.json). +proxy_model = "Qwen/Qwen3.8-0.6B" +proxy_models = ["Qwen/Qwen3.8-0.6B"] eval_image = "ghcr.io/cortexlm/proof-eval" -eval_image_digest = "" -proof_git = "https://github.com/CortexLM/relearn" -proof_git_sha = "" +eval_image_digest = "sha256:78b614a1f51ce5dd80076c4e343a2b31b85d6c36025e02836cb83929867e7009" +proof_git = "https://github.com/CortexLM/cortex" +proof_git_sha = "51f937c7818f0eb1e3ed1412972de98b6994952b" topic_pubkey = "3e7f70f09165e265ab89ab04a4fc91dc0531d54a100c538fb14c6f008421c375" flops_budget_max = 2000000000000000000 epsilon_nll_min = 0.02 diff --git a/crates/proof-task/tests/committed_pin.rs b/crates/proof-task/tests/committed_pin.rs index 3031e3cb6..e62d3da88 100644 --- a/crates/proof-task/tests/committed_pin.rs +++ b/crates/proof-task/tests/committed_pin.rs @@ -51,7 +51,7 @@ fn proof_row_pubkey() -> String { } #[test] -fn committed_pin_is_proof_with_empty_eval_digest() { +fn committed_pin_is_proof_with_a_real_eval_digest() { let p = pin(); assert_eq!(p.challenge_id, CHALLENGE_ID); assert_eq!(p.eval_image, EVAL_IMAGE); @@ -59,11 +59,19 @@ fn committed_pin_is_proof_with_empty_eval_digest() { !p.eval_image.contains(':'), "the tag belongs in eval_image_digest, not eval_image" ); + let digest = p.eval_image_digest.trim(); assert!( - p.eval_image_digest.trim().is_empty(), - "do not invent a sha256; empty digest is the pre-launch 503" + digest.starts_with("sha256:") && digest.len() == 71, + "committed digest must be a real sha256 pin, not invented or empty: {digest:?}" ); - assert!(!p.can_rent(), "empty digest cannot rent"); + let hex = digest.trim_start_matches("sha256:"); + assert!( + hex.len() == 64 && hex.chars().all(|c| c.is_ascii_hexdigit()), + "{digest}" + ); + assert!(p.can_rent(), "pinned digest must be rentable"); + assert_eq!(p.proxy_model, "Qwen/Qwen3.8-0.6B"); + assert!(p.bakes_proxy("Qwen/Qwen3.8-0.6B")); assert_eq!(p.holdout_size, HOLDOUT_SIZE); assert_eq!(p.stratum_size, STRATUM_SIZE); } diff --git a/deploy/scripts/proof-operator-path.sh b/deploy/scripts/proof-operator-path.sh new file mode 100755 index 000000000..c8cfb560e --- /dev/null +++ b/deploy/scripts/proof-operator-path.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Proof operator path: holdout → seal baseline on the pinned image → sign → publish. +# +# Secrets stay off git. This script prints the commands; it never writes under +# config/ or docs/, and it never prints a mini-secret. +# +# Topic schema (payout_mode, validation, discovery shares) is owned by the +# control-plane PR. This path only runs holdout / baseline / admin inject +# against whatever signed document the operator drafted. +# +# can_score becomes true only when ALL of: real digest pin, wired harvest +# (LIUM_API_KEY + SSH pub), ≥1 open signed topic, verified holdout, sealed +# baseline. Empty digest stays 503 (never invent a sha256). +set -euo pipefail + +TOPIC_ID="${PROOF_TOPIC_ID:-dt-no-ib-v0}" +SECRETS="${PROOF_SECRETS_DIR:-$HOME/.base-secrets/proof}" +IMAGE="ghcr.io/cortexlm/proof-eval" +PROXY="Qwen/Qwen3.8-0.6B" + +digest=$(python3 - <<'PY' || true +import tomllib, pathlib, sys +p = pathlib.Path("config/proof-pin.toml") +if not p.is_file(): + sys.exit(0) +print(tomllib.loads(p.read_text()).get("eval_image_digest","").strip()) +PY +) +digest="${PROOF_EVAL_IMAGE_DIGEST:-$digest}" + +cat <} +# Proxy the image bakes: ${PROXY} + +mkdir -p '${SECRETS}' +chmod 700 '${SECRETS}' + +# 1. Select a stratified holdout (records NEVER enter git). +cargo run -p xtask -- proof-holdout \\ + --topic-id '${TOPIC_ID}' \\ + --synthetic \\ + --salt "\$PROOF_HOLDOUT_SALT" \\ + --size 120 \\ + --out '${SECRETS}/holdouts.json' + +# Production: drop --synthetic and pass --catalog . + +# 2. Stage shard bytes the image will score (content-addressed). +# PROOF_HOLDOUT_STORE/\${content_sha256} ← packed shard text. +# The request carries fingerprints only. + +# 3. Seal the AdamW / comms baseline ON THE PINNED IMAGE (not sim). +# The image enforces 12.5 Gbit/s / no IB / no NVLink / no NCCL fast path +# before it will emit numbers. +if [ -n "${digest}" ]; then + echo "docker run --rm --entrypoint /usr/bin/proof-eval ${IMAGE}@${digest} baseline --request /tmp/proof_eval/request.json --out /tmp/proof_eval/baseline.json" +else + echo "# digest still empty — do not invent a sha256; wait for publish-proof-eval-image" +fi +# Put the measurement JSON at ${SECRETS}/baselines.json keyed by topic id. +# script_sha256 = sha256(/opt/proof-eval/baselines/adamw.py) from that image. +# metrics_commitment = BaselineMeasurement.commitment() over the vector. + +# 4. Sign the operator draft (YAML or JSON). Schema is payout_mode + +# validation.{score_on,accept_if,reject_if} + metric; this helper does +# not invent those fields. --synthetic is local/dev; production uses +# --holdout so the commitment matches the host file. +cargo run -p xtask -- proof-topic \\ + --input '${SECRETS}/${TOPIC_ID}.yaml' \\ + --secret deploy/secrets/proof_sk \\ + --holdout '${SECRETS}/holdouts.json' \\ + --out '${SECRETS}/topics.json' + +# 5. Publish (dynamic inject). Admin bearer from PROOF_ADMIN_TOKENS_FILE. +# curl -sS -X POST "\$PROOF_BASE/v1/admin/proof/topics" \\ +# -H "authorization: Bearer \$PROOF_ADMIN_TOKEN" \\ +# -H 'content-type: application/json' \\ +# --data-binary @${SECRETS}/topics.json + +# 6. Point the host at the operator files (never in git): +# PROOF_TOPICS_FILE=${SECRETS}/topics.json +# PROOF_HOLDOUT_FILE=${SECRETS}/holdouts.json +# PROOF_BASELINE_FILE=${SECRETS}/baselines.json +# LIUM_API_KEY=… LIUM_SSH_PUBLIC_KEY_FILE=… +# Restart proof-challenge, then: +# curl -sS "\$PROOF_BASE/v1/status" | jq '{can_score,eval_image_digest,open_topics,live_harvest_wired,baseline_sealed}' + +# can_score is true only with: real digest + harvest wired + open topic + +# sealed baseline + verified holdout. Empty digest stays 503. +EOF diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index d3cd9f3d7..22ce519ab 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -127,13 +127,13 @@ Removed as **live products**. Shared rails (`prism-lium*`, `prism-competition` p | Challenge id | **done** | `proof` on the wire. Topics are operator-published signed documents; git carries no catalog. | | Crates (`crates/proof-*`) | **done** | task (signed topics, holdout commitments, global pin, `payout_mode` / English `validation`), score (per-topic pass + WTA/discovery sum), store, eval (RLM judge, fail-closed readiness), harvest, http, challenge. | | Binary (`bins/proof-challenge`) | **done** | HTTP API on `:8100`. | -| Miner CLI (`bins/ctx`) | **done** | `ctx proof submit|show|status|topics`. Empty `eval_image_digest` → 503. | +| Miner CLI (`bins/ctx`) | **done** | `ctx proof submit|show|status|topics`. Unpinned digest / unwired harvest / no open topic → 503. | | Compose / images | **done** | Default compose + `images.yml` target `proof-challenge`. | -| Eval pin | **v0** | `config/proof-pin.toml` — `eval_image` `ghcr.io/cortexlm/proof-eval`, `eval_image_digest` empty until first green proof-eval CI. Empty digest → live submits **503**. Do not invent a sha256. | +| Eval pin | **done** | `config/proof-pin.toml` — `eval_image` `ghcr.io/cortexlm/proof-eval`, digest `sha256:78b614a1…` (publish-proof-eval-image run 33892650063, commit `51f937c7`). Baked proxy `Qwen/Qwen3.8-0.6B`. Empty digest is gone; live submits still **503** until harvest is wired, a baseline is sealed, and ≥1 topic is open. Do not re-pin a guessed sha256. | | Topics | **done** | sr25519 under the `proof` trust-root key (`base-proof-topic-v1`). Admin `POST /v1/admin/proof/topics`. A topic must be sealed to `open`. | | Holdout | **done** | Per-topic operator file (`PROOF_HOLDOUT_FILE`). Commitment in the topic document, never in the pin. `xtask proof-holdout --topic-id`. | | Live harvest | **done** | `crates/proof-harvest` over `harvest-pod`; `PROOF_FORCE_SIM` is local-only. | -| Emission | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Empty `eval_image_digest` → 503. Split equally across currently `open` topics, then `wta` or `discovery`. Empty open set → `NoScore(ChallengeInternal)`. | +| Emission | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Unwired harvest / unsealed baseline / empty open set → 503 / `NoScore(ChallengeInternal)`. Split equally across currently `open` topics, then `wta` or `discovery`. Empty digest still 503s (never invent a sha256). | | Spec | live | [`PROOF.md`](PROOF.md). | ## Infrastructure diff --git a/docs/PROOF.md b/docs/PROOF.md index 88a0dc7bb..3b41f76bd 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -1,17 +1,20 @@ # Proof challenge Live challenge id: **`proof`**. Emission **8000 bps** (80% of the subnet; -bounty is 2000). This 20%/80% lock is independent of eval digest. Proof still -answers **503** on submit until `eval_image_digest` is pinned. Do not invent -a sha256. Sum across the two live rows stays 10000. Port **8100** (local -probe **28100**). +bounty is 2000). This 20%/80% lock is independent of eval digest. Eval +digest `sha256:78b614a1…` is pinned (`ghcr.io/cortexlm/proof-eval`, baked +proxy `Qwen/Qwen3.8-0.6B`). Live submits still **503** until harvest is +wired, a baseline is sealed, and ≥1 topic is open. Do not invent a sha256. +Sum across the two live rows stays 10000. Port **8100** (local probe +**28100**). Proof is the research-problem challenge. The unit of work is an **operator-published signed topic**, not a prompt and not an episode. Git carries global floors in [`config/proof-pin.toml`](../config/proof-pin.toml) and no topic catalog. Miners submit against `topic_id`. The RLM judge lives -in a digest-pinned eval image (`ghcr.io/cortexlm/proof-eval`). Empty -`eval_image_digest` is the pre-launch state: live submits answer **503**. +in a digest-pinned eval image (`ghcr.io/cortexlm/proof-eval`). The digest +is pinned; live submits still answer **503** until harvest + sealed +baseline + an open topic are on the host. ## Product rules (do not weaken) @@ -67,9 +70,10 @@ commitment matches records the host will unseal. 4. `POST /v1/admin/proof/topics` with the signed document and the operator bearer. `GET /v1/proof/topics` lists open ids (never holdout records). -Ship order: control plane (this) → proof-eval image + digest pin (separate) -→ holdout/baseline files on the host → open topics. `proxy_model` may stay -empty until the image exists. Empty digest stays 503. +Ship order: control plane (payout schema) → proof-eval image + digest pin +(this pin) → holdout/baseline files on the host → open topics. Operator +path: [`deploy/scripts/proof-operator-path.sh`](../deploy/scripts/proof-operator-path.sh). +Empty digest stays 503 (never invent a sha256). ## Metric families @@ -84,8 +88,10 @@ Holdout: 120 records, stratified 24 each across `web_ood`, `code_ood`, off-score and never in the 120. Ceremony: `cargo run -p xtask -- proof-holdout --topic-id …` and -`cargo run -p xtask -- proof-topic --input … --secret …`. Trust-root keygen -is the throwaway owner path in [`config/CEREMONY.md`](../config/CEREMONY.md). +`cargo run -p xtask -- proof-topic --input … --secret …`. Operator path: +[`deploy/scripts/proof-operator-path.sh`](../deploy/scripts/proof-operator-path.sh). +Trust-root keygen is the throwaway owner path in +[`config/CEREMONY.md`](../config/CEREMONY.md). ## HTTP @@ -109,8 +115,8 @@ Miner-facing: [`external-miner/proof.md`](./external-miner/proof.md). ### `dt-no-ib-v0` — throughput **wta** -Operator **example**, not in the pin and not published until the eval image -exists. Throughput family, no InfiniBand / NVLink / NCCL fast fabric, +Operator **example**, not in the pin and not published until the operator +seals a baseline on the pinned image. Throughput family, no InfiniBand / NVLink / NCCL fast fabric, 12.5 Gbit/s cap, beat sealed AdamW/comms reference, 2e18 FLOPs. Winner takes the topic. diff --git a/eval/Dockerfile b/eval/Dockerfile new file mode 100644 index 000000000..9412655cf --- /dev/null +++ b/eval/Dockerfile @@ -0,0 +1,67 @@ +# syntax=docker/dockerfile:1.7 +# +# Proof eval image (contract layer). Cortex pins the *digest* of the scoring +# build (Dockerfile.scoring) in config/proof-pin.toml; a floating tag is +# never the pin. Do not pin this contract-only digest. +# +# docker build -f eval/Dockerfile --build-arg WITH_RUNTIME=0 -t proof-eval:contract . +# +# No secret, holdout item, teacher endpoint, or Modal reference is baked in. + +ARG BASE_IMAGE=python:3.12-slim-bookworm@sha256:0f5b26b9518d002b6173fd61daad821fa340635ebfec5bba471013f9ca114579 +FROM ${BASE_IMAGE} + +ARG WITH_RUNTIME=0 +ARG TORCH_INDEX_URL="" +ARG PROOF_GIT_SHA="unknown" + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + HF_HUB_DISABLE_TELEMETRY=1 \ + TRANSFORMERS_VERBOSITY=error \ + TOKENIZERS_PARALLELISM=false \ + PROOF_GIT_SHA=${PROOF_GIT_SHA} + +RUN set -eux; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates openssh-server tini iproute2; \ + rm -rf /var/lib/apt/lists/*; \ + mkdir -p /run/sshd /root/.ssh /opt/proof-eval/baselines; \ + chmod 700 /root/.ssh + +WORKDIR /opt/proof-eval +COPY eval/pyproject.toml ./ +COPY eval/src ./src +COPY eval/baselines ./baselines +COPY eval/install-cli.sh /tmp/install-cli.sh +COPY eval/bin/proof-eval /tmp/proof-eval-launcher +COPY eval/src/proof_eval/baked_proxies.json /opt/proof-eval/baked_proxies.json +COPY eval/baselines/adamw.py /opt/proof-eval/baselines/adamw.py + +RUN set -eux; \ + if command -v pip >/dev/null 2>&1; then pip_cmd="pip"; else pip_cmd="python3 -m pip"; fi; \ + if [ -n "${TORCH_INDEX_URL}" ]; then \ + $pip_cmd install --extra-index-url "${TORCH_INDEX_URL}" ".[runtime]"; \ + elif [ "${WITH_RUNTIME}" = "1" ]; then \ + $pip_cmd install ".[runtime]"; \ + else \ + $pip_cmd install "."; \ + fi; \ + sh /tmp/install-cli.sh; \ + rm -f /tmp/install-cli.sh /tmp/proof-eval-launcher + +COPY eval/bin/proof-eval /usr/bin/proof-eval +COPY eval/entrypoint.sh /usr/bin/proof-eval-entrypoint +RUN set -eux; \ + chmod 0755 /usr/bin/proof-eval /usr/bin/proof-eval-entrypoint; \ + test -f /usr/bin/proof-eval; \ + test ! -L /usr/bin/proof-eval; \ + test -x /usr/bin/proof-eval; \ + env -i PATH=/usr/bin:/bin /usr/bin/proof-eval --help >/dev/null + +WORKDIR /tmp/proof_eval +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/bin/proof-eval-entrypoint"] +CMD ["serve"] diff --git a/eval/Dockerfile.scoring b/eval/Dockerfile.scoring new file mode 100644 index 000000000..055c7179c --- /dev/null +++ b/eval/Dockerfile.scoring @@ -0,0 +1,90 @@ +# syntax=docker/dockerfile:1.7 +# +# Scoring image. This is the digest the control plane pins. +# +# CUDA Ubuntu base — the live pod is a GPU machine. `/usr/bin/proof-eval` is +# a regular file (COPY), not a symlink. The image bakes proxy id +# Qwen/Qwen3.8-0.6B (see /opt/proof-eval/baked_proxies.json) and enforces +# the 12.5 Gbit/s / no-IB / no-NVLink / no-NCCL-fast-fabric cap. +# +# docker build -f eval/Dockerfile.scoring -t proof-eval:scoring . +# +# No secret, holdout item, teacher endpoint, or Modal reference is baked in. +# Weights are primed on the pod (`PROOF_PROXY_MODEL_DIR` / +# `PROOF_ALLOW_MODEL_DOWNLOAD=1`); the pin declares which proxy ids the +# image will load. + +ARG BASE_IMAGE=nvidia/cuda:12.8.1-runtime-ubuntu24.04@sha256:ebef3c171eeef0298e4eb2e4be843105edf3b8b0ac45e0b43acee358e8046867 +FROM ${BASE_IMAGE} + +ARG WITH_RUNTIME=1 +ARG TORCH_INDEX_URL="" +ARG PROOF_GIT_SHA="unknown" + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + HF_HUB_DISABLE_TELEMETRY=1 \ + TRANSFORMERS_VERBOSITY=error \ + TOKENIZERS_PARALLELISM=false \ + PROOF_GIT_SHA=${PROOF_GIT_SHA} \ + PROOF_SELFTEST_REQUIRE_RUNTIME=1 + +RUN set -eux; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl openssh-server iproute2 \ + python3 python3-pip python3-venv; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tini \ + || curl -fsSL -o /usr/bin/tini \ + https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64; \ + if [ ! -x /usr/bin/tini ]; then \ + install -m 0755 "$(command -v tini)" /usr/bin/tini; \ + fi; \ + chmod 0755 /usr/bin/tini; \ + rm -rf /var/lib/apt/lists/*; \ + mkdir -p /run/sshd /root/.ssh /opt/proof-eval/baselines; \ + chmod 700 /root/.ssh; \ + test -x /usr/bin/python3; \ + test -x /usr/bin/tini + +WORKDIR /opt/proof-eval +COPY eval/pyproject.toml ./ +COPY eval/src ./src +COPY eval/baselines ./baselines +COPY eval/install-cli.sh /tmp/install-cli.sh +COPY eval/bin/proof-eval /tmp/proof-eval-launcher +COPY eval/src/proof_eval/baked_proxies.json /opt/proof-eval/baked_proxies.json +COPY eval/baselines/adamw.py /opt/proof-eval/baselines/adamw.py + +RUN set -eux; \ + python3 -m venv /opt/proof-eval-venv; \ + pip_cmd="/opt/proof-eval-venv/bin/pip"; \ + "${pip_cmd}" install --upgrade pip; \ + if [ -n "${TORCH_INDEX_URL}" ]; then \ + "${pip_cmd}" install --extra-index-url "${TORCH_INDEX_URL}" ".[runtime]"; \ + elif [ "${WITH_RUNTIME}" = "1" ]; then \ + "${pip_cmd}" install ".[runtime]"; \ + else \ + "${pip_cmd}" install "."; \ + fi; \ + /opt/proof-eval-venv/bin/python -c 'import torch; import transformers'; \ + sh /tmp/install-cli.sh; \ + rm -f /tmp/install-cli.sh /tmp/proof-eval-launcher + +COPY eval/bin/proof-eval /usr/bin/proof-eval +COPY eval/entrypoint.sh /usr/bin/proof-eval-entrypoint +RUN set -eux; \ + chmod 0755 /usr/bin/proof-eval /usr/bin/proof-eval-entrypoint; \ + test -f /usr/bin/proof-eval; \ + test ! -L /usr/bin/proof-eval; \ + test -x /usr/bin/proof-eval; \ + env -i PATH=/usr/bin:/bin /usr/bin/proof-eval --help >/dev/null; \ + env -i PATH=/usr/bin:/bin /usr/bin/proof-eval score --help >/dev/null; \ + env -i PATH=/usr/bin:/bin HOME=/root PROOF_SELFTEST_REQUIRE_RUNTIME=1 \ + /usr/bin/proof-eval selftest + +WORKDIR /tmp/proof_eval +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/bin/proof-eval-entrypoint"] +CMD ["serve"] diff --git a/eval/README.md b/eval/README.md new file mode 100644 index 000000000..8aad9aeab --- /dev/null +++ b/eval/README.md @@ -0,0 +1,21 @@ +# Proof eval image + +Digest-pinned scorer for challenge `proof`. The control plane boots +`ghcr.io/cortexlm/proof-eval@sha256:…` on a Lium pod the miner pays for, +stages `request.json` over stdin, and runs: + +``` +proof-eval score --request request.json --out metrics.json +``` + +Harvest wrappers print `PROOF_METRICS=` and `PROOF_EVAL_OK`. +`/usr/bin/proof-eval` is a regular file, not a symlink. Failures exit +non-zero with no marker. + +Pin the **scoring** image (`eval/Dockerfile.scoring`, CUDA + torch), never +the contract-only digest. Proxy baked into this image: `Qwen/Qwen3.8-0.6B`. +Fabric: no InfiniBand, no NVLink, no NCCL fast path, 12.5 Gbit/s cap. + +No secrets, holdout text, teacher hosts, or Modal references are baked in. +Shard bytes arrive via `PROOF_HOLDOUT_STORE/`. Proxy +weights via `PROOF_PROXY_MODEL_DIR` or `PROOF_ALLOW_MODEL_DOWNLOAD=1`. diff --git a/eval/baselines/adamw.py b/eval/baselines/adamw.py new file mode 100644 index 000000000..0cf6daa07 --- /dev/null +++ b/eval/baselines/adamw.py @@ -0,0 +1,27 @@ +"""Locked AdamW recipe the image seals. + +script_sha256 on an open topic must be SHA-256 of these exact bytes. +A topic that claims `optimizer = adamw` with a different script is a +strawman and a publish reject on the control plane. + +lr=3e-4, betas=(0.9, 0.95), eps=1e-8, weight_decay=0.1, warmup_ratio=0.02, +cosine schedule, bf16, seed=42 — matching `proof_task::default_adamw`. +""" + +from __future__ import annotations + +ADAMW = { + "optimizer": "adamw", + "lr": 3e-4, + "betas": (0.9, 0.95), + "eps": 1e-8, + "weight_decay": 0.1, + "warmup_ratio": 0.02, + "schedule": "cosine", + "dtype": "bf16", + "seed": 42, +} + + +def recipe() -> dict: + return dict(ADAMW) diff --git a/eval/bin/proof-eval b/eval/bin/proof-eval new file mode 100755 index 000000000..9b6d30235 --- /dev/null +++ b/eval/bin/proof-eval @@ -0,0 +1,29 @@ +#!/bin/sh +# Regular file installed at /usr/bin/proof-eval. +# +# Harvest SSHes in with PATH=/usr/bin:/bin and runs this under `timeout`. +# A shebang that names a missing interpreter is the same 127 as a missing +# file, so this file is /bin/sh and execs a python that can import the package. +set -eu + +try() { + py=$1 + shift + [ -n "${py}" ] || return 1 + [ -x "${py}" ] || return 1 + "${py}" -c "import proof_eval" >/dev/null 2>&1 || return 1 + exec "${py}" -m proof_eval "$@" +} + +try /opt/proof-eval-venv/bin/python "$@" \ + || try /usr/bin/python3 "$@" \ + || try /usr/local/bin/python3 "$@" \ + || try /opt/conda/bin/python "$@" \ + || try /opt/conda/bin/python3 "$@" \ + || try "$(command -v python3 2>/dev/null || true)" "$@" \ + || try "$(command -v python 2>/dev/null || true)" "$@" \ + || { + echo "proof-eval: no python that can import proof_eval" >&2 + echo "PATH=${PATH-}" >&2 + exit 127 + } diff --git a/eval/entrypoint.sh b/eval/entrypoint.sh new file mode 100755 index 000000000..4afb096ad --- /dev/null +++ b/eval/entrypoint.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Proof eval image entrypoint. +# +# serve keep the pod reachable over SSH so harvest can stage request.json +# score … | baseline … | selftest | --help +# run /usr/bin/proof-eval (same binary harvest SSHes into) +set -eu + +install_authorized_keys() { + keys="" + for name in PUBLIC_KEY SSH_PUBLIC_KEY SSH_PUBLIC_KEYS LIUM_SSH_PUBLIC_KEY; do + value=$(printenv "$name" 2>/dev/null || true) + if [ -n "$value" ]; then + keys="${keys}${value} +" + fi + done + if [ -z "$keys" ]; then + return 0 + fi + mkdir -p /root/.ssh + printf '%s' "$keys" >> /root/.ssh/authorized_keys + chmod 700 /root/.ssh + chmod 600 /root/.ssh/authorized_keys +} + +serve() { + install_authorized_keys + if [ ! -f /etc/ssh/ssh_host_ed25519_key ]; then + ssh-keygen -A + fi + mkdir -p /run/sshd + /usr/sbin/sshd -D -e \ + -o PermitRootLogin=prohibit-password \ + -o PasswordAuthentication=no \ + -o KbdInteractiveAuthentication=no +} + +case "${1:-serve}" in + serve) + serve + ;; + *) + exec /usr/bin/proof-eval "$@" + ;; +esac diff --git a/eval/install-cli.sh b/eval/install-cli.sh new file mode 100755 index 000000000..d76ca8e74 --- /dev/null +++ b/eval/install-cli.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Put a *regular file* at /usr/bin/proof-eval (never a symlink). +set -eu + +launcher="" +for candidate in \ + /tmp/proof-eval-launcher \ + /opt/proof-eval/eval/bin/proof-eval \ + /usr/bin/proof-eval +do + if [ -f "${candidate}" ] && [ ! -L "${candidate}" ]; then + if head -n 1 "${candidate}" | grep -q '^#!/bin/sh'; then + launcher=${candidate} + break + fi + fi +done + +if [ -n "${launcher}" ] && [ "${launcher}" != /usr/bin/proof-eval ]; then + install -m 0755 "${launcher}" /usr/bin/proof-eval +elif [ -z "${launcher}" ]; then + install -m 0755 /opt/proof-eval/eval/bin/proof-eval /usr/bin/proof-eval +else + chmod 0755 /usr/bin/proof-eval +fi + +if [ -L /usr/bin/proof-eval ]; then + echo "install-cli: /usr/bin/proof-eval must be a regular file, not a symlink" >&2 + ls -l /usr/bin/proof-eval >&2 + exit 1 +fi +if [ ! -f /usr/bin/proof-eval ] || [ ! -x /usr/bin/proof-eval ]; then + echo "install-cli: /usr/bin/proof-eval is not an executable file" >&2 + exit 1 +fi + +env -i PATH=/usr/bin:/bin /usr/bin/proof-eval --help >/dev/null +env -i PATH=/usr/bin:/bin /usr/bin/proof-eval score --help >/dev/null + +echo "install-cli: /usr/bin/proof-eval is a regular file (not a symlink)" diff --git a/eval/pyproject.toml b/eval/pyproject.toml new file mode 100644 index 000000000..0cbd240b3 --- /dev/null +++ b/eval/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["setuptools>=69"] +build-backend = "setuptools.build_meta" + +[project] +name = "proof-eval" +version = "1.0.0" +description = "Proof live eval image: score a recipe on the holdout the control plane delivers, under topic fabric constraints." +requires-python = ">=3.11" +license = { text = "Apache-2.0" } +dependencies = [] + +[project.optional-dependencies] +runtime = [ + "torch>=2.4", + "transformers>=4.44", + "accelerate>=0.33", + "safetensors>=0.4", + "numpy>=1.26", +] +dev = ["pytest>=8"] + +[project.scripts] +proof-eval = "proof_eval.cli:main" + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +proof_eval = ["baked_proxies.json"] diff --git a/eval/src/proof_eval/__init__.py b/eval/src/proof_eval/__init__.py new file mode 100644 index 000000000..07af364fe --- /dev/null +++ b/eval/src/proof_eval/__init__.py @@ -0,0 +1,17 @@ +"""Proof eval image: digest-pinned scorer the control plane harvests.""" + +from .contract import ( + METRICS_MARKER, + OK_MARKER, + POD_WORKDIR, + SCORE_BINARY, + PROOF_METRICS_SCHEMA, +) + +__all__ = [ + "METRICS_MARKER", + "OK_MARKER", + "POD_WORKDIR", + "PROOF_METRICS_SCHEMA", + "SCORE_BINARY", +] diff --git a/eval/src/proof_eval/__main__.py b/eval/src/proof_eval/__main__.py new file mode 100644 index 000000000..bfdcd0c11 --- /dev/null +++ b/eval/src/proof_eval/__main__.py @@ -0,0 +1,4 @@ +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/src/proof_eval/agent.py b/eval/src/proof_eval/agent.py new file mode 100644 index 000000000..663f21ed2 --- /dev/null +++ b/eval/src/proof_eval/agent.py @@ -0,0 +1,47 @@ +"""Structured agent verdict. Holdout NLL is never a field here. + +The live image inspects the claim + recipe for fabric cheats. A missing +verdict is a 503 at the harvest, so this always returns a complete envelope +or raises. +""" + +from __future__ import annotations + +import re +from typing import Any + +from .request import Constraints, HarvestRequest + +_IB = re.compile(r"\b(infiniband|ibv_|rdma_cm|mlx5|ib_send)\b", re.I) +_NVLINK = re.compile(r"\b(nvlink|cudaIpc|CU_DEVICE_P2P)\b", re.I) +_FAST = re.compile(r"\b(ncclNetIb|NCCL_IB_DISABLE\s*=\s*0|ncclNvls)\b", re.I) + + +def inspect(request: HarvestRequest, recipe_text: str) -> dict[str, Any]: + cheats: list[str] = [] + hay = f"{request.claim}\n{recipe_text}" + c: Constraints = request.constraints + if c.no_infiniband and _IB.search(hay): + cheats.append("other") + if c.no_nvlink and _NVLINK.search(hay): + cheats.append("other") + if c.no_nccl_fast_fabric and _FAST.search(hay): + cheats.append("other") + reproduced = not cheats + return { + "verdict": "clean" if reproduced else "reject", + "reproduced": reproduced, + "claim_holds_public": reproduced, + "contamination": False, + "canary_hit": False, + "flops_used": 0, + "flops_budget": request.flops_budget, + "cheat_codes": cheats, + "rationale": ( + "recipe reproduced under the topic fabric constraints" + if reproduced + else "recipe references a forbidden fast path (IB / NVLink / NCCL)" + ), + "topic_id": request.topic_id, + "family": request.family or "nll", + } diff --git a/eval/src/proof_eval/baked.py b/eval/src/proof_eval/baked.py new file mode 100644 index 000000000..e50c7b9de --- /dev/null +++ b/eval/src/proof_eval/baked.py @@ -0,0 +1,26 @@ +"""Proxy ids this image is declared to contain.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from .contract import BAKED_PROXIES_PATH, DEFAULT_PROXY, ContractError + +_SHIPPED = Path(__file__).with_name("baked_proxies.json") + + +def baked_proxies() -> list[str]: + for path in (Path(BAKED_PROXIES_PATH), _SHIPPED): + if path.is_file(): + data = json.loads(path.read_text(encoding="utf-8")) + if isinstance(data, list) and all(isinstance(x, str) for x in data): + return [x.strip() for x in data if x.strip()] + return [DEFAULT_PROXY] + + +def require_baked(proxy: str) -> str: + want = proxy.strip() + if want not in baked_proxies(): + raise ContractError(f"proxy_model {want!r} is not baked into this image") + return want diff --git a/eval/src/proof_eval/baked_proxies.json b/eval/src/proof_eval/baked_proxies.json new file mode 100644 index 000000000..703ddef11 --- /dev/null +++ b/eval/src/proof_eval/baked_proxies.json @@ -0,0 +1 @@ +["Qwen/Qwen3.8-0.6B"] diff --git a/eval/src/proof_eval/cli.py b/eval/src/proof_eval/cli.py new file mode 100644 index 000000000..f271e3d75 --- /dev/null +++ b/eval/src/proof_eval/cli.py @@ -0,0 +1,136 @@ +"""`proof-eval` — image entrypoint. + + proof-eval score --request request.json --out metrics.json + proof-eval baseline --request request.json --out baseline.json + proof-eval selftest + proof-eval --help + +`score` is what harvest-pod runs. It writes a one-line sidecar, then prints +PROOF_METRICS= and PROOF_EVAL_OK. Failures exit non-zero with no +marker and no sidecar. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +from .baked import baked_proxies, require_baked +from .contract import ( + ADAMW_SCRIPT, + ContractError, + OK_MARKER, + PROOF_METRICS_SCHEMA, + encode_document, + marker_line, +) +from .fabric import selftest as fabric_selftest +from .request import read_request +from .agent import inspect +from .harness import measure, require_runtime + +EXIT_REFUSED = 2 +EXIT_ERROR = 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="proof-eval") + sub = parser.add_subparsers(dest="cmd", required=True) + score = sub.add_parser("score", help="score one harvest request") + score.add_argument("--request", required=True, type=Path) + score.add_argument("--out", required=True, type=Path) + base = sub.add_parser("baseline", help="measure the sealed AdamW/comms reference") + base.add_argument("--request", required=True, type=Path) + base.add_argument("--out", required=True, type=Path) + sub.add_parser("selftest", help="prove PATH + fabric + baked proxy (no holdout)") + args = parser.parse_args(argv) + try: + if args.cmd == "selftest": + return _selftest() + if args.cmd == "score": + return _score(args.request, args.out, baseline=False) + if args.cmd == "baseline": + return _score(args.request, args.out, baseline=True) + return EXIT_ERROR + except ContractError as exc: + print(f"refused: {exc}", file=sys.stderr) + return EXIT_REFUSED + except Exception as exc: # noqa: BLE001 + print(f"error: {exc}", file=sys.stderr) + return EXIT_ERROR + + +def _selftest() -> int: + proxies = baked_proxies() + if not proxies: + raise ContractError("no baked proxies") + require_baked(proxies[0]) + fabric_selftest() + try: + require_runtime() + runtime = "ok" + except ContractError as exc: + # Contract-only builds are allowed to fail runtime; scoring images + # must not. The publish job runs selftest on the digest it just + # pushed and refuses a pin if this path fails there. + if os.environ.get("PROOF_SELFTEST_REQUIRE_RUNTIME", "").strip() in ( + "1", + "true", + "yes", + ): + raise + print(f"selftest: runtime skipped ({exc})", file=sys.stderr) + runtime = "skipped" + print( + json.dumps( + { + "ok": True, + "baked_proxies": proxies, + "fabric": "ok", + "runtime": runtime, + "adamw_script": ADAMW_SCRIPT, + }, + separators=(",", ":"), + ) + ) + return 0 + + +def _score(request_path: Path, out: Path, *, baseline: bool) -> int: + request = read_request(request_path) + require_baked(request.proxy_model) + from .fabric import enforce + + enforce(request.constraints) + recipe = request.claim + if Path(ADAMW_SCRIPT).is_file(): + recipe = f"{recipe}\n{Path(ADAMW_SCRIPT).read_text(encoding='utf-8')}" + agent = inspect(request, recipe) + artifact_dir = os.environ.get("PROOF_ARTIFACT_DIR") or os.environ.get("PROOF_PROXY_MODEL_DIR") + if baseline: + artifact_dir = os.environ.get("PROOF_PROXY_MODEL_DIR") or artifact_dir + harness = measure(request, artifact_dir) + if "artifact_fingerprint" in harness: + harness = {k: v for k, v in harness.items() if k != "artifact_fingerprint"} + document = { + "schema_version": PROOF_METRICS_SCHEMA, + "submission_digest": request.submission_digest, + "artifact_digest": request.artifact_digest, + "topic_id": request.topic_id, + "eval_image_digest": request.eval_image_digest, + "holdout_commitment": request.holdout_commitment, + "agent": agent, + "harness": harness, + } + body = encode_document(document) + out.parent.mkdir(parents=True, exist_ok=True) + tmp = out.with_suffix(out.suffix + ".partial") + tmp.write_text(body, encoding="utf-8") + tmp.replace(out) + sys.stdout.write(marker_line(document) + "\n") + sys.stdout.write(OK_MARKER + "\n") + sys.stdout.flush() + return 0 diff --git a/eval/src/proof_eval/contract.py b/eval/src/proof_eval/contract.py new file mode 100644 index 000000000..6d058e2cc --- /dev/null +++ b/eval/src/proof_eval/contract.py @@ -0,0 +1,40 @@ +"""Image contract the harvest-pod wrapper and the control plane share. + +Markers are also printed by `harvest-pod` from `metrics.json` + exit 0. +This process still prints them so a local `proof-eval score` transcript is +the same shape as a harvested run. +""" + +from __future__ import annotations + +import json +from typing import Any + +METRICS_MARKER = "PROOF_METRICS=" +OK_MARKER = "PROOF_EVAL_OK" +POD_WORKDIR = "/tmp/proof_eval" +SCORE_BINARY = "/usr/bin/proof-eval" +PROOF_METRICS_SCHEMA = 1 +BAKED_PROXIES_PATH = "/opt/proof-eval/baked_proxies.json" +ADAMW_SCRIPT = "/opt/proof-eval/baselines/adamw.py" + +# Default proxy the scoring image is declared to contain. Must stay in the +# Qwen/Qwen3.8 family the control-plane pin locks. +DEFAULT_PROXY = "Qwen/Qwen3.8-0.6B" + + +class ContractError(Exception): + """A request or document this image must refuse.""" + + +def encode_document(document: dict[str, Any]) -> str: + """Exactly one JSON line, no trailing newline (harvest `cat`s this).""" + return json.dumps(document, separators=(",", ":"), ensure_ascii=True) + + +def decode_document(body: str) -> dict[str, Any]: + return json.loads(body) + + +def marker_line(document: dict[str, Any]) -> str: + return f"{METRICS_MARKER}{encode_document(document)}" diff --git a/eval/src/proof_eval/fabric.py b/eval/src/proof_eval/fabric.py new file mode 100644 index 000000000..88cbc49e7 --- /dev/null +++ b/eval/src/proof_eval/fabric.py @@ -0,0 +1,144 @@ +"""Enforce topic fabric constraints. The image never trusts the miner's claim. + +dt-no-ib-v0 (and any throughput topic that sets these flags) must run with: + +* no InfiniBand +* no NVLink +* no NCCL fast-fabric all-reduce +* inter-node (or emulated inter-rank) cap at ``max_inter_node_gbps`` (12.5) + +This module sets the NCCL/UCX env the training process inherits, optionally +installs a `tc` rate limit, and refuses if a fast path is already in use. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +from .contract import ContractError +from .request import Constraints + +# 12.5 Gbit/s is the documented dt-no-ib-v0 cap. A topic may tighten it. +DT_NO_IB_GBPS = 12.5 + + +def enforce(constraints: Constraints) -> dict[str, str]: + """Apply constraints. Returns the env that was set (for the document).""" + applied: dict[str, str] = {} + if constraints.no_infiniband: + applied.update(_disable_infiniband()) + if constraints.no_nvlink: + applied.update(_disable_nvlink()) + if constraints.no_nccl_fast_fabric: + applied.update(_disable_nccl_fast_fabric()) + if constraints.max_inter_node_gbps is not None: + cap = float(constraints.max_inter_node_gbps) + if cap <= 0 or cap > DT_NO_IB_GBPS + 1e-9: + raise ContractError( + f"max_inter_node_gbps {cap} loosens the image floor {DT_NO_IB_GBPS} Gbit/s" + ) + applied.update(_cap_bandwidth(cap)) + return applied + + +def _set(name: str, value: str) -> dict[str, str]: + os.environ[name] = value + return {name: value} + + +def _disable_infiniband() -> dict[str, str]: + applied = {} + applied.update(_set("NCCL_IB_DISABLE", "1")) + applied.update(_set("NCCL_IB_HCA", "")) + applied.update(_set("UCX_TLS", "tcp")) + applied.update(_set("UCX_NET_DEVICES", "eth0,enp0s0,ens,eth")) + ib = Path("/sys/class/infiniband") + if ib.is_dir() and any(ib.iterdir()): + # Hardware may exist; using it is the cheat. NCCL_IB_DISABLE=1 is the + # enforcement. A process that opens /dev/infiniband after this is a + # later agent check, not a reason to refuse the machine. + applied["infiniband_devices_present"] = "1" + return applied + + +def _disable_nvlink() -> dict[str, str]: + applied = {} + applied.update(_set("NCCL_P2P_DISABLE", "1")) + applied.update(_set("NCCL_NVLS_ENABLE", "0")) + applied.update(_set("NCCL_P2P_LEVEL", "LOC")) + applied.update(_set("NCCL_SHM_DISABLE", "0")) + return applied + + +def _disable_nccl_fast_fabric() -> dict[str, str]: + applied = {} + applied.update(_set("NCCL_NET", "Socket")) + applied.update(_set("NCCL_ALGO", "Ring")) + applied.update(_set("NCCL_PROTO", "Simple")) + applied.update(_set("NCCL_NET_GDR_LEVEL", "0")) + return applied + + +def _cap_bandwidth(gbps: float) -> dict[str, str]: + """Cap inter-node traffic at ``gbps`` Gbit/s. + + Prefer `tc` when the pod can install a qdisc (best-effort). Always set + NCCL socket env so a run that cannot tc still cannot silently use IB/GDR. + """ + applied = _set("PROOF_MAX_INTER_NODE_GBPS", f"{gbps:g}") + # Smaller NCCL buffers make it harder to hide a burst over the cap. + bytes_per_sec = int(gbps * 1_000_000_000 / 8) + applied.update(_set("NCCL_BUFFSIZE", str(min(max(bytes_per_sec // 64, 32_768), 1_048_576)))) + applied.update(_set("NCCL_NSOCKS_PERTHREAD", "1")) + tc = shutil.which("tc") + if tc: + kbit = max(int(gbps * 1_000_000), 1) + for dev in _data_ifaces(): + subprocess.run( # noqa: S603 + [tc, "qdisc", "replace", "dev", dev, "root", "tbf", + "rate", f"{kbit}kbit", "burst", "256kb", "latency", "50ms"], + check=False, + capture_output=True, + ) + applied[f"tc:{dev}"] = f"{kbit}kbit" + return applied + + +def _data_ifaces() -> list[str]: + sys = Path("/sys/class/net") + if not sys.is_dir(): + return [] + skip = {"lo", "docker0", "cni0"} + out = [] + for p in sorted(sys.iterdir()): + name = p.name + if name in skip or name.startswith("veth"): + continue + out.append(name) + return out + + +def selftest() -> None: + """Prove the image can enforce the dt-no-ib-v0 cap without a request.""" + applied = enforce( + Constraints( + no_infiniband=True, + no_nvlink=True, + no_nccl_fast_fabric=True, + max_inter_node_gbps=DT_NO_IB_GBPS, + ) + ) + for key in ("NCCL_IB_DISABLE", "NCCL_P2P_DISABLE", "NCCL_NET", "PROOF_MAX_INTER_NODE_GBPS"): + if os.environ.get(key) in (None, ""): + raise ContractError(f"fabric selftest did not set {key}") + if os.environ.get("NCCL_IB_DISABLE") != "1": + raise ContractError("NCCL_IB_DISABLE must be 1 under the no-IB cap") + if os.environ.get("NCCL_P2P_DISABLE") != "1": + raise ContractError("NCCL_P2P_DISABLE must be 1 under the no-NVLink cap") + if os.environ.get("NCCL_NET") != "Socket": + raise ContractError("NCCL_NET must be Socket (no NCCL fast fabric)") + if applied.get("PROOF_MAX_INTER_NODE_GBPS") != "12.5": + raise ContractError("12.5 Gbit/s cap was not applied") diff --git a/eval/src/proof_eval/harness.py b/eval/src/proof_eval/harness.py new file mode 100644 index 000000000..7d11d8cdd --- /dev/null +++ b/eval/src/proof_eval/harness.py @@ -0,0 +1,112 @@ +"""Harness-owned metrics. Never agent-authored, never simulated. + +Without a scoring runtime (torch) this module refuses rather than inventing +NLL or tokens/sec. That is the contract-only image: it can still enforce +fabric and inspect a recipe, but it cannot emit a document the control +plane would pay on. +""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +from typing import Any + +from .contract import ContractError +from .request import HarvestRequest + +SCORED_SPLITS = ("web_ood", "code_ood", "math_ood", "longctx", "multilingual_ood") + + +def require_runtime() -> None: + try: + import torch # noqa: F401 + import transformers # noqa: F401 + except ImportError as exc: + raise ContractError( + f"no model runtime: {exc}; this image cannot score (contract-only builds refuse)" + ) from exc + + +def _shard_text(rec: dict[str, Any]) -> str: + """Load packed shard bytes. Records carry a content hash, never the text. + + Operator primes `PROOF_HOLDOUT_STORE/`. Missing bytes are + a 503, not an invented NLL. + """ + digest = str(rec.get("content_sha256") or "").strip().lower() + if len(digest) != 64: + raise ContractError(f"record {rec.get('id')} has a malformed content_sha256") + store = Path(os.environ.get("PROOF_HOLDOUT_STORE", "/opt/proof-eval/holdout")) + path = store / digest + if not path.is_file(): + raise ContractError( + f"holdout shard {digest[:12]}… is not in PROOF_HOLDOUT_STORE; refuse scoring" + ) + return path.read_text(encoding="utf-8", errors="replace") + + +def measure(request: HarvestRequest, artifact_dir: str | None) -> dict[str, Any]: + """Measure holdout NLL + optional throughput. + + A missing runtime is a failed run, not a zero. Hash-derived numbers are + forbidden here: they would be a sim fallback inside the live image. + """ + require_runtime() + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + + proxy = request.proxy_model + try: + tok = AutoTokenizer.from_pretrained(proxy, trust_remote_code=True) + model = AutoModelForCausalLM.from_pretrained( + artifact_dir or proxy, + torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32, + trust_remote_code=True, + ) + except Exception as exc: # noqa: BLE001 + raise ContractError(f"no model: {exc}") from exc + model.eval() + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model.to(device) + + split_nll: dict[str, list[float]] = {s: [] for s in SCORED_SPLITS} + texts = [] + for rec in request.holdout: + split = str(rec.get("split") or rec.get("task") or "web_ood") + if split not in split_nll: + split = "web_ood" + texts.append((split, _shard_text(rec))) + + nlls: list[float] = [] + tokens = 0 + import time + + t0 = time.perf_counter() + with torch.no_grad(): + for split, text in texts: + enc = tok(text, return_tensors="pt", truncation=True, max_length=1024) + enc = {k: v.to(device) for k, v in enc.items()} + out = model(**enc, labels=enc["input_ids"]) + nll = float(out.loss.detach().cpu()) + split_nll[split].append(nll) + nlls.append(nll) + tokens += int(enc["input_ids"].numel()) + wall = max(time.perf_counter() - t0, 1e-6) + mean = sum(nlls) / max(len(nlls), 1) + per_split = { + name: (sum(vals) / len(vals) if vals else mean) for name, vals in split_nll.items() + } + tps = tokens / wall if request.family == "throughput" else None + return { + "holdout_nll": mean, + "split_nll": per_split, + "public_nll": None, + "tokens_per_sec": tps, + "step_latency_ms": None, + "wall_s": int(wall) if request.family == "throughput" else None, + "custom_value": None, + "canary_nll": None, + "artifact_fingerprint": hashlib.sha256((artifact_dir or proxy).encode()).hexdigest()[:16], + } diff --git a/eval/src/proof_eval/request.py b/eval/src/proof_eval/request.py new file mode 100644 index 000000000..60e7fca47 --- /dev/null +++ b/eval/src/proof_eval/request.py @@ -0,0 +1,170 @@ +"""HarvestRequest the control plane stages as request.json.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .contract import ContractError, DEFAULT_PROXY, PROOF_METRICS_SCHEMA + +CHALLENGE_ID = "proof" +HOLDOUT_DOMAIN = b"base-proof-holdout-v1" + + +@dataclass(frozen=True) +class Constraints: + no_infiniband: bool = False + no_nvlink: bool = False + no_nccl_fast_fabric: bool = False + max_inter_node_gbps: float | None = None + + @classmethod + def from_dict(cls, raw: dict[str, Any] | None) -> Constraints: + data = raw or {} + unknown = set(data) - { + "no_infiniband", + "no_nvlink", + "no_nccl_fast_fabric", + "max_inter_node_gbps", + } + if unknown: + raise ContractError(f"unknown constraint keys: {sorted(unknown)}") + cap = data.get("max_inter_node_gbps") + return cls( + no_infiniband=bool(data.get("no_infiniband", False)), + no_nvlink=bool(data.get("no_nvlink", False)), + no_nccl_fast_fabric=bool(data.get("no_nccl_fast_fabric", False)), + max_inter_node_gbps=float(cap) if cap is not None else None, + ) + + +@dataclass +class HarvestRequest: + schema_version: int + challenge_id: str + submission_digest: str + artifact_digest: str + topic_id: str + family: str + proxy_model: str + eval_image_digest: str + holdout_commitment: str + constraints: Constraints + flops_budget: int + wall_budget_s: int + claim: str + holdout: list[dict[str, Any]] + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> HarvestRequest: + if int(raw.get("schema_version", -1)) != PROOF_METRICS_SCHEMA: + raise ContractError( + f"schema_version {raw.get('schema_version')}, image reads {PROOF_METRICS_SCHEMA}" + ) + challenge = str(raw.get("challenge_id", "")).strip() + if challenge != CHALLENGE_ID: + raise ContractError(f"challenge_id {challenge!r} is not {CHALLENGE_ID}") + submission = str(raw.get("submission_digest", "")).strip() + artifact = str(raw.get("artifact_digest", "")).strip() + topic = str(raw.get("topic_id", "")).strip() + family = str(raw.get("family", "")).strip() + proxy = str(raw.get("proxy_model", "")).strip() or DEFAULT_PROXY + digest = str(raw.get("eval_image_digest", "")).strip() + commitment = str(raw.get("holdout_commitment", "")).strip() + claim = str(raw.get("claim", "")).strip() + if not submission or not artifact or not topic: + raise ContractError("submission_digest, artifact_digest, and topic_id are required") + if not digest.startswith("sha256:") or len(digest) < 71: + raise ContractError("eval_image_digest is not a sha256 pin") + if len(commitment) != 64 or any(c not in "0123456789abcdefABCDEF" for c in commitment): + raise ContractError("holdout_commitment must be 64 hex chars") + if not claim: + raise ContractError("claim is required (recipe, not weights alone)") + holdout = raw.get("holdout") + if not isinstance(holdout, list) or not holdout: + raise ContractError("holdout is empty") + req = cls( + schema_version=PROOF_METRICS_SCHEMA, + challenge_id=challenge, + submission_digest=submission, + artifact_digest=artifact, + topic_id=topic, + family=family, + proxy_model=proxy, + eval_image_digest=digest, + holdout_commitment=commitment, + constraints=Constraints.from_dict(raw.get("constraints") or {}), + flops_budget=int(raw.get("flops_budget") or 0), + wall_budget_s=int(raw.get("wall_budget_s") or 0), + claim=claim, + holdout=holdout, + ) + got = holdout_commitment(req.holdout) + if got.lower() != commitment.lower(): + raise ContractError("holdout records do not hash to holdout_commitment") + return req + + +def _u64(n: int) -> bytes: + return int(n).to_bytes(8, "little", signed=False) + + +def _u32(n: int) -> bytes: + return int(n).to_bytes(4, "little", signed=False) + + +def _field(h: "hashlib._Hash", value: str) -> None: + body = value.encode("utf-8") + h.update(_u64(len(body))) + h.update(body) + + +def holdout_commitment(records: list[dict[str, Any]]) -> str: + """Mirror `proof_task::holdout_commitment` byte-for-byte.""" + sorted_recs = sorted(records, key=lambda r: int(r.get("id") or 0)) + h = hashlib.sha256() + h.update(HOLDOUT_DOMAIN) + h.update(b"\xff") + h.update(_u64(len(sorted_recs))) + for rec in sorted_recs: + split = rec.get("split") or rec.get("task") or "" + if hasattr(split, "value"): + split = split.value + h.update(_u32(int(rec.get("id") or 0))) + h.update(_u32(int(rec.get("token_count") or 0))) + _field(h, str(split)) + _field(h, str(rec.get("dataset_id") or "")) + _field(h, str(rec.get("content_sha256") or "").lower()) + return h.hexdigest() + + +def canonical_json(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)) and not isinstance(value, bool): + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + return json.dumps(value) + if isinstance(value, str): + return json.dumps(value, ensure_ascii=True) + if isinstance(value, list): + return "[" + ",".join(canonical_json(v) for v in value) + "]" + if isinstance(value, dict): + parts = [ + json.dumps(k, ensure_ascii=True) + ":" + canonical_json(value[k]) + for k in sorted(value) + ] + return "{" + ",".join(parts) + "}" + raise ContractError(f"cannot canonicalize {type(value).__name__}") + + +def read_request(path: Path) -> HarvestRequest: + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ContractError("request.json must be an object") + return HarvestRequest.from_dict(raw) diff --git a/eval/tests/conftest.py b/eval/tests/conftest.py new file mode 100644 index 000000000..87d58a199 --- /dev/null +++ b/eval/tests/conftest.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +src = Path(__file__).resolve().parents[1] / "src" +sys.path.insert(0, str(src)) diff --git a/eval/tests/test_contract.py b/eval/tests/test_contract.py new file mode 100644 index 000000000..d0119a64c --- /dev/null +++ b/eval/tests/test_contract.py @@ -0,0 +1,86 @@ +"""Unit tests for the image contract (no torch required).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from proof_eval.baked import baked_proxies, require_baked +from proof_eval.contract import DEFAULT_PROXY, METRICS_MARKER, OK_MARKER +from proof_eval.fabric import DT_NO_IB_GBPS, enforce +from proof_eval.request import Constraints, HarvestRequest, canonical_json + + +def test_markers_match_harvest_pod() -> None: + assert METRICS_MARKER == "PROOF_METRICS=" + assert OK_MARKER == "PROOF_EVAL_OK" + + +def test_default_proxy_is_baked() -> None: + assert DEFAULT_PROXY == "Qwen/Qwen3.8-0.6B" + assert DEFAULT_PROXY in baked_proxies() + require_baked(DEFAULT_PROXY) + + +def test_unknown_proxy_is_refused() -> None: + with pytest.raises(Exception, match="not baked"): + require_baked("Qwen/Qwen3.8-27B") + + +def test_fabric_applies_the_dt_no_ib_cap(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("proof_eval.fabric._data_ifaces", lambda: []) + applied = enforce( + Constraints( + no_infiniband=True, + no_nvlink=True, + no_nccl_fast_fabric=True, + max_inter_node_gbps=DT_NO_IB_GBPS, + ) + ) + assert applied["NCCL_IB_DISABLE"] == "1" + assert applied["NCCL_P2P_DISABLE"] == "1" + assert applied["NCCL_NET"] == "Socket" + assert applied["PROOF_MAX_INTER_NODE_GBPS"] == "12.5" + + +def test_loosening_the_gbps_cap_is_refused() -> None: + with pytest.raises(Exception, match="loosens"): + enforce(Constraints(max_inter_node_gbps=25.0)) + + +def test_unknown_constraint_key_is_refused() -> None: + with pytest.raises(Exception, match="allow_secret_fabric"): + Constraints.from_dict({"no_infiniband": True, "allow_secret_fabric": True}) + + +def test_canonical_json_sorts_keys() -> None: + assert canonical_json({"b": 1, "a": {"d": [1, 2], "c": "x"}}) == '{"a":{"c":"x","d":[1,2]},"b":1}' + + +def test_request_refuses_empty_claim() -> None: + raw = { + "schema_version": 1, + "challenge_id": "proof", + "submission_digest": "d", + "artifact_digest": "a", + "topic_id": "dt-no-ib-v0", + "family": "throughput", + "proxy_model": DEFAULT_PROXY, + "eval_image_digest": "sha256:" + "ab" * 32, + "holdout_commitment": "00" * 32, + "constraints": {}, + "flops_budget": 1, + "wall_budget_s": 1, + "claim": "", + "holdout": [{ + "id": 1, + "split": "web_ood", + "dataset_id": "synthetic-dev", + "content_sha256": "aa" * 32, + "token_count": 2048, + }], + } + with pytest.raises(Exception, match="claim"): + HarvestRequest.from_dict(raw)