From b28c0f87a1448c289eca322b975797faebde3069 Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 18 Sep 2026 09:55:17 +1000 Subject: [PATCH 1/3] fix(basert): accept the sweep's largest capacity as params.ctx BaseRT 0.2.5 runs each prefill at its own capacity, so its harness publishes the largest one the sweep used as the flat params.ctx (baseRT #525, the last commit before the tag). The client still expected the headline's capacity, 4096 for the defaults, so every default run on the released pair ended with "BaseRT returned incompatible headline measurement metadata" after the benchmark had already been spent: the default sweep reaches PP16384. Expect the sweep maximum instead. Per-workload capacities are checked as before, so pre-release harnesses that published 4096 for the full sweep are still refused. Co-Authored-By: Claude Fable 5.1 --- .../computearena-cli/src/adapters/basert.rs | 72 ++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/crates/computearena-cli/src/adapters/basert.rs b/crates/computearena-cli/src/adapters/basert.rs index ecabf77..cb60ee0 100644 --- a/crates/computearena-cli/src/adapters/basert.rs +++ b/crates/computearena-cli/src/adapters/basert.rs @@ -288,6 +288,10 @@ fn normalize_headline_protocol( { bail!("BaseRT returned incompatible headline capacity/order metadata"); } + // Each prefill runs at its own capacity, so the flat `params.ctx` names + // the largest one the sweep used (BaseRT 0.2.5): 16384 for the default + // sweep, not the headline's 4096. + let mut sweep_capacity = capacity; for prompt in request.pp.split(',') { let tokens = prompt.parse::()?; let expected = if tokens == headline { @@ -295,6 +299,7 @@ fn normalize_headline_protocol( } else { tokens.max(HEADLINE_CONTEXT_CAPACITY) }; + sweep_capacity = sweep_capacity.max(expected); if runtime["prefill"][prompt]["initial_context_tokens"] != 0 || runtime["prefill"][prompt]["context_capacity_tokens"].as_u64() != Some(expected) { @@ -306,7 +311,7 @@ fn normalize_headline_protocol( || runtime["decode"]["seed_prefill_in_timing"] != false || runtime["measurement"]["timed_repetitions"] != request.reps || runtime["measurement"]["requested_warmup_repetitions"] != request.warmup - || benchmark["params"]["ctx"] != capacity + || benchmark["params"]["ctx"] != sweep_capacity { bail!("BaseRT returned incompatible headline measurement metadata"); } @@ -445,6 +450,71 @@ mod tests { } } + /// What the BaseRT 0.2.5 harness returns for the default sweep: every + /// prefill at its own capacity, and `params.ctx` naming the largest. + fn default_sweep_report(ctx: u64) -> Value { + let pp = crate::config::DEFAULT_PREFILL_TOKENS; + let prefill: serde_json::Map = pp + .split(',') + .map(|size| { + let tokens: u64 = size.parse().unwrap(); + let capacity = if size == "512" { + 4096 + } else { + tokens.max(4096) + }; + ( + size.to_string(), + json!({"initial_context_tokens":0,"context_capacity_tokens":capacity}), + ) + }) + .collect(); + json!({"params":{"ctx":ctx},"protocol":{ + "schema":"basert-throughput-protocol/2","profile":"basert-bench-capacity/1", + "context_isolation":"headline_then_per_prefill","context_capacity_policy":"basert_bench_default", + "model_load_in_timing":false,"execution_layout":"headline_then_prefill_processes", + "execution_order":crate::protocol::headline_order(pp, 128), + "prefill":prefill, + "decode":{"initial_context_tokens":1,"context_capacity_tokens":4096,"seed_prefill_in_timing":false}, + "measurement":{"timed_repetitions":3,"requested_warmup_repetitions":3} + }}) + } + + #[test] + fn the_default_sweep_reports_its_largest_capacity_as_ctx() { + let request = BenchmarkRequest { + model: Path::new("test.base"), + pp: crate::config::DEFAULT_PREFILL_TOKENS, + tg: 128, + reps: 3, + warmup: 3, + cooldown: false, + }; + // BaseRT 0.2.5 publishes the 16384 the PP16384 workload ran at. + let mut report = default_sweep_report(16384); + normalize_protocol(&mut report, &request, true, true).unwrap(); + assert_eq!( + report["protocol"]["id"], + crate::protocol::HEADLINE_PROTOCOL_ID + ); + assert_eq!(report["params"]["ctx"], 16384); + // The decode capacity is still the headline's. + assert_eq!( + report["protocol"]["decode"]["context_capacity_tokens"], + 4096 + ); + + // Pre-release harnesses that published the headline's capacity + // instead misdescribe the sweep and are not signed. + let error = normalize_protocol(&mut default_sweep_report(4096), &request, true, true) + .unwrap_err() + .to_string(); + assert!( + error.contains("incompatible headline measurement metadata"), + "{error}" + ); + } + #[test] fn replay_based_harnesses_use_external_observation() { let descriptor = json!({"telemetry_schema":"basert-telemetry/3", From bfaa53efbf789401f85b009bb3016c000b56854e Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 18 Sep 2026 10:35:45 +1000 Subject: [PATCH 2/3] chore: start 0.1.3 release candidate Co-Authored-By: Claude Fable 5.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e70214f..ae6cb40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -329,7 +329,7 @@ dependencies = [ [[package]] name = "computearena-cli" -version = "0.1.2" +version = "0.1.3" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index b3b8621..86a5a01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/computearena-cli"] [workspace.package] -version = "0.1.2" +version = "0.1.3" edition = "2021" license = "Apache-2.0" rust-version = "1.85" From a62201e057e1afaefec06cda5c363a2cf9eaf58a Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Fri, 18 Sep 2026 10:38:42 +1000 Subject: [PATCH 3/3] docs: release notes for 0.1.3 Co-Authored-By: Claude Fable 5.1 --- docs/release-notes/0.1.3.md | 72 +++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 docs/release-notes/0.1.3.md diff --git a/docs/release-notes/0.1.3.md b/docs/release-notes/0.1.3.md new file mode 100644 index 0000000..4dd40c2 --- /dev/null +++ b/docs/release-notes/0.1.3.md @@ -0,0 +1,72 @@ +# ComputeArena CLI 0.1.3 + +This release fixes benchmarking with BaseRT 0.2.5. On 0.1.2, every default +run with the released BaseRT 0.2.5 ended, after the benchmark had completed, +with `BaseRT returned incompatible headline measurement metadata`, and no +report was saved. Nothing else changes. + +## Install or upgrade + +On macOS or Linux: + +```sh +curl -LsSf https://computearena.ai/install.sh | sh +``` + +The same command upgrades an existing installation. Saved reports, the +installation signing key, login sessions, model receipts, and installed +runtimes are kept. Release archives and checksums are also available below for +manual installation. + +BaseRT 0.2.5 itself needs no change. If BaseRT is not installed yet, or is +older than 0.2.5, `computearena basert install` fetches the current release. + +## What changed + +- **Runs with BaseRT 0.2.5 complete and submit.** BaseRT 0.2.5 runs each + prefill size at its own context capacity and records the largest one the + sweep used as the run's context size: 16384 for the default sweep, which + reaches PP16384. ComputeArena 0.1.2 expected the headline's 4096 there and + refused the run once it had finished. The client now expects the value + BaseRT 0.2.5 records. The measurement is unchanged: PP512 and TG128 still + run first in a freshly loaded model with a 4K reservation, every workload's + capacity, order, and repetitions are still checked before signing, and the + reports still normalize to `computearena-throughput/3`. computearena.ai + applies the same rule. + +## Reports and compatibility + +0.1.2 could not save a BaseRT 0.2.5 report, so there is nothing to resubmit +from it: run the benchmark again with 0.1.3. Reports saved by 0.1.0 to 0.1.2 +with BaseRT 0.2.4 or llama.cpp are unaffected and remain submittable when +they contain the full default sweep. Benchmarks already published on +computearena.ai are not changed. + +A BaseRT harness built from the development branch before the 0.2.5 release +reports version 0.2.5 but records the headline's capacity instead. 0.1.3 does +not sign its runs, and computearena.ai does not accept reports from it; +replace it with `computearena basert install`. BaseRT 0.2.4 keeps working as +before: its runs use the older protocol, the report records that, and the +client says so before the run. + +## Supported systems + +Prebuilt ComputeArena binaries are available for: + +- macOS on Apple Silicon; +- Linux x86_64 with glibc 2.31 or newer; and +- Linux arm64 with glibc 2.31 or newer. + +Windows and Intel Macs are not supported in this release. ComputeArena does not +bundle an inference runtime or model; install BaseRT or llama.cpp and obtain a +compatible model before benchmarking. Prebuilt BaseRT runtimes exist for macOS +on Apple Silicon and for Linux arm64 with CUDA; on Linux x86-64, use llama.cpp +or a BaseRT benchmark harness you built yourself. + +The macOS binary is not signed with an Apple Developer ID. The release archive +is checksum-verified and carries a Sigstore keyless signature from the release +workflow. Browser-downloaded copies may need their quarantine attribute removed +as described in the README. + +Accuracy benchmarks, MLX, vLLM, Windows, and Intel macOS support are not part +of v0.1.3.