RC 0.1.3

Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/computearena-cli"]
 
 [workspace.package]
-version = "0.1.2"
+version = "0.1.3"
 edition = "2021"
 license = "Apache-2.0"
 rust-version = "1.85"

crates/computearena-cli/src/adapters/basert.rs

@@ -288,13 +288,18 @@ fn normalize_headline_protocol(
     {
         bail!("BaseRT returned incompatible headline capacity/order metadata");
     }
+    // Each prefill runs at its own capacity, so the flat `params.ctx` names
+    // the largest one the sweep used (BaseRT 0.2.5): 16384 for the default
+    // sweep, not the headline's 4096.
+    let mut sweep_capacity = capacity;
     for prompt in request.pp.split(',') {
         let tokens = prompt.parse::<u64>()?;
         let expected = if tokens == headline {
             capacity
         } else {
             tokens.max(HEADLINE_CONTEXT_CAPACITY)
         };
+        sweep_capacity = sweep_capacity.max(expected);
         if runtime["prefill"][prompt]["initial_context_tokens"] != 0
             || runtime["prefill"][prompt]["context_capacity_tokens"].as_u64() != Some(expected)
         {
@@ -306,7 +311,7 @@ fn normalize_headline_protocol(
         || runtime["decode"]["seed_prefill_in_timing"] != false
         || runtime["measurement"]["timed_repetitions"] != request.reps
         || runtime["measurement"]["requested_warmup_repetitions"] != request.warmup
-        || benchmark["params"]["ctx"] != capacity
+        || benchmark["params"]["ctx"] != sweep_capacity
     {
         bail!("BaseRT returned incompatible headline measurement metadata");
     }
@@ -445,6 +450,71 @@ mod tests {
         }
     }
 
+    /// What the BaseRT 0.2.5 harness returns for the default sweep: every
+    /// prefill at its own capacity, and `params.ctx` naming the largest.
+    fn default_sweep_report(ctx: u64) -> Value {
+        let pp = crate::config::DEFAULT_PREFILL_TOKENS;
+        let prefill: serde_json::Map<String, Value> = pp
+            .split(',')
+            .map(|size| {
+                let tokens: u64 = size.parse().unwrap();
+                let capacity = if size == "512" {
+                    4096
+                } else {
+                    tokens.max(4096)
+                };
+                (
+                    size.to_string(),
+                    json!({"initial_context_tokens":0,"context_capacity_tokens":capacity}),
+                )
+            })
+            .collect();
+        json!({"params":{"ctx":ctx},"protocol":{
+            "schema":"basert-throughput-protocol/2","profile":"basert-bench-capacity/1",
+            "context_isolation":"headline_then_per_prefill","context_capacity_policy":"basert_bench_default",
+            "model_load_in_timing":false,"execution_layout":"headline_then_prefill_processes",
+            "execution_order":crate::protocol::headline_order(pp, 128),
+            "prefill":prefill,
+            "decode":{"initial_context_tokens":1,"context_capacity_tokens":4096,"seed_prefill_in_timing":false},
+            "measurement":{"timed_repetitions":3,"requested_warmup_repetitions":3}
+        }})
+    }
+
+    #[test]
+    fn the_default_sweep_reports_its_largest_capacity_as_ctx() {
+        let request = BenchmarkRequest {
+            model: Path::new("test.base"),
+            pp: crate::config::DEFAULT_PREFILL_TOKENS,
+            tg: 128,
+            reps: 3,
+            warmup: 3,
+            cooldown: false,
+        };
+        // BaseRT 0.2.5 publishes the 16384 the PP16384 workload ran at.
+        let mut report = default_sweep_report(16384);
+        normalize_protocol(&mut report, &request, true, true).unwrap();
+        assert_eq!(
+            report["protocol"]["id"],
+            crate::protocol::HEADLINE_PROTOCOL_ID
+        );
+        assert_eq!(report["params"]["ctx"], 16384);
+        // The decode capacity is still the headline's.
+        assert_eq!(
+            report["protocol"]["decode"]["context_capacity_tokens"],
+            4096
+        );
+
+        // Pre-release harnesses that published the headline's capacity
+        // instead misdescribe the sweep and are not signed.
+        let error = normalize_protocol(&mut default_sweep_report(4096), &request, true, true)
+            .unwrap_err()
+            .to_string();
+        assert!(
+            error.contains("incompatible headline measurement metadata"),
+            "{error}"
+        );
+    }
+
     #[test]
     fn replay_based_harnesses_use_external_observation() {
         let descriptor = json!({"telemetry_schema":"basert-telemetry/3",

docs/release-notes/0.1.3.md

@@ -0,0 +1,72 @@
+# ComputeArena CLI 0.1.3
+
+This release fixes benchmarking with BaseRT 0.2.5. On 0.1.2, every default
+run with the released BaseRT 0.2.5 ended, after the benchmark had completed,
+with `BaseRT returned incompatible headline measurement metadata`, and no
+report was saved. Nothing else changes.
+
+## Install or upgrade
+
+On macOS or Linux:
+
+```sh
+curl -LsSf https://computearena.ai/install.sh | sh
+```
+
+The same command upgrades an existing installation. Saved reports, the
+installation signing key, login sessions, model receipts, and installed
+runtimes are kept. Release archives and checksums are also available below for
+manual installation.
+
+BaseRT 0.2.5 itself needs no change. If BaseRT is not installed yet, or is
+older than 0.2.5, `computearena basert install` fetches the current release.
+
+## What changed
+
+- **Runs with BaseRT 0.2.5 complete and submit.** BaseRT 0.2.5 runs each
+  prefill size at its own context capacity and records the largest one the
+  sweep used as the run's context size: 16384 for the default sweep, which
+  reaches PP16384. ComputeArena 0.1.2 expected the headline's 4096 there and
+  refused the run once it had finished. The client now expects the value
+  BaseRT 0.2.5 records. The measurement is unchanged: PP512 and TG128 still
+  run first in a freshly loaded model with a 4K reservation, every workload's
+  capacity, order, and repetitions are still checked before signing, and the
+  reports still normalize to `computearena-throughput/3`. computearena.ai
+  applies the same rule.
+
+## Reports and compatibility
+
+0.1.2 could not save a BaseRT 0.2.5 report, so there is nothing to resubmit
+from it: run the benchmark again with 0.1.3. Reports saved by 0.1.0 to 0.1.2
+with BaseRT 0.2.4 or llama.cpp are unaffected and remain submittable when
+they contain the full default sweep. Benchmarks already published on
+computearena.ai are not changed.
+
+A BaseRT harness built from the development branch before the 0.2.5 release
+reports version 0.2.5 but records the headline's capacity instead. 0.1.3 does
+not sign its runs, and computearena.ai does not accept reports from it;
+replace it with `computearena basert install`. BaseRT 0.2.4 keeps working as
+before: its runs use the older protocol, the report records that, and the
+client says so before the run.
+
+## Supported systems
+
+Prebuilt ComputeArena binaries are available for:
+
+- macOS on Apple Silicon;
+- Linux x86_64 with glibc 2.31 or newer; and
+- Linux arm64 with glibc 2.31 or newer.
+
+Windows and Intel Macs are not supported in this release. ComputeArena does not
+bundle an inference runtime or model; install BaseRT or llama.cpp and obtain a
+compatible model before benchmarking. Prebuilt BaseRT runtimes exist for macOS
+on Apple Silicon and for Linux arm64 with CUDA; on Linux x86-64, use llama.cpp
+or a BaseRT benchmark harness you built yourself.
+
+The macOS binary is not signed with an Apple Developer ID. The release archive
+is checksum-verified and carries a Sigstore keyless signature from the release
+workflow. Browser-downloaded copies may need their quarantine attribute removed
+as described in the README.
+
+Accuracy benchmarks, MLX, vLLM, Windows, and Intel macOS support are not part
+of v0.1.3.