diff --git a/Cargo.lock b/Cargo.lock index e70214f..ae6cb40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -329,7 +329,7 @@ dependencies = [ [[package]] name = "computearena-cli" -version = "0.1.2" +version = "0.1.3" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index b3b8621..86a5a01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/computearena-cli"] [workspace.package] -version = "0.1.2" +version = "0.1.3" edition = "2021" license = "Apache-2.0" rust-version = "1.85" diff --git a/crates/computearena-cli/src/adapters/basert.rs b/crates/computearena-cli/src/adapters/basert.rs index ecabf77..cb60ee0 100644 --- a/crates/computearena-cli/src/adapters/basert.rs +++ b/crates/computearena-cli/src/adapters/basert.rs @@ -288,6 +288,10 @@ fn normalize_headline_protocol( { bail!("BaseRT returned incompatible headline capacity/order metadata"); } + // Each prefill runs at its own capacity, so the flat `params.ctx` names + // the largest one the sweep used (BaseRT 0.2.5): 16384 for the default + // sweep, not the headline's 4096. + let mut sweep_capacity = capacity; for prompt in request.pp.split(',') { let tokens = prompt.parse::()?; let expected = if tokens == headline { @@ -295,6 +299,7 @@ fn normalize_headline_protocol( } else { tokens.max(HEADLINE_CONTEXT_CAPACITY) }; + sweep_capacity = sweep_capacity.max(expected); if runtime["prefill"][prompt]["initial_context_tokens"] != 0 || runtime["prefill"][prompt]["context_capacity_tokens"].as_u64() != Some(expected) { @@ -306,7 +311,7 @@ fn normalize_headline_protocol( || runtime["decode"]["seed_prefill_in_timing"] != false || runtime["measurement"]["timed_repetitions"] != request.reps || runtime["measurement"]["requested_warmup_repetitions"] != request.warmup - || benchmark["params"]["ctx"] != capacity + || benchmark["params"]["ctx"] != sweep_capacity { bail!("BaseRT returned incompatible headline measurement metadata"); } @@ -445,6 +450,71 @@ mod tests { } } + /// What the BaseRT 0.2.5 harness returns for the default sweep: every + /// prefill at its own capacity, and `params.ctx` naming the largest. + fn default_sweep_report(ctx: u64) -> Value { + let pp = crate::config::DEFAULT_PREFILL_TOKENS; + let prefill: serde_json::Map = pp + .split(',') + .map(|size| { + let tokens: u64 = size.parse().unwrap(); + let capacity = if size == "512" { + 4096 + } else { + tokens.max(4096) + }; + ( + size.to_string(), + json!({"initial_context_tokens":0,"context_capacity_tokens":capacity}), + ) + }) + .collect(); + json!({"params":{"ctx":ctx},"protocol":{ + "schema":"basert-throughput-protocol/2","profile":"basert-bench-capacity/1", + "context_isolation":"headline_then_per_prefill","context_capacity_policy":"basert_bench_default", + "model_load_in_timing":false,"execution_layout":"headline_then_prefill_processes", + "execution_order":crate::protocol::headline_order(pp, 128), + "prefill":prefill, + "decode":{"initial_context_tokens":1,"context_capacity_tokens":4096,"seed_prefill_in_timing":false}, + "measurement":{"timed_repetitions":3,"requested_warmup_repetitions":3} + }}) + } + + #[test] + fn the_default_sweep_reports_its_largest_capacity_as_ctx() { + let request = BenchmarkRequest { + model: Path::new("test.base"), + pp: crate::config::DEFAULT_PREFILL_TOKENS, + tg: 128, + reps: 3, + warmup: 3, + cooldown: false, + }; + // BaseRT 0.2.5 publishes the 16384 the PP16384 workload ran at. + let mut report = default_sweep_report(16384); + normalize_protocol(&mut report, &request, true, true).unwrap(); + assert_eq!( + report["protocol"]["id"], + crate::protocol::HEADLINE_PROTOCOL_ID + ); + assert_eq!(report["params"]["ctx"], 16384); + // The decode capacity is still the headline's. + assert_eq!( + report["protocol"]["decode"]["context_capacity_tokens"], + 4096 + ); + + // Pre-release harnesses that published the headline's capacity + // instead misdescribe the sweep and are not signed. + let error = normalize_protocol(&mut default_sweep_report(4096), &request, true, true) + .unwrap_err() + .to_string(); + assert!( + error.contains("incompatible headline measurement metadata"), + "{error}" + ); + } + #[test] fn replay_based_harnesses_use_external_observation() { let descriptor = json!({"telemetry_schema":"basert-telemetry/3", diff --git a/docs/release-notes/0.1.3.md b/docs/release-notes/0.1.3.md new file mode 100644 index 0000000..4dd40c2 --- /dev/null +++ b/docs/release-notes/0.1.3.md @@ -0,0 +1,72 @@ +# ComputeArena CLI 0.1.3 + +This release fixes benchmarking with BaseRT 0.2.5. On 0.1.2, every default +run with the released BaseRT 0.2.5 ended, after the benchmark had completed, +with `BaseRT returned incompatible headline measurement metadata`, and no +report was saved. Nothing else changes. + +## Install or upgrade + +On macOS or Linux: + +```sh +curl -LsSf https://computearena.ai/install.sh | sh +``` + +The same command upgrades an existing installation. Saved reports, the +installation signing key, login sessions, model receipts, and installed +runtimes are kept. Release archives and checksums are also available below for +manual installation. + +BaseRT 0.2.5 itself needs no change. If BaseRT is not installed yet, or is +older than 0.2.5, `computearena basert install` fetches the current release. + +## What changed + +- **Runs with BaseRT 0.2.5 complete and submit.** BaseRT 0.2.5 runs each + prefill size at its own context capacity and records the largest one the + sweep used as the run's context size: 16384 for the default sweep, which + reaches PP16384. ComputeArena 0.1.2 expected the headline's 4096 there and + refused the run once it had finished. The client now expects the value + BaseRT 0.2.5 records. The measurement is unchanged: PP512 and TG128 still + run first in a freshly loaded model with a 4K reservation, every workload's + capacity, order, and repetitions are still checked before signing, and the + reports still normalize to `computearena-throughput/3`. computearena.ai + applies the same rule. + +## Reports and compatibility + +0.1.2 could not save a BaseRT 0.2.5 report, so there is nothing to resubmit +from it: run the benchmark again with 0.1.3. Reports saved by 0.1.0 to 0.1.2 +with BaseRT 0.2.4 or llama.cpp are unaffected and remain submittable when +they contain the full default sweep. Benchmarks already published on +computearena.ai are not changed. + +A BaseRT harness built from the development branch before the 0.2.5 release +reports version 0.2.5 but records the headline's capacity instead. 0.1.3 does +not sign its runs, and computearena.ai does not accept reports from it; +replace it with `computearena basert install`. BaseRT 0.2.4 keeps working as +before: its runs use the older protocol, the report records that, and the +client says so before the run. + +## Supported systems + +Prebuilt ComputeArena binaries are available for: + +- macOS on Apple Silicon; +- Linux x86_64 with glibc 2.31 or newer; and +- Linux arm64 with glibc 2.31 or newer. + +Windows and Intel Macs are not supported in this release. ComputeArena does not +bundle an inference runtime or model; install BaseRT or llama.cpp and obtain a +compatible model before benchmarking. Prebuilt BaseRT runtimes exist for macOS +on Apple Silicon and for Linux arm64 with CUDA; on Linux x86-64, use llama.cpp +or a BaseRT benchmark harness you built yourself. + +The macOS binary is not signed with an Apple Developer ID. The release archive +is checksum-verified and carries a Sigstore keyless signature from the release +workflow. Browser-downloaded copies may need their quarantine attribute removed +as described in the README. + +Accuracy benchmarks, MLX, vLLM, Windows, and Intel macOS support are not part +of v0.1.3.