Runner execution policy: the consolidated doc; --only reports deselected

AGENTS.md

@@ -79,6 +79,9 @@ js/viewer/            results viewer + live browser harness (#46/#5): the
 examples/compose/     wac composition walkthrough (bundle-then-plug)
 expected/             golden outputs diffed by the verify recipes
 docs/findings.md      toolchain findings log
+docs/runner-policy.md runner execution policy (#22): granularity,
+                      replication, budgets, wizening — guidance with
+                      the measurement campaigns, not contract
 ```
 
 **WIT deps are symlinks; suites have none.** `components/provider` and

components/sample-suite/README.md

@@ -85,4 +85,6 @@ snapshot freezes whatever init observed (env, entropy, clocks), and
 the wizened artifact must be the one used everywhere downstream
 (runners, `lock --check`). Suites that import a system-under-test
 drive `component_test_runner::wizen::wizen_with` with their own
-linker instead.
+linker instead. When wizening pays — and when instance granularity or
+corpus preprocessing is the better lever — is the runner policy doc's
+call: see `docs/runner-policy.md`.

crates/component-test-cli/tests/cli.rs

@@ -179,6 +179,38 @@ fn fold_duplicate_case_fails_coverage() {
     assert!(out.stdout.contains("more than once"), "{}", out.stdout);
 }
 
+/// Deselected rows (a runner's `--only` subsetting, #22) satisfy
+/// coverage: the whole census is reported, the subsetting is visible,
+/// and nothing failed — exit 0.
+#[test]
+fn fold_deselected_rows_cover() {
+    let lock = tmpfile("deselected.lock", LOCK_XY);
+    let jsonl = stream(
+        &[
+            r#"{"case":"a/x","status":"pass"}"#,
+            r#"{"case":"a/y","status":"deselected","detail":"--only x"}"#,
+        ],
+        true,
+    );
+    let out = fold_with(&lock, &jsonl);
+    assert_eq!(
+        out.code, 0,
+        "stdout: {}\nstderr: {}",
+        out.stdout, out.stderr
+    );
+    assert!(
+        out.stdout.contains("DESELECTED: a/y — --only x"),
+        "{}",
+        out.stdout
+    );
+    assert!(
+        out.stdout
+            .contains("2 results (terminated): 1 deselected, 1 pass"),
+        "{}",
+        out.stdout
+    );
+}
+
 #[test]
 fn fold_unknown_status_fails() {
     let lock = tmpfile("unknown.lock", LOCK_XY);

crates/component-test-formats/src/aggregate.rs

@@ -622,6 +622,55 @@ mod tests {
         assert_eq!(agg.targets, ["native", "sim"]);
     }
 
+    /// Corpus subsetting is selection policy (#22): an applicable case
+    /// reported `deselected` (a runner's `--only`, a smoke tier) is
+    /// coverage-complete, tolerated by applicability policing, and
+    /// non-failing — the subset target aggregates cleanly with the
+    /// subsetting visible. Capability still cannot hide behind
+    /// selection: `deselected` on a tags-excluded case remains the
+    /// applicability error (runners must report those not-applicable,
+    /// which takes precedence over deselection).
+    #[test]
+    fn deselected_is_selection_policy_not_capability() {
+        let sim_subset = doc(
+            "sim",
+            vec![
+                result("suite/add", Status::Deselected, Some("--only declined")),
+                result("suite/hsm/attest", Status::NotApplicable, Some("hsm")),
+                result("suite/hsm/declined", Status::Pass, None),
+            ],
+        );
+        let agg = aggregate(
+            &corpus_lock(),
+            &manifest(MANIFEST),
+            &[("native".into(), native_doc()), ("sim".into(), sim_subset)],
+        );
+        assert!(agg.errors.is_empty(), "{:?}", agg.errors);
+        assert!(agg.ok());
+
+        let sim_hiding = doc(
+            "sim",
+            vec![
+                result("suite/add", Status::Pass, None),
+                result("suite/hsm/attest", Status::Deselected, Some("--only x")),
+                result("suite/hsm/declined", Status::Pass, None),
+            ],
+        );
+        let agg = aggregate(
+            &corpus_lock(),
+            &manifest(MANIFEST),
+            &[("native".into(), native_doc()), ("sim".into(), sim_hiding)],
+        );
+        assert!(
+            agg.errors
+                .iter()
+                .any(|e| e.contains("suite/hsm/attest") && e.contains("not applicable")),
+            "{:?}",
+            agg.errors
+        );
+        assert!(!agg.ok());
+    }
+
     #[test]
     fn artifact_hashes_are_provenance_not_identity() {
         // Suite builds are not reproducible across environments, so a

crates/component-test-runner/src/lib.rs

@@ -6,6 +6,11 @@
 //! (component-model-async), so calls go through wasmtime's concurrent
 //! API; host-side `diagnostic` is a concurrent host function.
 //!
+//! The execution policy implemented here — instance granularity,
+//! replication/striping, layered budgets, no retries, selection vs
+//! capability — is documented with its measurements in
+//! `docs/runner-policy.md`.
+//!
 //! Env knobs: `COMPONENT_TEST_PROFILE=1` prints per-session enumeration
 //! timings to stderr (double-enumerates each session to separate
 //! registry construction from lifting cost).
@@ -187,14 +192,17 @@ pub enum OutputMode {
 #[derive(Debug, Default)]
 pub struct Summary {
     pub not_applicable: usize,
+    /// Census cases outside the `only` selection: reported (as
+    /// `deselected` wire events), never executed.
+    pub deselected: usize,
     pub passed: usize,
     pub failed: usize,
     pub skipped: usize,
 }
 
 impl Summary {
     pub fn total(&self) -> usize {
-        self.passed + self.failed + self.skipped + self.not_applicable
+        self.passed + self.failed + self.skipped + self.not_applicable + self.deselected
     }
 }
 
@@ -650,9 +658,12 @@ impl<D: RunnerView + 'static> Runner<D> {
         .await
     }
 
-    /// `only`: run only cases whose name contains the substring (a
-    /// dev-loop filter; filtered cases are omitted from output, so
-    /// filtered runs will not aggregate cleanly — by design).
+    /// `only`: run only cases whose name contains the substring. The
+    /// rest of the census is reported as `deselected` (never executed),
+    /// so filtered runs still fold and aggregate cleanly, with the
+    /// subsetting visible as selection policy — never conflated with
+    /// capability (`not-applicable`, which takes precedence for cases
+    /// excluded by tags). A filter matching nothing is a run error.
     ///
     /// `case_execution_budget_secs`: budget on actual wasm *execution*
     /// per case — the executing thread's CPU time, sampled at epoch
@@ -850,16 +861,31 @@ impl<D: RunnerView + 'static> Runner<D> {
             }
         }
 
-        // Scheduler pre-pass: census order, each entry either runs or
-        // is not-applicable (with the excluding tag).
+        // Scheduler pre-pass: census order, each entry runs, is
+        // not-applicable (with the excluding tag), or is deselected by
+        // `--only`. Applicability wins over deselection: capability is
+        // a property of the target and is reported truthfully
+        // regardless of selection, while deselection only narrows the
+        // runnable set (#22's cost-tier rule — subsetting is selection
+        // policy, visible as such; it is also what aggregate's
+        // applicability policing expects of scheduled streams).
         enum Action {
             Run,
             NotApplicable(String),
+            Deselected,
+        }
+        // Same normative rule as the zero-enumeration guard above: a
+        // `--only` substring matching nothing is an empty selection (a
+        // typo'd filter must not exit green with the whole census
+        // deselected).
+        if let Some(o) = only {
+            if !names.iter().any(|n| n.contains(o)) {
+                bail!("--only `{o}` matches no cases (empty selection is a run error)");
+            }
         }
         let plan: Vec<(usize, &String, Action)> = names
             .iter()
             .enumerate()
-            .filter(|(_, name)| only.is_none_or(|o| name.contains(o)))
             .map(|(index, name)| {
                 let action = match (&inventory, tags_of(name)) {
                     (Some(_), Some(tags)) if !tags.applies(missing_features) => {
@@ -869,20 +895,12 @@ impl<D: RunnerView + 'static> Runner<D> {
                                 .unwrap_or_default(),
                         )
                     }
+                    _ if !only.is_none_or(|o| name.contains(o)) => Action::Deselected,
                     _ => Action::Run,
                 };
                 (index, name, action)
             })
             .collect();
-        // Same normative rule as the zero-enumeration guard above: a
-        // `--only` substring matching nothing is an empty selection (a
-        // typo'd filter must not exit green with "0 total").
-        if plan.is_empty() {
-            bail!(
-                "--only `{}` matches no cases (empty selection is a run error)",
-                only.unwrap_or_default()
-            );
-        }
 
         // Parallel path: workers own stores; results are collected and
         // emitted in census order below.
@@ -934,24 +952,51 @@ impl<D: RunnerView + 'static> Runner<D> {
 
         let mut session: Option<Session<D>> = None;
         for (index, enumerated_name, action) in plan {
-            if let Action::NotApplicable(mark) = action {
-                summary.not_applicable += 1;
-                if human {
-                    println!("test {enumerated_name}: N/A ({mark})");
-                } else {
-                    let event = Event::Case(CaseResult {
-                        case: enumerated_name.clone(),
-                        status: Status::NotApplicable,
-                        provenance: None,
-                        detail: Some(mark),
-                        seed: None,
-                        duration_ms: None,
-                        diagnostics: vec![],
-                        diagnostics_complete: true,
-                    });
-                    println!("{}", serde_json::to_string(&event)?);
+            match action {
+                Action::NotApplicable(mark) => {
+                    summary.not_applicable += 1;
+                    if human {
+                        println!("test {enumerated_name}: N/A ({mark})");
+                    } else {
+                        let event = Event::Case(CaseResult {
+                            case: enumerated_name.clone(),
+                            status: Status::NotApplicable,
+                            provenance: None,
+                            detail: Some(mark),
+                            seed: None,
+                            duration_ms: None,
+                            diagnostics: vec![],
+                            diagnostics_complete: true,
+                        });
+                        println!("{}", serde_json::to_string(&event)?);
+                    }
+                    continue;
                 }
-                continue;
+                // Deselected cases stay silent per-case in human mode
+                // (a narrow `--only` over a large corpus must not bury
+                // the selected cases under thousands of lines; the
+                // summary carries the count) but every one is a wire
+                // event: coverage folds and aggregation need the whole
+                // census reported, and the rows make the subsetting
+                // visible as selection policy rather than silence.
+                Action::Deselected => {
+                    summary.deselected += 1;
+                    if !human {
+                        let event = Event::Case(CaseResult {
+                            case: enumerated_name.clone(),
+                            status: Status::Deselected,
+                            provenance: None,
+                            detail: only.map(|o| format!("--only {o}")),
+                            seed: None,
+                            duration_ms: None,
+                            diagnostics: vec![],
+                            diagnostics_complete: true,
+                        });
+                        println!("{}", serde_json::to_string(&event)?);
+                    }
+                    continue;
+                }
+                Action::Run => {}
             }
 
             if human {
@@ -1025,8 +1070,15 @@ impl<D: RunnerView + 'static> Runner<D> {
         }
 
         if human {
+            // The deselected segment appears only when a selection was
+            // in force, so unfiltered output stays byte-identical.
+            let deselected = if summary.deselected > 0 {
+                format!(", {} deselected", summary.deselected)
+            } else {
+                String::new()
+            };
             println!(
-                "\nresult: {} passed, {} failed, {} skipped, {} not applicable, {} total",
+                "\nresult: {} passed, {} failed, {} skipped, {} not applicable{deselected}, {} total",
                 summary.passed,
                 summary.failed,
                 summary.skipped,

crates/component-test-runner/tests/runner.rs

@@ -156,6 +156,65 @@ fn missing_feature_flips_applicability() {
     assert_eq!(status_of(&cases, "fixture/hsm/declined")["status"], "pass");
 }
 
+/// `--only` reports the unselected census as `deselected` instead of
+/// omitting it (#22's cost-tier rule: subsetting is selection policy,
+/// visible as such), so filtered runs keep full coverage. Capability
+/// wins over selection: a tags-excluded case stays `not-applicable`
+/// even outside the filter (what aggregate's applicability policing
+/// requires of scheduled streams).
+#[test]
+#[ignore = "needs built components: run via `just test-wasm`"]
+fn only_reports_the_rest_of_the_census_deselected() {
+    let wasm = fixture_wasm();
+    let run = ct_runner(&[
+        wasm.to_str().unwrap(),
+        "--jsonl",
+        "--missing",
+        "hsm",
+        "--only",
+        "gen",
+    ]);
+    // The deliberate trap is outside the selection: nothing failed.
+    assert_eq!(run.code, 0, "stderr: {}", run.stderr);
+    let (_, cases, terminated) = parse_jsonl(&run.stdout);
+    assert!(terminated);
+
+    // Full census: 8 events, in census order.
+    assert_eq!(cases.len(), 8, "full census reported");
+    assert_eq!(status_of(&cases, "fixture/gen/tc1")["status"], "pass");
+    assert_eq!(status_of(&cases, "fixture/gen/tc2")["status"], "pass");
+    let boom = status_of(&cases, "fixture/trap/boom");
+    assert_eq!(boom["status"], "deselected");
+    assert_eq!(boom["detail"], "--only gen");
+    assert!(boom.get("provenance").is_none(), "never executed");
+    // Applicability beats selection.
+    let attest = status_of(&cases, "fixture/hsm/attest");
+    assert_eq!(attest["status"], "not-applicable");
+    assert_eq!(attest["detail"], "hsm");
+
+    // Human mode: deselected cases are silent per-case; the summary
+    // carries the count and the census total.
+    let run = ct_runner(&[wasm.to_str().unwrap(), "--missing", "hsm", "--only", "gen"]);
+    assert_eq!(run.code, 0, "stderr: {}", run.stderr);
+    assert!(
+        !run.stdout.contains("fixture/trap/boom"),
+        "deselected cases stay silent in human mode:\n{}",
+        run.stdout
+    );
+    assert!(
+        run.stdout.contains(
+            "result: 2 passed, 0 failed, 0 skipped, 1 not applicable, 5 deselected, 8 total"
+        ),
+        "{}",
+        run.stdout
+    );
+
+    // A filter matching nothing remains an empty selection: run error.
+    let run = ct_runner(&[wasm.to_str().unwrap(), "--only", "zzz"]);
+    assert_eq!(run.code, 2, "stderr: {}", run.stderr);
+    assert!(run.stderr.contains("empty selection"), "{}", run.stderr);
+}
+
 #[test]
 #[ignore = "needs built components: run via `just test-wasm`"]
 fn suite_artifact_flag_rebinds_the_envelope() {

docs/findings.md

@@ -230,4 +230,27 @@ bench-suite artifact built with its `wizer-init` feature.
     one-off). Wizening pays on JS legs only when enumeration is
     genuinely expensive; instance-granularity K>1 remains the lever
     there.
+25. **wasm-opt post-pass on wizened artifacts: no meaningful
+    clawback, and blocked at the component level anyway.** binaryen
+    v124 refuses components outright (binaryen#6728). The bound holds
+    regardless: the wizened bench artifact's growth is snapshot
+    *data* — 1.18MB across 10,002 data segments vs 93KB of code in
+    the 1.29MB artifact (`wasm-tools objdump`) — so even total code
+    deletion reclaims ≤7.2%. Measured on the extracted core module
+    (byte range from objdump; no supported round-trip back into a
+    component exists): `wasm-opt -Oz` saves 19.5KB, 1.5% of the
+    artifact. #25's "init-only code goes dead after snapshotting"
+    hypothesis is immaterial in this shape: the case bodies stay live
+    through the snapshotted case table — only the one-shot
+    registration driver dies, and it is small. CoW already makes the
+    on-disk growth near-free at runtime (finding 23); transport
+    compression covers the wire. `wasm-tools strip` is no substitute
+    and a trap besides: it *does* process components, but saves only
+    4.9KB (0.38% — `[profile.release] strip = true` already removed
+    the heavy custom sections at build time), and its default keep
+    list is `name`/`component-type`/`dylink.0` only, so it deletes
+    `component-test:tags@0.1` — scheduling and `lock --check` break
+    exactly as the CLI's "sections stripped" error anticipates. If it
+    must run, name sections explicitly with `--delete`; never the
+    default form.