diff --git a/.gitignore b/.gitignore index 5db6562..d03acac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ target/ *.wasm +# The components baked into the CLI are committed (regenerate with +# `just embed-update`; `just verify-cli` gates their behavior). +!crates/component-test-cli/embedded/*.wasm node_modules/ # Prebuilt CLI/runner binaries for downstream consumers (their build diff --git a/AGENTS.md b/AGENTS.md index 6923360..2e1895f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,7 +51,13 @@ crates/ host-side Rust (tested natively at root) component-test-results canonical results schema + fold (guest-linkable) component-test-sdk guest SDK (registry, prelude, verdict ergonomics) component-test-sdk-macro #[suite] attribute (case discovery, tags section) - component-test-cli `component-test` bin: lock, fold, aggregate + component-test-cli `component-test` bin: lock, fold, aggregate, + pins, wizen, compose-runner, run (the last + three embed wasmtime/wac-graph plus + size-optimized runner-cli + provider builds + from embedded/ — regenerate those with + `just embed-update`, commit the diff; + `just verify-cli` gates their behavior) component-test-runner wasmtime host-embed runner (`ct-runner` bin) components/ guest components (build with --target wasm32-wasip2) provider reference context provider diff --git a/Cargo.lock b/Cargo.lock index c16fa79..54faba2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "gimli", ] +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "allocator-api2" version = "0.2.21" @@ -23,6 +29,56 @@ version = "0.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -55,6 +111,18 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "auditable-serde" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d026218ae25ba5c72834245412dd1338f6d270d2c5109ee03a4badec288d4056" +dependencies = [ + "semver", + "serde", + "serde_json", + "topological-sort", +] + [[package]] name = "base64" version = "0.22.1" @@ -175,6 +243,46 @@ dependencies = [ "rand_core", ] +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cobs" version = "0.3.0" @@ -184,15 +292,26 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "component-test-cli" version = "0.1.0" dependencies = [ "anyhow", + "clap", "component-test-core", "component-test-formats", + "component-test-runner", "serde_json", "toml 0.9.12+spec-1.1.0", + "wac-graph", + "wasmtime", + "wasmtime-wasi", ] [[package]] @@ -610,6 +729,16 @@ dependencies = [ "component-test-sdk", ] +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -990,6 +1119,12 @@ version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1174,6 +1309,16 @@ dependencies = [ "rustix", ] +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -1203,6 +1348,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1522,6 +1673,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "slab" version = "0.4.12" @@ -1547,12 +1704,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spdx" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "081670c233dfbed55690cc0cd38424e0e24ac1b2673d0b408b3f7b684738dfa9" +dependencies = [ + "smallvec", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "2.0.119" @@ -1747,6 +1919,12 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +[[package]] +name = "topological-sort" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d" + [[package]] name = "tracing" version = "0.1.44" @@ -1829,6 +2007,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.24.0" @@ -1855,6 +2039,39 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wac-graph" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdc8c5ffe944e002a24ba9d0fa67f51d4a761c1fc97718a17d1d7379753a6f8e" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "petgraph", + "semver", + "thiserror 1.0.69", + "wac-types", + "wasm-encoder 0.247.0", + "wasm-metadata 0.247.0", + "wasmparser 0.247.0", +] + +[[package]] +name = "wac-types" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b90654c08294d879e85f0b8b2c5a7efc3a7a10df3009dd1578d98902d1ec8a2" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "semver", + "wasm-encoder 0.247.0", + "wasmparser 0.247.0", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1923,6 +2140,16 @@ dependencies = [ "wat", ] +[[package]] +name = "wasm-encoder" +version = "0.247.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b6733b8b91d010a6ac5b0fb237dc46a19650bc4c67db66857e2e787d437204" +dependencies = [ + "leb128fmt", + "wasmparser 0.247.0", +] + [[package]] name = "wasm-encoder" version = "0.252.0" @@ -1953,6 +2180,25 @@ dependencies = [ "wasmparser 0.255.0", ] +[[package]] +name = "wasm-metadata" +version = "0.247.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "665fe59e56cc9b419ca6fcca56673e3421d1a5011e3b65caf6b726fd9e041d10" +dependencies = [ + "anyhow", + "auditable-serde", + "flate2", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "url", + "wasm-encoder 0.247.0", + "wasmparser 0.247.0", +] + [[package]] name = "wasm-metadata" version = "0.254.0" @@ -1990,6 +2236,19 @@ dependencies = [ "serde", ] +[[package]] +name = "wasmparser" +version = "0.247.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6fb4c2bee46c5ea4d40f8cdb5c131725cd976718ec56f1c8e82fbde5fa2a80" +dependencies = [ + "bitflags", + "hashbrown 0.17.1", + "indexmap", + "semver", + "serde", +] + [[package]] name = "wasmparser" version = "0.252.0" @@ -2047,6 +2306,7 @@ dependencies = [ "async-trait", "bitflags", "bumpalo", + "bytes", "cc", "cfg-if", "encoding_rs", @@ -2649,7 +2909,7 @@ dependencies = [ "indexmap", "prettyplease", "syn 2.0.119", - "wasm-metadata", + "wasm-metadata 0.254.0", "wit-bindgen-core", "wit-component", ] @@ -2684,7 +2944,7 @@ dependencies = [ "serde_derive", "serde_json", "wasm-encoder 0.254.0", - "wasm-metadata", + "wasm-metadata 0.254.0", "wasmparser 0.254.0", "wit-parser 0.254.0", ] diff --git a/Cargo.toml b/Cargo.toml index 15eb1a7..da9d7d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ component-test-formats = { path = "crates/component-test-formats" } component-test-results = { path = "crates/component-test-results" } component-test-sdk = { path = "crates/component-test-sdk" } component-test-sdk-macro = { path = "crates/component-test-sdk-macro" } +component-test-runner = { path = "crates/component-test-runner" } wit-bindgen = "0.60" futures = "0.3" serde = { version = "1", features = ["derive"] } @@ -53,7 +54,24 @@ toml = "0.9" tokio = { version = "1", default-features = false } anyhow = "1" trybuild = "1" +clap = { version = "4", features = ["derive"] } +# One wasmtime everywhere: the host-embed runner and the CLI's `run` +# embedding must agree (and stay in step with the wasmtime CLI the +# verify recipes drive). +wasmtime = { version = "47", features = ["component-model-async"] } +wasmtime-wasi = { version = "47", features = ["p3"] } +# Composition as a library (the CLI's compose-runner/run): same +# release line as the wac CLI the compose examples document. +wac-graph = "0.10" [profile.release] opt-level = "s" strip = true + +# The components baked into the CLI (`just embed-update`): size over +# speed — they ship inside every `component-test` binary. +[profile.embed] +inherits = "release" +opt-level = "z" +lto = true +codegen-units = 1 diff --git a/README.md b/README.md index 1a7a3bb..4ed0242 100644 --- a/README.md +++ b/README.md @@ -269,4 +269,6 @@ WPT, `go test`, LLVM lit) is tracked in the issues. viewer. - Interop emitters: JUnit XML, TAP, GitHub Actions annotations. - Reusable GitHub Actions workflows. -- A `component-test` CLI wrapping composition, execution, and aggregation. +- A `component-test` CLI wrapping composition, execution, and aggregation + (`compose-runner`/`run` embed the reference provider + wasi:cli runner + core), plus wizer pre-initialization for large suites (`wizen`). diff --git a/components/sample-suite/README.md b/components/sample-suite/README.md index bbcb54b..7c3564d 100644 --- a/components/sample-suite/README.md +++ b/components/sample-suite/README.md @@ -63,3 +63,26 @@ ct-runner target/wasm32-wasip2/release/my_suite.wasm # or --jsonl Commit `tests.lock` and regenerate it (`lock ... -o`) after any case change: the diff is the review surface, and runners/aggregation cross-check the inventory against what the suite actually enumerates. + +## Large suites + +Enumeration builds the whole registry per fresh instance, so +instance-per-case isolation gets expensive at scale (10k cases ≈ 3ms +per instance). Instead of relaxing isolation, pre-initialize the +artifact once after building: + +```sh +component-test wizen target/wasm32-wasip2/release/my_suite.wasm -o my_suite.wasm +``` + +The suite needs no changes (the contract's own `all()` is the init +function), inventory and tag scheduling survive, and every fresh +instance is born with the registry built — a 10k-case synthetic's +K=1 run went 30.8s → 7.1s sequential, 1.14s at `--jobs 8` +(findings.md 22–24; JS legs benefit far less — no copy-on-write +memory images). Two caveats: the +snapshot freezes whatever init observed (env, entropy, clocks), and +the wizened artifact must be the one used everywhere downstream +(runners, `lock --check`). Suites that import a system-under-test +drive `component_test_runner::wizen::wizen_with` with their own +linker instead. diff --git a/crates/component-test-cli/Cargo.toml b/crates/component-test-cli/Cargo.toml index a9d1860..1f2307a 100644 --- a/crates/component-test-cli/Cargo.toml +++ b/crates/component-test-cli/Cargo.toml @@ -13,6 +13,16 @@ path = "src/main.rs" [dependencies] component-test-core = { workspace = true } component-test-formats = { workspace = true } +# The wizen entry points (feature `wizer`); the CLI is the plain-suite +# driver, SUT-importing suites call the library directly (#85). +component-test-runner = { workspace = true, features = ["wizer"] } anyhow = { workspace = true } +clap = { workspace = true } serde_json = { workspace = true } toml = { workspace = true } +# compose-runner/run: composition as a library (the shared-provider +# topology of examples/compose, without shelling out to wac). +wac-graph = { workspace = true } +# run: the minimal embedding executing composed wasi:cli components. +wasmtime = { workspace = true } +wasmtime-wasi = { workspace = true } diff --git a/crates/component-test-cli/embedded/provider.wasm b/crates/component-test-cli/embedded/provider.wasm new file mode 100644 index 0000000..eaddae4 Binary files /dev/null and b/crates/component-test-cli/embedded/provider.wasm differ diff --git a/crates/component-test-cli/embedded/runner-cli.wasm b/crates/component-test-cli/embedded/runner-cli.wasm new file mode 100644 index 0000000..fd94f48 Binary files /dev/null and b/crates/component-test-cli/embedded/runner-cli.wasm differ diff --git a/crates/component-test-cli/src/compose.rs b/crates/component-test-cli/src/compose.rs new file mode 100644 index 0000000..7738605 --- /dev/null +++ b/crates/component-test-cli/src/compose.rs @@ -0,0 +1,229 @@ +//! `compose-runner`: compose a suite with a context provider and the +//! wasi:cli runner core, as a library (wac-graph) instead of shelled-out +//! `wac` — the same graph as `examples/compose/compose.wac`. +//! +//! The topology needs a *shared* provider instance — the suite's +//! `test-context` import and the runner core's `factory`/`test-context` +//! imports must resolve to the same instance so the `context` resource +//! type is one type — which is exactly what `wac plug` cannot express +//! (see examples/compose/README.md). Pre-composed *bundles* (a suite +//! already `wac compose`d with its own providers, re-exporting `tests` +//! plus `test-context` plus `factory`) skip the provider and plug +//! straight into the runner core. +//! +//! The default provider and runner core are baked in at build time from +//! `embedded/` (size-optimized `embed`-profile builds of +//! `components/provider` and `components/runner-cli`; regenerate with +//! `just embed-update` and commit the diff — `just verify-cli` gates +//! their behavior against the same goldens as the wac-composed path). +//! Both are overridable per invocation. +//! +//! Composition strips custom sections (findings #14): the result is +//! execute-everything, its envelope says `scheduling: none`, and +//! lockfiles must be generated from the suite artifact, not from +//! anything this module produces. + +use anyhow::{bail, Context as _, Result}; +use wac_graph::types::Package; +use wac_graph::{CompositionGraph, EncodeOptions, NodeId, PackageId}; + +/// The wasi:cli runner core (`components/runner-cli`), `embed` profile. +pub const EMBEDDED_RUNNER: &[u8] = include_bytes!("../embedded/runner-cli.wasm"); +/// The reference context provider (`components/provider`), `embed` profile. +pub const EMBEDDED_PROVIDER: &[u8] = include_bytes!("../embedded/provider.wasm"); + +/// The frozen contract interfaces (wit/tests.wit; L1) and the provider +/// interface the reference runner core consumes. +const TESTS: &str = "polymorph:test/tests@0.1.0"; +const CONTEXT: &str = "polymorph:test/test-context@0.1.0"; +const FACTORY: &str = "polymorph:test-provider/factory@0.1.0"; +/// The composed entry point (version-prefix matched: the runner core +/// override decides the exact wasi:cli version). +const RUN_PREFIX: &str = "wasi:cli/run@"; + +/// What a composition input turned out to be, by its exports. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Input { + /// Exports `tests` only: needs a provider and the runner core. + Suite, + /// Exports `tests` + `test-context` + `factory` (a suite already + /// bundled with its providers): needs only the runner core. + Bundle, + /// Exports `wasi:cli/run`: already composed, nothing to do. + Composed, +} + +/// Read a component from disk, with the same crisp non-wasm error the +/// other subcommands give. +pub fn read_component(path: &str) -> Result> { + let wasm = std::fs::read(path).with_context(|| format!("reading {path}"))?; + if !wasm.starts_with(b"\0asm") { + bail!("{path} is not a WebAssembly binary (bad magic)"); + } + Ok(wasm) +} + +/// Classify `input` by its exports (see [`Input`]). +pub fn classify(input: &[u8]) -> Result { + let mut graph = CompositionGraph::new(); + let pkg = register(&mut graph, "input", input)?; + Ok(classify_registered(&graph, pkg)) +} + +fn classify_registered(graph: &CompositionGraph, pkg: PackageId) -> Input { + let exports = &graph.types()[graph[pkg].ty()].exports; + if exports.keys().any(|n| n.starts_with(RUN_PREFIX)) { + Input::Composed + } else if exports.contains_key(FACTORY) && exports.contains_key(CONTEXT) { + Input::Bundle + } else { + Input::Suite + } +} + +fn register(graph: &mut CompositionGraph, name: &str, bytes: &[u8]) -> Result { + let package = Package::from_bytes(name, None, bytes.to_vec(), graph.types_mut()) + .with_context(|| format!("parsing the {name} component"))?; + graph + .register_package(package) + .with_context(|| format!("registering the {name} component")) +} + +/// Wire every import of `to` that `from`'s instance exports by exact +/// name. Unwired imports (WASI) stay imports of the composition. +/// `aliases` caches export aliases so one exported instance feeds any +/// number of importers (the shared-provider requirement). +fn wire( + graph: &mut CompositionGraph, + aliases: &mut std::collections::HashMap<(NodeId, String), NodeId>, + from: (NodeId, PackageId, &str), + to: (NodeId, PackageId, &str), +) -> Result<()> { + let (from_inst, from_pkg, from_name) = from; + let (to_inst, to_pkg, to_name) = to; + let exports: Vec = graph.types()[graph[from_pkg].ty()] + .exports + .keys() + .cloned() + .collect(); + for name in exports { + if !graph.types()[graph[to_pkg].ty()] + .imports + .contains_key(&name) + { + continue; + } + let alias = match aliases.entry((from_inst, name.clone())) { + std::collections::hash_map::Entry::Occupied(e) => *e.get(), + std::collections::hash_map::Entry::Vacant(e) => *e.insert( + graph + .alias_instance_export(from_inst, &name) + .with_context(|| format!("aliasing `{name}` from the {from_name}"))?, + ), + }; + graph + .set_instantiation_argument(to_inst, &name, alias) + .with_context(|| { + format!("plugging the {to_name}'s `{name}` import from the {from_name}") + })?; + } + Ok(()) +} + +/// Compose `input` (a suite or a bundle; see [`Input`]) with `provider` +/// and the `runner` core into a runnable wasi:cli component. +pub fn compose(input: &[u8], provider: &[u8], runner: &[u8]) -> Result> { + let mut graph = CompositionGraph::new(); + let input_pkg = register(&mut graph, "input", input)?; + let runner_pkg = register(&mut graph, "runner-cli", runner)?; + + let input_kind = classify_registered(&graph, input_pkg); + if input_kind == Input::Composed { + bail!( + "input already exports wasi:cli/run — it is already composed \ + (`component-test run` executes it directly)" + ); + } + if !graph.types()[graph[input_pkg].ty()] + .exports + .contains_key(TESTS) + { + bail!("input does not export `{TESTS}`: not a test suite (or bundle of one)"); + } + // A partial bundle would wire ambiguously (whose context feeds the + // runner core?); a real bundle exports both or neither. + if input_kind == Input::Suite + && (graph.types()[graph[input_pkg].ty()] + .exports + .contains_key(CONTEXT) + || graph.types()[graph[input_pkg].ty()] + .exports + .contains_key(FACTORY)) + { + bail!( + "input exports one of `{CONTEXT}`/`{FACTORY}` but not both: \ + a bundle must re-export both (see examples/compose/bundle.wac)" + ); + } + + let mut aliases = std::collections::HashMap::new(); + let runner_inst = graph.instantiate(runner_pkg); + match input_kind { + Input::Composed => unreachable!("rejected above: composed inputs do not export tests"), + // Bundle: everything the runner core imports re-exports from + // the one bundle instance (whose innards already share their + // provider). + Input::Bundle => { + let bundle_inst = graph.instantiate(input_pkg); + wire( + &mut graph, + &mut aliases, + (bundle_inst, input_pkg, "bundle"), + (runner_inst, runner_pkg, "runner core"), + )?; + } + // Bare suite: one shared provider instance feeds the suite's + // `test-context` import and the runner core's `factory` + + // `test-context` imports (examples/compose/compose.wac). + Input::Suite => { + let provider_pkg = register(&mut graph, "provider", provider)?; + let provider_inst = graph.instantiate(provider_pkg); + let suite_inst = graph.instantiate(input_pkg); + wire( + &mut graph, + &mut aliases, + (provider_inst, provider_pkg, "provider"), + (suite_inst, input_pkg, "suite"), + )?; + wire( + &mut graph, + &mut aliases, + (provider_inst, provider_pkg, "provider"), + (runner_inst, runner_pkg, "runner core"), + )?; + wire( + &mut graph, + &mut aliases, + (suite_inst, input_pkg, "suite"), + (runner_inst, runner_pkg, "runner core"), + )?; + } + } + + let run_name = graph.types()[graph[runner_pkg].ty()] + .exports + .keys() + .find(|n| n.starts_with(RUN_PREFIX)) + .cloned() + .with_context(|| format!("runner core exports no `{RUN_PREFIX}...` — not a runner core"))?; + let run = graph + .alias_instance_export(runner_inst, &run_name) + .context("aliasing the runner core's run export")?; + graph + .export(run, &run_name) + .context("exporting wasi:cli/run")?; + + graph + .encode(EncodeOptions::default()) + .context("encoding the composition") +} diff --git a/crates/component-test-cli/src/main.rs b/crates/component-test-cli/src/main.rs index cc1afc1..5357ff9 100644 --- a/crates/component-test-cli/src/main.rs +++ b/crates/component-test-cli/src/main.rs @@ -1,34 +1,34 @@ //! `component-test`: tooling entry point. //! -//! Subcommands (v0): -//! lock [-o tests.lock] [--check existing.lock] [--leaves names.txt] -//! Execution-free inventory from the suite's tags section. -//! `--leaves` (a runner's `--enumerate` output) additionally pins -//! each generated row's leaves, making coverage exact for them. -//! fold [tests.lock] < results.jsonl -//! Fold a JSONL results stream into the document form + summary. -//! aggregate --lock tests.lock --manifest targets.toml -//! [--results target=path.jsonl]... [-o matrix.md] -//! Cross-target validation + markdown matrix (#30). -//! emit junit [--lock tests.lock] [--results target=path.jsonl]... -//! [-o results.xml] -//! Convert results streams to JUnit XML for CI test UIs (#11). -//! A converter, not a gate: always exits 0 after writing. -//! pins --cargo-lock Cargo.lock [--js-lock lockfile]... [--expect rev] -//! One-rev-everywhere gate for downstream pin sets (#54): the -//! component-test-* crates in Cargo.lock and the JS facade's -//! npm/pnpm pin must agree on a single commit. -//! pins bump [--cargo-toml Cargo.toml]... -//! [--package-json package.json]... [--workflow ci.yml]... -//! The write half of the gate (#60): rewrite the declared pins -//! in place and print the lockfile-regeneration follow-ups. +//! Subcommands: +//! lock Execution-free inventory from the suite's tags +//! section; `--check` is the drift gate, `--leaves` +//! pins generated rows from a runner enumeration. +//! fold Fold a JSONL results stream into the document form +//! + summary (the acceptance gate for one stream). +//! aggregate Cross-target validation + markdown matrix (#30). +//! emit Convert results streams for foreign consumers +//! (JUnit XML, #11). A converter, not a gate: always +//! exits 0 after writing. +//! pins One-rev-everywhere gate for downstream pin sets +//! (#54); `pins bump` is the write half (#60). +//! wizen Wizer pre-initialization of a suite artifact (#25, +//! #85): snapshot after the contract's own `all()`. +//! compose-runner Compose a suite (or bundle) with a context +//! provider and the wasi:cli runner core (#13, #85); +//! embedded defaults, overridable. +//! run compose-runner + execute under an embedded +//! wasmtime; the exit code is the guest's verdict. +mod compose; mod junit; mod pins; +mod run; use std::io::{IsTerminal, Read}; use anyhow::{bail, Context as _}; +use clap::{Args, Parser, Subcommand}; use component_test_formats::{ aggregate, inventory, lockfile::{Lockfile, SuiteRef}, @@ -36,72 +36,203 @@ use component_test_formats::{ matrix, results, sha256_hex, }; -fn usage() -> String { - "usage: component-test lock [-o tests.lock] [--check tests.lock] [--leaves names.txt]\n \ - component-test fold [tests.lock] < results.jsonl\n \ - component-test aggregate --lock tests.lock --manifest targets.toml \ - [--results target=path.jsonl]... [-o matrix.md]\n \ - component-test emit junit [--lock tests.lock] \ - [--results target=path.jsonl]... [-o results.xml]\n \ - component-test pins --cargo-lock Cargo.lock \ - [--js-lock lockfile]... [--expect rev]\n \ - component-test pins bump [--cargo-toml Cargo.toml]... \ - [--package-json package.json]... [--workflow ci.yml]..." - .into() +#[derive(Parser)] +#[command( + name = "component-test", + version, + about = "polymorph:test tooling: composition, execution, inventory, and aggregation" +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Execution-free inventory from the suite's tags section + Lock(LockArgs), + /// Fold a JSONL results stream (stdin) into the document form + summary + Fold(FoldArgs), + /// Cross-target validation + markdown matrix + Aggregate(AggregateArgs), + /// Convert results streams for foreign consumers (a converter, not a gate) + Emit(EmitArgs), + /// One-rev-everywhere gate for downstream pin sets + Pins(pins::PinsArgs), + /// Pre-initialize a suite with wizer: its own `all()` runs at build + /// time, so every fresh instance is born with the registry built + #[command(alias = "wizer")] + Wizen(WizenArgs), + /// Compose a suite with a context provider and the wasi:cli runner + /// core into a runnable component + ComposeRunner(ComposeRunnerArgs), + /// Compose (unless already composed) and execute under an embedded + /// wasmtime; the exit code is the guest's verdict + Run(RunArgs), +} + +#[derive(Args)] +struct LockArgs { + /// Suite artifact (the suite itself, not a composed bundle: + /// composition strips the tags section) + #[arg(value_name = "suite.wasm")] + suite: String, + /// Write the lockfile here (default: stdout) + #[arg(short, long, value_name = "tests.lock")] + output: Option, + /// Check an existing lockfile for drift instead of writing + #[arg(long, value_name = "tests.lock")] + check: Option, + /// A runner's --enumerate output (one case name per line): pins + /// each generated row's leaves, making coverage exact for them + #[arg(long, value_name = "names.txt")] + leaves: Option, +} + +#[derive(Args)] +struct FoldArgs { + /// With a lockfile, coverage is part of acceptance: every case + /// reported exactly once, generated leaves under their prefix + #[arg(value_name = "tests.lock")] + lock: Option, +} + +#[derive(Args)] +struct AggregateArgs { + /// The suite inventory (coverage + applicability baseline) + #[arg(long, value_name = "tests.lock")] + lock: String, + /// The targets manifest (features, targets, expected failures) + #[arg(long, value_name = "targets.toml")] + manifest: String, + /// A target's results stream (repeatable) + #[arg( + long = "results", + value_name = "target=path.jsonl", + value_parser = parse_target_results + )] + results: Vec<(String, String)>, + /// Write the markdown matrix here (default: stdout) + #[arg(short, long, value_name = "matrix.md")] + output: Option, +} + +#[derive(Args)] +struct EmitArgs { + /// Output format + #[arg(value_parser = ["junit"])] + format: String, + /// Fold each stream against this lockfile (not-reached synthesis + /// for dropped cases); without it, the stream speaks for itself + #[arg(long, value_name = "tests.lock")] + lock: Option, + /// A target's results stream (repeatable, at least one) + #[arg( + long = "results", + value_name = "target=path.jsonl", + value_parser = parse_target_results, + required = true + )] + results: Vec<(String, String)>, + /// Write here (default: stdout) + #[arg(short, long, value_name = "results.xml")] + output: Option, +} + +#[derive(Args)] +struct WizenArgs { + /// Suite artifact to pre-initialize (plain suites — WASI + + /// test-context imports only; SUT-importing suites drive + /// component_test_runner::wizen::wizen_with with their own linker) + #[arg(value_name = "suite.wasm")] + suite: String, + /// Where to write the pre-initialized artifact (run it everywhere + /// downstream — runners, lockfile checks — instead of mixing + /// artifacts) + #[arg(short, long, value_name = "out.wasm")] + output: String, + /// NAME=VAL visible during initialization, baked into the snapshot + /// (as are entropy and clocks: init must not leave the registry + /// parameterized on any of them; repeatable) + #[arg(long = "env", value_name = "NAME=VAL", value_parser = parse_env)] + env: Vec<(String, String)>, +} + +#[derive(Args)] +struct ComposeRunnerArgs { + /// Suite artifact, or a bundle re-exporting tests + test-context + + /// factory (a suite pre-composed with its own providers) + #[arg(value_name = "suite.wasm")] + input: String, + /// Where to write the composed component + #[arg(short, long, value_name = "composed.wasm")] + output: String, + /// Context provider component (default: the embedded reference + /// provider; unused when the input is a bundle) + #[arg(long, value_name = "provider.wasm")] + provider: Option, + /// wasi:cli runner core component (default: the embedded + /// components/runner-cli build) + #[arg(long, value_name = "runner.wasm")] + runner: Option, +} + +#[derive(Args)] +struct RunArgs { + /// Suite, bundle, or already-composed component (anything exporting + /// wasi:cli/run is executed as-is) + #[arg(value_name = "suite.wasm")] + input: String, + /// Context provider component (default: the embedded reference + /// provider) + #[arg(long, value_name = "provider.wasm")] + provider: Option, + /// wasi:cli runner core component (default: the embedded + /// components/runner-cli build) + #[arg(long, value_name = "runner.wasm")] + runner: Option, + /// Emit the JSONL results wire format (sets COMPONENT_TEST_JSONL=1 + /// in the guest; pipe into `component-test fold`) + #[arg(long)] + jsonl: bool, + /// NAME=VAL in the guest environment (the host environment is not + /// inherited, matching `wasmtime run`; repeatable) + #[arg(long = "env", value_name = "NAME=VAL", value_parser = parse_env)] + env: Vec<(String, String)>, +} + +fn parse_target_results(s: &str) -> Result<(String, String), String> { + s.split_once('=') + .map(|(t, p)| (t.to_string(), p.to_string())) + .ok_or_else(|| "must be target=path.jsonl".to_string()) +} + +fn parse_env(s: &str) -> Result<(String, String), String> { + s.split_once('=') + .map(|(k, v)| (k.to_string(), v.to_string())) + .ok_or_else(|| "must be NAME=VAL".to_string()) } fn main() -> anyhow::Result<()> { - let args: Vec = std::env::args().skip(1).collect(); - match args.first().map(|s| s.as_str()) { - Some("lock") => lock(&args[1..]), - Some("fold") => fold(&args[1..]), - Some("aggregate") => aggregate_cmd(&args[1..]), - Some("emit") => emit_cmd(&args[1..]), - Some("pins") => pins::pins_cmd(&args[1..]), - Some("-h" | "--help" | "help") => { - println!("{}", usage()); - Ok(()) - } - Some("-V" | "--version") => { - println!("component-test {}", env!("CARGO_PKG_VERSION")); - Ok(()) - } - _ => { - eprintln!("{}", usage()); - std::process::exit(2); - } + match Cli::parse().command { + Command::Lock(args) => lock(&args), + Command::Fold(args) => fold(&args), + Command::Aggregate(args) => aggregate_cmd(&args), + Command::Emit(args) => emit_cmd(&args), + Command::Pins(args) => pins::pins_cmd(&args), + Command::Wizen(args) => wizen_cmd(&args), + Command::ComposeRunner(args) => compose_runner_cmd(&args), + Command::Run(args) => run_cmd(&args), } } -fn lock(args: &[String]) -> anyhow::Result<()> { - let mut suite_path = None; - let mut out_path = None; - let mut check_path = None; - let mut leaves_path = None; - let mut it = args.iter(); - while let Some(arg) = it.next() { - match arg.as_str() { - "-o" => out_path = Some(it.next().context("-o needs a path")?.clone()), - "--check" => check_path = Some(it.next().context("--check needs a path")?.clone()), - "--leaves" => leaves_path = Some(it.next().context("--leaves needs a path")?.clone()), - "-h" | "--help" => { - println!("{}", usage()); - return Ok(()); - } - s if s.starts_with('-') => bail!("unknown flag `{s}`\n{}", usage()), - _ if suite_path.is_none() => suite_path = Some(arg.clone()), - other => bail!("unexpected argument `{other}`"), - } - } - let suite_path = suite_path.context("missing suite.wasm path")?; - let wasm = std::fs::read(&suite_path).with_context(|| format!("reading {suite_path}"))?; - if !wasm.starts_with(b"\0asm") { - bail!("{suite_path} is not a WebAssembly binary (bad magic)"); - } +fn lock(args: &LockArgs) -> anyhow::Result<()> { + let suite_path = &args.suite; + let wasm = compose::read_component(suite_path)?; let inv = inventory::inventory(&wasm)?; let artifact_sha256 = sha256_hex(&wasm); - let name = std::path::Path::new(&suite_path) + let name = std::path::Path::new(suite_path) .file_stem() .and_then(|s| s.to_str()) .unwrap_or("suite") @@ -118,7 +249,7 @@ fn lock(args: &[String]) -> anyhow::Result<()> { // case name per line) pins each generated row's leaves. Exact // static cases pass through; a name matching neither is inventory // drift, as is a generated row the enumeration never touched. - if let Some(path) = &leaves_path { + if let Some(path) = &args.leaves { let text = std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?; let static_names: std::collections::BTreeSet> = lf.case.iter().map(|c| c.name.as_str()).collect(); @@ -155,9 +286,9 @@ fn lock(args: &[String]) -> anyhow::Result<()> { lf.validate()?; let toml = lf.to_toml()?; - if let Some(check) = check_path { + if let Some(check) = &args.check { let existing = Lockfile::from_toml( - &std::fs::read_to_string(&check).with_context(|| format!("reading {check}"))?, + &std::fs::read_to_string(check).with_context(|| format!("reading {check}"))?, )?; existing.validate()?; // Inventory equality: names + tags (artifact hash may differ), @@ -170,7 +301,7 @@ fn lock(args: &[String]) -> anyhow::Result<()> { .iter() .zip(&lf.generated) .all(|(a, b)| a.prefix == b.prefix && a.tags == b.tags); - let leaves_ok = leaves_path.is_none() + let leaves_ok = args.leaves.is_none() || existing .generated .iter() @@ -182,7 +313,7 @@ fn lock(args: &[String]) -> anyhow::Result<()> { (regenerate with `component-test lock` and review the diff)" ); } - if leaves_path.is_none() && existing.generated.iter().any(|g| !g.cases.is_empty()) { + if args.leaves.is_none() && existing.generated.iter().any(|g| !g.cases.is_empty()) { println!( "note: {check} pins generated leaves; static check only (pass --leaves \ for the full comparison)" @@ -192,9 +323,9 @@ fn lock(args: &[String]) -> anyhow::Result<()> { return Ok(()); } - match out_path { + match &args.output { Some(path) => { - std::fs::write(&path, toml)?; + std::fs::write(path, toml)?; println!("wrote {} cases to {path}", lf.case.len()); } None => print!("{toml}"), @@ -202,16 +333,7 @@ fn lock(args: &[String]) -> anyhow::Result<()> { Ok(()) } -fn fold(args: &[String]) -> anyhow::Result<()> { - match args.first().map(|s| s.as_str()) { - Some("-h" | "--help") => { - println!("{}", usage()); - return Ok(()); - } - Some(s) if s.starts_with('-') => bail!("unknown flag `{s}`\n{}", usage()), - _ => {} - } - +fn fold(args: &FoldArgs) -> anyhow::Result<()> { // Reading stdin to EOF is the first thing that happens; without a // pipe it blocks forever, so say so. if std::io::stdin().is_terminal() { @@ -223,7 +345,7 @@ fn fold(args: &[String]) -> anyhow::Result<()> { let mut stream = String::new(); std::io::stdin().read_to_string(&mut stream)?; - let lockfile = match args.first() { + let lockfile = match &args.lock { Some(path) => { let lf = Lockfile::from_toml( &std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?, @@ -304,46 +426,20 @@ fn fold(args: &[String]) -> anyhow::Result<()> { std::process::exit(if failed { 1 } else { 0 }); } -fn aggregate_cmd(args: &[String]) -> anyhow::Result<()> { - let mut lock_path = None; - let mut manifest_path = None; - let mut result_args: Vec<(String, String)> = Vec::new(); - let mut out_path = None; - let mut it = args.iter(); - while let Some(arg) = it.next() { - match arg.as_str() { - "--lock" => lock_path = Some(it.next().context("--lock needs a path")?.clone()), - "--manifest" => { - manifest_path = Some(it.next().context("--manifest needs a path")?.clone()) - } - "--results" => { - let spec = it.next().context("--results needs target=path.jsonl")?; - let (target, path) = spec - .split_once('=') - .context("--results argument must be target=path.jsonl")?; - result_args.push((target.to_string(), path.to_string())); - } - "-o" => out_path = Some(it.next().context("-o needs a path")?.clone()), - "-h" | "--help" => { - println!("{}", usage()); - return Ok(()); - } - other => bail!("unexpected argument `{other}`\n{}", usage()), - } - } - let lock_path = lock_path.context("missing --lock")?; - let manifest_path = manifest_path.context("missing --manifest")?; +fn aggregate_cmd(args: &AggregateArgs) -> anyhow::Result<()> { + let lock_path = &args.lock; + let manifest_path = &args.manifest; let lf = Lockfile::from_toml( - &std::fs::read_to_string(&lock_path).with_context(|| format!("reading {lock_path}"))?, + &std::fs::read_to_string(lock_path).with_context(|| format!("reading {lock_path}"))?, )?; let manifest = Manifest::from_toml( - &std::fs::read_to_string(&manifest_path) + &std::fs::read_to_string(manifest_path) .with_context(|| format!("reading {manifest_path}"))?, )?; let mut docs = Vec::new(); - for (target, path) in &result_args { + for (target, path) in &args.results { let stream = std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?; let doc = results::fold_jsonl( &stream, @@ -355,7 +451,7 @@ fn aggregate_cmd(args: &[String]) -> anyhow::Result<()> { let agg = aggregate::aggregate(&lf, &manifest, &docs); let md = matrix::render(&agg); - match &out_path { + match &args.output { Some(path) => { std::fs::write(path, &md).with_context(|| format!("writing {path}"))?; } @@ -388,7 +484,7 @@ fn aggregate_cmd(args: &[String]) -> anyhow::Result<()> { "{} targets, {total} results, {failures} failing{expected}, {} validation error(s){}", agg.targets.len(), agg.errors.len(), - out_path + args.output .as_deref() .map(|p| format!("; wrote {p}")) .unwrap_or_default() @@ -403,47 +499,15 @@ fn aggregate_cmd(args: &[String]) -> anyhow::Result<()> { /// *in* the output where the consuming UI wants them, and the exit /// code only reflects I/O; `fold`/`aggregate` remain the verdict /// authorities. -fn emit_cmd(args: &[String]) -> anyhow::Result<()> { - let format = match args.first().map(|s| s.as_str()) { - Some("junit") => "junit", - other => bail!( - "emit: unknown or missing format {other:?} (supported: junit)\n{}", - usage() - ), - }; - let mut lock_path = None; - let mut result_args: Vec<(String, String)> = Vec::new(); - let mut out_path = None; - let mut it = args[1..].iter(); - while let Some(arg) = it.next() { - match arg.as_str() { - "--lock" => lock_path = Some(it.next().context("--lock needs a path")?.clone()), - "--results" => { - let spec = it.next().context("--results needs target=path.jsonl")?; - let (target, path) = spec - .split_once('=') - .context("--results argument must be target=path.jsonl")?; - result_args.push((target.to_string(), path.to_string())); - } - "-o" => out_path = Some(it.next().context("-o needs a path")?.clone()), - "-h" | "--help" => { - println!("{}", usage()); - return Ok(()); - } - other => bail!("unexpected argument `{other}`\n{}", usage()), - } - } - if result_args.is_empty() { - bail!("emit {format}: at least one --results target=path.jsonl required"); - } - let lockfile = match &lock_path { +fn emit_cmd(args: &EmitArgs) -> anyhow::Result<()> { + let lockfile = match &args.lock { Some(p) => Some(Lockfile::from_toml( &std::fs::read_to_string(p).with_context(|| format!("reading {p}"))?, )?), None => None, }; let mut docs = Vec::new(); - for (target, path) in &result_args { + for (target, path) in &args.results { let stream = std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?; let doc = results::fold_jsonl( &stream, @@ -453,9 +517,73 @@ fn emit_cmd(args: &[String]) -> anyhow::Result<()> { docs.push((target.clone(), doc)); } let xml = junit::junit(&docs); - match &out_path { + match &args.output { Some(path) => std::fs::write(path, &xml).with_context(|| format!("writing {path}"))?, None => print!("{xml}"), } Ok(()) } + +/// `wizen`: pre-initialize a plain suite (#25/#85). The library entry +/// points live in component-test-runner (`wizen`/`wizen_with`) so +/// SUT-importing embedders reuse their own linker setup; this is the +/// convenience driver for suites with no imports beyond WASI + +/// test-context. +fn wizen_cmd(args: &WizenArgs) -> anyhow::Result<()> { + let wasm = compose::read_component(&args.suite)?; + let wizened = component_test_runner::wizen::wizen(&wasm, &args.env) + .map_err(anyhow::Error::from) + .with_context(|| format!("wizening {}", args.suite))?; + std::fs::write(&args.output, &wizened).with_context(|| format!("writing {}", args.output))?; + println!( + "wizened: {} -> {} bytes ({})", + wasm.len(), + wizened.len(), + args.output + ); + Ok(()) +} + +fn compose_runner_cmd(args: &ComposeRunnerArgs) -> anyhow::Result<()> { + let input = compose::read_component(&args.input)?; + let composed = compose_from_args(&input, args.provider.as_deref(), args.runner.as_deref())?; + std::fs::write(&args.output, &composed).with_context(|| format!("writing {}", args.output))?; + println!( + "wrote composed component to {} ({} bytes)", + args.output, + composed.len() + ); + Ok(()) +} + +fn run_cmd(args: &RunArgs) -> anyhow::Result<()> { + let input = compose::read_component(&args.input)?; + let composed = match compose::classify(&input)? { + compose::Input::Composed => input, + _ => compose_from_args(&input, args.provider.as_deref(), args.runner.as_deref())?, + }; + let mut env = args.env.clone(); + if args.jsonl { + env.push(("COMPONENT_TEST_JSONL".into(), "1".into())); + } + let code = run::execute(&composed, &env)?; + std::process::exit(code); +} + +/// Shared compose step: resolve the provider/runner overrides (embedded +/// defaults otherwise) and compose. +fn compose_from_args( + input: &[u8], + provider: Option<&str>, + runner: Option<&str>, +) -> anyhow::Result> { + let provider = match provider { + Some(path) => compose::read_component(path)?, + None => compose::EMBEDDED_PROVIDER.to_vec(), + }; + let runner = match runner { + Some(path) => compose::read_component(path)?, + None => compose::EMBEDDED_RUNNER.to_vec(), + }; + compose::compose(input, &provider, &runner) +} diff --git a/crates/component-test-cli/src/pins.rs b/crates/component-test-cli/src/pins.rs index a90661a..cef85e3 100644 --- a/crates/component-test-cli/src/pins.rs +++ b/crates/component-test-cli/src/pins.rs @@ -10,41 +10,76 @@ //! which broke every downstream grep-based gate at once — names do not. use anyhow::{bail, Context as _}; +use clap::{Args, Subcommand}; /// This workspace's crates, as named in a consumer's Cargo.lock. const CRATE_PREFIX: &str = "component-test"; /// The JS runner-core facade packaged at the repository root. const JS_PACKAGE: &str = "@polymorph/component-test-js"; -pub(crate) fn pins_cmd(args: &[String]) -> anyhow::Result<()> { - if args.first().map(|s| s.as_str()) == Some("bump") { - return bump_cmd(&args[1..]); - } - let mut cargo_lock = None; - let mut js_locks: Vec = Vec::new(); - let mut expect = None; - let mut it = args.iter(); - while let Some(arg) = it.next() { - match arg.as_str() { - "--cargo-lock" => { - cargo_lock = Some(it.next().context("--cargo-lock needs a path")?.clone()) - } - "--js-lock" => js_locks.push(it.next().context("--js-lock needs a path")?.clone()), - "--expect" => expect = Some(it.next().context("--expect needs a rev")?.clone()), - "-h" | "--help" => { - println!("{}", crate::usage()); - return Ok(()); - } - other => bail!("unexpected argument `{other}`\n{}", crate::usage()), - } +#[derive(Args)] +#[command(args_conflicts_with_subcommands = true, subcommand_negates_reqs = true)] +pub(crate) struct PinsArgs { + #[command(subcommand)] + command: Option, + /// Consumer Cargo.lock: every git-sourced component-test-* crate's + /// resolved commit is a pin + #[arg(long, required = true, value_name = "Cargo.lock")] + cargo_lock: Option, + /// npm/pnpm lockfile carrying the @polymorph/component-test-js pin + /// (repeatable) + #[arg(long = "js-lock", value_name = "lockfile")] + js_locks: Vec, + /// Additionally require the single rev to be exactly this + #[arg(long, value_name = "rev")] + expect: Option, +} + +#[derive(Subcommand)] +enum PinsCommand { + /// Rewrite the declared pins in place (the write half of the gate); + /// lockfile regeneration follow-ups are printed, not run + Bump(BumpArgs), +} + +#[derive(Args)] +struct BumpArgs { + /// The commit to pin (a full 40-hex hash) + #[arg(value_name = "rev", value_parser = parse_rev)] + rev: String, + /// Cargo.toml with git-pinned component-test-* dependency lines + /// (repeatable) + #[arg(long = "cargo-toml", value_name = "Cargo.toml")] + cargo_tomls: Vec, + /// package.json naming @polymorph/component-test-js (repeatable) + #[arg(long = "package-json", value_name = "package.json")] + package_jsons: Vec, + /// Workflow file with polymorph-test action refs (repeatable) + #[arg(long = "workflow", value_name = "ci.yml")] + workflows: Vec, +} + +fn parse_rev(s: &str) -> Result { + let rev = s.to_ascii_lowercase(); + if is_rev(&rev) { + Ok(rev) + } else { + Err("rev must be a full commit hash (40 hex)".to_string()) } - let cargo_lock = cargo_lock.context("missing --cargo-lock")?; +} + +pub(crate) fn pins_cmd(args: &PinsArgs) -> anyhow::Result<()> { + if let Some(PinsCommand::Bump(bump)) = &args.command { + return bump_cmd(bump); + } + // Required by clap whenever no subcommand was given. + let cargo_lock = args.cargo_lock.as_deref().expect("required by clap"); // Every place a pin was found: (source description, rev). let mut findings: Vec<(String, String)> = Vec::new(); let text = - std::fs::read_to_string(&cargo_lock).with_context(|| format!("reading {cargo_lock}"))?; + std::fs::read_to_string(cargo_lock).with_context(|| format!("reading {cargo_lock}"))?; let crates = cargo_lock_revs(&text)?; if crates.is_empty() { bail!( @@ -56,7 +91,7 @@ pub(crate) fn pins_cmd(args: &[String]) -> anyhow::Result<()> { findings.push((format!("{cargo_lock}: {name}"), rev)); } - for path in &js_locks { + for path in &args.js_locks { let text = std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?; let revs = js_lock_revs(&text); if revs.is_empty() { @@ -67,7 +102,7 @@ pub(crate) fn pins_cmd(args: &[String]) -> anyhow::Result<()> { } } - if let Some(rev) = expect { + if let Some(rev) = &args.expect { findings.push(("--expect".into(), rev.to_ascii_lowercase())); } @@ -94,41 +129,19 @@ pub(crate) fn pins_cmd(args: &[String]) -> anyhow::Result<()> { /// regenerating them belongs to cargo/npm/pnpm, so the follow-up /// commands are printed instead of run, and the check half verifies /// the result. -fn bump_cmd(args: &[String]) -> anyhow::Result<()> { - let mut rev = None; - let mut cargo_tomls: Vec = Vec::new(); - let mut package_jsons: Vec = Vec::new(); - let mut workflows: Vec = Vec::new(); - let mut it = args.iter(); - while let Some(arg) = it.next() { - match arg.as_str() { - "--cargo-toml" => { - cargo_tomls.push(it.next().context("--cargo-toml needs a path")?.clone()) - } - "--package-json" => { - package_jsons.push(it.next().context("--package-json needs a path")?.clone()) - } - "--workflow" => workflows.push(it.next().context("--workflow needs a path")?.clone()), - "-h" | "--help" => { - println!("{}", crate::usage()); - return Ok(()); - } - other if rev.is_none() && is_rev(&other.to_ascii_lowercase()) => { - rev = Some(other.to_ascii_lowercase()) - } - other => bail!("unexpected argument `{other}` (rev must be 40 lowercase hex)"), - } - } - let rev = rev.context("missing (40-hex commit)")?; +fn bump_cmd(args: &BumpArgs) -> anyhow::Result<()> { + let rev = &args.rev; + let (cargo_tomls, package_jsons, workflows) = + (&args.cargo_tomls, &args.package_jsons, &args.workflows); if cargo_tomls.is_empty() && package_jsons.is_empty() && workflows.is_empty() { bail!("nothing to bump: name at least one --cargo-toml/--package-json/--workflow"); } let mut cargo_crates: std::collections::BTreeSet = std::collections::BTreeSet::new(); let mut touched_js = false; - for path in &cargo_tomls { + for path in cargo_tomls { let text = std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?; - let (new, crates) = bump_cargo_toml(&text, &rev); + let (new, crates) = bump_cargo_toml(&text, rev); if crates.is_empty() { bail!("{path}: no git-pinned `{CRATE_PREFIX}-*` dependency lines found"); } @@ -136,9 +149,9 @@ fn bump_cmd(args: &[String]) -> anyhow::Result<()> { cargo_crates.extend(crates); std::fs::write(path, new).with_context(|| format!("writing {path}"))?; } - for path in &package_jsons { + for path in package_jsons { let text = std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?; - let (new, n) = bump_named_lines(&text, JS_PACKAGE, &rev); + let (new, n) = bump_named_lines(&text, JS_PACKAGE, rev); if n == 0 { bail!("{path}: no rev-pinned {JS_PACKAGE} line found"); } @@ -146,9 +159,9 @@ fn bump_cmd(args: &[String]) -> anyhow::Result<()> { touched_js = true; std::fs::write(path, new).with_context(|| format!("writing {path}"))?; } - for path in &workflows { + for path in workflows { let text = std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?; - let (new, n) = bump_action_refs(&text, &rev); + let (new, n) = bump_action_refs(&text, rev); if n == 0 { bail!("{path}: no `{ACTION_PATH_ANCHOR}@` refs found"); } diff --git a/crates/component-test-cli/src/run.rs b/crates/component-test-cli/src/run.rs new file mode 100644 index 0000000..116015b --- /dev/null +++ b/crates/component-test-cli/src/run.rs @@ -0,0 +1,81 @@ +//! `run`: execute a composed wasi:cli component — the minimal embedding +//! equivalent of `wasmtime run -W component-model-async -S p3`, so +//! `component-test run` is compose-runner + wasmtime in one step with +//! no external tools. +//! +//! Scope is exactly "sufficient for the runner core": WASI p2 (the +//! suite's and provider's std imports) + WASI p3 (the runner core's +//! `wasi:cli/stdout@0.3.0` and its `wasi:cli/run@0.3.0` entry point), +//! stdio inherited, environment explicit (wasmtime run's behavior: the +//! host env is not inherited). Known divergences from `wasmtime run`, +//! irrelevant to the runner core: stdin is closed, and no argv is +//! passed. No epoch budgets, no pooling — hangs and +//! traps are the wasmtime CLI's problem too; the serious execution +//! policy lives in the host-embed runner. + +use anyhow::Result; +use wasmtime::component::{Component, Linker}; +use wasmtime::error::Context as _; +use wasmtime::{Config, Engine, Store}; +use wasmtime_wasi::p3::bindings::Command; +use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; + +struct Ctx { + wasi: WasiCtx, + table: ResourceTable, +} + +impl WasiView for Ctx { + fn ctx(&mut self) -> WasiCtxView<'_> { + WasiCtxView { + ctx: &mut self.wasi, + table: &mut self.table, + } + } +} + +/// Execute `composed` (exporting `wasi:cli/run@0.3.0`) and return the +/// process exit code: the guest's `run` result (0/1), or its explicit +/// exit status. Traps and instantiation failures surface as errors. +pub fn execute(composed: &[u8], env: &[(String, String)]) -> Result { + let mut config = Config::new(); + config.wasm_component_model(true); + config.wasm_component_model_async(true); + let engine = Engine::new(&config)?; + let component = Component::new(&engine, composed).context("loading composed component")?; + + let mut linker: Linker = Linker::new(&engine); + wasmtime_wasi::p2::add_to_linker_async(&mut linker)?; + wasmtime_wasi::p3::add_to_linker(&mut linker)?; + + let mut wasi = WasiCtxBuilder::new(); + wasi.inherit_stdout().inherit_stderr(); + for (k, v) in env { + wasi.env(k, v); + } + let mut store = Store::new( + &engine, + Ctx { + wasi: wasi.build(), + table: ResourceTable::new(), + }, + ); + + let result: wasmtime::Result> = wasmtime_wasi::runtime::in_tokio(async { + let command = Command::instantiate_async(&mut store, &component, &linker).await?; + store + .run_concurrent(async move |store| command.wasi_cli_run().call_run(store).await) + .await? + }); + + match result { + Ok(Ok(())) => Ok(0), + Ok(Err(())) => Ok(1), + // An explicit guest exit unwinds as an error carrying the + // status (both the p2 and p3 exit interfaces). + Err(e) => match e.downcast_ref::() { + Some(exit) => Ok(exit.0), + None => Err(e.context("executing composed component").into()), + }, + } +} diff --git a/crates/component-test-cli/tests/cli.rs b/crates/component-test-cli/tests/cli.rs index e1c2b4a..5d664c7 100644 --- a/crates/component-test-cli/tests/cli.rs +++ b/crates/component-test-cli/tests/cli.rs @@ -90,7 +90,7 @@ fn fold_with(lock: &Path, jsonl: &str) -> Output { fn usage_and_help() { let out = run(&[], None); assert_eq!(out.code, 2); - assert!(out.stderr.contains("usage:"), "{}", out.stderr); + assert!(out.stderr.contains("Usage:"), "{}", out.stderr); for args in [ &["--help"][..], @@ -98,22 +98,26 @@ fn usage_and_help() { &["fold", "--help"], &["aggregate", "--help"], &["pins", "--help"], + &["wizen", "--help"], + &["compose-runner", "--help"], + &["run", "--help"], ] { let out = run(args, None); assert_eq!(out.code, 0, "{args:?}"); - assert!(out.stdout.contains("usage:"), "{args:?}"); + assert!(out.stdout.contains("Usage:"), "{args:?}"); } let out = run(&["frobnicate"], None); assert_eq!(out.code, 2); let out = run(&["lock", "--bogus"], None); - assert_eq!(out.code, 1); - assert!( - out.stderr.contains("unknown flag `--bogus`"), - "{}", - out.stderr - ); + assert_eq!(out.code, 2); + assert!(out.stderr.contains("--bogus"), "{}", out.stderr); + + // `wizer` is an alias for `wizen`. + let out = run(&["wizer", "--help"], None); + assert_eq!(out.code, 0); + assert!(out.stdout.contains("Usage:"), "{}", out.stdout); } #[test] @@ -517,9 +521,10 @@ fn aggregate_applies_applicability_for_unscheduled_streams() { #[test] fn aggregate_missing_args() { + // Required-argument enforcement is clap's: exit 2, naming the flag. let out = run(&["aggregate", "--manifest", "x.toml"], None); - assert_eq!(out.code, 1); - assert!(out.stderr.contains("missing --lock"), "{}", out.stderr); + assert_eq!(out.code, 2); + assert!(out.stderr.contains("--lock"), "{}", out.stderr); } const AGG_XFAIL_MANIFEST: &str = r#" @@ -735,6 +740,123 @@ fn lock_emits_inventory_to_stdout() { assert!(out.stdout.contains("artifact_sha256"), "{}", out.stdout); } +// ---------------------------------- wizen / compose-runner / run +// (need built wasm) + +fn target_tmp(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(name) +} + +/// Wizening must leave the inventory intact: the tags section survives +/// the rewrite (finding #22), so the committed lockfile still checks +/// against the wizened artifact. +#[test] +#[ignore = "needs built components: run via `just test-wasm`"] +fn wizen_preserves_inventory() { + let wasm = suite_artifact("sample_suite"); + let wizened = target_tmp("sample-wizened.wasm"); + let out = run( + &[ + "wizen", + wasm.to_str().unwrap(), + "-o", + wizened.to_str().unwrap(), + ], + None, + ); + assert_eq!(out.code, 0, "stderr: {}", out.stderr); + assert!(out.stdout.contains("wizened:"), "{}", out.stdout); + + let lock = workspace_root().join("components/sample-suite/tests.lock"); + let out = run( + &[ + "lock", + wizened.to_str().unwrap(), + "--check", + lock.to_str().unwrap(), + ], + None, + ); + assert_eq!(out.code, 0, "stderr: {}", out.stderr); + assert!(out.stdout.contains("ok: 3 cases match"), "{}", out.stdout); +} + +/// compose-runner emits a component; run executes the same composition +/// in memory and reproduces the composed runner's JSONL golden +/// byte-for-byte (the same bytes `just verify-compose` pins for the +/// wac-composed path — one wire format, however composed). +#[test] +#[ignore = "needs built components: run via `just test-wasm`"] +fn compose_runner_and_run_reproduce_the_composed_golden() { + let wasm = suite_artifact("sample_suite"); + let composed = target_tmp("sample-composed.wasm"); + let out = run( + &[ + "compose-runner", + wasm.to_str().unwrap(), + "-o", + composed.to_str().unwrap(), + ], + None, + ); + assert_eq!(out.code, 0, "stderr: {}", out.stderr); + let bytes = std::fs::read(&composed).unwrap(); + assert!(bytes.starts_with(b"\0asm"), "composed output is not wasm"); + + let golden = workspace_root().join("expected/verify-compose-sample.jsonl"); + let golden = std::fs::read_to_string(golden).unwrap(); + // The sample suite has a deliberate failure: exit 1, verdicts intact. + for input in [wasm.as_path(), composed.as_path()] { + let out = run(&["run", "--jsonl", input.to_str().unwrap()], None); + assert_eq!(out.code, 1, "stderr: {}", out.stderr); + assert_eq!(out.stdout, golden, "run {input:?} diverged from golden"); + } +} + +/// Inputs that cannot compose are rejected by name, not by wac +/// internals: an already-composed component, and a non-suite component. +#[test] +#[ignore = "needs built components: run via `just test-wasm`"] +fn compose_runner_rejects_non_suites() { + let wasm = suite_artifact("sample_suite"); + let composed = target_tmp("sample-composed-reject.wasm"); + let out = run( + &[ + "compose-runner", + wasm.to_str().unwrap(), + "-o", + composed.to_str().unwrap(), + ], + None, + ); + assert_eq!(out.code, 0, "stderr: {}", out.stderr); + + let out = run( + &[ + "compose-runner", + composed.to_str().unwrap(), + "-o", + target_tmp("never-written.wasm").to_str().unwrap(), + ], + None, + ); + assert_eq!(out.code, 1); + assert!(out.stderr.contains("already composed"), "{}", out.stderr); + + let provider = suite_artifact("provider"); + let out = run( + &[ + "compose-runner", + provider.to_str().unwrap(), + "-o", + target_tmp("never-written2.wasm").to_str().unwrap(), + ], + None, + ); + assert_eq!(out.code, 1); + assert!(out.stderr.contains("not a test suite"), "{}", out.stderr); +} + // ------------------------------------------------------ pins const PIN_REV: &str = "b80c13be7b9fecfed8ec10a91d23d8cf8349defe"; diff --git a/crates/component-test-runner/Cargo.toml b/crates/component-test-runner/Cargo.toml index edf087d..9c0069e 100644 --- a/crates/component-test-runner/Cargo.toml +++ b/crates/component-test-runner/Cargo.toml @@ -15,10 +15,11 @@ futures = { workspace = true } # on the wasmtime-wasi runtime (which powers wasi clock pollables, so # time is enabled there). tokio = { workspace = true, features = ["time"] } -wasmtime = { version = "47", features = ["component-model-async"] } -wasmtime-wasi = "47" -# For the feature-gated `wizer-preinit` bin only (component-level -# pre-initialization experiments, #25 / findings.md #22). +wasmtime = { workspace = true } +wasmtime-wasi = { workspace = true } +# For the feature-gated `wizen` module (component-level +# pre-initialization, #25 / findings.md #22; `component-test wizen` +# is the plain-suite driver). wasmtime-wizer = { version = "47", features = ["wasmtime", "component-model"], optional = true } [features] @@ -32,10 +33,5 @@ path = "src/bin/ct-runner.rs" name = "bench-mint" path = "src/bin/bench-mint.rs" -[[bin]] -name = "wizer-preinit" -path = "src/bin/wizer-preinit.rs" -required-features = ["wizer"] - [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/crates/component-test-runner/src/bin/wizer-preinit.rs b/crates/component-test-runner/src/bin/wizer-preinit.rs deleted file mode 100644 index 415dfe6..0000000 --- a/crates/component-test-runner/src/bin/wizer-preinit.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! Component-level wizer pre-initialization driver (`wizer-preinit`, -//! feature `wizer`): snapshots a suite component after running its own -//! `tests.all()`, so every fresh instance is born with the case table -//! built and `all()` costs only mint+lift (#25). -//! -//! Works on any unmodified suite artifact: the init function is the -//! contract's own `all()` — named in the version-last invoke syntax -//! `polymorph:test/tests.all@0.1.0()` (the parenthesized wave-call -//! form; the bare item-name form requires a `[] -> []` signature). -//! The handles `all()` returns are per-call state; only the guest heap -//! (the built registry) lands in the snapshot, at no measurable size -//! cost over a dedicated no-op init export (findings.md #22). -//! -//! Exists because the `wasmtime wizer` CLI cannot wizen a *suite*: -//! the suite world imports `test-context`, whose `context` resource -//! cannot be synthesized by unknown-import stubbing ("resource -//! implementation is missing"), and composed bundles hit "nested -//! components with modules not currently supported". Driving -//! wasmtime-wizer as a library with our own linker — WASI plus a host -//! `context` resource whose methods init never calls — sidesteps both. -//! -//! cargo run --release -p component-test-runner --features wizer \ -//! --bin wizer-preinit -- [ENV=VAL ...] - -use wasmtime::component::{Linker, Resource, ResourceType}; -use wasmtime::error::format_err; -use wasmtime::{Config, Engine, Result, Store}; -use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; - -struct Ctx { - wasi: WasiCtx, - table: ResourceTable, -} - -impl WasiView for Ctx { - fn ctx(&mut self) -> WasiCtxView<'_> { - WasiCtxView { - ctx: &mut self.wasi, - table: &mut self.table, - } - } -} - -struct HostContext; - -fn main() -> Result<()> { - let mut args = std::env::args().skip(1); - let usage = "usage: wizer-preinit [ENV=VAL ...]"; - let input = args.next().ok_or_else(|| format_err!("{usage}"))?; - let output = args.next().ok_or_else(|| format_err!("{usage}"))?; - let env: Vec<(String, String)> = args - .map(|kv| { - kv.split_once('=') - .map(|(k, v)| (k.to_string(), v.to_string())) - .ok_or_else(|| format_err!("bad env pair `{kv}` (want NAME=VAL)\n{usage}")) - }) - .collect::>()?; - - let wasm = std::fs::read(&input)?; - - let mut config = Config::new(); - config.wasm_component_model(true); - config.wasm_component_model_async(true); - let engine = Engine::new(&config)?; - - let mut linker: Linker = Linker::new(&engine); - wasmtime_wasi::p2::add_to_linker_async(&mut linker)?; - let mut ctx_instance = linker.instance("polymorph:test/test-context@0.1.0")?; - ctx_instance.resource( - "context", - ResourceType::host::(), - |_, _| Ok(()), - )?; - // Present so the component type-checks; never called during init - // (registry builds are pure — anything else is a suite bug this - // driver would surface as a snapshot-time diagnostic call). - ctx_instance.func_wrap_concurrent( - "[method]context.diagnostic", - |_accessor, (_this, _msg): (Resource, String)| Box::pin(async move { Ok(()) }), - )?; - - let mut wasi = WasiCtxBuilder::new(); - wasi.inherit_stderr(); - for (k, v) in &env { - wasi.env(k, v); - } - let mut store = Store::new( - &engine, - Ctx { - wasi: wasi.build(), - table: ResourceTable::new(), - }, - ); - - let mut wizer = wasmtime_wizer::Wizer::new(); - // The contract's own enumeration is the init function (version-last - // invoke syntax; the parens select the wave-call path, which - // permits results). - wizer.init_func("polymorph:test/tests.all@0.1.0()"); - // Mandatory: "stripping" the init function would remove the - // `tests.all` export itself. (Stripping a dedicated init export is - // also currently broken upstream — dangling core-instance export - // reference, bytecodealliance/wasmtime#13168.) - wizer.keep_init_func(true); - let (wizened, _rets) = wasmtime_wasi::runtime::in_tokio(wizer.run_component( - &mut store, - &wasm, - async |store: &mut Store, component| linker.instantiate_async(store, component).await, - ))?; - - std::fs::write(&output, &wizened)?; - eprintln!( - "wizened: {} -> {} bytes ({output})", - wasm.len(), - wizened.len(), - ); - Ok(()) -} diff --git a/crates/component-test-runner/src/lib.rs b/crates/component-test-runner/src/lib.rs index df138bb..d9e4a42 100644 --- a/crates/component-test-runner/src/lib.rs +++ b/crates/component-test-runner/src/lib.rs @@ -12,6 +12,9 @@ use std::path::Path; +#[cfg(feature = "wizer")] +pub mod wizen; + use component_test_core::{Provenance, Tags}; use component_test_formats::results::{ CaseResult, Envelope, Event, RunInfo, Status, SuiteInfo, RESULTS_VERSION, TERMINATOR, @@ -590,6 +593,10 @@ impl Runner { /// unlimited (single instance for the whole run; cheap suites); /// K = fresh instance every K cases. A trap always abandons the /// current instance regardless. + /// For K=1 at scale, wizening the suite removes most of the + /// per-instance enumeration cost instead of relaxing isolation + /// ([`wizen`](crate::wizen), `component-test wizen`; findings + /// 22–24). /// `jobs`: worker parallelism. Workers share the compiled engine, /// own their stores, and run the modulo stripe /// `runnable-index % jobs == worker` (expensive cases cluster, so diff --git a/crates/component-test-runner/src/wizen.rs b/crates/component-test-runner/src/wizen.rs new file mode 100644 index 0000000..32ef569 --- /dev/null +++ b/crates/component-test-runner/src/wizen.rs @@ -0,0 +1,138 @@ +//! Component-level wizer pre-initialization (#25, findings.md 22–24): +//! snapshot a suite component after running its own `tests.all()`, so +//! every fresh instance is born with the case table built and `all()` +//! costs only mint+lift. +//! +//! Works on any unmodified suite artifact: the init function is the +//! contract's own `all()` — named in the version-last invoke syntax +//! `polymorph:test/tests.all@0.1.0()` (the parenthesized wave-call +//! form; the bare item-name form requires a `[] -> []` signature). +//! The handles `all()` returns are per-call state; only the guest heap +//! (the built registry) lands in the snapshot, at no measurable size +//! cost over a dedicated no-op init export (findings.md #22). Custom +//! sections survive, so tags scheduling, drift checks, and +//! `lock --check` all work unchanged on the wizened artifact. +//! +//! Exists because the `wasmtime wizer` CLI cannot wizen a *suite*: +//! the suite world imports `test-context`, whose `context` resource +//! cannot be synthesized by unknown-import stubbing ("resource +//! implementation is missing"), and composed bundles hit "nested +//! components with modules not currently supported". Driving +//! wasmtime-wizer as a library with our own linker — WASI plus a host +//! `context` resource whose methods init never calls — sidesteps both. +//! +//! Two entry points, mirroring [`Runner::new`]/[`Runner::with_data`]: +//! [`wizen`] for plain suites (WASI + `test-context` only), and +//! [`wizen_with`] for suites with SUT imports — component +//! instantiation is eager, so a SUT-importing suite fails to wizen +//! without its host module even though init never calls it; reuse the +//! same linker setup the embedding runner already has. +//! +//! Caveat the caller must own: the snapshot freezes whatever registry +//! construction observed (env, entropy, clocks — baked at wizen-time +//! values for every future instance). For contract-conforming suites +//! that is #25's determinism *feature*, but an env-parameterized +//! registry stops being parameterizable; run the wizened artifact +//! everywhere downstream instead of mixing artifacts. +//! +//! [`Runner::new`]: crate::Runner::new +//! [`Runner::with_data`]: crate::Runner::with_data + +use wasmtime::component::{Linker, Resource, ResourceType}; +use wasmtime::{Config, Engine, Result, Store}; +use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView}; + +use crate::CONTEXT_INSTANCE; + +/// The init function: the contract's own enumeration, in version-last +/// invoke syntax (the parens select the wave-call path, which permits +/// results — finding #22). +const INIT_FUNC: &str = "polymorph:test/tests.all@0.1.0()"; + +struct Ctx { + wasi: WasiCtx, + table: ResourceTable, +} + +impl WasiView for Ctx { + fn ctx(&mut self) -> WasiCtxView<'_> { + WasiCtxView { + ctx: &mut self.wasi, + table: &mut self.table, + } + } +} + +struct HostContext; + +/// Wizen a plain suite (WASI + `test-context` imports only), returning +/// the pre-initialized component bytes. `env` is the environment +/// visible during initialization — and therefore baked into the +/// snapshot for every future instance. +pub fn wizen(wasm: &[u8], env: &[(String, String)]) -> Result> { + wizen_with( + wasm, + || { + let mut wasi = WasiCtxBuilder::new(); + wasi.inherit_stderr(); + for (k, v) in env { + wasi.env(k, v); + } + Ctx { + wasi: wasi.build(), + table: ResourceTable::new(), + } + }, + |_| Ok(()), + ) +} + +/// Wizen a suite with SUT imports: `make_data` builds the store data +/// for the single init store; `configure_linker` wires the SUT's +/// `add_to_linker` (instantiation is eager, so every import must be +/// satisfiable even though init never calls the SUT). WASI and the +/// host `context` resource are provided here, exactly as +/// [`Runner::with_data`](crate::Runner::with_data) does for execution. +pub fn wizen_with( + wasm: &[u8], + make_data: impl FnOnce() -> D, + configure_linker: impl FnOnce(&mut Linker) -> Result<()>, +) -> Result> { + let mut config = Config::new(); + config.wasm_component_model(true); + config.wasm_component_model_async(true); + let engine = Engine::new(&config)?; + + let mut linker: Linker = Linker::new(&engine); + wasmtime_wasi::p2::add_to_linker_async(&mut linker)?; + configure_linker(&mut linker)?; + let mut ctx_instance = linker.instance(CONTEXT_INSTANCE)?; + ctx_instance.resource( + "context", + ResourceType::host::(), + |_, _| Ok(()), + )?; + // Present so the component type-checks; never called during init + // (registry builds are pure — anything else is a suite bug this + // driver would surface as a snapshot-time diagnostic call). + ctx_instance.func_wrap_concurrent( + "[method]context.diagnostic", + |_accessor, (_this, _msg): (Resource, String)| Box::pin(async move { Ok(()) }), + )?; + + let mut store = Store::new(&engine, make_data()); + + let mut wizer = wasmtime_wizer::Wizer::new(); + wizer.init_func(INIT_FUNC); + // Mandatory: "stripping" the init function would remove the + // `tests.all` export itself. (Stripping a dedicated init export is + // also currently broken upstream — dangling core-instance export + // reference, bytecodealliance/wasmtime#13168.) + wizer.keep_init_func(true); + let (wizened, _rets) = wasmtime_wasi::runtime::in_tokio(wizer.run_component( + &mut store, + wasm, + async |store: &mut Store, component| linker.instantiate_async(store, component).await, + ))?; + Ok(wizened) +} diff --git a/docs/findings.md b/docs/findings.md index 18b7f67..f02011c 100644 --- a/docs/findings.md +++ b/docs/findings.md @@ -178,7 +178,9 @@ x86_64 Linux), wasmtime 47.0.3 / deltic pre-83fff30 / Node 24. Follow-up to the bench above: pre-build the registry at build time so fresh instances are born initialized. `wasmtime-wizer` 47 as a -library; drivers: `wizer-preinit` bin (feature `wizer`) over the +library; drivers: originally the `wizer-preinit` bin, since #85 the +`wizen` module in component-test-runner (feature `wizer`) behind +`component-test wizen`; measured over the bench-suite artifact built with its `wizer-init` feature. 22. **Component-level pre-init works today (wasmtime-wizer 47) — #25's diff --git a/examples/compose/README.md b/examples/compose/README.md index daeae6c..e0a1471 100644 --- a/examples/compose/README.md +++ b/examples/compose/README.md @@ -1,5 +1,12 @@ # Composition examples +`component-test compose-runner suite.wasm -o composed.wasm` (and +`component-test run suite.wasm`, which composes in memory and +executes) wrap this entire flow with an embedded reference provider +and runner core. The raw commands below remain the recipe for custom +topologies — SUT providers, negative composition gates, debugging — +and are what the wrapper mirrors. + Two-step (canonical): bundle the suite with the reference context provider, then plug the bundle into a runner core. diff --git a/justfile b/justfile index d6c92e4..f38e0c3 100644 --- a/justfile +++ b/justfile @@ -19,8 +19,7 @@ default: ci: (gha::host-checks) (gha::verify) # Everything: host tests, component builds, all verification paths. -all: build test test-wasm lock-check verify-embed verify-compose verify-deltic verify-pipeline verify-aggregate verify-viewer verify-imports verify-emit - +all: build test test-wasm lock-check verify-embed verify-compose verify-cli verify-deltic verify-pipeline verify-aggregate verify-viewer verify-imports verify-emit # The fast pre-commit checks: formatting, clippy, host tests, WIT # validation. The CI job of the same name runs the identical set # through gha::host-checks. @@ -99,6 +98,42 @@ verify-compose: build diff -u expected/verify-pipeline-sample-fold.txt "$tmp/fold.txt" echo "verify-compose: output matches expected/ (incl. JSONL + cross-runner fold)" +# Path 2b: the CLI's composition/execution subcommands (#85). +# compose-runner (embedded provider + runner core) must reproduce Path +# 2's goldens under the wasmtime CLI; run is the same composition under +# the embedded wasmtime (human + JSONL legs); wizen pre-initializes +# with inventory, scheduling, and runnability intact (findings 22–24). +# This is also the embedded artifacts' freshness gate: behavioral drift +# between components/{runner-cli,provider} and the committed +# crates/component-test-cli/embedded/ copies fails the diffs +# (byte-comparing builds across environments is off the table, #44) — +# after changing those components run `just embed-update` and commit. +verify-cli: build + #!/usr/bin/env bash + set -euo pipefail + cli() { cargo run -q -p component-test-cli -- "$@"; } + ct() { cargo run -q -p component-test-runner --bin ct-runner -- "$@"; } + tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT + cli compose-runner {{release_dir}}/sample_suite.wasm -o "$tmp/composed.wasm" + out=$(wasmtime run {{wasmtime_flags}} "$tmp/composed.wasm") && code=0 || code=$? + test "$code" -eq 1 + diff -u expected/verify-run-sample.txt <(printf '%s\n' "$out") + out=$(cli run {{release_dir}}/sample_suite.wasm) && code=0 || code=$? + test "$code" -eq 1 + diff -u expected/verify-run-sample.txt <(printf '%s\n' "$out") + jsonl=$(cli run --jsonl {{release_dir}}/sample_suite.wasm) && code=0 || code=$? + test "$code" -eq 1 + diff -u expected/verify-compose-sample.jsonl <(printf '%s\n' "$jsonl") + cli wizen {{release_dir}}/sample_suite.wasm -o "$tmp/wizened.wasm" + cli lock "$tmp/wizened.wasm" --check components/sample-suite/tests.lock + out=$(ct "$tmp/wizened.wasm") && code=0 || code=$? + test "$code" -eq 1 + diff -u expected/verify-embed-sample.txt <(printf '%s\n' "$out") + out=$(cli run "$tmp/wizened.wasm") && code=0 || code=$? + test "$code" -eq 1 + diff -u expected/verify-run-sample.txt <(printf '%s\n' "$out") + echo "verify-cli: output matches expected/ (compose-runner, run, wizen)" + # The one-version-everywhere gate for the deltic pin (successor to the # retired fetch script's `assertPinConsistency`): every jsr:@deltic/* # specifier in every deno.json, and every @deltic package the lock @@ -386,6 +421,18 @@ lock-update: build cargo run -q -p component-test-cli -- lock \ {{release_dir}}/fixture_suite.wasm -o components/fixture-suite/tests.lock +# Regenerate the components baked into the CLI (compose-runner/run +# defaults) after changing components/runner-cli or components/provider, +# and commit the diff. Size-optimized `embed` profile; freshness is +# gated behaviorally by verify-cli (no byte comparison — builds are not +# reproducible across environments, #44). +embed-update: + cargo build --target {{wasm_target}} --profile embed -p runner-cli -p provider + cp target/{{wasm_target}}/embed/runner_cli.wasm \ + crates/component-test-cli/embedded/runner-cli.wasm + cp target/{{wasm_target}}/embed/provider.wasm \ + crates/component-test-cli/embedded/provider.wasm + # --- WIT --------------------------------------------------------------- # Component WIT dirs are symlinks into the canonical copies (wit/ and