diff --git a/.gitignore b/.gitignore index d03acac..5db6562 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,5 @@ target/ *.wasm -# The components baked into the CLI are committed (regenerate with -# `just embed-update`; `just verify-cli` gates their behavior). -!crates/component-test-cli/embedded/*.wasm node_modules/ # Prebuilt CLI/runner binaries for downstream consumers (their build diff --git a/AGENTS.md b/AGENTS.md index 4b60f76..2a9b4fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,9 +55,10 @@ crates/ host-side Rust (tested natively at root) pins, wizen, compose-runner, run (the last three embed wasmtime/wac-graph plus size-optimized runner-cli + provider builds - from embedded/ — regenerate those with - `just embed-update`, commit the diff; - `just verify-cli` gates their behavior) + compiled from source by build.rs — always + current, needs the wasm32-wasip2 target; + `--no-default-features` for a host-only + build; `just verify-cli` gates behavior) component-test-runner wasmtime host-embed runner (`ct-runner` bin) components/ guest components (build with --target wasm32-wasip2) provider reference context provider diff --git a/Cargo.toml b/Cargo.toml index da9d7d7..0350530 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,8 +68,9 @@ wac-graph = "0.10" opt-level = "s" strip = true -# The components baked into the CLI (`just embed-update`): size over -# speed — they ship inside every `component-test` binary. +# The components the CLI builds into itself (crates/component-test-cli +# build.rs): size over speed — they ship inside every `component-test` +# binary. [profile.embed] inherits = "release" opt-level = "z" diff --git a/actions/README.md b/actions/README.md index 8d15336..7bec4d5 100644 --- a/actions/README.md +++ b/actions/README.md @@ -36,8 +36,15 @@ The installed CLI carries the whole composition/execution surface — `wizen` (pre-initialize large suites, #25/#85), `compose-runner`, and `run` (embedded reference provider, runner core, and wasmtime) — so a consumer pipeline that only needs the composed path installs no wac -and no wasmtime. Wizening in CI is one line after setup, on the built -artifact: +and no wasmtime. The embedded components are built from source by the +CLI's build script, so installing with default features needs the +`wasm32-wasip2` target (already in the toolchain of every repo that +builds suites; measured cost ~zero — the wasm builds in the shadow of +the CLI's own compile). A host-only CLI for wasm-free contexts +installs with `--no-default-features`: `compose-runner`/`run` then +require explicit `--runner`/`--provider`, everything else is +unchanged (the aggregate action's fallback install below uses exactly +this). Wizening in CI is one line after setup, on the built artifact: ```sh target/ct-tools/bin/component-test wizen suite.wasm -o suite.wasm diff --git a/actions/aggregate/action.yml b/actions/aggregate/action.yml index e25a3a7..e6e3cc0 100644 --- a/actions/aggregate/action.yml +++ b/actions/aggregate/action.yml @@ -84,7 +84,11 @@ runs: exit 0 fi root="$RUNNER_TEMP/component-test-cli" - cargo install -q --locked --root "$root" \ + # Host-only build: aggregation never composes, and the slim + # feature set keeps this fallback runnable on runners without + # the wasm32-wasip2 target (the default features build the + # embedded components from source at compile time). + cargo install -q --locked --root "$root" --no-default-features \ --git https://github.com/polymorph-components/polymorph-test \ --rev "${ACTION_REF:-main}" component-test-cli echo "CT_CLI=$root/bin/component-test" >> "$GITHUB_ENV" diff --git a/crates/component-test-cli/Cargo.toml b/crates/component-test-cli/Cargo.toml index 1f2307a..d55448a 100644 --- a/crates/component-test-cli/Cargo.toml +++ b/crates/component-test-cli/Cargo.toml @@ -10,6 +10,15 @@ repository.workspace = true name = "component-test" path = "src/main.rs" +[features] +default = ["embedded-components"] +# Build components/runner-cli and components/provider from source at +# compile time (wasm32-wasip2, `embed` profile) as the +# compose-runner/run defaults — see build.rs. Disable for a host-only +# CLI (reporting/aggregation consumers, no wasm target required); +# compose-runner/run then require explicit --runner/--provider. +embedded-components = [] + [dependencies] component-test-core = { workspace = true } component-test-formats = { workspace = true } diff --git a/crates/component-test-cli/build.rs b/crates/component-test-cli/build.rs new file mode 100644 index 0000000..37db3a8 --- /dev/null +++ b/crates/component-test-cli/build.rs @@ -0,0 +1,128 @@ +//! Builds the components baked into the CLI — `components/runner-cli` +//! and `components/provider`, the `compose-runner`/`run` defaults — +//! from source at compile time (feature `embedded-components`, +//! default), so the embedded bytes can never go stale against their +//! sources (#88; previously they were committed artifacts refreshed by +//! hand, gated only through sample-suite-visible behavior). +//! +//! Mechanics, each load-bearing: +//! +//! - **Nested cargo, separate target dir**: the inner build writes to +//! `$OUT_DIR/embed-target` — sharing the outer target dir would +//! deadlock on cargo's build-dir lock. The inner graph is small +//! (wit-bindgen runtime, serde_json, the results crate) and cargo's +//! own fingerprinting caches it there across rebuilds. +//! - **Broad rerun-if-changed** (whole `components/`, `crates/`, +//! `wit/` trees): over-firing costs a sub-second inner no-op, while +//! a curated file list would silently go stale when the dependency +//! closure grows — the exact failure mode this build script exists +//! to delete. The inner cargo rebuilds precisely what changed. +//! - **Curated environment**: the outer build's host-targeted +//! compilation vars (`RUSTFLAGS`, `RUSTC`, cargo's own `CARGO_*` +//! bookkeeping) must not leak into the wasm build. `$CARGO` is used +//! as the binary (the outer build's resolved toolchain, bypassing +//! rustup's cwd-based resolution); `CARGO_HOME` and network knobs +//! survive so registry caches and offline/vendored setups keep +//! working. +//! - `--locked --profile embed`: the workspace lockfile governs the +//! inner graph too, and the components ship size-optimized +//! regardless of the outer profile. +//! +//! Requires the `wasm32-wasip2` target (rust-toolchain.toml carries it +//! in-repo; consumers of this stack have it for their own suites). A +//! host-only CLI — reporting/aggregation consumers — builds with +//! `--no-default-features`; `compose-runner`/`run` then require +//! explicit `--runner`/`--provider`. +//! +//! When cargo's artifact-dependencies (`bindeps`) stabilize, this +//! whole script becomes two `[build-dependencies]` entries; it is the +//! stable-Rust approximation of exactly that. + +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() { + let manifest_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let workspace = manifest_dir + .parent() + .and_then(|p| p.parent()) + .expect("crates/component-test-cli sits two levels under the workspace root") + .to_path_buf(); + + for input in ["components", "crates", "wit", "Cargo.toml", "Cargo.lock"] { + println!("cargo:rerun-if-changed={}", workspace.join(input).display()); + } + + if env::var_os("CARGO_FEATURE_EMBEDDED_COMPONENTS").is_none() { + return; + } + + let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + let target_dir = out_dir.join("embed-target"); + let cargo = env::var_os("CARGO").expect("cargo sets $CARGO for build scripts"); + + let mut cmd = Command::new(cargo); + cmd.current_dir(&workspace) + .args([ + "build", + "--locked", + "--profile", + "embed", + "--target", + "wasm32-wasip2", + "-p", + "runner-cli", + "-p", + "provider", + ]) + .arg("--target-dir") + .arg(&target_dir); + for (key, _) in env::vars_os() { + let k = key.to_string_lossy().into_owned(); + if !keep_env(&k) { + cmd.env_remove(&key); + } + } + + let status = cmd.status().expect("spawning the inner cargo build"); + if !status.success() { + eprintln!( + "\nbuilding the embedded components (components/runner-cli, components/provider) \ + failed.\n\ + - missing target? `rustup target add wasm32-wasip2`\n\ + - host-only CLI (no compose-runner/run defaults): build with \ + `--no-default-features`\n" + ); + std::process::exit(1); + } + + let built = target_dir.join("wasm32-wasip2").join("embed"); + for artifact in ["runner_cli.wasm", "provider.wasm"] { + std::fs::copy(built.join(artifact), out_dir.join(artifact)) + .unwrap_or_else(|e| panic!("copying {artifact} out of the inner build: {e}")); + } +} + +/// The inner build keeps only what it needs: registry/network knobs +/// and the ambient environment. Everything cargo set for *this* build +/// script — and the host-targeted compiler overrides — is scrubbed. +/// Deliberately including `CARGO_TARGET_WASM32_WASIP2_RUSTFLAGS` (the +/// `CARGO` prefix): the embedded components build with the workspace's +/// own flags everywhere, ambient wasm-target overrides included. +fn keep_env(k: &str) -> bool { + if k == "CARGO_HOME" + || k.starts_with("CARGO_NET_") + || k.starts_with("CARGO_HTTP_") + || k.starts_with("CARGO_REGISTR") + { + return true; + } + if k.starts_with("CARGO") { + return false; + } + !matches!( + k, + "RUSTFLAGS" | "RUSTC" | "RUSTDOC" | "RUSTC_WORKSPACE_WRAPPER" | "OUT_DIR" + ) +} diff --git a/crates/component-test-cli/embedded/provider.wasm b/crates/component-test-cli/embedded/provider.wasm deleted file mode 100644 index eaddae4..0000000 Binary files a/crates/component-test-cli/embedded/provider.wasm and /dev/null differ diff --git a/crates/component-test-cli/embedded/runner-cli.wasm b/crates/component-test-cli/embedded/runner-cli.wasm deleted file mode 100644 index fd94f48..0000000 Binary files a/crates/component-test-cli/embedded/runner-cli.wasm and /dev/null differ diff --git a/crates/component-test-cli/src/compose.rs b/crates/component-test-cli/src/compose.rs index 7738605..928b025 100644 --- a/crates/component-test-cli/src/compose.rs +++ b/crates/component-test-cli/src/compose.rs @@ -11,12 +11,14 @@ //! plus `test-context` plus `factory`) skip the provider and plug //! straight into the runner core. //! -//! The default provider and runner core are baked in at build time from -//! `embedded/` (size-optimized `embed`-profile builds of -//! `components/provider` and `components/runner-cli`; regenerate with -//! `just embed-update` and commit the diff — `just verify-cli` gates -//! their behavior against the same goldens as the wac-composed path). -//! Both are overridable per invocation. +//! The default provider and runner core are built from their sources +//! (`components/provider`, `components/runner-cli`) at compile time by +//! build.rs — size-optimized `embed`-profile wasm32-wasip2 builds, so +//! they can never drift from the sources (#88). `just verify-cli` +//! gates their behavior against the same goldens as the wac-composed +//! path. Both are overridable per invocation, and a +//! `--no-default-features` build (host-only consumers) omits them +//! entirely — the flags become required. //! //! Composition strips custom sections (findings #14): the result is //! execute-everything, its envelope says `scheduling: none`, and @@ -27,10 +29,31 @@ use anyhow::{bail, Context as _, Result}; use wac_graph::types::Package; use wac_graph::{CompositionGraph, EncodeOptions, NodeId, PackageId}; -/// The wasi:cli runner core (`components/runner-cli`), `embed` profile. -pub const EMBEDDED_RUNNER: &[u8] = include_bytes!("../embedded/runner-cli.wasm"); -/// The reference context provider (`components/provider`), `embed` profile. -pub const EMBEDDED_PROVIDER: &[u8] = include_bytes!("../embedded/provider.wasm"); +/// The wasi:cli runner core (`components/runner-cli`), when this build +/// carries it (feature `embedded-components`). +pub fn embedded_runner() -> Option<&'static [u8]> { + #[cfg(feature = "embedded-components")] + { + Some(include_bytes!(concat!(env!("OUT_DIR"), "/runner_cli.wasm"))) + } + #[cfg(not(feature = "embedded-components"))] + { + None + } +} + +/// The reference context provider (`components/provider`), when this +/// build carries it (feature `embedded-components`). +pub fn embedded_provider() -> Option<&'static [u8]> { + #[cfg(feature = "embedded-components")] + { + Some(include_bytes!(concat!(env!("OUT_DIR"), "/provider.wasm"))) + } + #[cfg(not(feature = "embedded-components"))] + { + None + } +} /// The frozen contract interfaces (wit/tests.wit; L1) and the provider /// interface the reference runner core consumes. diff --git a/crates/component-test-cli/src/main.rs b/crates/component-test-cli/src/main.rs index 5357ff9..6333820 100644 --- a/crates/component-test-cli/src/main.rs +++ b/crates/component-test-cli/src/main.rs @@ -577,13 +577,21 @@ fn compose_from_args( provider: Option<&str>, runner: Option<&str>, ) -> anyhow::Result> { + let embedded = |bytes: Option<&'static [u8]>, flag: &str| { + bytes.map(<[u8]>::to_vec).with_context(|| { + format!( + "this component-test build carries no embedded components \ + (built with --no-default-features); pass {flag}" + ) + }) + }; let provider = match provider { Some(path) => compose::read_component(path)?, - None => compose::EMBEDDED_PROVIDER.to_vec(), + None => embedded(compose::embedded_provider(), "--provider")?, }; let runner = match runner { Some(path) => compose::read_component(path)?, - None => compose::EMBEDDED_RUNNER.to_vec(), + None => embedded(compose::embedded_runner(), "--runner")?, }; compose::compose(input, &provider, &runner) } diff --git a/justfile b/justfile index f38e0c3..ca3a393 100644 --- a/justfile +++ b/justfile @@ -99,15 +99,11 @@ verify-compose: build echo "verify-compose: output matches expected/ (incl. JSONL + cross-runner fold)" # Path 2b: the CLI's composition/execution subcommands (#85). -# compose-runner (embedded provider + runner core) must reproduce Path -# 2's goldens under the wasmtime CLI; run is the same composition under +# compose-runner (embedded provider + runner core, built from source +# by the CLI's build.rs — always current, #88) must reproduce Path 2's +# goldens under the wasmtime CLI; run is the same composition under # the embedded wasmtime (human + JSONL legs); wizen pre-initializes # with inventory, scheduling, and runnability intact (findings 22–24). -# This is also the embedded artifacts' freshness gate: behavioral drift -# between components/{runner-cli,provider} and the committed -# crates/component-test-cli/embedded/ copies fails the diffs -# (byte-comparing builds across environments is off the table, #44) — -# after changing those components run `just embed-update` and commit. verify-cli: build #!/usr/bin/env bash set -euo pipefail @@ -421,18 +417,6 @@ lock-update: build cargo run -q -p component-test-cli -- lock \ {{release_dir}}/fixture_suite.wasm -o components/fixture-suite/tests.lock -# Regenerate the components baked into the CLI (compose-runner/run -# defaults) after changing components/runner-cli or components/provider, -# and commit the diff. Size-optimized `embed` profile; freshness is -# gated behaviorally by verify-cli (no byte comparison — builds are not -# reproducible across environments, #44). -embed-update: - cargo build --target {{wasm_target}} --profile embed -p runner-cli -p provider - cp target/{{wasm_target}}/embed/runner_cli.wasm \ - crates/component-test-cli/embedded/runner-cli.wasm - cp target/{{wasm_target}}/embed/provider.wasm \ - crates/component-test-cli/embedded/provider.wasm - # --- WIT --------------------------------------------------------------- # Component WIT dirs are symlinks into the canonical copies (wit/ and