From 14ff83827040dba16e9da8fcdd86de94c1b6dddb Mon Sep 17 00:00:00 2001 From: James Devine Date: Thu, 6 Aug 2026 21:54:44 +0100 Subject: [PATCH 1/7] feat(compile): add typed ShellScript generation with a script registry Generated shell was built with `format!`, which forced three layers of escaping onto every script: `\n\` continuations to fake multi-line source, doubled braces to survive `format!` itself (so a Docker Go-template read `{{{{.State.Status}}}}`), and an escaped quote for every quoted word. A 200-line body written that way is not reviewable as shell, and reviewing it as shell is the only way to know it is correct. Add `src/compile/shell/`. A script is a raw-string const written exactly as it will run, registered by the `shell_script!` macro, with substitution restricted to a typed, shell-quoted prelude. A value can only land as the right-hand side of an assignment, so it cannot alter the structure of the script; the typed binders validate shape at render time. Registration via `inventory` makes the script set enumerable without compiling a pipeline, which closes the reachability gap in the existing bash lint: a generator that no fixture happened to exercise was linted by nothing. `ado-aw export-bash-scripts` materialises the same set as files for review. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: aa46b482-6121-45e2-8278-1a9465b4f73e --- Cargo.lock | 10 + Cargo.toml | 1 + src/compile/mod.rs | 1 + src/compile/shell/bindings.rs | 352 ++++++++++++++++++ src/compile/shell/export.rs | 185 ++++++++++ src/compile/shell/lint.rs | 236 ++++++++++++ src/compile/shell/mod.rs | 664 ++++++++++++++++++++++++++++++++++ src/compile/shell/registry.rs | 262 ++++++++++++++ src/main.rs | 23 ++ 9 files changed, 1734 insertions(+) create mode 100644 src/compile/shell/bindings.rs create mode 100644 src/compile/shell/export.rs create mode 100644 src/compile/shell/lint.rs create mode 100644 src/compile/shell/mod.rs create mode 100644 src/compile/shell/registry.rs diff --git a/Cargo.lock b/Cargo.lock index 3266fbd7..2c6ef214 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,7 @@ dependencies = [ "glob-match", "indexmap", "inquire", + "inventory", "log", "percent-encoding", "rand", @@ -1098,6 +1099,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.11.0" diff --git a/Cargo.toml b/Cargo.toml index 26cbc3ed..1fd753b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ sha2 = "0.11.0" indexmap = "2" zip = { version = "8.6.0", default-features = false, features = ["deflate"] } semver = "1.0.28" +inventory = "0.3.24" [dev-dependencies] reqwest = { version = "0.12", features = ["blocking"] } diff --git a/src/compile/mod.rs b/src/compile/mod.rs index 769cf729..a8ab6d70 100644 --- a/src/compile/mod.rs +++ b/src/compile/mod.rs @@ -26,6 +26,7 @@ mod onees; mod onees_ir; mod path_layout_check; pub(crate) mod pr_filters; +pub mod shell; pub mod source_path_guard; mod stage; mod stage_ir; diff --git a/src/compile/shell/bindings.rs b/src/compile/shell/bindings.rs new file mode 100644 index 00000000..4ce8e9d3 --- /dev/null +++ b/src/compile/shell/bindings.rs @@ -0,0 +1,352 @@ +//! Typed, validated substitution values for [`super::ShellScript`]. +//! +//! # Why a binding rather than an interpolation +//! +//! The generators this module replaces built shell with `format!`, which meant +//! a substituted value could land in *any* syntactic position — inside a +//! single-quoted string, inside a `docker run` argument list, inside a `sed` +//! expression. Whether that value could alter the structure of the script +//! depended on the position, and the position was invisible from the call +//! site. +//! +//! A [`Binding`] can only ever be emitted as the right-hand side of a shell +//! assignment in the generated prelude. That is the single position where a +//! value's own quoting fully determines its meaning, so the escaping is +//! decided once, here, rather than per call site. +//! +//! Each constructor validates the shape it accepts. The types are deliberately +//! narrow: [`Binding::words`] refuses a value containing whitespace because a +//! word list is expanded unquoted by its consumer, and [`Binding::ado_macro`] +//! accepts only a well-formed predefined-variable name. A caller that needs +//! something outside these shapes has to add a constructor and justify it, +//! which is the point. + +/// A validated value bound to a shell variable in the generated prelude. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Binding { + /// The rendered right-hand side of the assignment, already quoted for + /// the shell as the constructor determined appropriate. + rhs: String, + kind: BindingKind, +} + +/// How a [`Binding`]'s value was validated. Retained for diagnostics and so +/// tests can assert on intent rather than on rendered text. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BindingKind { + /// An arbitrary literal string, single-quoted. + Text, + /// An integer, emitted bare. + Number, + /// `true` or `false`, emitted bare. + Bool, + /// A whitespace-separated word list, single-quoted, intended for + /// unquoted `for` expansion. + Words, + /// An Azure DevOps macro such as `$(Agent.TempDirectory)`, single-quoted + /// so the shell treats the already-substituted text as a literal. + AdoMacro, + /// Bulk text carried in a quoted heredoc. + Document, +} + +/// Variable names that carry a credential. A credential must reach a step +/// through `env:` (as `EnvValue::secret`) so Azure DevOps can mask it in logs; +/// routing one through the prelude would print it verbatim into the generated +/// YAML committed to the repository. +const SECRET_NAMES: &[&str] = &[ + "SC_READ_TOKEN", + "SC_WRITE_TOKEN", + "System.AccessToken", + "SYSTEM_ACCESSTOKEN", + "GITHUB_TOKEN", + "AZURE_DEVOPS_EXT_PAT", + "ADO_PROXY_BEARER", +]; + +impl Binding { + /// An arbitrary literal string. + /// + /// Rejects newlines (which would break the one-assignment-per-line prelude + /// shape) and `$(` (which would smuggle either a command substitution or + /// an unreviewed Azure DevOps macro past the typed channel — use + /// [`Binding::ado_macro`] for the latter). + #[track_caller] + pub fn text(value: impl AsRef) -> Self { + let value = value.as_ref(); + assert!( + !value.contains('\n') && !value.contains('\r'), + "shell binding value must be a single line, got {value:?}" + ); + assert!( + !value.contains("$("), + "shell binding value must not contain `$(`; use Binding::ado_macro \ + for an Azure DevOps predefined variable, got {value:?}" + ); + assert_not_secret(value); + Self { + rhs: single_quote(value), + kind: BindingKind::Text, + } + } + + /// An unsigned integer, emitted bare so arithmetic contexts work. + pub fn number(value: u64) -> Self { + Self { + rhs: value.to_string(), + kind: BindingKind::Number, + } + } + + /// A boolean, emitted bare as `true` / `false` so `[ "$V" = true ]` reads + /// naturally. + pub fn boolean(value: bool) -> Self { + Self { + rhs: if value { "true" } else { "false" }.to_string(), + kind: BindingKind::Bool, + } + } + + /// A whitespace-separated word list, for bodies that iterate it with an + /// intentionally unquoted `for W in $LIST`. + /// + /// Each word is rejected if it contains whitespace or a glob metacharacter, + /// because the consumer's word splitting would otherwise silently produce a + /// different list than the caller wrote. + #[track_caller] + pub fn words>(values: impl IntoIterator) -> Self { + let mut joined = String::new(); + for value in values { + let word = value.as_ref(); + assert!(!word.is_empty(), "shell word-list entry must not be empty"); + assert!( + !word.contains(|c: char| c.is_whitespace()), + "shell word-list entry must not contain whitespace, got {word:?}" + ); + assert!( + !word.contains(['*', '?', '[', ']', '\'', '\\', '$', '`']), + "shell word-list entry must not contain a glob or quoting \ + metacharacter, got {word:?}" + ); + assert_not_secret(word); + if !joined.is_empty() { + joined.push(' '); + } + joined.push_str(word); + } + Self { + rhs: single_quote(&joined), + kind: BindingKind::Words, + } + } + + /// An Azure DevOps predefined variable, e.g. `Agent.TempDirectory`. + /// + /// Rendered as `'$(Agent.TempDirectory)'`. Azure DevOps substitutes the + /// macro before the shell ever sees the script, and the single quotes then + /// keep the substituted text literal — so a path containing a space or a + /// shell metacharacter cannot alter the script. + #[track_caller] + pub fn ado_macro(name: &str) -> Self { + assert!( + is_ado_macro_name(name), + "Azure DevOps macro name must be dotted alphanumeric, got {name:?}" + ); + assert_not_secret(name); + Self { + rhs: single_quote(&format!("$({name})")), + kind: BindingKind::AdoMacro, + } + } + + /// Bulk text — a JSON document, a prompt, a certificate — assigned through + /// a quoted heredoc so no expansion occurs and no escaping is needed. + /// + /// Rendered across multiple lines, unlike every other binding. + #[track_caller] + pub fn document(value: impl AsRef) -> Self { + let value = value.as_ref(); + assert!( + !value.lines().any(|line| line.trim() == DOCUMENT_DELIMITER), + "document binding must not contain the heredoc delimiter {DOCUMENT_DELIMITER}" + ); + assert_not_secret(value); + let trimmed = value.trim_end_matches('\n'); + Self { + rhs: format!("$(cat <<'{DOCUMENT_DELIMITER}'\n{trimmed}\n{DOCUMENT_DELIMITER}\n)"), + kind: BindingKind::Document, + } + } + + /// The rendered right-hand side of the assignment. + pub fn rhs(&self) -> &str { + &self.rhs + } + + /// How this binding was validated. + pub fn kind(&self) -> BindingKind { + self.kind + } +} + +/// Heredoc delimiter for [`Binding::document`]. Long and namespaced so it +/// cannot collide with real content by accident. +const DOCUMENT_DELIMITER: &str = "ADO_AW_SHELL_DOC_EOF"; + +/// Refuse a value that names a credential. See [`SECRET_NAMES`]. +#[track_caller] +fn assert_not_secret(value: &str) { + for secret in SECRET_NAMES { + assert!( + !value.contains(secret), + "a credential must not reach the generated prelude: {value:?} \ + mentions {secret}. Pass it through `with_env` / `EnvValue::secret` \ + so Azure DevOps masks it." + ); + } +} + +/// POSIX single-quoting: the only escape available inside `'…'` is to close +/// the quote, emit an escaped `'`, and reopen. Every other byte — including +/// `$`, backtick, backslash and newline — is literal. +pub(crate) fn single_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) +} + +/// A shell variable name the prelude may assign. +pub(crate) fn is_shell_var_name(name: &str) -> bool { + let mut chars = name.chars(); + match chars.next() { + Some(c) if c.is_ascii_uppercase() || c == '_' => {} + _ => return false, + } + chars.all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_') +} + +/// `Build.Repository.Name`-shaped predefined-variable names. +fn is_ado_macro_name(name: &str) -> bool { + !name.is_empty() + && name.split('.').all(|segment| { + !segment.is_empty() + && segment + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_') + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn text_is_single_quoted_and_closes_embedded_quotes() { + assert_eq!(Binding::text("plain").rhs(), "'plain'"); + // The classic escape: close, escaped quote, reopen. + assert_eq!(Binding::text("it's").rhs(), r"'it'\''s'"); + } + + #[test] + fn text_keeps_shell_metacharacters_literal() { + // Nothing inside '…' expands, so a value that looks like code stays + // data. This is the property that makes the prelude the only safe + // injection position. + let binding = Binding::text("rm -rf /; `id`; ${HOME}"); + assert_eq!(binding.rhs(), "'rm -rf /; `id`; ${HOME}'"); + } + + #[test] + #[should_panic(expected = "must not contain `$(`")] + fn text_refuses_an_untyped_command_substitution() { + Binding::text("$(Agent.TempDirectory)/work"); + } + + #[test] + #[should_panic(expected = "single line")] + fn text_refuses_a_multi_line_value() { + Binding::text("first\nsecond"); + } + + #[test] + #[should_panic(expected = "credential must not reach the generated prelude")] + fn text_refuses_a_credential() { + Binding::text("token=SC_READ_TOKEN"); + } + + #[test] + fn numbers_and_booleans_are_emitted_bare() { + assert_eq!(Binding::number(11080).rhs(), "11080"); + assert_eq!(Binding::boolean(true).rhs(), "true"); + assert_eq!(Binding::boolean(false).rhs(), "false"); + } + + #[test] + fn words_join_with_a_single_space() { + let binding = Binding::words(["dev.azure.com", "vssps.dev.azure.com"]); + assert_eq!(binding.rhs(), "'dev.azure.com vssps.dev.azure.com'"); + assert_eq!(binding.kind(), BindingKind::Words); + } + + #[test] + fn words_are_empty_when_the_list_is() { + assert_eq!(Binding::words(Vec::::new()).rhs(), "''"); + } + + #[test] + #[should_panic(expected = "must not contain whitespace")] + fn words_refuse_an_entry_that_would_split() { + // The consumer expands this unquoted; a space would silently produce + // two list entries where the caller wrote one. + Binding::words(["dev.azure.com", "two words"]); + } + + #[test] + #[should_panic(expected = "glob or quoting metacharacter")] + fn words_refuse_a_glob() { + Binding::words(["*.azure.com"]); + } + + #[test] + fn ado_macro_is_quoted_so_the_substituted_text_stays_literal() { + let binding = Binding::ado_macro("Agent.TempDirectory"); + assert_eq!(binding.rhs(), "'$(Agent.TempDirectory)'"); + assert_eq!(binding.kind(), BindingKind::AdoMacro); + } + + #[test] + #[should_panic(expected = "dotted alphanumeric")] + fn ado_macro_refuses_an_arbitrary_expression() { + Binding::ado_macro("Agent.TempDirectory)/x; rm -rf /; echo $("); + } + + #[test] + #[should_panic(expected = "credential must not reach the generated prelude")] + fn ado_macro_refuses_the_access_token() { + Binding::ado_macro("System.AccessToken"); + } + + #[test] + fn document_uses_a_quoted_heredoc_so_nothing_expands() { + let binding = Binding::document("{\"a\": \"$NOT_EXPANDED\"}\n"); + assert_eq!( + binding.rhs(), + "$(cat <<'ADO_AW_SHELL_DOC_EOF'\n{\"a\": \"$NOT_EXPANDED\"}\nADO_AW_SHELL_DOC_EOF\n)" + ); + } + + #[test] + #[should_panic(expected = "heredoc delimiter")] + fn document_refuses_content_that_would_close_the_heredoc() { + Binding::document("ok\nADO_AW_SHELL_DOC_EOF\nsmuggled"); + } + + #[test] + fn shell_var_names_are_screaming_snake_case() { + assert!(is_shell_var_name("PROXY_DIR")); + assert!(is_shell_var_name("_PRIVATE")); + assert!(is_shell_var_name("PORT2")); + assert!(!is_shell_var_name("proxy_dir")); + assert!(!is_shell_var_name("2PORT")); + assert!(!is_shell_var_name("PROXY-DIR")); + assert!(!is_shell_var_name("")); + } +} diff --git a/src/compile/shell/export.rs b/src/compile/shell/export.rs new file mode 100644 index 00000000..ab523ded --- /dev/null +++ b/src/compile/shell/export.rs @@ -0,0 +1,185 @@ +//! Materialise every registered shell script for review and analysis. +//! +//! Backs `ado-aw export-bash-scripts`. The registry already makes the set of +//! scripts enumerable in-process; this makes it enumerable *outside* the +//! process, so a reviewer, an agentic workflow, or any shell-analysis tool can +//! work on the scripts as ordinary files rather than by reading Rust. +//! +//! Two forms, because they answer different questions: +//! +//! * `--format files` (default) writes one `.sh` per script with a provenance +//! header. This is what you run `shellcheck`, `shfmt` or a diff over. +//! * `--format json` writes a single document carrying the same content plus +//! the declared binding surface, for tooling that wants structure. +//! +//! What is written is [`ShellScriptDef::lint_source`] — the body with declared +//! variables stub-assigned — not a rendered script. A rendered script needs +//! real bindings, which only the producing call site has; the lint source is +//! the form that stands alone and is what the shellcheck harness judges. + +use std::path::Path; + +use anyhow::{Context, Result}; +use serde::Serialize; + +use super::registry::all_scripts; + +/// Output shape for `ado-aw export-bash-scripts`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +pub enum ExportFormat { + /// One `.sh` file per script, with a provenance header. + Files, + /// A single JSON document describing every script. + Json, +} + +/// One script as exported. +#[derive(Debug, Serialize)] +struct ExportedScript { + name: String, + interpreter: &'static str, + file: &'static str, + line: u32, + bindings: &'static [&'static str], + externals: &'static [&'static str], + fragments: &'static [&'static str], + source: String, +} + +/// Write every registered script to `out_dir`. +/// +/// Returns the number of scripts exported. +pub fn export(out_dir: &Path, format: ExportFormat) -> Result { + let scripts = all_scripts(); + std::fs::create_dir_all(out_dir) + .with_context(|| format!("creating export directory {}", out_dir.display()))?; + + let exported: Vec = scripts + .iter() + .map(|def| ExportedScript { + name: def.name.to_string(), + interpreter: def.interpreter.shellcheck_dialect(), + file: def.file, + line: def.line, + bindings: def.bindings, + externals: def.externals, + fragments: def.fragments, + source: def.lint_source(), + }) + .collect(); + + match format { + ExportFormat::Json => { + let path = out_dir.join("bash-scripts.json"); + let json = serde_json::to_string_pretty(&exported)?; + std::fs::write(&path, json) + .with_context(|| format!("writing {}", path.display()))?; + } + ExportFormat::Files => { + for (def, script) in scripts.iter().zip(&exported) { + let path = out_dir.join(def.export_file_name()); + let contents = format!("{}{}", provenance_header(script), script.source); + std::fs::write(&path, contents) + .with_context(|| format!("writing {}", path.display()))?; + } + } + } + + Ok(exported.len()) +} + +/// A header that points a reader back at the producing Rust source, so a +/// finding in an exported file is actionable without a repository-wide grep. +fn provenance_header(script: &ExportedScript) -> String { + format!( + "# ado-aw generated export — do not edit.\n\ + # script: {}\n\ + # source: {}:{}\n\ + # interpreter: {}\n\ + # bindings: {}\n\ + # externals: {}\n\ + #\n\ + # Variables below the lint-stub marker are placeholders. The real\n\ + # values are bound at the call site in the source file above.\n", + script.name, + script.file, + script.line, + script.interpreter, + join_or_none(script.bindings), + join_or_none(script.externals), + ) +} + +fn join_or_none(values: &[&str]) -> String { + if values.is_empty() { + "(none)".to_string() + } else { + values.join(", ") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::shell_script; + + shell_script! { + /// Fixture for export behaviour. + EXPORT_FIXTURE { + interpreter: Bash, + bindings: [PROXY_CONTAINER], + externals: [], + fragments: [], + body: r#" +docker rm -f "$PROXY_CONTAINER" 2>/dev/null || true +"#, + } + } + + #[test] + fn files_mode_writes_one_script_per_registration() { + let dir = tempfile::tempdir().expect("temp dir"); + let count = export(dir.path(), ExportFormat::Files).expect("export"); + assert_eq!(count, all_scripts().len()); + assert!(count > 0, "the registry must not be empty"); + + let path = dir.path().join(EXPORT_FIXTURE.export_file_name()); + let contents = std::fs::read_to_string(&path).expect("read exported script"); + // Provenance first, so a finding is traceable without a grep. + assert!(contents.contains("# source: src")); + assert!(contents.contains("# bindings: PROXY_CONTAINER")); + // Then the shell itself, verbatim and unescaped. + assert!(contents.contains("docker rm -f \"$PROXY_CONTAINER\" 2>/dev/null || true")); + } + + #[test] + fn files_mode_creates_a_missing_directory() { + let dir = tempfile::tempdir().expect("temp dir"); + let nested = dir.path().join("a").join("b"); + export(&nested, ExportFormat::Files).expect("export into a fresh path"); + assert!(nested.join(EXPORT_FIXTURE.export_file_name()).exists()); + } + + #[test] + fn json_mode_carries_the_declared_surface() { + let dir = tempfile::tempdir().expect("temp dir"); + export(dir.path(), ExportFormat::Json).expect("export"); + let raw = std::fs::read_to_string(dir.path().join("bash-scripts.json")).expect("read"); + let parsed: serde_json::Value = serde_json::from_str(&raw).expect("valid JSON"); + let entry = parsed + .as_array() + .expect("array") + .iter() + .find(|e| e["name"].as_str().unwrap_or_default().ends_with("::EXPORT_FIXTURE")) + .expect("the fixture is exported"); + assert_eq!(entry["interpreter"], "bash"); + assert_eq!(entry["bindings"][0], "PROXY_CONTAINER"); + assert_eq!(entry["externals"].as_array().expect("array").len(), 0); + } + + #[test] + fn empty_declarations_render_as_none_rather_than_blank() { + assert_eq!(join_or_none(&[]), "(none)"); + assert_eq!(join_or_none(&["A", "B"]), "A, B"); + } +} diff --git a/src/compile/shell/lint.rs b/src/compile/shell/lint.rs new file mode 100644 index 00000000..3e7b9bcf --- /dev/null +++ b/src/compile/shell/lint.rs @@ -0,0 +1,236 @@ +//! Shellcheck every registered script, in isolation. +//! +//! # Why this sits next to the registry rather than in `tests/` +//! +//! `tests/bash_lint_tests.rs` lints the shell that *reached* the emitted YAML, +//! which is the right check for "is what we ship correct" but makes coverage a +//! function of fixture reachability. A generator no fixture exercises is +//! linted by nothing — which is how several hundred lines of `ado-proxy` and +//! `az` wrapper shell went unlinted. +//! +//! This harness reads [`super::registry::all_scripts`] directly, so it sees +//! every script whether or not any pipeline emits it. The two are +//! complementary and both are kept: this one proves the shell is *correct*, +//! the integration test proves it is *emitted*. +//! +//! Skips when `shellcheck` is absent unless `ENFORCE_BASH_LINT` is set, which +//! CI does — matching the integration test's behaviour exactly. + +use std::io::Write; +use std::process::{Command, Stdio}; + +use serde::Deserialize; + +use super::registry::{ShellScriptDef, all_scripts}; + +/// One shellcheck JSON finding. +#[derive(Debug, Deserialize)] +struct Finding { + line: u64, + level: String, + code: u64, + message: String, +} + +fn shellcheck_available() -> bool { + Command::new("shellcheck") + .arg("--version") + .output() + .map(|out| out.status.success()) + .unwrap_or(false) +} + +/// Run shellcheck over one script's lint source. +fn check(def: &ShellScriptDef) -> Vec { + let mut child = Command::new("shellcheck") + .arg(format!("--shell={}", def.interpreter.shellcheck_dialect())) + .arg("--format=json") + .arg("-") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn shellcheck"); + child + .stdin + .take() + .expect("shellcheck stdin") + .write_all(def.lint_source().as_bytes()) + .expect("write script to shellcheck"); + let output = child.wait_with_output().expect("await shellcheck"); + serde_json::from_slice(&output.stdout).unwrap_or_else(|e| { + panic!( + "shellcheck produced unparseable output for {}: {e}\nstdout: {}\nstderr: {}", + def.name, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ) + }) +} + +#[test] +fn every_registered_script_passes_shellcheck() { + if !shellcheck_available() { + assert!( + std::env::var_os("ENFORCE_BASH_LINT").is_none(), + "ENFORCE_BASH_LINT is set but shellcheck is not on PATH" + ); + eprintln!("note: shellcheck not found; skipping. Set ENFORCE_BASH_LINT to enforce."); + return; + } + + let mut report = String::new(); + for def in all_scripts() { + let findings: Vec = check(def) + .into_iter() + .filter(|f| f.level == "error" || f.level == "warning") + .collect(); + if findings.is_empty() { + continue; + } + // Line numbers are relative to the lint source, whose stub prelude is + // synthesised — the header below names the producing Rust source so a + // reader can map a finding back to the raw-string body. + report.push_str(&format!("\n{} ({}:{})\n", def.name, def.file, def.line)); + for finding in findings { + report.push_str(&format!( + " line {:>3} SC{} {} {}\n", + finding.line, finding.code, finding.level, finding.message + )); + } + } + + assert!( + report.is_empty(), + "shellcheck flagged registered scripts. Fix the raw-string body, or \ + add a per-line `# shellcheck disable=SCxxxx` comment above the \ + offending line with a justification.\n{report}" + ); +} + +#[test] +fn every_declared_fragment_has_a_marker_and_vice_versa() { + // `splice_fragments` enforces this at render time, but only for scripts a + // test or a compile actually renders. Checking the whole registry + // statically means a fragment that is declared and never marked — shell + // that was meant to run and silently would not — fails here instead. + let mut problems = String::new(); + for def in all_scripts() { + let marked: Vec<&str> = def + .body + .lines() + .filter_map(super::fragment_marker) + .collect(); + for name in def.fragments { + if !marked.contains(name) { + problems.push_str(&format!( + " {} declares fragment `{name}` with no marker in the body ({}:{})\n", + def.name, def.file, def.line + )); + } + } + for name in &marked { + if !def.fragments.contains(name) { + problems.push_str(&format!( + " {} marks fragment `{name}` without declaring it ({}:{})\n", + def.name, def.file, def.line + )); + } + } + } + assert!(problems.is_empty(), "fragment declaration drift:\n{problems}"); +} + +#[test] +fn every_registered_script_declares_the_variables_it_reads() { + // A body that reads `$FOO` without declaring FOO as a binding or an + // external is either a typo or a variable arriving through an + // undocumented channel. Both are worth failing on, and shellcheck's + // SC2154 only catches it when the variable is never assigned *anywhere* + // in the body — this catches the declaration gap directly. + let mut undeclared = String::new(); + for def in all_scripts() { + for name in referenced_vars(def.body) { + let declared = def.bindings.contains(&name.as_str()) + || def.externals.contains(&name.as_str()) + || assigned_in_body(def.body, &name); + if !declared { + undeclared.push_str(&format!( + " {} reads ${name} without declaring it ({}:{})\n", + def.name, def.file, def.line + )); + } + } + } + assert!( + undeclared.is_empty(), + "shell scripts must declare every variable they read as a `bindings:` \ + entry (compiler-supplied) or an `externals:` entry (env / fragment / \ + setvariable):\n{undeclared}" + ); +} + +/// Variable names a body references as `$NAME` or `${NAME…}`. +/// +/// Deliberately only SCREAMING_SNAKE names: lowercase locals are the body's +/// own business, and shell specials (`$1`, `$@`, `$?`) are not names. +fn referenced_vars(body: &str) -> Vec { + let mut out = Vec::new(); + let bytes: Vec = body.chars().collect(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] != '$' { + i += 1; + continue; + } + let mut j = i + 1; + if j < bytes.len() && bytes[j] == '{' { + j += 1; + } + let start = j; + while j < bytes.len() && (bytes[j].is_ascii_uppercase() || bytes[j] == '_' || (j > start && bytes[j].is_ascii_digit())) { + j += 1; + } + if j > start { + let name: String = bytes[start..j].iter().collect(); + if !out.contains(&name) { + out.push(name); + } + } + i = j.max(i + 1); + } + out +} + +/// Whether the body assigns `name` itself (`NAME=`, `for NAME in`, `read NAME`). +fn assigned_in_body(body: &str, name: &str) -> bool { + body.lines().any(|line| { + let line = line.trim(); + line.starts_with(&format!("{name}=")) + || line.starts_with(&format!("export {name}=")) + || line.starts_with(&format!("for {name} in")) + || line.starts_with(&format!("read {name}")) + || line.starts_with(&format!("read -r {name}")) + || line.contains(&format!("; {name}=")) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn referenced_vars_finds_both_spellings_and_ignores_specials() { + let vars = referenced_vars(r#"echo "$FOO ${BAR}x $1 $@ $lower $BAZ2""#); + assert_eq!(vars, vec!["FOO", "BAR", "BAZ2"]); + } + + #[test] + fn assigned_in_body_recognises_the_common_forms() { + assert!(assigned_in_body("PROXY_DIR=$(mktemp -d)", "PROXY_DIR")); + assert!(assigned_in_body("export PROXY_DIR=/tmp", "PROXY_DIR")); + assert!(assigned_in_body("for PROXY_HOST in $HOSTS; do", "PROXY_HOST")); + assert!(assigned_in_body("set -eu; UMASK=1", "UMASK")); + assert!(!assigned_in_body("echo \"$PROXY_DIR\"", "PROXY_DIR")); + } +} diff --git a/src/compile/shell/mod.rs b/src/compile/shell/mod.rs new file mode 100644 index 00000000..67ab1347 --- /dev/null +++ b/src/compile/shell/mod.rs @@ -0,0 +1,664 @@ +//! Structured generation of the shell scripts this compiler emits. +//! +//! # The problem this solves +//! +//! Generated shell used to be built with `format!`, which forced three +//! layers of escaping onto every script: `\n\` continuations to fake +//! multi-line source, `{{` / `}}` to survive `format!`'s own syntax (so a +//! Docker Go-template read `{{{{.State.Status}}}}`), and `\"` for every +//! quoted word. A 200-line body written that way is not reviewable as shell, +//! and reviewing it as shell is the only way to know it is correct. +//! +//! # The shape +//! +//! A script is a plain raw-string constant — the shell exactly as it will run, +//! with no escaping — registered with metadata by the [`shell_script!`] macro +//! and rendered through [`ShellScript`]: +//! +//! ```ignore +//! shell_script! { +//! /// Greppable: search `mkfifo` and you land on the producer. +//! START_ADO_PROXY { +//! interpreter: Bash, +//! bindings: [PROXY_IMAGE, PROXY_CONTAINER, AGENT_TEMP], +//! externals: [ADO_PROXY_BEARER], +//! fragments: [resolve_org], +//! body: r#" +//! set -euo pipefail +//! # ado-aw:fragment resolve_org +//! PROXY_DIR=$(mktemp -d "$AGENT_TEMP/ado-proxy.XXXXXX") +//! docker run -d --name "$PROXY_CONTAINER" "$PROXY_IMAGE" >/dev/null +//! docker inspect -f 'state={{.State.Status}}' "$PROXY_CONTAINER" +//! "#, +//! } +//! } +//! +//! ShellScript::new(&START_ADO_PROXY) +//! .bind("PROXY_IMAGE", Binding::text(ADO_PROXY_IMAGE)) +//! .bind("PROXY_CONTAINER", Binding::text(ADO_PROXY_CONTAINER_NAME)) +//! .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) +//! .fragment("resolve_org", common::resolve_ado_organization_bash()) +//! .into_step("Start ado-proxy policy engine") +//! ``` +//! +//! [`ShellScript::render`] emits a generated prelude of quoted assignments, +//! then the body with each fragment spliced at its marker. +//! +//! # The properties that matter +//! +//! * **One injection position.** A caller-supplied value can only be the +//! right-hand side of a prelude assignment. It can never land mid-command, +//! so it can never alter the structure of the script. See [`bindings`]. +//! * **Declared surface.** Every variable the body reads is declared as either +//! a `binding` (the compiler supplies it) or an `external` (the runtime +//! does: `env:`, a fragment, or an ADO `setvariable`). [`ShellScript::render`] +//! refuses to render if the bound set does not match the declared set, so a +//! forgotten binding fails loudly rather than emitting `$UNSET`. +//! * **Composable without becoming opaque.** A long script is assembled from +//! independently registered, independently shellchecked phases spliced at +//! `# ado-aw:fragment` markers. The markers are ordinary comments, so the +//! outline body remains valid shell, and the inter-phase variable contract +//! is forced into the `externals:` declaration where a reviewer can see it. +//! * **Static enumerability.** Registration is automatic via `inventory`, so +//! `ado-aw export-bash-scripts` and the shellcheck harness can reach *every* +//! script without a fixture having to exercise the generator first. Before +//! this, a generator no fixture reached was linted by nothing. +//! * **Single-hop editing.** The shell stays in the file that produces it: +//! grep the shell text, land on the producer, edit in place. +//! +//! # Secrets +//! +//! A credential must never appear in the prelude — the prelude is written into +//! the `*.lock.yml` committed to the repository. [`Binding`] rejects values +//! that name a known credential; credentials continue to arrive through +//! `env:` as `EnvValue::secret`, which Azure DevOps masks. + +pub mod bindings; +pub mod export; +pub mod registry; + +#[cfg(test)] +mod lint; + +use indexmap::IndexMap; + +use super::ir::step::BashStep; + +pub use bindings::{Binding, BindingKind}; +pub use registry::{Interpreter, ShellScriptDef, all_scripts}; + +/// Opening marker of the generated prelude. +const PRELUDE_OPEN: &str = "# --- ado-aw generated bindings (do not edit) ---"; +/// Closing marker of the generated prelude. +const PRELUDE_CLOSE: &str = "# --- end generated bindings ---"; + +/// A registered script plus the values bound to its declared variables. +#[derive(Debug, Clone)] +pub struct ShellScript { + def: &'static ShellScriptDef, + bindings: IndexMap<&'static str, Binding>, + fragments: IndexMap<&'static str, String>, +} + +impl ShellScript { + /// Begin binding values to a registered script. + pub fn new(def: &'static ShellScriptDef) -> Self { + Self { + def, + bindings: IndexMap::new(), + fragments: IndexMap::new(), + } + } + + /// Bind a declared variable. + /// + /// # Panics + /// + /// If `name` is not declared in the script's `bindings:` list. A typo + /// would otherwise emit an assignment the body never reads while leaving + /// the variable the body *does* read unset. + #[track_caller] + pub fn bind(mut self, name: &str, value: Binding) -> Self { + let declared = self + .def + .bindings + .iter() + .find(|d| **d == name) + .unwrap_or_else(|| { + panic!( + "{}: `{name}` is not a declared binding; declared: {:?}", + self.def.name, self.def.bindings + ) + }); + self.bindings.insert(*declared, value); + self + } + + /// Bind a declared variable to a literal string. Shorthand for + /// `bind(name, Binding::text(value))`, which is the common case. + #[track_caller] + pub fn text(self, name: &str, value: impl AsRef) -> Self { + self.bind(name, Binding::text(value)) + } + + /// Splice a declared fragment — a block of shell produced elsewhere — + /// into the body at its marker. + /// + /// The body marks the splice point with a comment line: + /// + /// ```sh + /// # ado-aw:fragment resolve_org + /// ``` + /// + /// A marker is an ordinary shell comment, so the body stays valid, + /// shellcheck-able shell whether or not the fragment is spliced — and the + /// splice point is visible in the source rather than implied by call + /// order. + /// + /// This is the composition escape hatch, and the only way arbitrary shell + /// text (rather than a quoted value) enters a script. It is deliberately + /// awkward: a fragment must be declared in the script's `fragments:` list + /// *and* marked in the body, and any variable it defines must be declared + /// in `externals:` so the shellcheck harness still sees a complete + /// variable surface. + #[track_caller] + pub fn fragment(mut self, name: &str, shell: impl Into) -> Self { + let declared = self + .def + .fragments + .iter() + .find(|d| **d == name) + .unwrap_or_else(|| { + panic!( + "{}: `{name}` is not a declared fragment; declared: {:?}", + self.def.name, self.def.fragments + ) + }); + self.fragments.insert(*declared, shell.into()); + self + } + + /// Render the complete script: shebang (if the body carries one), the + /// generated binding prelude, then the body with fragments spliced at + /// their markers. + /// + /// # Panics + /// + /// If any declared binding or fragment was not supplied. This is a + /// compiler bug, not a user error, and emitting a script with an unset + /// variable would fail far away from its cause. + #[track_caller] + pub fn render(&self) -> String { + let missing: Vec<&str> = self + .def + .bindings + .iter() + .copied() + .filter(|name| !self.bindings.contains_key(name)) + .collect(); + assert!( + missing.is_empty(), + "{}: declared bindings were never bound: {missing:?}", + self.def.name + ); + let missing: Vec<&str> = self + .def + .fragments + .iter() + .copied() + .filter(|name| !self.fragments.contains_key(name)) + .collect(); + assert!( + missing.is_empty(), + "{}: declared fragments were never supplied: {missing:?}", + self.def.name + ); + + let (shebang, body) = split_shebang(self.def.body); + let mut out = String::with_capacity(self.def.body.len() + 256); + if let Some(shebang) = shebang { + out.push_str(shebang); + out.push('\n'); + } + + if !self.def.bindings.is_empty() { + out.push_str(PRELUDE_OPEN); + out.push('\n'); + // Declared order, not insertion order: the prelude reads the same + // whatever order the producer happened to call `bind` in, so a + // reordered call site produces no diff. + for name in self.def.bindings { + out.push_str(name); + out.push('='); + out.push_str(self.bindings[name].rhs()); + out.push('\n'); + } + out.push_str(PRELUDE_CLOSE); + out.push('\n'); + } + + let body = dedent(body.trim_start_matches('\n')); + out.push_str(&splice_fragments(self.def, &body, |name| { + Some(self.fragments[name].as_str()) + })); + if !out.ends_with('\n') { + out.push('\n'); + } + out + } + + /// Render into an ADO bash step. + /// + /// # Panics + /// + /// If the script's interpreter is not [`Interpreter::Bash`] — a `sh` + /// script is a standalone artefact (the `az` wrapper) written to disk by + /// some *other* step, not a step in its own right. + #[track_caller] + pub fn into_step(self, display_name: impl Into) -> BashStep { + assert_eq!( + self.def.interpreter, + Interpreter::Bash, + "{}: only a bash script can become an ADO bash step", + self.def.name + ); + BashStep::new(display_name, self.render()) + } + + /// The registration this script was built from. + pub fn def(&self) -> &'static ShellScriptDef { + self.def + } + + /// The rendered right-hand side bound to `name`, for tests that want to + /// assert on the value a producer supplied rather than on a substring of + /// the rendered script. + pub fn binding(&self, name: &str) -> Option<&Binding> { + self.bindings.get(name) + } +} + +/// Split a leading `#!` line off a script body. +fn split_shebang(body: &str) -> (Option<&str>, &str) { + let trimmed = body.trim_start_matches('\n'); + if !trimmed.starts_with("#!") { + return (None, body); + } + match trimmed.split_once('\n') { + Some((shebang, rest)) => (Some(shebang.trim_end()), rest), + None => (Some(trimmed.trim_end()), ""), + } +} + +/// The comment that marks a fragment splice point inside a body. +pub(crate) const FRAGMENT_MARKER: &str = "# ado-aw:fragment "; + +/// The fragment named by `line`, if it is a marker line. +pub(crate) fn fragment_marker(line: &str) -> Option<&str> { + line.trim_start() + .strip_prefix(FRAGMENT_MARKER) + .map(str::trim) + .filter(|name| !name.is_empty()) +} + +/// Replace each fragment marker in `body` with the text `resolve` returns, +/// re-indented to the marker's own indentation. +/// +/// # Panics +/// +/// If the body marks a fragment the definition does not declare, or declares +/// one the body never marks. Either is a silent no-op otherwise: shell that +/// was meant to run simply would not. +#[track_caller] +fn splice_fragments<'a>( + def: &'static ShellScriptDef, + body: &str, + resolve: impl Fn(&str) -> Option<&'a str>, +) -> String { + let mut seen: Vec<&str> = Vec::new(); + let mut out = String::with_capacity(body.len()); + for line in body.lines() { + match fragment_marker(line) { + None => { + out.push_str(line); + out.push('\n'); + } + Some(name) => { + let declared = def.fragments.iter().find(|d| **d == name).unwrap_or_else(|| { + panic!( + "{}: body marks fragment `{name}`, which is not declared; declared: {:?}", + def.name, def.fragments + ) + }); + assert!( + !seen.contains(declared), + "{}: fragment `{name}` is marked more than once", + def.name + ); + seen.push(declared); + let indent: String = line.chars().take_while(|c| c.is_whitespace()).collect(); + let Some(shell) = resolve(name) else { + continue; + }; + for fragment_line in dedent(shell.trim_matches('\n')).lines() { + if fragment_line.is_empty() { + out.push('\n'); + } else { + out.push_str(&indent); + out.push_str(fragment_line); + out.push('\n'); + } + } + } + } + } + let unmarked: Vec<&&str> = def + .fragments + .iter() + .filter(|name| !seen.contains(name)) + .collect(); + assert!( + unmarked.is_empty(), + "{}: declared fragments have no `{FRAGMENT_MARKER}` marker in the body: {unmarked:?}", + def.name + ); + out +} + +/// Strip the common leading indentation from every non-empty line, and +/// trailing whitespace from every line. +/// +/// Raw-string bodies are normally written at column 0, in which case this is a +/// no-op. It exists for the ones that read better indented inside the +/// producing function, and because trailing whitespace makes `serde_yaml` fall +/// back to the double-quoted scalar form, which would make the emitted YAML +/// unreadable. +pub(crate) fn dedent(s: &str) -> String { + let min = s + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| l.chars().take_while(|c| *c == ' ').count()) + .min() + .unwrap_or(0); + let mut out = String::with_capacity(s.len()); + let mut first = true; + for line in s.lines() { + if !first { + out.push('\n'); + } + first = false; + let leading_spaces = line.chars().take_while(|c| *c == ' ').count(); + let strip = leading_spaces.min(min); + out.push_str(line[strip..].trim_end_matches([' ', '\t'])); + } + if s.ends_with('\n') { + out.push('\n'); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::shell_script; + + shell_script! { + /// Fixture: exercises bindings, externals and a fragment. + TEST_SCRIPT { + interpreter: Bash, + bindings: [CONTAINER, PORT], + externals: [FROM_ENV, ORG], + fragments: [resolve_org], + body: r#" +set -euo pipefail +# ado-aw:fragment resolve_org +docker run --name "$CONTAINER" -p "$PORT" >/dev/null +echo "$ORG $FROM_ENV" +"#, + } + } + + shell_script! { + /// Fixture: a bare script with no substitution surface at all. + TEST_BARE { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [], + body: r#" +echo hello +"#, + } + } + + /// A definition that is deliberately **not** registered. + /// + /// Negative-case fixtures below are malformed on purpose. Registering them + /// would make the registry-wide guards (shellcheck, fragment-marker drift) + /// fail on fixtures rather than on real scripts. + const fn unregistered( + name: &'static str, + interpreter: Interpreter, + externals: &'static [&'static str], + fragments: &'static [&'static str], + body: &'static str, + ) -> ShellScriptDef { + ShellScriptDef { + name, + interpreter, + bindings: &[], + externals, + fragments, + body, + file: file!(), + line: line!(), + } + } + + fn built() -> ShellScript { + ShellScript::new(&TEST_SCRIPT) + .text("CONTAINER", "awmg-ado-proxy") + .bind("PORT", Binding::number(11080)) + .fragment("resolve_org", "ORG=example") + } + + #[test] + fn renders_a_prelude_then_the_body_with_fragments_in_place() { + let rendered = built().render(); + let expected = concat!( + "# --- ado-aw generated bindings (do not edit) ---\n", + "CONTAINER='awmg-ado-proxy'\n", + "PORT=11080\n", + "# --- end generated bindings ---\n", + "set -euo pipefail\n", + "ORG=example\n", + "docker run --name \"$CONTAINER\" -p \"$PORT\" >/dev/null\n", + "echo \"$ORG $FROM_ENV\"\n", + ); + assert_eq!(rendered, expected); + } + + #[test] + fn a_fragment_is_reindented_to_its_marker() { + // Not registered: the outline is only valid shell *after* splicing, so + // linting it standalone would report a spurious empty `then` clause. + // This is exactly why a fragment must not carry a control-flow body in + // a real script. + static NESTED: ShellScriptDef = unregistered( + "test::NESTED", + Interpreter::Bash, + &["ORG"], + &["resolve_org"], + r#" +if true; then + # ado-aw:fragment resolve_org +fi +"#, + ); + let rendered = ShellScript::new(&NESTED) + .fragment("resolve_org", "ORG=example\necho \"$ORG\"") + .render(); + assert_eq!( + rendered, + "if true; then\n ORG=example\n echo \"$ORG\"\nfi\n" + ); + } + + #[test] + #[should_panic(expected = "have no `# ado-aw:fragment ` marker in the body")] + fn refuses_a_declared_fragment_the_body_never_marks() { + static UNMARKED: ShellScriptDef = unregistered( + "test::UNMARKED", + Interpreter::Bash, + &[], + &["orphan"], + "\necho hi\n", + ); + ShellScript::new(&UNMARKED) + .fragment("orphan", "echo spliced") + .render(); + } + + #[test] + #[should_panic(expected = "which is not declared")] + fn refuses_a_marker_the_definition_never_declares() { + static STRAY_MARKER: ShellScriptDef = unregistered( + "test::STRAY_MARKER", + Interpreter::Bash, + &[], + &[], + "\n# ado-aw:fragment ghost\n", + ); + ShellScript::new(&STRAY_MARKER).render(); + } + + #[test] + fn the_body_needs_no_escaping() { + // The whole point: a Go template survives verbatim. Under `format!` + // this had to be written `{{{{.State.Status}}}}`. + shell_script! { + GO_TEMPLATE { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [], + body: r#" +docker inspect -f 'state={{.State.Status}} exit={{.State.ExitCode}}' proxy +"#, + } + } + assert!( + ShellScript::new(&GO_TEMPLATE) + .render() + .contains("'state={{.State.Status}} exit={{.State.ExitCode}}'") + ); + } + + #[test] + fn prelude_order_follows_the_declaration_not_the_call_site() { + // Reordering `bind` calls must not produce a diff in generated YAML. + let reordered = ShellScript::new(&TEST_SCRIPT) + .bind("PORT", Binding::number(11080)) + .text("CONTAINER", "awmg-ado-proxy") + .fragment("resolve_org", "ORG=example"); + assert_eq!(reordered.render(), built().render()); + } + + #[test] + fn a_script_with_no_substitutions_gets_no_prelude() { + let rendered = ShellScript::new(&TEST_BARE).render(); + assert_eq!(rendered, "echo hello\n"); + assert!(!rendered.contains(PRELUDE_OPEN)); + } + + #[test] + #[should_panic(expected = "declared bindings were never bound: [\"PORT\"]")] + fn refuses_to_render_with_a_binding_missing() { + ShellScript::new(&TEST_SCRIPT) + .text("CONTAINER", "c") + .fragment("resolve_org", "ORG=example") + .render(); + } + + #[test] + #[should_panic(expected = "declared fragments were never supplied")] + fn refuses_to_render_with_a_fragment_missing() { + ShellScript::new(&TEST_SCRIPT) + .text("CONTAINER", "c") + .bind("PORT", Binding::number(1)) + .render(); + } + + #[test] + #[should_panic(expected = "is not a declared binding")] + fn refuses_an_undeclared_binding() { + ShellScript::new(&TEST_SCRIPT).text("CONTAINR", "typo"); + } + + #[test] + fn a_shebang_stays_first_and_the_prelude_follows_it() { + shell_script! { + WRAPPER { + interpreter: Sh, + bindings: [TARGET], + externals: [], + fragments: [], + body: r#" +#!/bin/sh +exec "$TARGET" "$@" +"#, + } + } + let rendered = ShellScript::new(&WRAPPER).text("TARGET", "/usr/bin/az").render(); + assert_eq!( + rendered, + concat!( + "#!/bin/sh\n", + "# --- ado-aw generated bindings (do not edit) ---\n", + "TARGET='/usr/bin/az'\n", + "# --- end generated bindings ---\n", + "exec \"$TARGET\" \"$@\"\n", + ) + ); + } + + #[test] + #[should_panic(expected = "only a bash script can become an ADO bash step")] + fn a_sh_script_is_not_a_step() { + shell_script! { + SH_ONLY { + interpreter: Sh, + bindings: [], + externals: [], + fragments: [], + body: r#" +echo hi +"#, + } + } + let _ = ShellScript::new(&SH_ONLY).into_step("nope"); + } + + #[test] + fn into_step_carries_the_rendered_script() { + let step = built().into_step("Start ado-proxy policy engine"); + assert_eq!(step.display_name, "Start ado-proxy policy engine"); + assert_eq!(step.script, built().render()); + } + + #[test] + fn tests_can_assert_on_a_binding_rather_than_a_substring() { + // Stronger than `script.contains("awmg-ado-proxy")`, which would also + // pass if the value appeared in a comment. + let script = built(); + assert_eq!(script.binding("CONTAINER").unwrap().rhs(), "'awmg-ado-proxy'"); + assert_eq!(script.binding("PORT").unwrap().kind(), BindingKind::Number); + assert!(script.binding("NOPE").is_none()); + } + + #[test] + fn dedent_strips_source_indentation_and_trailing_space() { + assert_eq!(dedent(" a\n b\n"), "a\n b\n"); + assert_eq!(dedent("a \nb\t\n"), "a\nb\n"); + } +} diff --git a/src/compile/shell/registry.rs b/src/compile/shell/registry.rs new file mode 100644 index 00000000..b5fcfc53 --- /dev/null +++ b/src/compile/shell/registry.rs @@ -0,0 +1,262 @@ +//! Static registration of every shell script the compiler can emit. +//! +//! # Why a registry +//! +//! The bash lint used to work by compiling a set of fixtures and walking the +//! emitted YAML for `bash:` bodies. That makes lint coverage a function of +//! *reachability*: a generator no fixture happens to exercise is linted by +//! nothing. The `ado-proxy` lifecycle steps and the `az` wrapper were in +//! exactly that position — several hundred lines of unlinted shell. +//! +//! Registration inverts it. Every script announces itself at link time via +//! `inventory`, so [`all_scripts`] enumerates the complete set without +//! compiling anything. `ado-aw export-bash-scripts` and the shellcheck +//! harness both read from here, which makes coverage total by construction +//! rather than by a hand-maintained list. +//! +//! There is deliberately no manual catalogue to keep in sync: the +//! [`shell_script!`](crate::shell_script) macro registers as a side effect of +//! declaring, so the two cannot drift. + +/// Which shell a script is written for. +/// +/// This is not cosmetic: it selects `shellcheck --shell`, and the two dialects +/// genuinely differ (`set -o pipefail`, `[[`, arrays and `local` are bash-only). +/// The `az` wrapper is `sh` because it runs on whatever image the agent pool +/// provides. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Interpreter { + Bash, + Sh, +} + +impl Interpreter { + /// The `--shell=` value shellcheck expects. + pub fn shellcheck_dialect(self) -> &'static str { + match self { + Interpreter::Bash => "bash", + Interpreter::Sh => "sh", + } + } +} + +/// A registered script: its verbatim body plus the surface it declares. +/// +/// Construct through [`shell_script!`](crate::shell_script) rather than +/// directly, so registration cannot be forgotten. +#[derive(Debug, Clone, Copy)] +pub struct ShellScriptDef { + /// Fully qualified name, `module::path::IDENT`. Unique by construction, + /// and used as the export filename. + pub name: &'static str, + pub interpreter: Interpreter, + /// Variables the **compiler** supplies through the generated prelude. + /// Every one must be bound before [`super::ShellScript::render`] will + /// produce a script. + pub bindings: &'static [&'static str], + /// Variables the **runtime** supplies: step `env:`, an ADO + /// `##vso[task.setvariable]` from an earlier step, or a spliced fragment. + /// Declaring them is what lets the shellcheck harness distinguish "arrives + /// from outside" from "genuinely never assigned" (SC2154). + pub externals: &'static [&'static str], + /// Named blocks of shell composed in from elsewhere. The composition + /// escape hatch — see [`super::ShellScript::fragment`]. + pub fragments: &'static [&'static str], + /// The script itself, verbatim, exactly as it will run. + pub body: &'static str, + /// Source file, for the export provenance header. + pub file: &'static str, + /// Source line, for the export provenance header. + pub line: u32, +} + +impl ShellScriptDef { + /// The script as shellcheck should see it in isolation. + /// + /// Declared bindings and externals are stub-assigned so SC2154 + /// ("referenced but not assigned") still fires for a variable the body + /// reads without declaring — which is the bug worth catching — while not + /// firing for every legitimately-injected value. + /// + /// Fragment markers are left in place: they are ordinary comments, and the + /// fragment's own shell is linted where it is produced. Splicing an + /// unknown block in here would only produce noise. + pub fn lint_source(&self) -> String { + let (shebang, body) = super::split_shebang(self.body); + let mut out = String::with_capacity(self.body.len() + 256); + if let Some(shebang) = shebang { + out.push_str(shebang); + out.push('\n'); + } + out.push_str("# --- ado-aw lint stubs (not emitted) ---\n"); + for name in self.bindings.iter().chain(self.externals.iter()) { + // A non-empty stub: an empty one makes `[ -z "$V" ]` branches + // unreachable to shellcheck's flow analysis. + out.push_str(name); + out.push_str("='ado-aw-lint-stub'\n"); + } + out.push_str("# --- end lint stubs ---\n"); + out.push_str(&super::dedent(body.trim_start_matches('\n'))); + if !out.ends_with('\n') { + out.push('\n'); + } + out + } + + /// Filename this script exports to, e.g. `compile__shell__START_PROXY.sh`. + pub fn export_file_name(&self) -> String { + format!("{}.sh", self.name.replace("::", "__")) + } +} + +inventory::collect!(ShellScriptDef); + +/// Every registered script, in a stable order (sorted by [`ShellScriptDef::name`]). +/// +/// `inventory` iteration order is link-order and therefore not stable across +/// builds; sorting keeps the export output and any lint report diffable. +pub fn all_scripts() -> Vec<&'static ShellScriptDef> { + let mut scripts: Vec<&'static ShellScriptDef> = + inventory::iter::.into_iter().collect(); + scripts.sort_by_key(|def| def.name); + scripts +} + +/// Declare and register a shell script. +/// +/// The body is a raw string containing the shell **exactly as it will run** — +/// no `format!`, no `\n\` continuations, no escaping of quotes or `{{`. +/// Substitution happens only through the declared `bindings`, which +/// [`ShellScript`](super::ShellScript) renders into a quoted prelude. +/// +/// ```ignore +/// shell_script! { +/// /// One line on why this script exists. +/// STOP_ADO_PROXY { +/// interpreter: Bash, +/// bindings: [PROXY_CONTAINER], +/// externals: [], +/// fragments: [], +/// body: r#" +/// docker rm -f "$PROXY_CONTAINER" 2>/dev/null || true +/// "#, +/// } +/// } +/// ``` +#[macro_export] +macro_rules! shell_script { + ( + $(#[$meta:meta])* + $ident:ident { + interpreter: $interpreter:ident, + bindings: [$($binding:ident),* $(,)?], + externals: [$($external:ident),* $(,)?], + fragments: [$($fragment:ident),* $(,)?], + body: $body:expr $(,)? + } + ) => { + $(#[$meta])* + #[allow(dead_code)] + pub const $ident: $crate::compile::shell::ShellScriptDef = + $crate::compile::shell::ShellScriptDef { + name: concat!(module_path!(), "::", stringify!($ident)), + interpreter: $crate::compile::shell::Interpreter::$interpreter, + bindings: &[$(stringify!($binding)),*], + externals: &[$(stringify!($external)),*], + fragments: &[$(stringify!($fragment)),*], + body: $body, + file: file!(), + line: line!(), + }; + + $crate::inventory::submit! { $ident } + }; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::shell_script; + + shell_script! { + /// Fixture for registry behaviour. + REGISTRY_FIXTURE { + interpreter: Sh, + bindings: [TARGET], + externals: [FROM_ENV, ORG], + fragments: [resolve_org], + body: r#" +#!/bin/sh +# ado-aw:fragment resolve_org +echo "$TARGET $FROM_ENV $ORG" +"#, + } + } + + #[test] + fn a_declared_script_registers_itself() { + // No manual catalogue: declaring is registering, so the two cannot + // drift apart. + let found = all_scripts() + .into_iter() + .find(|def| def.name.ends_with("::REGISTRY_FIXTURE")) + .expect("the fixture must appear in the registry"); + assert_eq!(found.interpreter, Interpreter::Sh); + assert_eq!(found.bindings, &["TARGET"]); + assert_eq!(found.externals, &["FROM_ENV", "ORG"]); + assert_eq!(found.fragments, &["resolve_org"]); + } + + #[test] + fn names_are_module_qualified_and_unique() { + let mut names: Vec<&str> = all_scripts().iter().map(|def| def.name).collect(); + let total = names.len(); + names.sort_unstable(); + names.dedup(); + assert_eq!( + names.len(), + total, + "two scripts registered under the same name; \ + `ado-aw export-bash-scripts` would overwrite one with the other" + ); + } + + #[test] + fn the_registry_is_sorted_so_exports_stay_diffable() { + let names: Vec<&str> = all_scripts().iter().map(|def| def.name).collect(); + let mut sorted = names.clone(); + sorted.sort_unstable(); + assert_eq!(names, sorted); + } + + #[test] + fn lint_source_stubs_every_declared_variable() { + let source = REGISTRY_FIXTURE.lint_source(); + assert!(source.starts_with("#!/bin/sh\n"), "shebang stays first: {source}"); + assert!(source.contains("TARGET='ado-aw-lint-stub'")); + // An external is stubbed too: it genuinely arrives from outside, so + // SC2154 on it would be noise. + assert!(source.contains("FROM_ENV='ado-aw-lint-stub'")); + // `ORG` reaches the body from the fragment, and is declared external + // for exactly that reason. + assert!(source.contains("ORG='ado-aw-lint-stub'")); + // A fragment marker stays in the body as an ordinary comment: the + // fragment's own shell is linted where it is produced. + assert!(source.contains("# ado-aw:fragment resolve_org")); + // Nothing undeclared is invented. + assert!(!source.contains("UNDECLARED='ado-aw-lint-stub'")); + assert!(source.trim_end().ends_with("echo \"$TARGET $FROM_ENV $ORG\"")); + } + + #[test] + fn export_file_names_are_path_safe() { + assert!(REGISTRY_FIXTURE.export_file_name().ends_with("__REGISTRY_FIXTURE.sh")); + assert!(!REGISTRY_FIXTURE.export_file_name().contains(':')); + } + + #[test] + fn interpreters_map_to_shellcheck_dialects() { + assert_eq!(Interpreter::Bash.shellcheck_dialect(), "bash"); + assert_eq!(Interpreter::Sh.shellcheck_dialect(), "sh"); + } +} diff --git a/src/main.rs b/src/main.rs index db4f5ff5..9cb5003a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -34,6 +34,10 @@ mod update_check; pub mod validate; mod version; +/// Re-exported so the [`shell_script!`](crate::shell_script) macro can name +/// `$crate::inventory` from any module without every call site importing it. +pub use inventory; + use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use std::path::{Path, PathBuf}; @@ -600,6 +604,20 @@ enum Commands { #[arg(short, long)] output: Option, }, + /// Export every registered shell script the compiler can emit, so the + /// generated shell can be reviewed and analysed as ordinary files rather + /// than read out of Rust source. Unlike the fixture-driven bash lint this + /// reaches *every* script, including ones no pipeline currently emits. + #[command(hide = true)] + ExportBashScripts { + /// Directory to write into; created if it does not exist. + #[arg(short, long)] + output: std::path::PathBuf, + /// `files` writes one `.sh` per script; `json` writes a single + /// document carrying the declared binding surface too. + #[arg(long, value_enum, default_value_t = compile::shell::export::ExportFormat::Files)] + format: compile::shell::export::ExportFormat, + }, /// Inspect an agent source file's typed IR: jobs, stages, steps, outputs, derived `dependsOn`. Inspect { /// Path to the agent markdown source. @@ -666,6 +684,7 @@ impl Commands { Commands::ExportFactCatalog { .. } => "export-fact-catalog", Commands::ExportAdoProxyCatalogSchema { .. } => "export-ado-proxy-catalog-schema", Commands::ExportAdoProxyCatalog { .. } => "export-ado-proxy-catalog", + Commands::ExportBashScripts { .. } => "export-bash-scripts", Commands::Inspect { .. } => "inspect", Commands::Graph { .. } => "graph", Commands::Whatif { .. } => "whatif", @@ -1633,6 +1652,10 @@ async fn main() -> Result<()> { let catalog = ado_proxy::catalog::generate_catalog_json(); write_or_print(&catalog, output)?; } + Commands::ExportBashScripts { output, format } => { + let count = compile::shell::export::export(&output, format)?; + println!("Exported {count} shell scripts to {}", output.display()); + } Commands::Inspect { source, json } => { inspect::dispatch_inspect(inspect::InspectOptions { source: &source, From c01a3f751f3df792b41d5d1114375d6656c128f9 Mon Sep 17 00:00:00 2001 From: James Devine Date: Mon, 10 Aug 2026 13:09:36 +0100 Subject: [PATCH 2/7] refactor(compile): structure the ado-proxy and az wrapper scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #1833. `start_ado_proxy_step` was a ~200-line body built by `format!`, and `render_az_wrapper` a ~120-line interpolated standalone executable. Neither was reviewable as shell, and neither was reachable by the bash lint — coverage was a function of fixture reachability, so a generator no fixture exercised was linted by nothing. Both now go through `ShellScript`. `start_ado_proxy_step` is decomposed into eight registered phases (work directory, policy, material minting, material assembly, container start, handover, destruction, readiness) spliced into an outline at `# ado-aw:fragment` markers. It remains a single Bash task: the phases are Rust-level and shell-level structure, not additional ADO steps, so the bearer, CA private key and leaf keys still never touch a runner path, argv, environment or container layer. The container's nested `sh -c` entrypoint is registered as its own `Sh` script and is therefore linted for the first time. A composed script is linted with its phases spliced in, so cross-phase variable flow — the one new risk decomposition introduces — is checked. Two fixes to the lint itself, both cases of the tool distorting the source rather than the source being wrong: * `referenced_vars` now skips single-quoted spans. It was demanding a declaration for awk's `$NF`, which had been renamed to `$n` to satisfy it; the original expression is restored. * Shell-provided variables (`PATH`, `HOME`, …) are no longer stub- assigned. Assigning `PATH` trips SC2123, reporting a bug the real script does not have. An `$(…)` binding carries a targeted `# shellcheck disable=SC2016`: the single quotes are the point, since ADO substitutes the macro before bash runs and the quoting keeps the result literal. Also migrated: the `az` wrapper, every `extensions/` generator, the four runtimes, `cache_memory`, `common.rs`, `filter_ir.rs` and `engine.rs`. Substring assertions that a literal appears somewhere are replaced by assertions that the producer bound it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: aa46b482-6121-45e2-8278-1a9465b4f73e --- .github/workflows/bash-lint-auditor.md | 46 +- .github/workflows/review-compiler-contract.md | 35 +- AGENTS.md | 65 +- docs/cli.md | 6 + docs/extending.md | 144 ++- src/compile/agentic_pipeline.rs | 907 +++++++++++++----- src/compile/az_wrapper.rs | 202 ++-- src/compile/common.rs | 180 +++- src/compile/extensions/ado_aw_marker.rs | 73 +- src/compile/extensions/ado_script.rs | 237 ++++- src/compile/extensions/azure_cli.rs | 185 +++- .../extensions/exec_context/ci_push.rs | 48 +- src/compile/extensions/exec_context/manual.rs | 43 +- .../extensions/exec_context/pipeline.rs | 35 +- src/compile/extensions/exec_context/pr.rs | 55 +- .../extensions/exec_context/pr_checks.rs | 61 +- src/compile/extensions/exec_context/repo.rs | 24 +- .../extensions/exec_context/schedule.rs | 35 +- .../extensions/exec_context/workitem.rs | 74 +- src/compile/filter_ir.rs | 32 +- src/compile/shell/bindings.rs | 15 + src/compile/shell/lint.rs | 113 ++- src/compile/shell/mod.rs | 84 +- src/compile/shell/registry.rs | 89 +- src/engine.rs | 270 ++++-- src/runtimes/dotnet/extension.rs | 59 +- src/runtimes/dotnet/mod.rs | 192 ---- src/runtimes/lean/extension.rs | 44 +- src/runtimes/lean/mod.rs | 45 - src/runtimes/node/extension.rs | 41 +- src/runtimes/node/mod.rs | 118 --- src/runtimes/python/mod.rs | 76 -- src/tools/cache_memory/extension.rs | 79 +- tests/compiler_tests.rs | 41 +- 34 files changed, 2596 insertions(+), 1157 deletions(-) diff --git a/.github/workflows/bash-lint-auditor.md b/.github/workflows/bash-lint-auditor.md index 9f318c30..b6ec466b 100644 --- a/.github/workflows/bash-lint-auditor.md +++ b/.github/workflows/bash-lint-auditor.md @@ -79,13 +79,23 @@ Confirm `shellcheck --version` runs and the version is `>= 0.9`. ## Step 3 — Baseline the Lint -Run the existing integration test under enforce mode and capture the result: +Shell is linted at two levels. Run both under enforce mode and capture the results: ```bash +# 1. Every registered script, in isolation, straight from the registry. +ENFORCE_BASH_LINT=1 cargo test --bin ado-aw compile::shell -- --nocapture 2>&1 | tee /tmp/lint-registry.log + +# 2. Every bash body that actually reaches the emitted YAML. ENFORCE_BASH_LINT=1 cargo test --test bash_lint_tests -- --nocapture 2>&1 | tee /tmp/lint-baseline.log echo "exit=$?" ``` +The registry lint (`src/compile/shell/lint.rs`) proves the shell is *correct*; +the integration test proves it is *emitted*. Coverage of the first is total by +construction, so a "no fixture reaches this generator" gap can no longer hide a +broken script — but it can still hide a script that is never emitted at all, +which is what the second catches. + There are three possible outcomes; each takes a different path. **A. Lint is green (exit 0).** The PR gate is doing its job. Move to Step 4 and look for proactive improvements. @@ -100,28 +110,48 @@ When the lint is already green, audit the *quality* of the bash hygiene story. D ### 4a. Stale disable directives -Find `# shellcheck disable=SCxxxx` directives that no longer fire on the bash body that contains them: +Find `# shellcheck disable=SCxxxx` directives that no longer fire on the shell body that contains them: ```bash -grep -rn "shellcheck disable=" src/data/ src/runtimes/ src/compile/ src/tools/ src/engine.rs 2>/dev/null +grep -rn "shellcheck disable=" src/data/ src/runtimes/ src/compile/ src/tools/ src/safe_outputs/ src/engine.rs 2>/dev/null ``` -For each hit, temporarily delete the directive, rerun `cargo test --test bash_lint_tests -- --nocapture` (with `ENFORCE_BASH_LINT=1`), and check whether the test still passes. If the directive is now unnecessary (test still passes), remove it permanently. Restore the source file if the test fails. +For each hit, temporarily delete the directive, rerun both lint commands from Step 3 (with `ENFORCE_BASH_LINT=1`), and check whether they still pass. If the directive is now unnecessary (both still pass), remove it permanently. Restore the source file if either fails. ### 4b. Lint exclude-list audit -The lint excludes `SC1090,SC1091` globally (documented in `tests/bash_lint_tests.rs`). Check whether tightening would surface new findings: +The integration lint keeps a deliberately minimal global exclude list (documented in `tests/bash_lint_tests.rs`). Check whether tightening would surface new findings: ```bash # Probe a stricter rule set ENFORCE_BASH_LINT=1 cargo test --test bash_lint_tests 2>&1 | head -50 ``` -If you propose tightening, add a per-line `# shellcheck disable=` comment inside the offending bash body rather than expanding the global exclude list. Keep the exclude list minimal. +If you propose tightening, add a per-line `# shellcheck disable=` comment inside the offending body rather than expanding the global exclude list. Keep the exclude list minimal. + +### 4c. Unstructured shell + +Generated shell must go through `ShellScript` (`src/compile/shell/`, see the +*Generated shell scripts* section of `docs/extending.md`), which registers it +for linting and restricts substitution to a typed, quoted prelude. Shell built +with `format!` is invisible to the registry lint and reintroduces the escaping +that made these bodies unreviewable. + +Look for shell still being assembled by hand: + +```bash +# `\n\` continuations are the signature of a format!-built shell body +grep -rn '\\n\\' src/ --include=*.rs | grep -v '^src/compile/shell/' +``` + +If you find any, migrate it: move the body into a `shell_script!` const written +verbatim, and pass each interpolated value as a typed `Binding`. Do one file +per run — this is a mechanical change but a reviewable diff matters more than +volume. -### 4c. Expand fixture coverage +### 4d. Expand fixture coverage -Walk `src/runtimes/`, `src/tools/`, `src/compile/extensions/` and check whether every code path that emits a `- bash: |` step is exercised by some fixture. A generator that the lint never reaches is a generator with no quality story. Add a fixture (or extend an existing one) only if you find a real, currently-unreached generator. +Walk `src/runtimes/`, `src/tools/`, `src/compile/extensions/` and check whether every code path that emits a `- bash: |` step is exercised by some fixture. The registry lint already proves each script is *correct*; a fixture proves it is actually *emitted*. Add a fixture (or extend an existing one) only if you find a real, currently-unreached generator. If none of 4a / 4b / 4c finds anything, **exit cleanly** — use the `noop` safe output with the message "Bash hygiene is current; no actionable findings." diff --git a/.github/workflows/review-compiler-contract.md b/.github/workflows/review-compiler-contract.md index 8b6ca25f..5b5f28c3 100644 --- a/.github/workflows/review-compiler-contract.md +++ b/.github/workflows/review-compiler-contract.md @@ -160,12 +160,35 @@ New `CompilerExtension` implementations must be registered in `collect_extensions()`, and new runtimes/tools documented in `docs/runtimes.md` / `docs/tools.md` per `docs/extending.md`. -### Generated bash - -Any new literal `bash:` body in generated pipeline YAML must survive -`cargo test --test bash_lint_tests` (shellcheck). Watch for `cd "$X"` without -`|| exit`, tilde inside double quotes, and masked return codes in assignments — -ADO's "fail on last command" default hides all three. +### Generated shell + +Compiler-generated shell must go through `ShellScript` (`src/compile/shell/`, +documented in `docs/extending.md` under *Generated shell scripts*). Findings: + +- **Shell built with `format!`.** A `\n\` continuation, a doubled `{{` brace or + an escaped `\"` inside a shell body means the script is not registered, so + neither the registry lint nor `export-bash-scripts` can see it — and the + escaping is what made these bodies unreviewable in the first place. +- **An interpolated value that is not a `Binding`.** Substitution must be a + `Binding::text` / `::number` / `::boolean` / `::words` / `::ado_macro` / + `::document`. Those land in exactly one position — the right-hand side of a + prelude assignment — and therefore cannot alter the structure of the script. + A value spliced anywhere else is an injection surface. +- **A credential in a binding.** The prelude is written verbatim into the + committed `*.lock.yml`. Credentials belong on + `.with_env(…, EnvValue::secret(…))`, which ADO masks. +- **An undeclared variable.** Everything a body reads must be declared in + `bindings:` or `externals:`. A missing `externals:` entry hides an + undocumented runtime coupling. +- **A `fragment` carrying control flow that the outline body depends on.** The + outline must remain valid shell without the fragment, or it cannot be linted. + +Any body must survive both `ENFORCE_BASH_LINT=1 cargo test --bin ado-aw +compile::shell` (every registered script, in isolation) and +`ENFORCE_BASH_LINT=1 cargo test --test bash_lint_tests` (every body that +reaches emitted YAML). Watch for `cd "$X"` without `|| exit`, tilde inside +double quotes, and masked return codes in assignments — ADO's "fail on last +command" default hides all three. ## Step 4 — Documentation sync diff --git a/AGENTS.md b/AGENTS.md index 820d61cb..bcc0a05b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,6 +76,12 @@ fail-closed and only pauses when the agent actually proposed a reviewed output. │ │ ├── stage_ir.rs # Stage target typed-IR builder │ │ ├── az_wrapper.rs # Renders the `az` CLI redirect wrapper installed into the agent sandbox (env-based `HTTPS_PROXY` redirect, not argument rewriting) │ │ ├── source_path_guard.rs # Validation guard for untrusted workflow source-path inputs used by audit + mcp_author +│ │ ├── shell/ # Typed generation of every shell script the compiler emits (see docs/extending.md "Generated shell scripts") +│ │ │ ├── mod.rs # ShellScript: raw-string bodies + a typed shell-quoted binding prelude; `# ado-aw:fragment` splicing; into_step() +│ │ │ ├── bindings.rs # Binding constructors (text/number/boolean/words/ado_macro/document) — the single injection chokepoint; rejects credentials +│ │ │ ├── registry.rs # shell_script! macro + `inventory` auto-registration; ShellScriptDef; all_scripts() +│ │ │ ├── export.rs # `ado-aw export-bash-scripts` — materializes every registered script as reviewable .sh / JSON +│ │ │ └── lint.rs # Registry-wide shellcheck + declared-variable-surface guards (reaches scripts no fixture emits) │ │ ├── gitattributes.rs # .gitattributes management for compiled pipelines │ │ ├── filter_ir.rs # Filter expression IR: Fact/Predicate types, lowering, validation, codegen │ │ ├── pr_filters.rs # PR trigger filter generation (native ADO + gate steps) @@ -398,8 +404,8 @@ index to jump to the right page. `remove`, `list`, `status`, `run`, `audit`, `mcp-author`, `trace`, `inspect`, `graph`, `whatif`, `lint`, `catalog`; `configure` is a deprecated hidden alias; `export-gate-schema`, `export-fact-catalog`, - `export-ado-proxy-catalog-schema`, and `export-ado-proxy-catalog` are hidden - build-time tools). + `export-ado-proxy-catalog-schema`, `export-ado-proxy-catalog`, and + `export-bash-scripts` are hidden build-time tools). - [`docs/agency-plugin.md`](docs/agency-plugin.md) — the Agency / Claude Code plugin (`agency/plugins/ado-aw/`): canonical layout, six skills, `mcp-author` wiring, the self-contained root marketplace catalogs, `init --agency` @@ -536,24 +542,63 @@ cargo test cargo clippy ``` +### Generated shell + +Compiler-generated shell is **not** built with `format!`. Every script is a +raw-string constant registered with `shell_script!` in the module that +produces it, with substitution restricted to a typed, shell-quoted prelude — +see `src/compile/shell/` and the *Generated shell scripts* section of +[`docs/extending.md`](docs/extending.md). + +The body is the shell exactly as it runs: no `\n\` continuations, no doubled +braces, no escaped quotes. A value reaches a script only as `Binding::text`, +`::number`, `::boolean`, `::words`, `::ado_macro` or `::document`, all of +which land in a single position (the right-hand side of a prelude assignment) +and therefore cannot alter the structure of the script. A credential must +never become a binding — the prelude is committed to the repository — so it +stays on `.with_env(…, EnvValue::secret(…))`. + +Every variable a body reads must be declared as a `binding` or an `external`. +Both the render path and a registry-wide test enforce it. + ### Bash step lint +Shell is linted at two levels. + +`src/compile/shell/lint.rs` shellchecks **every registered script in +isolation**, straight from the registry. Coverage is total by construction: +before this, lint coverage was a function of fixture reachability, so a +generator no fixture exercised was linted by nothing. + The `tests/bash_lint_tests.rs` integration test compiles a representative set of fixtures and runs `shellcheck` against every literal `bash:` body in the -generated YAML. It catches silent-failure patterns that ADO's "fail on last -command" default would let through (e.g. `cd "$X"` without `|| exit`, tilde -inside double quotes, masked-return assignments). +generated YAML — proving scripts are *emitted*, where the registry lint proves +they are *correct*. It catches silent-failure patterns that ADO's "fail on +last command" default would let through (e.g. `cd "$X"` without `|| exit`, +tilde inside double quotes, masked-return assignments). -The test is skipped if `shellcheck` is not on PATH. Install locally with +Both are skipped if `shellcheck` is not on PATH. Install locally with `brew install shellcheck` (macOS) or `apt-get install -y shellcheck` (Debian / Ubuntu); CI installs it in `.github/workflows/rust-tests.yml` and sets `ENFORCE_BASH_LINT=1` so a missing shellcheck becomes a hard failure rather than a silent skip. -When adding a new bash step, run `cargo test --test bash_lint_tests` and fix -anything it flags. If a finding is genuinely intentional, add a -`# shellcheck disable=SCxxxx` comment immediately above the offending line in -the bash body — shellcheck honours the directive and it's inert at runtime. +When adding a new shell script, run both and fix anything they flag: + +```bash +ENFORCE_BASH_LINT=1 cargo test --bin ado-aw compile::shell +ENFORCE_BASH_LINT=1 cargo test --test bash_lint_tests +``` + +If a finding is genuinely intentional, add a `# shellcheck disable=SCxxxx` +comment immediately above the offending line in the body — shellcheck honours +the directive and it's inert at runtime. + +To review the generated shell as ordinary files: + +```bash +cargo run -- export-bash-scripts --out /tmp/ado-aw-shell +``` ### Markdown-only smoke suite diff --git a/docs/cli.md b/docs/cli.md index 46d7c75f..7aead421 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -198,6 +198,12 @@ These commands are not shown in `--help` but are available for contributors work - `export-ado-proxy-catalog` - Export the `ado-proxy` catalog data as JSON. Build-time drift guard for the bundle's committed catalog snapshot (see [`docs/ado-proxy-design.md`](ado-proxy-design.md)). - `--output, -o ` - Write the catalog to a file instead of stdout. +- `export-bash-scripts` - Materialize every shell script the compiler can emit as ordinary files, so generated shell can be reviewed and analysed without reading Rust. Reads the `src/compile/shell/` registry directly, so unlike the fixture-driven bash lint it reaches *every* script — including ones no pipeline currently emits. + - `--output, -o ` - Directory to write into. Created if it does not exist. Required. + - `--format ` - `files` (default) writes one `.sh` per script with a provenance header naming the producing Rust source; `json` writes a single `bash-scripts.json` carrying the same content plus each script's declared binding surface. + - What is written is the *lint source*: the body with declared variables stub-assigned, which is the form that stands alone and the form the shellcheck harness judges. A rendered script needs real bindings, which only the producing call site has. + - Typical use: `cargo run -- export-bash-scripts --output /tmp/ado-aw-shell && shellcheck /tmp/ado-aw-shell/*.sh` + ### Hidden Pipeline-Internal Commands These commands are started by the pipeline itself (or by AWF on its behalf) and are not part of the authoring surface: diff --git a/docs/extending.md b/docs/extending.md index f88fa3af..ea068cc2 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -106,6 +106,8 @@ Compiler-owned steps should be `Step` variants from `src/compile/ir/step.rs`. ### Bash steps +Compiler-generated shell must go through `ShellScript` — see [Generated shell scripts](#generated-shell-scripts) below. Construct `BashStep::new` directly only for a body with no substitution at all. + ```rust use crate::compile::ir::env::EnvValue; use crate::compile::ir::ids::StepId; @@ -286,6 +288,120 @@ First-class tools live under `src/tools//`. 6. Extend `ToolsConfig` in `src/compile/types.rs` and `collect_extensions()`. 7. Add tests for config parsing, declarations, and emitted pipeline behavior. +## Generated shell scripts + +Every shell script the compiler emits lives in `src/compile/shell/` as a +registered raw-string constant, not as a `format!` template. `format!` forced +three layers of escaping onto a script at once — `\n\` continuations, doubled +braces to survive `format!` itself, and an escaped quote for every quoted word +— which made a long body impossible to review as shell. Reviewing it as shell +is the only way to know it is correct. + +### Declaring a script + +```rust +use crate::shell_script; +use crate::compile::shell::{Binding, ShellScript}; + +shell_script! { + /// One line on why this script exists. + STOP_ADO_PROXY { + interpreter: Bash, // or Sh + bindings: [PROXY_CONTAINER], // the compiler supplies these + externals: [SOME_ENV_VAR], // the runtime supplies these + fragments: [], // shell composed in from elsewhere + body: r#" +docker rm -f "$PROXY_CONTAINER" 2>/dev/null || true +"#, + } +} + +ShellScript::new(&STOP_ADO_PROXY) + .text("PROXY_CONTAINER", ADO_PROXY_CONTAINER_NAME) + .into_step("Stop ado-proxy") +``` + +The `body:` is the shell **exactly as it will run**. Nothing is escaped: a +Docker Go-template is written literally as `{{.State.Status}}`. + +### Bindings + +A value can only ever be the right-hand side of an assignment in the generated +prelude. That is the one position where a value's own quoting fully determines +its meaning, so it cannot alter the structure of the script no matter what it +contains. Pick the constructor that matches the shape: + +| Constructor | Renders as | Use for | +|---|---|---| +| `Binding::text(s)` | `'s'` | any single-line literal | +| `Binding::number(n)` | `n` | ports, counts, timeouts | +| `Binding::boolean(b)` | `true` / `false` | flags read as `[ "$V" = true ]` | +| `Binding::words([…])` | `'a b c'` | a list the body expands unquoted in `for` | +| `Binding::ado_macro("Agent.TempDirectory")` | `'$(Agent.TempDirectory)'` | ADO predefined variables | +| `Binding::document(text)` | quoted heredoc | JSON, prompts, certificates | + +Each validates its own shape: `words` rejects an entry containing whitespace +or a glob (the consumer expands it unquoted, so that would silently change the +list), and `ado_macro` accepts only a well-formed dotted name. + +### Declaring the variable surface + +Every variable the body reads must be declared as a `binding` (the compiler +supplies it) or an `external` (the runtime does — step `env:`, an ADO +`##vso[task.setvariable]` from an earlier step, or a fragment). Anything the +body assigns itself needs no declaration. + +This is enforced two ways: `ShellScript::render` refuses to render with a +declared binding unbound, and a registry-wide test fails on a body that reads +an undeclared variable. + +### Secrets + +A credential must never become a binding. The prelude is written verbatim into +the `*.lock.yml` committed to the repository. `Binding` rejects values naming a +known credential; credentials arrive through `env:` as `EnvValue::secret`, +which Azure DevOps masks in logs. + +```rust +ShellScript::new(&START_ADO_PROXY) + .text("PROXY_CONTAINER", ADO_PROXY_CONTAINER_NAME) + .into_step("Start ado-proxy policy engine") + .with_env("ADO_PROXY_BEARER", EnvValue::secret("SC_READ_TOKEN")) +``` + +### Composing a long script from phases + +A script too long to review whole is assembled from registered phases spliced +at markers: + +```rust +body: r#" +set -euo pipefail +# ado-aw:fragment resolve_org +echo "$ADO_PROXY_ORGANIZATION" +"#, +``` + +```rust +.fragment("resolve_org", common::resolve_ado_organization_bash()) +``` + +A marker is an ordinary shell comment, so the outline body stays valid, +shellcheck-able shell whether or not the fragment is spliced. Any variable a +fragment defines must be declared in the consumer's `externals:`, which forces +the inter-phase contract somewhere a reviewer can see it. Declaring a fragment +without marking it (or vice versa) is a test failure, not a silent no-op. + +### Reviewing the scripts as files + +```bash +cargo run -- export-bash-scripts --out /tmp/ado-aw-shell +cargo run -- export-bash-scripts --out /tmp/ado-aw-shell --format json +``` + +Writes one `.sh` per registered script with a provenance header naming the +producing Rust source, for review with ordinary shell tooling. + ## Filter IR (`src/compile/filter_ir.rs`) Trigger filter expressions still use the separate filter IR. It lowers `PrFilters` / `PipelineFilters` into typed checks, validates conflicts, and emits bash consumed by `AdoScriptExtension` declarations. The generated gate steps are now returned as typed IR steps instead of being spliced into YAML templates. @@ -300,10 +416,28 @@ To add a new filter type: ## Bash step linting -`tests/bash_lint_tests.rs` compiles representative fixtures and runs `shellcheck` against every literal `bash:` body in generated YAML. When adding or modifying bash: +Generated shell is linted at two levels, and both matter. + +**Every registered script, in isolation** (`src/compile/shell/lint.rs`). Reads +the registry directly, so it reaches every script whether or not any pipeline +emits it. Declared bindings and externals are stub-assigned so SC2154 still +fires for a variable the body reads without declaring. This closes a real gap: +lint coverage used to be a function of fixture reachability, so a generator no +fixture happened to exercise — including several hundred lines of `ado-proxy` +and `az` wrapper shell — was linted by nothing. + +**Every bash body that reaches the emitted YAML** (`tests/bash_lint_tests.rs`). +Compiles representative fixtures and shellchecks what actually ships. This +proves scripts are *emitted*, where the registry lint proves they are *correct*. + +When adding or modifying shell: -1. Run `cargo test --test bash_lint_tests` if `shellcheck` is available locally. -2. Fix findings such as unquoted variables, `cd` without failure handling, masked exit codes, and tilde-in-double-quotes. -3. If a finding is intentional, add a `# shellcheck disable=SCxxxx` comment immediately above the line in the bash body. +1. Run `ENFORCE_BASH_LINT=1 cargo test --test bash_lint_tests` and + `cargo test --bin ado-aw compile::shell` if `shellcheck` is available locally. +2. Fix findings such as unquoted variables, `cd` without failure handling, + masked exit codes, and tilde-in-double-quotes. +3. If a finding is intentional, add a `# shellcheck disable=SCxxxx` comment + immediately above the line in the body. -Do not add blanket `set -eo pipefail` to every step just to satisfy lint. Use targeted fail-fast behavior only when the step requires it. +Do not add blanket `set -eo pipefail` to every step just to satisfy lint. Use +targeted fail-fast behavior only when the step requires it. diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index 7960ca0d..5c5feddb 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -72,8 +72,10 @@ use super::common::{ HEADER_MARKER, MCPG_CONTAINER_NAME, MCPG_DOMAIN, MCPG_IMAGE, MCPG_PORT, MCPG_VERSION, image_ref, }; use super::extensions::ado_script as paths; +use super::shell::{Binding, ShellScript}; use crate::ado_proxy::catalog; use crate::ado_proxy::policy::PolicyDocument; +use crate::shell_script; use super::custom_tools::{CustomToolDefinition, collect_custom_tool_definitions}; use super::extensions::{CompileContext, CompilerExtension, Declarations, Extension, McpgConfig}; use super::ir::condition::{Condition, Expr}; @@ -4202,204 +4204,520 @@ fn stop_mcpg_step() -> BashStep { /// read them from a file would hand the agent the exact credential this whole /// design exists to withhold. /// +/// # Structure +/// +/// The step's ~200-line body is composed from ordered phases, each registered +/// as its own [`shell_script!`] so shellcheck sees it in isolation and the +/// declared variable surface at each phase boundary is visible in the source. +/// See [`docs/ado-script.md`] and issue #1833 for the design rationale. All +/// phases still emit into a **single ADO Bash task**: the credential-custody +/// contract (bearer via `env:` only, private material streamed on stdin, +/// destroyed before readiness polling) requires atomic execution. +/// /// Not yet emitted: see [`stop_ado_proxy_step`]. fn start_ado_proxy_step(front_matter: &FrontMatter) -> BashStep { let policy = PolicyDocument::new(front_matter).to_json(); - let hosts = catalog::catalog().protected_hosts; - // Mint one leaf per catalogued protected host. A host without a leaf - // cannot be intercepted, so this list must track the catalog rather than - // be maintained separately. - let leaf_loop = hosts - .iter() - .map(|host| format!("\"{host}\"")) - .collect::>() - .join(" "); - - let script = format!( - "# Start the ado-proxy policy engine.\n\ - #\n\ - # The agent never receives an Azure DevOps credential. This container\n\ - # holds it, and serves only the operations in the versioned catalog.\n\ - set -euo pipefail\n\ - \n\ - # Generate into the agent work directory, NOT /tmp: AWF mounts /tmp\n\ - # into the agent chroot, so /tmp is readable by the agent.\n\ - umask 077\n\ - PROXY_DIR=$(mktemp -d \"$(Agent.TempDirectory)/ado-proxy.XXXXXX\")\n\ - cleanup_material() {{ rm -rf \"$PROXY_DIR\"; }}\n\ - trap cleanup_material EXIT\n\ - \n\ - # Policy document. Non-secret, so it is mounted rather than streamed.\n\ - # Scope is substituted here rather than at compile time so the same\n\ - # compiled pipeline can be queued against a different project.\n\ - #\n\ - # Both the name and the GUID of the project and repository are\n\ - # supplied: clients address them either way — `az` substitutes\n\ - # whichever it cached — and the bundle treats an absent identifier as\n\ - # matching nothing, so omitting one is a silent denial.\n\ -{org_resolve}\ - ADO_PROXY_PROJECT=\"$(System.TeamProject)\"\n\ - ADO_PROXY_PROJECT_ID=\"$(System.TeamProjectId)\"\n\ - ADO_PROXY_REPOSITORY=\"$(Build.Repository.Name)\"\n\ - ADO_PROXY_REPOSITORY_ID=\"$(Build.Repository.ID)\"\n\ - mkdir -p \"$PROXY_DIR/policy\"\n\ - cat > \"$PROXY_DIR/policy/policy.json\" <<'ADO_PROXY_POLICY_EOF'\n\ - {policy}\n\ - ADO_PROXY_POLICY_EOF\n\ - sed -i \\\n \ - -e \"s|\\${{ADO_PROXY_ORGANIZATION}}|$ADO_PROXY_ORGANIZATION|g\" \\\n \ - -e \"s|\\${{ADO_PROXY_PROJECT}}|$ADO_PROXY_PROJECT|g\" \\\n \ - -e \"s|\\${{ADO_PROXY_PROJECT_ID}}|$ADO_PROXY_PROJECT_ID|g\" \\\n \ - -e \"s|\\${{ADO_PROXY_REPOSITORY}}|$ADO_PROXY_REPOSITORY|g\" \\\n \ - -e \"s|\\${{ADO_PROXY_REPOSITORY_ID}}|$ADO_PROXY_REPOSITORY_ID|g\" \\\n \ - \"$PROXY_DIR/policy/policy.json\"\n\ - \n\ - # A surviving placeholder would be read as a literal organization or\n\ - # repository name, matching nothing — a total denial that reads as a\n\ - # policy decision rather than a bug.\n\ - if grep -q 'ADO_PROXY_' \"$PROXY_DIR/policy/policy.json\"; then\n \ - echo \"##vso[task.complete result=Failed]ado-proxy policy still contains an unsubstituted placeholder\"\n \ - exit 1\n\ - fi\n\ - echo \"ado-proxy policy:\"\n\ - python3 -m json.tool < \"$PROXY_DIR/policy/policy.json\"\n\ - \n\ - # Interception certificate authority and one leaf per protected host.\n\ - openssl req -x509 -newkey rsa:2048 -nodes -days 2 \\\n \ - -subj \"/CN=ado-aw ado-proxy interception CA\" \\\n \ - -keyout \"$PROXY_DIR/ca.key\" -out \"$PROXY_DIR/ca.pem\" \\\n \ - -addext \"basicConstraints=critical,CA:TRUE,pathlen:0\" \\\n \ - -addext \"keyUsage=critical,keyCertSign,cRLSign\" 2>/dev/null\n\ - for PROXY_HOST in {leaf_loop}; do\n \ - printf 'basicConstraints=CA:FALSE\\nkeyUsage=critical,digitalSignature,keyEncipherment\\nextendedKeyUsage=serverAuth\\nsubjectAltName=DNS:%s\\n' \"$PROXY_HOST\" > \"$PROXY_DIR/leaf.ext\"\n \ - openssl req -new -newkey rsa:2048 -nodes -subj \"/CN=$PROXY_HOST\" \\\n \ - -keyout \"$PROXY_DIR/$PROXY_HOST.key\" -out \"$PROXY_DIR/$PROXY_HOST.csr\" 2>/dev/null\n \ - openssl x509 -req -in \"$PROXY_DIR/$PROXY_HOST.csr\" \\\n \ - -CA \"$PROXY_DIR/ca.pem\" -CAkey \"$PROXY_DIR/ca.key\" -CAcreateserial \\\n \ - -days 2 -extfile \"$PROXY_DIR/leaf.ext\" -out \"$PROXY_DIR/$PROXY_HOST.pem\" 2>/dev/null\n\ - done\n\ - \n\ - # The proxy publishes its own interception CA certificate for clients\n\ - # to trust. It goes under /tmp deliberately: AWF mounts /tmp into the\n\ - # agent chroot, so this one file is what the az wrapper reads and what\n\ - # the MCP container mounts. Publishing once means no client can trust\n\ - # a stale copy. The matching private key never leaves $PROXY_DIR and\n\ - # is destroyed below.\n\ - mkdir -p {az_wrapper_dir}\n\ - echo \"##vso[task.setvariable variable=ADO_PROXY_CA_FILE]{ca_host_path}\"\n\ - \n\ - # Build the material document. jq assembles it so that a value\n\ - # containing JSON metacharacters cannot alter the document shape.\n\ - PROXY_MATERIAL=$(jq -n \\\n \ - --arg schema 'ado-aw/ado-proxy-material/v1' \\\n \ - --arg ca_cert \"$(base64 -w0 < \"$PROXY_DIR/ca.pem\")\" \\\n \ - --arg token \"$(printf '%s' \"$ADO_PROXY_BEARER\" | base64 -w0)\" \\\n \ - '{{schema: $schema, ca_cert: $ca_cert, token: $token, leaves: {{}}}}')\n\ - for PROXY_HOST in {leaf_loop}; do\n \ - PROXY_MATERIAL=$(printf '%s' \"$PROXY_MATERIAL\" | jq \\\n \ - --arg host \"$PROXY_HOST\" \\\n \ - --arg key \"$(base64 -w0 < \"$PROXY_DIR/$PROXY_HOST.key\")\" \\\n \ - --arg cert \"$(base64 -w0 < \"$PROXY_DIR/$PROXY_HOST.pem\")\" \\\n \ - '.leaves[$host] = {{key: $key, cert: $cert}}')\n\ - done\n\ - \n\ - # Remove any container left behind by an interrupted run.\n\ - docker rm -f {ADO_PROXY_CONTAINER_NAME} 2>/dev/null || true\n\ - mkdir -p /tmp/gh-aw/ado-proxy-logs\n\ - \n\ - # Start detached so the container lifetime belongs to Docker, not to\n\ - # this Bash task's attached STDIO. Azure Pipelines cleans up inherited\n\ - # child streams between tasks; an attached `docker run -i ... &` was\n\ - # observed to exit and `--rm` itself before AWF could attach it.\n\ - #\n\ - # A container-local FIFO preserves the stdin-only custody contract:\n\ - # material is streamed through `docker exec -i`, never written to a\n\ - # runner path, container layer, argv, or environment.\n\ - docker run -d \\\n \ - --name {ADO_PROXY_CONTAINER_NAME} \\\n \ - --network {ADO_PROXY_NETWORK_NAME} \\\n \ - --entrypoint sh \\\n \ - -v \"{ado_proxy_path}:/app/ado-proxy.js:ro\" \\\n \ - -v \"$PROXY_DIR/policy:/etc/ado-proxy:ro\" \\\n \ - -v /tmp/ado-aw-lib:/var/lib/ado-proxy \\\n \ - -v /tmp/gh-aw/ado-proxy-logs:/var/log/ado-proxy \\\n \ - {ado_proxy_image} \\\n \ - -c 'set -eu; umask 077; MATERIAL_FIFO=/tmp/ado-proxy-material; mkfifo \"$MATERIAL_FIFO\"; exec node /app/ado-proxy.js --policy-file /etc/ado-proxy/policy.json --public-ca-file /var/lib/ado-proxy/ado-proxy-ca.pem --upstream-proxy {squid_url} --listen-port {listen_port} --tls-port {tls_port} --log-dir /var/log/ado-proxy < \"$MATERIAL_FIFO\"' \\\n \ - >/dev/null\n\ - \n\ - # Wait until the detached container is blocked on its private FIFO,\n\ - # then hand over the one-shot material. A transfer failure prints the\n\ - # durable Docker log and container state before failing the pipeline.\n\ - FIFO_READY=false\n\ - for _i in $(seq 1 30); do\n \ - if docker exec {ADO_PROXY_CONTAINER_NAME} test -p /tmp/ado-proxy-material 2>/dev/null; then\n \ - FIFO_READY=true\n \ - break\n \ - fi\n \ - sleep 1\n\ - done\n\ - if [ \"$FIFO_READY\" != \"true\" ]; then\n \ - echo \"##vso[task.logissue type=error]ado-proxy container did not create its private material channel\"\n \ - docker inspect -f 'state={{{{.State.Status}}}} exit={{{{.State.ExitCode}}}} error={{{{.State.Error}}}}' {ADO_PROXY_CONTAINER_NAME} 2>/dev/null || true\n \ - docker logs --tail 200 {ADO_PROXY_CONTAINER_NAME} 2>&1 || true\n \ - exit 1\n\ - fi\n\ - if ! printf '%s' \"$PROXY_MATERIAL\" | docker exec -i {ADO_PROXY_CONTAINER_NAME} sh -c 'cat > /tmp/ado-proxy-material'; then\n \ - echo \"##vso[task.logissue type=error]ado-proxy material handover failed\"\n \ - docker inspect -f 'state={{{{.State.Status}}}} exit={{{{.State.ExitCode}}}} error={{{{.State.Error}}}}' {ADO_PROXY_CONTAINER_NAME} 2>/dev/null || true\n \ - docker logs --tail 200 {ADO_PROXY_CONTAINER_NAME} 2>&1 || true\n \ - exit 1\n\ - fi\n\ - \n\ - # Drop the private material as soon as it has been handed over. The\n\ - # container has it in memory; nothing else needs it again.\n\ - PROXY_MATERIAL=\"\"\n\ - unset PROXY_MATERIAL\n\ - shred -u \"$PROXY_DIR/ca.key\" \"$PROXY_DIR\"/*.key 2>/dev/null || rm -f \"$PROXY_DIR/ca.key\" \"$PROXY_DIR\"/*.key\n\ - \n\ - # Resolve the container IP only after the engine has parsed policy,\n\ - # published its public CA and reached its listening state.\n\ - PROXY_READY=false\n\ - for _i in $(seq 1 30); do\n \ - PROXY_STATE=$(docker inspect -f '{{{{.State.Status}}}}' {ADO_PROXY_CONTAINER_NAME} 2>/dev/null || true)\n \ - if [ \"$PROXY_STATE\" = \"exited\" ] || [ \"$PROXY_STATE\" = \"dead\" ]; then\n \ - break\n \ - fi\n \ - ADO_PROXY_IP=$(docker inspect -f '{{{{range .NetworkSettings.Networks}}}}{{{{.IPAddress}}}}{{{{end}}}}' {ADO_PROXY_CONTAINER_NAME} 2>/dev/null || true)\n \ - if [ -n \"$ADO_PROXY_IP\" ] \\\n \ - && [ -f {ca_host_path} ] \\\n \ - && docker logs {ADO_PROXY_CONTAINER_NAME} 2>&1 | grep -q '\\[ado-proxy\\] listening'; then\n \ - PROXY_READY=true\n \ - break\n \ - fi\n \ - sleep 1\n\ - done\n\ - if [ \"$PROXY_READY\" != \"true\" ]; then\n \ - echo \"##vso[task.logissue type=error]ado-proxy did not become ready within 30s (state=${{PROXY_STATE:-missing}})\"\n \ - docker inspect -f 'state={{{{.State.Status}}}} exit={{{{.State.ExitCode}}}} error={{{{.State.Error}}}}' {ADO_PROXY_CONTAINER_NAME} 2>/dev/null || true\n \ - docker logs --tail 200 {ADO_PROXY_CONTAINER_NAME} 2>&1 || true\n \ - exit 1\n\ - fi\n\ - echo \"ado-proxy is ready at $ADO_PROXY_IP\"\n\ - docker logs --tail 1 {ADO_PROXY_CONTAINER_NAME} 2>&1 || true\n\ - echo \"##vso[task.setvariable variable=ADO_PROXY_IP]$ADO_PROXY_IP\"\n", - org_resolve = common::resolve_ado_organization_bash(), - ado_proxy_path = paths::ADO_PROXY_PATH, - ca_host_path = ADO_PROXY_PUBLIC_CA_HOST_PATH, - az_wrapper_dir = AZ_WRAPPER_DIR, - ado_proxy_image = ADO_PROXY_IMAGE, - squid_url = AWF_SQUID_URL, - listen_port = ADO_PROXY_LISTEN_PORT, - tls_port = ADO_PROXY_TLS_PORT, - ); - - bash("Start ado-proxy policy engine", script) + let hosts: Vec<&str> = catalog::catalog().protected_hosts.to_vec(); + + ShellScript::new(&START_ADO_PROXY) + .text("PROXY_CONTAINER", ADO_PROXY_CONTAINER_NAME) + .text("PROXY_IMAGE", ADO_PROXY_IMAGE) + .text("PROXY_NETWORK", ADO_PROXY_NETWORK_NAME) + .text("PROXY_SCRIPT_PATH", paths::ADO_PROXY_PATH) + .text("AZ_WRAPPER_DIR", AZ_WRAPPER_DIR) + .text("CA_HOST_PATH", ADO_PROXY_PUBLIC_CA_HOST_PATH) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .bind( + "ADO_PROXY_PROJECT", + Binding::ado_macro("System.TeamProject"), + ) + .bind( + "ADO_PROXY_PROJECT_ID", + Binding::ado_macro("System.TeamProjectId"), + ) + .bind( + "ADO_PROXY_REPOSITORY", + Binding::ado_macro("Build.Repository.Name"), + ) + .bind( + "ADO_PROXY_REPOSITORY_ID", + Binding::ado_macro("Build.Repository.ID"), + ) + .bind("LEAF_HOSTS", Binding::words(&hosts)) + .bind("POLICY", Binding::document(policy)) + .bind( + "CONTAINER_ENTRYPOINT", + Binding::text(ado_proxy_container_entrypoint_flattened()), + ) + .fragment("resolve_org", common::resolve_ado_organization_bash()) + .fragment( + "setup_workdir", + phase_body(&START_ADO_PROXY_SETUP_WORKDIR), + ) + .fragment("write_policy", phase_body(&START_ADO_PROXY_WRITE_POLICY)) + .fragment( + "mint_material", + phase_body(&START_ADO_PROXY_MINT_MATERIAL), + ) + .fragment( + "build_material", + phase_body(&START_ADO_PROXY_BUILD_MATERIAL), + ) + .fragment( + "run_container", + phase_body(&START_ADO_PROXY_RUN_CONTAINER), + ) + .fragment( + "handover_material", + phase_body(&START_ADO_PROXY_HANDOVER_MATERIAL), + ) + .fragment( + "destroy_private", + phase_body(&START_ADO_PROXY_DESTROY_PRIVATE), + ) + .fragment("wait_ready", phase_body(&START_ADO_PROXY_WAIT_READY)) + .into_step("Start ado-proxy policy engine") // The bearer is read from the environment here and immediately // base64-encoded into the stdin document; it is never written to a // file and never reaches the container's `Env`. .with_env("ADO_PROXY_BEARER", EnvValue::secret("SC_READ_TOKEN")) } +/// Return a phase's body verbatim (trimmed and dedented) so it can be +/// spliced as a `# ado-aw:fragment` in [`START_ADO_PROXY`]. Each phase is +/// still registered — and therefore shellchecked — in isolation. +fn phase_body(def: &crate::compile::shell::ShellScriptDef) -> String { + // Skip a leading shebang line if present (phases don't carry one; guard + // is defence-in-depth), then dedent the raw body. + let body = def.body.trim_start_matches('\n'); + crate::compile::shell::dedent(body).trim().to_string() +} + +/// The one-liner passed to the container's `sh -c`. Kept in sync with the +/// registered [`START_ADO_PROXY_CONTAINER_ENTRYPOINT_SH`] script via +/// `container_entrypoint_matches_registered_body`. +fn ado_proxy_container_entrypoint_flattened() -> String { + format!( + "set -eu; umask 077; MATERIAL_FIFO=/tmp/ado-proxy-material; \ + mkfifo \"$MATERIAL_FIFO\"; \ + exec node /app/ado-proxy.js \ + --policy-file /etc/ado-proxy/policy.json \ + --public-ca-file /var/lib/ado-proxy/ado-proxy-ca.pem \ + --upstream-proxy {url} \ + --listen-port {lp} \ + --tls-port {tp} \ + --log-dir /var/log/ado-proxy < \"$MATERIAL_FIFO\"", + url = AWF_SQUID_URL, + lp = ADO_PROXY_LISTEN_PORT, + tp = ADO_PROXY_TLS_PORT, + ) +} + +// ── ado-proxy phase scripts ───────────────────────────────────────────── +// +// The start_ado_proxy step's ~200-line body is composed from these ordered +// phases. Each is registered so `src/compile/shell/lint.rs` shellchecks it in +// isolation, and each phase's declared `externals:` list makes the +// inter-phase variable contract visible in the source. +// +// The full script still runs as a **single trusted Bash task** because the +// credential-custody contract (bearer via env only, private material on +// stdin, destroyed before polling) requires atomic execution. + +shell_script! { + /// Phase 1: create the agent-private work directory outside `/tmp` and + /// register a cleanup trap. AWF mounts `/tmp` into the agent chroot, so + /// any private material generated under `/tmp` would be agent-readable. + START_ADO_PROXY_SETUP_WORKDIR { + interpreter: Bash, + bindings: [], + externals: [AGENT_TEMP], + fragments: [], + body: r###" +set -euo pipefail + +# Generate into the agent work directory, NOT /tmp: AWF mounts /tmp +# into the agent chroot, so /tmp is readable by the agent. +umask 077 +PROXY_DIR=$(mktemp -d "$AGENT_TEMP/ado-proxy.XXXXXX") +cleanup_material() { rm -rf "$PROXY_DIR"; } +trap cleanup_material EXIT +"###, + } +} + +shell_script! { + /// Phase 3: write the policy document, substitute the scope identifiers + /// resolved at pipeline runtime, refuse to start if any placeholder + /// survives, then dump the fully substituted policy for auditability. + /// + /// Both name and GUID of project and repository are supplied because + /// clients may address either — `az` substitutes whichever it cached. + /// The bundle treats an absent identifier as matching nothing, so + /// omitting one is a silent denial. + START_ADO_PROXY_WRITE_POLICY { + interpreter: Bash, + bindings: [], + externals: [ + PROXY_DIR, POLICY, + ADO_PROXY_ORGANIZATION, + ADO_PROXY_PROJECT, ADO_PROXY_PROJECT_ID, + ADO_PROXY_REPOSITORY, ADO_PROXY_REPOSITORY_ID + ], + fragments: [], + body: r###" +# Policy document. Non-secret, so it is mounted rather than streamed. +# Scope is substituted here rather than at compile time so the same +# compiled pipeline can be queued against a different project. +mkdir -p "$PROXY_DIR/policy" +printf '%s\n' "$POLICY" > "$PROXY_DIR/policy/policy.json" +sed -i \ + -e "s|\${ADO_PROXY_ORGANIZATION}|$ADO_PROXY_ORGANIZATION|g" \ + -e "s|\${ADO_PROXY_PROJECT}|$ADO_PROXY_PROJECT|g" \ + -e "s|\${ADO_PROXY_PROJECT_ID}|$ADO_PROXY_PROJECT_ID|g" \ + -e "s|\${ADO_PROXY_REPOSITORY}|$ADO_PROXY_REPOSITORY|g" \ + -e "s|\${ADO_PROXY_REPOSITORY_ID}|$ADO_PROXY_REPOSITORY_ID|g" \ + "$PROXY_DIR/policy/policy.json" + +# A surviving placeholder would be read as a literal organization or +# repository name, matching nothing — a total denial that reads as a +# policy decision rather than a bug. +if grep -q 'ADO_PROXY_' "$PROXY_DIR/policy/policy.json"; then + echo "##vso[task.complete result=Failed]ado-proxy policy still contains an unsubstituted placeholder" + exit 1 +fi +echo "ado-proxy policy:" +python3 -m json.tool < "$PROXY_DIR/policy/policy.json" +"###, + } +} + +shell_script! { + /// Phase 4: mint the interception CA plus one leaf per catalogued + /// protected host, then publish the CA path via `task.setvariable` for + /// clients (the `az` wrapper, the ADO MCP mount). The matching private + /// key never leaves `$PROXY_DIR` and is destroyed in + /// [`START_ADO_PROXY_DESTROY_PRIVATE`]. + /// + /// `keyUsage=critical,keyCertSign,cRLSign` is load-bearing: OpenSSL 3 + /// (as used by Python `requests`) rejects `pathlen` without an explicit + /// `keyCertSign`, so a lax CA would fail strict verifiers. + START_ADO_PROXY_MINT_MATERIAL { + interpreter: Bash, + bindings: [], + externals: [PROXY_DIR, LEAF_HOSTS, AZ_WRAPPER_DIR, CA_HOST_PATH], + fragments: [], + body: r###" +# Interception certificate authority and one leaf per protected host. +openssl req -x509 -newkey rsa:2048 -nodes -days 2 \ + -subj "/CN=ado-aw ado-proxy interception CA" \ + -keyout "$PROXY_DIR/ca.key" -out "$PROXY_DIR/ca.pem" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" 2>/dev/null +# shellcheck disable=SC2086 # LEAF_HOSTS is Binding::words; unquoted expansion is the documented word-list contract. +for PROXY_HOST in $LEAF_HOSTS; do + printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:%s\n' "$PROXY_HOST" > "$PROXY_DIR/leaf.ext" + openssl req -new -newkey rsa:2048 -nodes -subj "/CN=$PROXY_HOST" \ + -keyout "$PROXY_DIR/$PROXY_HOST.key" -out "$PROXY_DIR/$PROXY_HOST.csr" 2>/dev/null + openssl x509 -req -in "$PROXY_DIR/$PROXY_HOST.csr" \ + -CA "$PROXY_DIR/ca.pem" -CAkey "$PROXY_DIR/ca.key" -CAcreateserial \ + -days 2 -extfile "$PROXY_DIR/leaf.ext" -out "$PROXY_DIR/$PROXY_HOST.pem" 2>/dev/null +done + +# The proxy publishes its own interception CA certificate for clients +# to trust. It goes under /tmp deliberately: AWF mounts /tmp into the +# agent chroot, so this one file is what the az wrapper reads and what +# the MCP container mounts. Publishing once means no client can trust +# a stale copy. The matching private key never leaves $PROXY_DIR and +# is destroyed below. +mkdir -p "$AZ_WRAPPER_DIR" +echo "##vso[task.setvariable variable=ADO_PROXY_CA_FILE]$CA_HOST_PATH" +"###, + } +} + +shell_script! { + /// Phase 5: assemble the material document with `jq` so a value + /// containing JSON metacharacters cannot alter the document shape. The + /// bearer is read from the environment here and immediately base64- + /// encoded into the JSON string — it never lands in argv, in a file, or + /// in the container `Env`. + START_ADO_PROXY_BUILD_MATERIAL { + interpreter: Bash, + bindings: [], + externals: [PROXY_DIR, LEAF_HOSTS, ADO_PROXY_BEARER], + fragments: [], + body: r###" +# Build the material document. jq assembles it so that a value +# containing JSON metacharacters cannot alter the document shape. +PROXY_MATERIAL=$(jq -n \ + --arg schema 'ado-aw/ado-proxy-material/v1' \ + --arg ca_cert "$(base64 -w0 < "$PROXY_DIR/ca.pem")" \ + --arg token "$(printf '%s' "$ADO_PROXY_BEARER" | base64 -w0)" \ + '{schema: $schema, ca_cert: $ca_cert, token: $token, leaves: {}}') +# shellcheck disable=SC2086 # LEAF_HOSTS is Binding::words; unquoted expansion is the documented word-list contract. +for PROXY_HOST in $LEAF_HOSTS; do + PROXY_MATERIAL=$(printf '%s' "$PROXY_MATERIAL" | jq \ + --arg host "$PROXY_HOST" \ + --arg key "$(base64 -w0 < "$PROXY_DIR/$PROXY_HOST.key")" \ + --arg cert "$(base64 -w0 < "$PROXY_DIR/$PROXY_HOST.pem")" \ + '.leaves[$host] = {key: $key, cert: $cert}') +done +"###, + } +} + +shell_script! { + /// Phase 6: start the proxy container detached, so the container + /// lifetime belongs to Docker, not to this Bash task's attached STDIO. + /// Azure Pipelines cleans up inherited child streams between tasks; an + /// attached `docker run -i ... &` was observed to exit and `--rm` itself + /// before AWF could attach it. + /// + /// A container-local FIFO preserves the stdin-only custody contract: + /// material is streamed through `docker exec -i`, never written to a + /// runner path, container layer, argv, or environment. + START_ADO_PROXY_RUN_CONTAINER { + interpreter: Bash, + bindings: [], + externals: [ + PROXY_CONTAINER, PROXY_NETWORK, PROXY_SCRIPT_PATH, + PROXY_DIR, PROXY_IMAGE, CONTAINER_ENTRYPOINT, AZ_WRAPPER_DIR + ], + fragments: [], + body: r###" +# Remove any container left behind by an interrupted run. +docker rm -f "$PROXY_CONTAINER" 2>/dev/null || true +mkdir -p /tmp/gh-aw/ado-proxy-logs + +# Start detached so the container lifetime belongs to Docker, not to +# this Bash task's attached STDIO. Azure Pipelines cleans up inherited +# child streams between tasks; an attached `docker run -i ... &` was +# observed to exit and `--rm` itself before AWF could attach it. +# +# A container-local FIFO preserves the stdin-only custody contract: +# material is streamed through `docker exec -i`, never written to a +# runner path, container layer, argv, or environment. +docker run -d \ + --name "$PROXY_CONTAINER" \ + --network "$PROXY_NETWORK" \ + --entrypoint sh \ + -v "$PROXY_SCRIPT_PATH:/app/ado-proxy.js:ro" \ + -v "$PROXY_DIR/policy:/etc/ado-proxy:ro" \ + -v "$AZ_WRAPPER_DIR:/var/lib/ado-proxy" \ + -v /tmp/gh-aw/ado-proxy-logs:/var/log/ado-proxy \ + "$PROXY_IMAGE" \ + -c "$CONTAINER_ENTRYPOINT" \ + >/dev/null +"###, + } +} + +shell_script! { + /// Phase 7: wait for the container to open its private material FIFO, + /// then stream the assembled material in over `docker exec -i`. A + /// failed transfer prints durable Docker log + inspect state before + /// failing the pipeline so the true cause reaches the audit trail. + START_ADO_PROXY_HANDOVER_MATERIAL { + interpreter: Bash, + bindings: [], + externals: [PROXY_CONTAINER, PROXY_MATERIAL], + fragments: [], + body: r###" +# Wait until the detached container is blocked on its private FIFO, +# then hand over the one-shot material. A transfer failure prints the +# durable Docker log and container state before failing the pipeline. +FIFO_READY=false +for _i in $(seq 1 30); do + if docker exec "$PROXY_CONTAINER" test -p /tmp/ado-proxy-material 2>/dev/null; then + FIFO_READY=true + break + fi + sleep 1 +done +if [ "$FIFO_READY" != "true" ]; then + echo "##vso[task.logissue type=error]ado-proxy container did not create its private material channel" + docker inspect -f 'state={{.State.Status}} exit={{.State.ExitCode}} error={{.State.Error}}' "$PROXY_CONTAINER" 2>/dev/null || true + docker logs --tail 200 "$PROXY_CONTAINER" 2>&1 || true + exit 1 +fi +if ! printf '%s' "$PROXY_MATERIAL" | docker exec -i "$PROXY_CONTAINER" sh -c 'cat > /tmp/ado-proxy-material'; then + echo "##vso[task.logissue type=error]ado-proxy material handover failed" + docker inspect -f 'state={{.State.Status}} exit={{.State.ExitCode}} error={{.State.Error}}' "$PROXY_CONTAINER" 2>/dev/null || true + docker logs --tail 200 "$PROXY_CONTAINER" 2>&1 || true + exit 1 +fi +"###, + } +} + +shell_script! { + /// Phase 8: destroy the private material as soon as the container has + /// consumed it. The container holds material in memory only; nothing + /// else needs it. Ordering matters: this must run before readiness + /// polling so a hung poll cannot keep the CA private key on disk. + START_ADO_PROXY_DESTROY_PRIVATE { + interpreter: Bash, + bindings: [], + externals: [PROXY_DIR], + fragments: [], + body: r###" +# Drop the private material as soon as it has been handed over. The +# container has it in memory; nothing else needs it again. +# +# Blanking before `unset` is deliberate: `unset` alone removes the name +# binding but a shell is free to leave the value in the freed slot, and this +# value is the ADO bearer plus the CA private key. Assigning "" overwrites it +# first. +# shellcheck disable=SC2034 # write-only by design; the assignment *is* the erasure +PROXY_MATERIAL="" +unset PROXY_MATERIAL +shred -u "$PROXY_DIR/ca.key" "$PROXY_DIR"/*.key 2>/dev/null || rm -f "$PROXY_DIR/ca.key" "$PROXY_DIR"/*.key +"###, + } +} + +shell_script! { + /// Phase 9: resolve the container IP after the engine has parsed + /// policy, published its public CA, and reached its listening state. + /// Publish the IP so downstream MCPG config substitution can redirect + /// clients at the engine. + START_ADO_PROXY_WAIT_READY { + interpreter: Bash, + bindings: [], + externals: [PROXY_CONTAINER, CA_HOST_PATH], + fragments: [], + body: r###" +# Resolve the container IP only after the engine has parsed policy, +# published its public CA and reached its listening state. +PROXY_READY=false +PROXY_STATE="" +ADO_PROXY_IP="" +for _i in $(seq 1 30); do + PROXY_STATE=$(docker inspect -f '{{.State.Status}}' "$PROXY_CONTAINER" 2>/dev/null || true) + if [ "$PROXY_STATE" = "exited" ] || [ "$PROXY_STATE" = "dead" ]; then + break + fi + ADO_PROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PROXY_CONTAINER" 2>/dev/null || true) + if [ -n "$ADO_PROXY_IP" ] \ + && [ -f "$CA_HOST_PATH" ] \ + && docker logs "$PROXY_CONTAINER" 2>&1 | grep -q '\[ado-proxy\] listening'; then + PROXY_READY=true + break + fi + sleep 1 +done +if [ "$PROXY_READY" != "true" ]; then + echo "##vso[task.logissue type=error]ado-proxy did not become ready within 30s (state=${PROXY_STATE:-missing})" + docker inspect -f 'state={{.State.Status}} exit={{.State.ExitCode}} error={{.State.Error}}' "$PROXY_CONTAINER" 2>/dev/null || true + docker logs --tail 200 "$PROXY_CONTAINER" 2>&1 || true + exit 1 +fi +echo "ado-proxy is ready at $ADO_PROXY_IP" +docker logs --tail 1 "$PROXY_CONTAINER" 2>&1 || true +echo "##vso[task.setvariable variable=ADO_PROXY_IP]$ADO_PROXY_IP" +"###, + } +} + +shell_script! { + /// The container's `sh -c` entrypoint. Registered as its own `Sh` + /// script so shellcheck lints the multi-line source; the actual + /// `docker run` invocation embeds the semicolon-flattened form + /// returned by [`ado_proxy_container_entrypoint_flattened`], which + /// `container_entrypoint_matches_registered_body` keeps in sync. + ADO_PROXY_CONTAINER_ENTRYPOINT_SH { + interpreter: Sh, + bindings: [], + externals: [], + fragments: [], + body: r###" +set -eu +umask 077 +MATERIAL_FIFO=/tmp/ado-proxy-material +mkfifo "$MATERIAL_FIFO" +exec node /app/ado-proxy.js \ + --policy-file /etc/ado-proxy/policy.json \ + --public-ca-file /var/lib/ado-proxy/ado-proxy-ca.pem \ + --upstream-proxy http://172.30.0.10:3128 \ + --listen-port 11080 \ + --tls-port 443 \ + --log-dir /var/log/ado-proxy < "$MATERIAL_FIFO" +"###, + } +} + +shell_script! { + /// The **atomic** `start_ado_proxy` step: composed from + /// independently-registered phases, still emitted as a single trusted + /// Bash task so the credential-custody contract holds. See + /// [`start_ado_proxy_step`] for the phase catalogue and issue #1833 for + /// the design. + START_ADO_PROXY { + interpreter: Bash, + bindings: [ + PROXY_CONTAINER, PROXY_IMAGE, PROXY_NETWORK, + PROXY_SCRIPT_PATH, AZ_WRAPPER_DIR, CA_HOST_PATH, + AGENT_TEMP, + ADO_PROXY_PROJECT, ADO_PROXY_PROJECT_ID, + ADO_PROXY_REPOSITORY, ADO_PROXY_REPOSITORY_ID, + LEAF_HOSTS, POLICY, CONTAINER_ENTRYPOINT + ], + externals: [], + fragments: [ + setup_workdir, + resolve_org, + write_policy, + mint_material, + build_material, + run_container, + handover_material, + destroy_private, + wait_ready + ], + // Every phase but `resolve_org` is a registered script, so the lint + // shellchecks the *composed* body rather than an outline of markers. + // That is what catches a variable one phase sets and another reads — + // the one new risk splitting a script into phases introduces. + // `resolve_org` is supplied at runtime by `common`, so it keeps its + // marker and is linted where it is produced. + phases: [ + setup_workdir = START_ADO_PROXY_SETUP_WORKDIR, + write_policy = START_ADO_PROXY_WRITE_POLICY, + mint_material = START_ADO_PROXY_MINT_MATERIAL, + build_material = START_ADO_PROXY_BUILD_MATERIAL, + run_container = START_ADO_PROXY_RUN_CONTAINER, + handover_material = START_ADO_PROXY_HANDOVER_MATERIAL, + destroy_private = START_ADO_PROXY_DESTROY_PRIVATE, + wait_ready = START_ADO_PROXY_WAIT_READY, + ], + body: r###" +# Start the ado-proxy policy engine. +# +# The agent never receives an Azure DevOps credential. This container +# holds it, and serves only the operations in the versioned catalog. + +# ado-aw:fragment setup_workdir + +# ado-aw:fragment resolve_org + +# ado-aw:fragment write_policy + +# ado-aw:fragment mint_material + +# ado-aw:fragment build_material + +# ado-aw:fragment run_container + +# ado-aw:fragment handover_material + +# ado-aw:fragment destroy_private + +# ado-aw:fragment wait_ready +"###, + } +} + /// Stop the `ado-proxy` container. /// /// `--rm` only fires on a clean exit, so an OOM or SIGKILL would otherwise @@ -4410,26 +4728,43 @@ fn start_ado_proxy_step(front_matter: &FrontMatter) -> BashStep { /// once AWF is also told to attach the container. Landing the lifecycle first /// keeps that change to the wiring alone. fn stop_ado_proxy_step() -> BashStep { - let script = format!( - "# Preserve auditable lifecycle output before stopping the policy engine\n\ - mkdir -p /tmp/gh-aw/ado-proxy-logs\n\ - if docker inspect {ADO_PROXY_CONTAINER_NAME} >/dev/null 2>&1; then\n \ - docker inspect -f 'state={{{{.State.Status}}}} exit={{{{.State.ExitCode}}}} error={{{{.State.Error}}}}' {ADO_PROXY_CONTAINER_NAME} \\\n \ - > /tmp/gh-aw/ado-proxy-logs/container-state.txt 2>&1 || true\n \ - docker logs {ADO_PROXY_CONTAINER_NAME} \\\n \ - > /tmp/gh-aw/ado-proxy-logs/container.log 2>&1 || true\n\ - else\n \ - echo 'state=missing before teardown' > /tmp/gh-aw/ado-proxy-logs/container-state.txt\n\ - echo \"##vso[task.logissue type=warning]ado-proxy container was already missing at teardown; inspect the preflight/AWF step and ado-proxy log artifact\"\n\ - fi\n\ - \n\ - # Stop the ado-proxy policy engine\n\ - echo \"Stopping ado-proxy...\"\n\ - docker stop {ADO_PROXY_CONTAINER_NAME} 2>/dev/null || true\n\ - docker rm -f {ADO_PROXY_CONTAINER_NAME} 2>/dev/null || true\n\ - echo \"ado-proxy stopped\"\n" - ); - bash("Stop ado-proxy", script).with_condition(Condition::Always) + ShellScript::new(&STOP_ADO_PROXY) + .text("PROXY_CONTAINER", ADO_PROXY_CONTAINER_NAME) + .into_step("Stop ado-proxy") + .with_condition(Condition::Always) +} + +shell_script! { + /// Stop the ado-proxy container, preserving auditable lifecycle output + /// (`docker inspect` state + `docker logs` tail) even when the + /// container disappeared between the start step and here — that + /// scenario is precisely the one worth debugging, so a missing + /// container publishes an explicit warning rather than a silent skip. + STOP_ADO_PROXY { + interpreter: Bash, + bindings: [PROXY_CONTAINER], + externals: [], + fragments: [], + body: r###" +# Preserve auditable lifecycle output before stopping the policy engine +mkdir -p /tmp/gh-aw/ado-proxy-logs +if docker inspect "$PROXY_CONTAINER" >/dev/null 2>&1; then + docker inspect -f 'state={{.State.Status}} exit={{.State.ExitCode}} error={{.State.Error}}' "$PROXY_CONTAINER" \ + > /tmp/gh-aw/ado-proxy-logs/container-state.txt 2>&1 || true + docker logs "$PROXY_CONTAINER" \ + > /tmp/gh-aw/ado-proxy-logs/container.log 2>&1 || true +else + echo 'state=missing before teardown' > /tmp/gh-aw/ado-proxy-logs/container-state.txt + echo "##vso[task.logissue type=warning]ado-proxy container was already missing at teardown; inspect the preflight/AWF step and ado-proxy log artifact" +fi + +# Stop the ado-proxy policy engine +echo "Stopping ado-proxy..." +docker stop "$PROXY_CONTAINER" 2>/dev/null || true +docker rm -f "$PROXY_CONTAINER" 2>/dev/null || true +echo "ado-proxy stopped" +"###, + } } /// Verify externally-launched peers still exist immediately before AWF tries @@ -4440,37 +4775,56 @@ fn stop_ado_proxy_step() -> BashStep { /// Reporting the peer's Docker state and logs here turns AWF's otherwise opaque /// "No such container" error into an actionable startup/lifecycle failure. fn verify_trusted_topology_peers_step() -> BashStep { - let script = format!( - "set -euo pipefail\n\ - mkdir -p /tmp/gh-aw/ado-proxy-logs\n\ - for PEER in {MCPG_CONTAINER_NAME} {ADO_PROXY_CONTAINER_NAME}; do\n \ - PEER_STATE=$(docker inspect -f '{{{{.State.Status}}}}' \"$PEER\" 2>/dev/null || true)\n \ - if [ \"$PEER_STATE\" != \"running\" ]; then\n \ - echo \"##vso[task.logissue type=error]trusted topology peer $PEER is not running before AWF attachment (state=${{PEER_STATE:-missing}})\"\n \ - docker ps -a --filter \"name=^/${{PEER}}$\" --no-trunc || true\n \ - if [ \"$PEER\" = \"{ADO_PROXY_CONTAINER_NAME}\" ]; then\n \ - docker logs --tail 200 \"$PEER\" 2>&1 \\\n \ - | tee /tmp/gh-aw/ado-proxy-logs/container.log || true\n \ - else\n \ - docker logs --tail 200 \"$PEER\" 2>&1 || true\n \ - fi\n \ - exit 1\n \ - fi\n \ - echo \"Trusted topology peer $PEER is running\"\n \ - done\n\ - if [ ! -r {ca_host_path} ]; then\n \ - echo \"##vso[task.logissue type=error]ado-proxy public CA is not readable by the runner/agent identity: {ca_host_path}\"\n \ - ls -l {ca_host_path} 2>&1 || true\n \ - echo \"The proxy publishes this intentionally public certificate for the wrapped az process and Azure DevOps MCP. A restrictive container umask must not leave it owner-only.\"\n \ - docker logs --tail 200 {ADO_PROXY_CONTAINER_NAME} 2>&1 || true\n \ - exit 1\n \ - fi\n\ - CA_MODE=$(stat -c '%a' {ca_host_path} 2>/dev/null || echo unknown)\n\ - echo \"ado-proxy public CA is readable (mode=$CA_MODE)\"\n\ - echo \"ado-proxy policy and client configuration are ready; runtime denials will include the policy reason and sanitized decision logs\"\n", - ca_host_path = ADO_PROXY_PUBLIC_CA_HOST_PATH - ); - bash("Verify trusted topology peers", script) + ShellScript::new(&VERIFY_TRUSTED_TOPOLOGY_PEERS) + .text("MCPG_CONTAINER", MCPG_CONTAINER_NAME) + .text("PROXY_CONTAINER", ADO_PROXY_CONTAINER_NAME) + .text("CA_HOST_PATH", ADO_PROXY_PUBLIC_CA_HOST_PATH) + .into_step("Verify trusted topology peers") +} + +shell_script! { + /// Verify externally-launched trusted peers (MCPG, the ado-proxy) are + /// still running immediately before AWF attaches them to its internal + /// network. Turns AWF's opaque "No such container" into an actionable + /// startup / lifecycle failure with `docker ps -a` and `docker logs` + /// tails; also asserts the ado-proxy public CA is readable, since a + /// too-restrictive container umask leaves it owner-only and clients + /// then can't verify the intercepted chain. + VERIFY_TRUSTED_TOPOLOGY_PEERS { + interpreter: Bash, + bindings: [MCPG_CONTAINER, PROXY_CONTAINER, CA_HOST_PATH], + externals: [], + fragments: [], + body: r###" +set -euo pipefail +mkdir -p /tmp/gh-aw/ado-proxy-logs +for PEER in "$MCPG_CONTAINER" "$PROXY_CONTAINER"; do + PEER_STATE=$(docker inspect -f '{{.State.Status}}' "$PEER" 2>/dev/null || true) + if [ "$PEER_STATE" != "running" ]; then + echo "##vso[task.logissue type=error]trusted topology peer $PEER is not running before AWF attachment (state=${PEER_STATE:-missing})" + docker ps -a --filter "name=^/${PEER}$" --no-trunc || true + if [ "$PEER" = "$PROXY_CONTAINER" ]; then + docker logs --tail 200 "$PEER" 2>&1 \ + | tee /tmp/gh-aw/ado-proxy-logs/container.log || true + else + docker logs --tail 200 "$PEER" 2>&1 || true + fi + exit 1 + fi + echo "Trusted topology peer $PEER is running" +done +if [ ! -r "$CA_HOST_PATH" ]; then + echo "##vso[task.logissue type=error]ado-proxy public CA is not readable by the runner/agent identity: $CA_HOST_PATH" + ls -l "$CA_HOST_PATH" 2>&1 || true + echo "The proxy publishes this intentionally public certificate for the wrapped az process and Azure DevOps MCP. A restrictive container umask must not leave it owner-only." + docker logs --tail 200 "$PROXY_CONTAINER" 2>&1 || true + exit 1 +fi +CA_MODE=$(stat -c '%a' "$CA_HOST_PATH" 2>/dev/null || echo unknown) +echo "ado-proxy public CA is readable (mode=$CA_MODE)" +echo "ado-proxy policy and client configuration are ready; runtime denials will include the policy reason and sanitized decision logs" +"###, + } } fn copy_logs_step(engine_log_dir: &str, is_detection: bool) -> BashStep { @@ -5824,6 +6178,10 @@ safe-outputs: // was previously omitted entirely, which killed all twelve catalogued // repository operations without any test noticing. let script = start_ado_proxy_step(&proxy_fm()).script; + // `ShellScript` renders ADO predefined macros as `Binding::ado_macro` + // — single-quoted so the value can never break out of the RHS — so the + // prelude carries `NAME='$(macro)'` rather than the older + // double-quoted `NAME="$(macro)"` form. for (variable, macro_name) in [ ("ADO_PROXY_PROJECT", "System.TeamProject"), ("ADO_PROXY_PROJECT_ID", "System.TeamProjectId"), @@ -5831,7 +6189,7 @@ safe-outputs: ("ADO_PROXY_REPOSITORY_ID", "Build.Repository.ID"), ] { assert!( - script.contains(&format!("{variable}=\"$({macro_name})\"")), + script.contains(&format!("{variable}='$({macro_name})'")), "{variable} must be sourced from $({macro_name}): {script}" ); assert!( @@ -5964,9 +6322,15 @@ safe-outputs: let script = start_ado_proxy_step(&proxy_fm()).script; assert!( - script.contains("mktemp -d \"$(Agent.TempDirectory)/ado-proxy."), + script.contains("mktemp -d \"$AGENT_TEMP/ado-proxy."), "private material must be generated outside /tmp: {script}" ); + // AGENT_TEMP is bound from the ADO `Agent.TempDirectory` macro, which + // is confined to the agent work directory — never under /tmp. + assert!( + script.contains("AGENT_TEMP='$(Agent.TempDirectory)'"), + "AGENT_TEMP must come from Agent.TempDirectory: {script}" + ); for private in ["ca.key", "$ADO_PROXY_BEARER", "PROXY_MATERIAL"] { for line in script.lines().filter(|line| line.contains(private)) { let container_private_fifo = line.contains("docker exec -i") @@ -5984,9 +6348,17 @@ safe-outputs: fn ado_proxy_streams_material_on_stdin_rather_than_via_env_or_argv() { let step = start_ado_proxy_step(&proxy_fm()); + // The container name reaches the body through the `PROXY_CONTAINER` + // binding, so the docker invocations reference `$PROXY_CONTAINER` + // rather than the literal `awmg-ado-proxy`. + assert!( + step.script.contains("PROXY_CONTAINER='awmg-ado-proxy'"), + "PROXY_CONTAINER must be bound to the ado-proxy container name: {}", + step.script + ); assert!( step.script.contains( - "printf '%s' \"$PROXY_MATERIAL\" | docker exec -i awmg-ado-proxy" + "printf '%s' \"$PROXY_MATERIAL\" | docker exec -i \"$PROXY_CONTAINER\"" ) && step.script.contains("cat > /tmp/ado-proxy-material"), "material must stream through the container-private FIFO: {}", step.script @@ -6068,12 +6440,19 @@ safe-outputs: // `--public-ca-file` is an *output*: the proxy writes its interception // CA there so clients can trust it. It must land somewhere the agent // can read (AWF mounts /tmp into the chroot) — unlike the signing key. + // The flag lives inside the flattened container entrypoint that the + // `CONTAINER_ENTRYPOINT` binding carries into the prelude. assert!(script.contains("--public-ca-file /var/lib/ado-proxy/ado-proxy-ca.pem")); - assert!(script.contains(&format!("-v {AZ_WRAPPER_DIR}:/var/lib/ado-proxy"))); assert!( - script.contains(&format!( - "##vso[task.setvariable variable=ADO_PROXY_CA_FILE]{ADO_PROXY_PUBLIC_CA_HOST_PATH}" - )), + script.contains(&format!("AZ_WRAPPER_DIR='{AZ_WRAPPER_DIR}'")) + && script.contains("-v \"$AZ_WRAPPER_DIR:/var/lib/ado-proxy\""), + "the wrapper directory must be bound and mounted at /var/lib/ado-proxy: {script}" + ); + assert!( + script.contains(&format!("CA_HOST_PATH='{ADO_PROXY_PUBLIC_CA_HOST_PATH}'")) + && script.contains( + "##vso[task.setvariable variable=ADO_PROXY_CA_FILE]$CA_HOST_PATH" + ), "clients need the published certificate's path: {script}" ); assert!( @@ -6117,8 +6496,21 @@ safe-outputs: // runner, so it must not introduce an image to build, pin or mirror. let script = start_ado_proxy_step(&proxy_fm()).script; assert_eq!(ADO_PROXY_IMAGE, common::ADO_MCP_IMAGE); - assert!(script.contains(&format!("{ADO_PROXY_IMAGE} \\"))); - assert!(script.contains(&format!("{}:/app/ado-proxy.js:ro", paths::ADO_PROXY_PATH))); + // The image and the bundle path both reach the body through bindings, + // so the docker invocation references them as `$PROXY_IMAGE` and + // `$PROXY_SCRIPT_PATH` while the concrete values live in the prelude. + assert!( + script.contains(&format!("PROXY_IMAGE='{ADO_PROXY_IMAGE}'")) + && script.contains("\"$PROXY_IMAGE\" \\"), + "docker run must reuse the bound $PROXY_IMAGE: {script}" + ); + assert!( + script.contains(&format!( + "PROXY_SCRIPT_PATH='{}'", + paths::ADO_PROXY_PATH + )) && script.contains("\"$PROXY_SCRIPT_PATH:/app/ado-proxy.js:ro\""), + "docker run must mount the bound ado-proxy bundle: {script}" + ); } #[test] @@ -6145,16 +6537,19 @@ safe-outputs: #[test] fn ado_proxy_recovers_from_an_interrupted_previous_run() { // --rm only fires on clean exit; an OOM or SIGKILL leaves the - // container, and with it a live credential, behind. + // container, and with it a live credential, behind. Both scripts now + // read the container name through the `PROXY_CONTAINER` binding. + let start = start_ado_proxy_step(&proxy_fm()).script; assert!( - start_ado_proxy_step(&proxy_fm()) - .script - .contains(&format!("docker rm -f {ADO_PROXY_CONTAINER_NAME}")) + start.contains(&format!("PROXY_CONTAINER='{ADO_PROXY_CONTAINER_NAME}'")) + && start.contains("docker rm -f \"$PROXY_CONTAINER\""), + "start_ado_proxy_step must reap a stale container: {start}" ); + let stop = stop_ado_proxy_step().script; assert!( - stop_ado_proxy_step() - .script - .contains(&format!("docker rm -f {ADO_PROXY_CONTAINER_NAME}")) + stop.contains(&format!("PROXY_CONTAINER='{ADO_PROXY_CONTAINER_NAME}'")) + && stop.contains("docker rm -f \"$PROXY_CONTAINER\""), + "stop_ado_proxy_step must reap the container: {stop}" ); } diff --git a/src/compile/az_wrapper.rs b/src/compile/az_wrapper.rs index f3e2db72..0ab0fa13 100644 --- a/src/compile/az_wrapper.rs +++ b/src/compile/az_wrapper.rs @@ -33,32 +33,22 @@ //! declines the certificate fails closed rather than escaping the policy. use super::common::{AZ_WRAPPER_CA_PATH, AZ_WRAPPER_DIR, az_allowed_groups}; +use super::shell::{Binding, ShellScript}; use crate::ado_proxy::catalog::Capability; +use crate::shell_script; -/// Render the wrapper script. -/// -/// `engine_host` is the policy engine's container name, which AWF registers in -/// the agent's `/etc/hosts` when it attaches the container to the internal -/// network. `capabilities` are the ones the policy actually grants, so the -/// wrapper refuses a command group the engine would refuse anyway — with an -/// explanation, rather than an opaque `403` several layers down. -#[allow(dead_code)] -pub fn render_az_wrapper( - engine_host: &str, - connect_port: u16, - sentinel: &str, - capabilities: &[Capability], -) -> String { - let groups = az_allowed_groups(capabilities); - let allowed_list = groups.join(" "); - let allowed_display = if groups.is_empty() { - "none".to_string() - } else { - groups.join(", ") - }; - - format!( - r##"#!/bin/sh +shell_script! { + /// Wrapper script the agent invokes as `az`. Written to disk by + /// `extensions/azure_cli.rs::install_az_wrapper_step` rather than run as a + /// step, so this is an `Sh` script rendered to a String; every value that + /// used to be interpolated with `format!` is now a validated binding, and + /// the body is the shell exactly as it will run. + AZ_WRAPPER { + interpreter: Sh, + bindings: [ALLOWED_GROUPS, ALLOWED_DISPLAY, ENGINE_HOST, ENGINE_PORT, CA_PATH, SENTINEL, WRAPPER_DIR], + externals: [TMPDIR, PATH], + fragments: [], + body: r#"#!/bin/sh # Azure CLI wrapper installed by ado-aw. # # The agent has no Azure DevOps credential. This wrapper points `az` at the @@ -72,8 +62,8 @@ set -eu # otherwise fail somewhere far less legible: no Azure credential is present, so # `az vm` or `az storage` would surface an authentication error that looks like # a broken pipeline rather than a deliberate boundary. -AZ_GROUP="${{1:-}}" -case " {allowed_list} " in +AZ_GROUP="${1:-}" +case " $ALLOWED_GROUPS " in *" $AZ_GROUP "*) ;; *) case "$AZ_GROUP" in @@ -85,7 +75,7 @@ case " {allowed_list} " in echo "" >&2 echo "This workflow reaches Azure DevOps through a policy proxy that" >&2 echo "serves read-only operations for the current project. Available" >&2 - echo "command groups: {allowed_display}." >&2 + echo "command groups: $ALLOWED_DISPLAY." >&2 echo "" >&2 echo "To act outside that boundary, use a safe output instead:" >&2 echo "https://github.com/githubnext/ado-aw/blob/main/docs/safe-outputs.md" >&2 @@ -99,7 +89,7 @@ esac # redirect independent of how the organization was specified — --organization, # --org, AZURE_DEVOPS_ORG and stored defaults all resolve to the same canonical # host, and that host is what gets intercepted. -HTTPS_PROXY="http://{engine_host}:{connect_port}" +HTTPS_PROXY="http://$ENGINE_HOST:$ENGINE_PORT" export HTTPS_PROXY https_proxy="$HTTPS_PROXY" export https_proxy @@ -107,14 +97,14 @@ export https_proxy # Trust the engine's interception certificate for this process only. Python's # requests ignores the OS trust store, so this variable — not the system CA # bundle — is what `az` actually consults. -REQUESTS_CA_BUNDLE="{AZ_WRAPPER_CA_PATH}" +REQUESTS_CA_BUNDLE="$CA_PATH" export REQUESTS_CA_BUNDLE # Azure CLI writes extension metadata, command indexes and defaults beneath its # config directory. The rootless AWF agent cannot write the runner user's # default ~/.azure, so establish a private, writable sandbox-local default. # Honour an explicit caller override for tests and advanced use. -AZURE_CONFIG_DIR="${{AZURE_CONFIG_DIR:-${{TMPDIR:-/tmp}}/ado-aw-az-config}}" +AZURE_CONFIG_DIR="${AZURE_CONFIG_DIR:-${TMPDIR:-/tmp}/ado-aw-az-config}" export AZURE_CONFIG_DIR mkdir -p "$AZURE_CONFIG_DIR" if [ ! -w "$AZURE_CONFIG_DIR" ]; then @@ -125,7 +115,7 @@ fi # A non-secret placeholder. `az` requires *some* credential to attempt a call; # the engine strips whatever the client sent and attaches the real bearer only # after a complete allow decision. -AZURE_DEVOPS_EXT_PAT="{sentinel}" +AZURE_DEVOPS_EXT_PAT="$SENTINEL" export AZURE_DEVOPS_EXT_PAT # Locate the real binary. `exec az` would re-enter this wrapper, because the @@ -134,7 +124,7 @@ AZ_REAL="" IFS=: for dir in $PATH; do case "$dir" in - ""|{AZ_WRAPPER_DIR}) continue ;; + ""|"$WRAPPER_DIR") continue ;; esac if [ -x "$dir/az" ]; then AZ_REAL="$dir/az" @@ -149,8 +139,57 @@ if [ -z "$AZ_REAL" ]; then fi exec "$AZ_REAL" "$@" -"## - ) +"#, + } +} + +/// Build the wrapper as a typed [`ShellScript`], with each interpolated value +/// carried as a validated [`Binding`] rather than a `format!` substitution. +/// +/// The list of `az` command groups the wrapper advertises is derived from +/// [`az_allowed_groups`] rather than hand-maintained, so a capability the +/// policy does not grant cannot be advertised to the agent. `allowed_display` +/// is a comma-joined form of the same list; the deny-branch message consumes +/// it verbatim, so the two shapes come from the same source and cannot drift +/// per site. +pub(crate) fn az_wrapper_script( + engine_host: &str, + connect_port: u16, + sentinel: &str, + capabilities: &[Capability], +) -> ShellScript { + let groups = az_allowed_groups(capabilities); + let allowed_display = if groups.is_empty() { + "none".to_string() + } else { + groups.join(", ") + }; + + ShellScript::new(&AZ_WRAPPER) + .bind("ALLOWED_GROUPS", Binding::words(groups.iter().copied())) + .text("ALLOWED_DISPLAY", allowed_display) + .text("ENGINE_HOST", engine_host) + .bind("ENGINE_PORT", Binding::number(connect_port as u64)) + .text("CA_PATH", AZ_WRAPPER_CA_PATH) + .text("SENTINEL", sentinel) + .text("WRAPPER_DIR", AZ_WRAPPER_DIR) +} + +/// Render the wrapper script. +/// +/// `engine_host` is the policy engine's container name, which AWF registers in +/// the agent's `/etc/hosts` when it attaches the container to the internal +/// network. `capabilities` are the ones the policy actually grants, so the +/// wrapper refuses a command group the engine would refuse anyway — with an +/// explanation, rather than an opaque `403` several layers down. +#[allow(dead_code)] +pub fn render_az_wrapper( + engine_host: &str, + connect_port: u16, + sentinel: &str, + capabilities: &[Capability], +) -> String { + az_wrapper_script(engine_host, connect_port, sentinel, capabilities).render() } #[cfg(test)] @@ -167,10 +206,29 @@ mod tests { ) } + fn wrapper_script() -> ShellScript { + az_wrapper_script( + "awmg-ado-proxy", + 11080, + ADO_MCP_TOKEN_SENTINEL, + Capability::ALL, + ) + } + #[test] fn routes_azure_devops_traffic_through_the_engine() { - let script = wrapper(); - assert!(script.contains("HTTPS_PROXY=\"http://awmg-ado-proxy:11080\"")); + let s = wrapper_script(); + // The producer supplied the engine host and port; the body glues them + // into the HTTPS_PROXY URL. Asserting on the bindings is a + // strengthening: it verifies the values reached the prelude, rather + // than that the concatenated URL happens to appear as a substring. + assert_eq!(s.binding("ENGINE_HOST").unwrap().rhs(), "'awmg-ado-proxy'"); + assert_eq!(s.binding("ENGINE_PORT").unwrap().rhs(), "11080"); + let script = s.render(); + assert!( + script.contains(r#"HTTPS_PROXY="http://$ENGINE_HOST:$ENGINE_PORT""#), + "the body must glue the two together into the proxy URL: {script}" + ); assert!( script.contains("export HTTPS_PROXY") && script.contains("export https_proxy"), "both spellings matter: tooling reads one or the other" @@ -179,8 +237,13 @@ mod tests { #[test] fn trusts_the_interception_certificate_for_this_process_only() { - let script = wrapper(); - assert!(script.contains(&format!("REQUESTS_CA_BUNDLE=\"{AZ_WRAPPER_CA_PATH}\""))); + let s = wrapper_script(); + assert_eq!( + s.binding("CA_PATH").unwrap().rhs(), + format!("'{AZ_WRAPPER_CA_PATH}'") + ); + let script = s.render(); + assert!(script.contains(r#"REQUESTS_CA_BUNDLE="$CA_PATH""#)); // A system-wide install would not help: Python's requests uses its own // certifi bundle. It would also widen trust beyond this one client. assert!(!script.contains("update-ca-certificates")); @@ -202,10 +265,14 @@ mod tests { #[test] fn carries_a_sentinel_rather_than_a_credential() { - let script = wrapper(); - assert!(script.contains(&format!( - "AZURE_DEVOPS_EXT_PAT=\"{ADO_MCP_TOKEN_SENTINEL}\"" - ))); + let s = wrapper_script(); + assert_eq!( + s.binding("SENTINEL").unwrap().rhs(), + format!("'{ADO_MCP_TOKEN_SENTINEL}'"), + "the wrapper's PAT value must be the compiler-supplied sentinel" + ); + let script = s.render(); + assert!(script.contains(r#"AZURE_DEVOPS_EXT_PAT="$SENTINEL""#)); assert!( !script.contains("SC_READ_TOKEN") && !script.contains("System.AccessToken"), "no real credential may appear in an agent-readable file: {script}" @@ -255,21 +322,31 @@ mod tests { fn advertises_only_what_the_policy_actually_grants() { // `az artifacts` was briefly permitted while no catalogued operation // backed it, so the call passed the wrapper and was refused by the - // engine. The allow-list is now derived from the granted capabilities. - assert!(!wrapper().contains("artifacts")); + // engine. The allow-list is now derived from the granted capabilities, + // and reaches the prelude verbatim as `ALLOWED_GROUPS` / + // `ALLOWED_DISPLAY` — not as a substring smuggled through the body. + let s = wrapper_script(); + assert!(!s.binding("ALLOWED_GROUPS").unwrap().rhs().contains("artifacts")); + assert!( + !s.binding("ALLOWED_DISPLAY") + .unwrap() + .rhs() + .contains("artifacts") + ); // Narrowing the policy narrows the wrapper with it. - let repos_only = render_az_wrapper( + let repos_only = az_wrapper_script( "awmg-ado-proxy", 11080, ADO_MCP_TOKEN_SENTINEL, &[Capability::Discovery, Capability::Repos], ); - assert!(repos_only.contains(" repos ")); + let groups = repos_only.binding("ALLOWED_GROUPS").unwrap().rhs(); + assert!(groups.contains("repos"), "repos must appear: {groups}"); for absent in ["devops", "boards", "pipelines"] { assert!( - !repos_only.contains(&format!(" {absent} ")), - "{absent} is not granted and must not be advertised: {repos_only}" + !groups.contains(absent), + "{absent} is not granted and must not be advertised: {groups}" ); } } @@ -280,23 +357,27 @@ mod tests { // against a live engine it completed a catalogued read and was refused // 403 for a denied route family and for a POST. Excluding it would // also contradict `az devops invoke`, which reaches the same surface. - assert!(wrapper().contains(" rest ")); - let narrow = render_az_wrapper( + let full = wrapper_script(); + assert!(full.binding("ALLOWED_GROUPS").unwrap().rhs().contains("rest")); + let narrow = az_wrapper_script( "awmg-ado-proxy", 11080, ADO_MCP_TOKEN_SENTINEL, &[Capability::Discovery], ); - assert!(narrow.contains(" rest ")); + assert!(narrow.binding("ALLOWED_GROUPS").unwrap().rhs().contains("rest")); } #[test] fn execs_the_real_binary_without_re_entering_itself() { - let script = wrapper(); + let s = wrapper_script(); + assert_eq!(s.binding("WRAPPER_DIR").unwrap().rhs(), format!("'{AZ_WRAPPER_DIR}'")); + let script = s.render(); // The wrapper directory is prepended to PATH, so a bare `exec az` - // would loop until the process ran out of file descriptors. - assert!(script.contains(&format!("\"\"|{AZ_WRAPPER_DIR}) continue"))); - assert!(script.contains("exec \"$AZ_REAL\" \"$@\"")); + // would loop until the process ran out of file descriptors. The body + // consults `$WRAPPER_DIR` in a `case` pattern to skip its own dir. + assert!(script.contains(r#"""|"$WRAPPER_DIR") continue"#)); + assert!(script.contains(r#"exec "$AZ_REAL" "$@""#)); } #[test] @@ -306,4 +387,15 @@ mod tests { assert!(script.contains("exit 127")); } + #[test] + fn is_a_standalone_sh_script_with_a_shebang() { + // The wrapper is written to disk and invoked as `az`, so it must be a + // self-contained script starting with `#!/bin/sh`. `render()` splits + // the shebang off before the prelude and emits it first. + let script = wrapper(); + assert!( + script.starts_with("#!/bin/sh\n"), + "the shebang must come first: {script}" + ); + } } diff --git a/src/compile/common.rs b/src/compile/common.rs index c536b428..c3fe6348 100644 --- a/src/compile/common.rs +++ b/src/compile/common.rs @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf}; use super::extensions::{ CompilerExtension, Declarations, McpgConfig, McpgGatewayConfig, McpgServerConfig, }; +use super::shell::{Binding, ShellScript}; use super::types::{ CheckoutFetchOpts, CompileTarget, FrontMatter, PipelineParameter, PoolConfig, ReposItem, Repository, SELF_CHECKOUT_ALIAS, @@ -17,6 +18,7 @@ use crate::compile::types::McpConfig; use crate::ecosystem_domains::{ get_ecosystem_domains, is_ecosystem_identifier, is_known_ecosystem, }; +use crate::shell_script; use crate::validate; /// Atomically write `contents` to `path`. @@ -1811,21 +1813,52 @@ pub const ADO_MCP_CA_MOUNT: &str = "/etc/ado-proxy/ca.pem"; /// is. Getting this wrong is not cosmetic — in a policy document a wrong /// organization matches nothing, denying every request in a way that reads as /// a deliberate policy decision. +/// +/// # Contract for consumers +/// +/// This function returns raw shell text (no YAML wrapping). Callers that +/// splice the text into a larger script must know it assigns two variables: +/// `ADO_PROXY_COLLECTION` and `ADO_PROXY_ORGANIZATION`. See +/// [`crate::compile::shell`] for the fragment convention (`# ado-aw:fragment` +/// marker + `externals:` list) other producers use when embedding this +/// fragment. pub fn resolve_ado_organization_bash() -> String { - "# $(System.CollectionUri) is expanded by ADO before bash runs. Two\n\ - # shapes are in use: \"https://dev.azure.com/myorg/\" (organization in\n\ - # the path) and the legacy \"https://myorg.visualstudio.com/\"\n\ - # (organization in the host). Handle both — a fixed-prefix strip or a\n\ - # bare last-segment rule is silently wrong for one of them.\n\ - ADO_PROXY_COLLECTION=\"$(System.CollectionUri)\"\n\ - ADO_PROXY_ORGANIZATION=$(printf '%s' \"$ADO_PROXY_COLLECTION\" \\\n\ - | sed -e 's#^https\\?://##' -e 's#/*$##' \\\n\ - | awk -F/ '{ if (NF>1) print $NF; else { sub(/\\..*$/, \"\", $1); print $1 } }')\n\ - if [ -z \"$ADO_PROXY_ORGANIZATION\" ]; then\n\ - echo \"##vso[task.complete result=Failed]cannot determine the Azure DevOps organization from System.CollectionUri\"\n\ - exit 1\n\ - fi\n" - .to_string() + ShellScript::new(&RESOLVE_ADO_ORGANIZATION).render() +} + +shell_script! { + /// Derive `$ADO_PROXY_ORGANIZATION` from `$(System.CollectionUri)`. + /// + /// Rendered as a standalone fragment (no bindings, no prelude) so it can + /// be spliced verbatim into a larger step body via + /// [`ShellScript::fragment`]. Every variable the body reads is assigned + /// by the body itself, so no bindings or externals are needed and the + /// splice contributes nothing to the parent's declared surface except + /// the two variables it assigns. + RESOLVE_ADO_ORGANIZATION { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [], + // Uses r###"..."### so the "##vso[...]" line inside the body cannot + // prematurely close the raw string literal (the sequence `"#` would + // otherwise terminate an r#"..."# body). + body: r###" +# $(System.CollectionUri) is expanded by ADO before bash runs. Two +# shapes are in use: "https://dev.azure.com/myorg/" (organization in +# the path) and the legacy "https://myorg.visualstudio.com/" +# (organization in the host). Handle both — a fixed-prefix strip or a +# bare last-segment rule is silently wrong for one of them. +ADO_PROXY_COLLECTION="$(System.CollectionUri)" +ADO_PROXY_ORGANIZATION=$(printf '%s' "$ADO_PROXY_COLLECTION" \ + | sed -e 's#^https\?://##' -e 's#/*$##' \ + | awk -F/ '{ if (NF>1) print $NF; else { sub(/\..*$/, "", $1); print $1 } }') +if [ -z "$ADO_PROXY_ORGANIZATION" ]; then + echo "##vso[task.complete result=Failed]cannot determine the Azure DevOps organization from System.CollectionUri" + exit 1 +fi +"###, + } } /// Whether this workflow routes Azure DevOps access through the policy engine. @@ -2038,14 +2071,56 @@ pub fn generate_integrity_check(skip: bool) -> String { return String::new(); } - // Indentation is handled by replace_with_indent at the call site. - r#"- bash: | - AGENTIC_PIPELINES_PATH="$(Pipeline.Workspace)/agentic-pipeline-compiler/ado-aw" - chmod +x "$AGENTIC_PIPELINES_PATH" - $AGENTIC_PIPELINES_PATH check "{{ pipeline_path }}" - workingDirectory: {{ trigger_repo_directory }} - displayName: "Verify pipeline integrity""# - .to_string() + let script = ShellScript::new(&VERIFY_PIPELINE_INTEGRITY) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .render(); + + // Indent every body line by 4 spaces to satisfy the `- bash: |` literal + // block scalar shape. `workingDirectory:` uses the `{{ trigger_repo_directory }}` + // template placeholder resolved by `replace_with_indent` at the call site. + let indented: String = script + .lines() + .map(|line| { + if line.is_empty() { + "\n".to_string() + } else { + format!(" {line}\n") + } + }) + .collect(); + format!( + "- bash: |\n{indented} workingDirectory: {{{{ trigger_repo_directory }}}}\n displayName: \"Verify pipeline integrity\"" + ) + .trim_end_matches('\n') + .to_string() +} + +shell_script! { + /// The bash body of the "Verify pipeline integrity" step: downloads the + /// pipelined `ado-aw` binary and re-runs `ado-aw check` against the + /// authored source path. Called with `--skip-integrity` returns an empty + /// string from [`generate_integrity_check`] and the step is omitted; the + /// registered body always describes the enabled case. + /// + /// `{{ pipeline_path }}` remains a compile-time template placeholder + /// resolved by `replace_with_indent` at the call site (it names the + /// authored `.md` source path relative to the trigger repo). It is inert + /// under shellcheck because it appears only inside a double-quoted string + /// argument. + VERIFY_PIPELINE_INTEGRITY { + interpreter: Bash, + bindings: [PIPELINE_WORKSPACE], + externals: [], + fragments: [], + body: r#" +AGENTIC_PIPELINES_PATH="$PIPELINE_WORKSPACE/agentic-pipeline-compiler/ado-aw" +chmod +x "$AGENTIC_PIPELINES_PATH" +"$AGENTIC_PIPELINES_PATH" check "{{ pipeline_path }}" +"#, + } } /// Validate the `ado-aw-debug:` section. @@ -3390,24 +3465,52 @@ pub fn generate_awf_path_step(awf_paths: &[String]) -> String { return String::new(); } - let path_lines = awf_paths - .iter() - .map(|p| format!(" {p}")) - .collect::>() - .join("\n"); + let path_lines: String = awf_paths.join("\n"); + + let body = ShellScript::new(&GENERATE_GITHUB_PATH) + .bind("PATH_LINES", Binding::document(&path_lines)) + .render(); + // Wrap in a `- bash: |` YAML step. Each body line indented by 4 spaces + // so it survives the literal-block scalar shape emitted downstream. + let indented: String = body + .lines() + .map(|line| { + if line.is_empty() { + "\n".to_string() + } else { + format!(" {line}\n") + } + }) + .collect(); format!( - "\ -- bash: | - AWF_PATH_FILE=\"/tmp/awf-tools/ado-path-entries\" - cat > \"$AWF_PATH_FILE\" << AWF_PATH_EOF -{path_lines} - AWF_PATH_EOF - echo \"##vso[task.setvariable variable=GITHUB_PATH]$AWF_PATH_FILE\" - displayName: \"Generate GITHUB_PATH file\"" + "- bash: |\n{indented} displayName: \"Generate GITHUB_PATH file\"" ) } +shell_script! { + /// Bash body of the "Generate GITHUB_PATH file" step. + /// + /// AWF reads `$GITHUB_PATH` as a file path at startup and merges its + /// entries into the chroot PATH. `PATH_LINES` arrives as a + /// [`Binding::document`] — a quoted heredoc — because the path list may + /// contain multiple lines; the heredoc keeps them literal without any + /// per-entry escaping. The rendered file is then advertised to AWF via + /// `##vso[task.setvariable]`. + GENERATE_GITHUB_PATH { + interpreter: Bash, + bindings: [PATH_LINES], + externals: [], + fragments: [], + // r###"..."### to protect against the "##vso sequence. + body: r###" +AWF_PATH_FILE="/tmp/awf-tools/ado-path-entries" +printf '%s\n' "$PATH_LINES" > "$AWF_PATH_FILE" +echo "##vso[task.setvariable variable=GITHUB_PATH]$AWF_PATH_FILE" +"###, + } +} + /// Generates the `env:` block entry that passes `GITHUB_PATH` to the AWF /// invocation step. /// @@ -6603,9 +6706,12 @@ safe-outputs: result.contains("displayName"), "should be a complete pipeline step" ); + // Paths flow through a `Binding::document` heredoc so multi-line + // content stays literal without per-entry escaping — the marker is + // the shell module's own document delimiter, not a caller-picked one. assert!( - result.contains("AWF_PATH_EOF"), - "should use heredoc markers" + result.contains("ADO_AW_SHELL_DOC_EOF"), + "should use the shell::Binding::document heredoc marker: {result}" ); } diff --git a/src/compile/extensions/ado_aw_marker.rs b/src/compile/extensions/ado_aw_marker.rs index d1f9cb5d..5a259c1c 100644 --- a/src/compile/extensions/ado_aw_marker.rs +++ b/src/compile/extensions/ado_aw_marker.rs @@ -25,8 +25,51 @@ use super::{CompileContext, CompilerExtension, Declarations, ExtensionPhase}; use crate::compile::ir::condition::Condition; use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::shell::{Binding, ShellScript}; +use crate::shell_script; use serde::Serialize; +shell_script! { + /// Discovery marker: a `# ado-aw-metadata: {json}` shell comment plus a + /// human-readable echo. Both lines are spliced as a single fragment + /// because the JSON is dynamic and the echo values still flow through + /// `bash_single_quote_escape` — the fragment lets that quoting stay + /// verbatim without piping it through Binding::text (which would reject + /// `$(` in a user-supplied filename). + ADO_AW_MARKER { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [marker], + body: r#" +# ado-aw:fragment marker +"#, + } +} + +shell_script! { + /// Write `aw_info.json` to Agent.TempDirectory/staging. + /// + /// The JSON is spliced as a fragment because it may (harmlessly) contain + /// substrings that `Binding::document`'s SECRET_NAMES check would reject + /// as false-positives, and the quoted heredoc delimiter here means the + /// splice is *shell data*, not shell to execute. + EMIT_AW_INFO { + interpreter: Bash, + bindings: [AGENT_TEMP], + externals: [], + fragments: [aw_info_json], + body: r#" +set -eo pipefail + +mkdir -p "$AGENT_TEMP/staging" +cat >"$AGENT_TEMP/staging/aw_info.json" <<'AW_INFO_EOF' +# ado-aw:fragment aw_info_json +AW_INFO_EOF +"#, + } +} + // ─── ado-aw marker (always-on, internal) ───────────────────────────── /// Always-on internal extension that embeds machine-readable @@ -147,9 +190,9 @@ fn marker_bash_step(metadata: &CompileMetadata) -> BashStep { let echo_repo = bash_single_quote_escape(&crate::sanitize::neutralize_pipeline_commands( &metadata.repo, )); - let script = format!( + let fragment = format!( "# ado-aw-metadata: {metadata_json}\n\ - echo 'ado-aw metadata: source={echo_source} org={echo_org} repo={echo_repo} version={version} target={target}'\n", + echo 'ado-aw metadata: source={echo_source} org={echo_org} repo={echo_repo} version={version} target={target}'", metadata_json = metadata.marker_json(), echo_source = echo_source, echo_org = echo_org, @@ -157,21 +200,18 @@ fn marker_bash_step(metadata: &CompileMetadata) -> BashStep { version = metadata.compiler_version.as_str(), target = metadata.target.as_str(), ); - BashStep::new("ado-aw", script) + ShellScript::new(&ADO_AW_MARKER) + .fragment("marker", fragment) + .into_step("ado-aw") } /// Build the typed [`BashStep`] form of the `aw_info.json` emit step. fn aw_info_bash_step(metadata: &CompileMetadata) -> BashStep { - let script = format!( - "set -eo pipefail\n\ - \n\ - mkdir -p \"$(Agent.TempDirectory)/staging\"\n\ - cat >\"$(Agent.TempDirectory)/staging/aw_info.json\" <<'AW_INFO_EOF'\n\ - {aw_info_json}\n\ - AW_INFO_EOF\n", - aw_info_json = metadata.aw_info_json(), - ); - BashStep::new("Emit aw_info.json", script).with_condition(Condition::Always) + ShellScript::new(&EMIT_AW_INFO) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .fragment("aw_info_json", metadata.aw_info_json()) + .into_step("Emit aw_info.json") + .with_condition(Condition::Always) } struct CompileMetadata { @@ -525,10 +565,15 @@ mod tests { assert!(matches!(step.condition, Some(Condition::Always))); assert!( step.script - .contains("cat >\"$(Agent.TempDirectory)/staging/aw_info.json\" <<'AW_INFO_EOF'"), + .contains("cat >\"$AGENT_TEMP/staging/aw_info.json\" <<'AW_INFO_EOF'"), "step missing quoted heredoc write:\n{}", step.script ); + assert!( + step.script.contains("AGENT_TEMP='$(Agent.TempDirectory)'"), + "step missing AGENT_TEMP binding to $(Agent.TempDirectory):\n{}", + step.script + ); assert!( step.script.contains("\"schema\":\"ado-aw/aw_info/1\""), "step missing aw_info schema:\n{}", diff --git a/src/compile/extensions/ado_script.rs b/src/compile/extensions/ado_script.rs index a57d9573..161d8ae7 100644 --- a/src/compile/extensions/ado_script.rs +++ b/src/compile/extensions/ado_script.rs @@ -34,7 +34,160 @@ use crate::compile::ir::ids::StepId; use crate::compile::ir::output::OutputDecl; use crate::compile::ir::step::{BashStep, Step}; use crate::compile::ir::tasks::use_node::UseNode; +use crate::compile::shell::{Binding, ShellScript}; use crate::compile::types::{PipelineFilters, PrFilters, SupplyChainConfig}; +use crate::shell_script; + +shell_script! { + /// Stage the `ado-script` bundle downloaded from an ADO Artifacts feed + /// (or a raw `.nupkg`), verify checksums, and unpack it to + /// `/tmp/ado-aw-scripts/`. + /// + /// The find/copy dance handles both delivery shapes: an extracted tree + /// (an earlier task already unzipped the nupkg) and a raw `.nupkg` (which + /// this step unzips first). The `##vso[task.complete result=Failed]` + /// line is load-bearing — without it, a missing archive would leave the + /// rest of the pipeline running against a partially-staged bundle. + STAGE_ADO_AW_SCRIPTS_FEED { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [], + body: r###" +set -eo pipefail +mkdir -p /tmp/ado-aw-scripts +STAGING=/tmp/ado-aw-scripts/_pkg +if [ -z "$(find "$STAGING" -name 'ado-script.zip' -print -quit)" ]; then + NUPKG="$(find "$STAGING" -name '*.nupkg' -print -quit)" + if [ -n "$NUPKG" ]; then + unzip -o "$NUPKG" -d "$STAGING" >/dev/null + fi +fi +ZIP="$(find "$STAGING" -name 'ado-script.zip' -print -quit)" +CHK="$(find "$STAGING" -name 'checksums.txt' -print -quit)" +if [ -z "$ZIP" ] || [ -z "$CHK" ]; then + echo "##vso[task.complete result=Failed]ado-script.zip or checksums.txt not found in package" + exit 1 +fi +cp "$ZIP" /tmp/ado-aw-scripts/ado-script.zip +cp "$CHK" /tmp/ado-aw-scripts/checksums.txt +cd /tmp/ado-aw-scripts && grep "ado-script.zip" checksums.txt | sha256sum -c - +unzip -o /tmp/ado-aw-scripts/ado-script.zip -d /tmp/ado-aw-scripts/ +"###, + } +} + +shell_script! { + /// Download the `ado-script` bundle from GitHub Releases, verify + /// checksums, and unpack it to `/tmp/ado-aw-scripts/`. + DOWNLOAD_ADO_AW_SCRIPTS_RELEASE { + interpreter: Bash, + bindings: [RELEASE_BASE_URL, VERSION], + externals: [], + fragments: [], + body: r#" +set -eo pipefail +mkdir -p /tmp/ado-aw-scripts +curl -fsSL "$RELEASE_BASE_URL/v$VERSION/checksums.txt" -o /tmp/ado-aw-scripts/checksums.txt +curl -fsSL "$RELEASE_BASE_URL/v$VERSION/ado-script.zip" -o /tmp/ado-aw-scripts/ado-script.zip +cd /tmp/ado-aw-scripts && grep "ado-script.zip" checksums.txt | sha256sum -c - +unzip -o /tmp/ado-aw-scripts/ado-script.zip -d /tmp/ado-aw-scripts/ +"#, + } +} + +shell_script! { + /// Resolve `{{#runtime-import}}` markers in the agent prompt file. The + /// argv list is dynamically composed by the compiler (see + /// `resolver_step_typed`) and spliced as a fragment because each + /// `--var "=$()"` flag embeds an ADO macro expansion that + /// must reach `import.js` verbatim. + RESOLVE_RUNTIME_IMPORTS { + interpreter: Bash, + bindings: [IMPORT_EVAL_PATH, BASE], + externals: [], + fragments: [var_flags], + body: r#" +set -eo pipefail +node "$IMPORT_EVAL_PATH" /tmp/awf-tools/agent-prompt.md --base "$BASE" \ +# ado-aw:fragment var_flags +"#, + } +} + +shell_script! { + /// Mint a GitHub App installation access token via the + /// `github-app-token` ado-script bundle. The argv (non-secret inputs) + /// is composed by the compiler; the private key stays in the masked + /// env (`GH_APP_PRIVATE_KEY`) and never appears on the command line. + /// + /// The step runs outside the AWF sandbox and reaches `api.github.com` + /// (or the configured `--api-url`) over the build-agent pool's normal + /// network — no AWF allowlist entry is required. + MINT_GITHUB_APP_TOKEN { + interpreter: Bash, + bindings: [GITHUB_APP_TOKEN_PATH], + externals: [], + fragments: [args], + body: r#" +set -eo pipefail +node "$GITHUB_APP_TOKEN_PATH" \ +# ado-aw:fragment args +"#, + } +} + +shell_script! { + /// Revoke the minted GitHub App installation token, best-effort. Runs + /// with `condition: always()` and `continueOnError: true`; the bundle's + /// `revoke` mode always exits 0 (downgrading failures to warnings), so + /// no `set -eo pipefail` — aborting the shell early on a non-zero would + /// only risk turning a benign revoke hiccup into a timeline error. + REVOKE_GITHUB_APP_TOKEN { + interpreter: Bash, + bindings: [GITHUB_APP_TOKEN_PATH], + externals: [], + fragments: [args], + body: r#" +node "$GITHUB_APP_TOKEN_PATH" revoke \ +# ado-aw:fragment args +"#, + } +} + +shell_script! { + /// Run the `prepare-pr-base` bundle in one of two modes: + /// `PatchBase` (Agent-side merge-base recovery) or `TargetWorktree` + /// (SafeOutputs-side single-tip fetch). Per-repo argv is dynamic and + /// spliced as a fragment. + PREPARE_PR_BASE { + interpreter: Bash, + bindings: [PREPARE_PR_BASE_PATH, MODE], + externals: [], + fragments: [repo_flags], + body: r#" +set -eo pipefail +node "$PREPARE_PR_BASE_PATH" --mode "$MODE" \ +# ado-aw:fragment repo_flags +"#, + } +} + +shell_script! { + /// The Setup-job synthetic-PR context resolver. Runs the + /// `exec-context-pr-synth` bundle; its outputs are declared on the + /// step so downstream consumers reach them via `OutputRef`. + RESOLVE_SYNTHETIC_PR { + interpreter: Bash, + bindings: [BUNDLE], + externals: [], + fragments: [], + body: r#" +set -euo pipefail +node "$BUNDLE" +"#, + } +} pub(crate) const GATE_EVAL_PATH: &str = "/tmp/ado-aw-scripts/ado-script/gate.js"; pub(crate) const IMPORT_EVAL_PATH: &str = "/tmp/ado-aw-scripts/ado-script/import.js"; @@ -451,28 +604,8 @@ pub(crate) fn install_and_download_steps_typed( // Locate ado-script.zip + checksums.txt within the package staging // dir (handling both extracted-tree and raw-.nupkg delivery), // verify, then unzip the bundle into /tmp/ado-aw-scripts/. - let script = "\ - set -eo pipefail\n\ - mkdir -p /tmp/ado-aw-scripts\n\ - STAGING=/tmp/ado-aw-scripts/_pkg\n\ - if [ -z \"$(find \"$STAGING\" -name 'ado-script.zip' -print -quit)\" ]; then\n \ - NUPKG=\"$(find \"$STAGING\" -name '*.nupkg' -print -quit)\"\n \ - if [ -n \"$NUPKG\" ]; then\n \ - unzip -o \"$NUPKG\" -d \"$STAGING\" >/dev/null\n \ - fi\n\ - fi\n\ - ZIP=\"$(find \"$STAGING\" -name 'ado-script.zip' -print -quit)\"\n\ - CHK=\"$(find \"$STAGING\" -name 'checksums.txt' -print -quit)\"\n\ - if [ -z \"$ZIP\" ] || [ -z \"$CHK\" ]; then\n \ - echo \"##vso[task.complete result=Failed]ado-script.zip or checksums.txt not found in package\"\n \ - exit 1\n\ - fi\n\ - cp \"$ZIP\" /tmp/ado-aw-scripts/ado-script.zip\n\ - cp \"$CHK\" /tmp/ado-aw-scripts/checksums.txt\n\ - cd /tmp/ado-aw-scripts && grep \"ado-script.zip\" checksums.txt | sha256sum -c -\n\ - unzip -o /tmp/ado-aw-scripts/ado-script.zip -d /tmp/ado-aw-scripts/\n" - .to_string(); - let mut b = BashStep::new(format!("Stage ado-aw scripts (v{version})"), script) + let mut b = ShellScript::new(&STAGE_ADO_AW_SCRIPTS_FEED) + .into_step(format!("Stage ado-aw scripts (v{version})")) .with_condition(Condition::Succeeded); b.timeout = Some(std::time::Duration::from_secs(300)); return vec![ @@ -484,15 +617,10 @@ pub(crate) fn install_and_download_steps_typed( } let download = { - let script = format!( - "set -eo pipefail\n\ - mkdir -p /tmp/ado-aw-scripts\n\ - curl -fsSL \"{RELEASE_BASE_URL}/v{version}/checksums.txt\" -o /tmp/ado-aw-scripts/checksums.txt\n\ - curl -fsSL \"{RELEASE_BASE_URL}/v{version}/ado-script.zip\" -o /tmp/ado-aw-scripts/ado-script.zip\n\ - cd /tmp/ado-aw-scripts && grep \"ado-script.zip\" checksums.txt | sha256sum -c -\n\ - unzip -o /tmp/ado-aw-scripts/ado-script.zip -d /tmp/ado-aw-scripts/\n" - ); - let mut b = BashStep::new(format!("Download ado-aw scripts (v{version})"), script) + let mut b = ShellScript::new(&DOWNLOAD_ADO_AW_SCRIPTS_RELEASE) + .text("RELEASE_BASE_URL", RELEASE_BASE_URL) + .text("VERSION", version) + .into_step(format!("Download ado-aw scripts (v{version})")) .with_condition(Condition::Succeeded); b.timeout = Some(std::time::Duration::from_secs(300)); b @@ -535,12 +663,12 @@ fn resolver_step_typed() -> Step { .iter() .map(|name| format!(" --var \"{name}=$({name})\"")) .collect(); - let script = format!( - "set -eo pipefail\n\ - node '{IMPORT_EVAL_PATH}' /tmp/awf-tools/agent-prompt.md --base \"$(Build.SourcesDirectory)\"{var_flags}\n" - ); Step::Bash( - BashStep::new("Resolve runtime imports (agent prompt)", script) + ShellScript::new(&RESOLVE_RUNTIME_IMPORTS) + .text("IMPORT_EVAL_PATH", IMPORT_EVAL_PATH) + .bind("BASE", Binding::ado_macro("Build.SourcesDirectory")) + .fragment("var_flags", var_flags) + .into_step("Resolve runtime imports (agent prompt)") .with_condition(Condition::Succeeded), ) } @@ -1199,9 +1327,16 @@ mod tests { other => panic!("expected download bash step, got {other:?}"), } match &steps[2] { + // The evaluator path is now supplied as a `ShellScript` binding + // rather than interpolated into the command, so assert on the + // generated prelude: that proves the producer supplied this path, + // where a bare `contains` would also pass on a comment. Step::Bash(b) => assert!( b.script - .contains("node '/tmp/ado-aw-scripts/ado-script/gate.js'") + .contains(&format!("EVALUATOR_PATH='{GATE_EVAL_PATH}'")) + && b.script.contains(r#"node "$EVALUATOR_PATH""#), + "gate step must invoke the bound evaluator path: {}", + b.script ), other => panic!("expected gate bash step, got {other:?}"), } @@ -1785,24 +1920,38 @@ mod tests { let Step::Bash(resolver) = &steps[2] else { panic!("expected resolver bash step, got {:?}", steps[2]); }; + // The resolver path is now supplied as a `ShellScript` binding rather + // than interpolated into the command, so assert on the generated + // prelude: that proves the producer supplied this path, where a bare + // `contains` would also pass on a comment. assert!( resolver .script - .contains("node '/tmp/ado-aw-scripts/ado-script/import.js'") + .contains(&format!("IMPORT_EVAL_PATH='{IMPORT_EVAL_PATH}'")) + && resolver.script.contains(r#"node "$IMPORT_EVAL_PATH""#), + "resolver must invoke the bound import path: {}", + resolver.script ); assert_eq!( resolver.display_name, "Resolve runtime imports (agent prompt)" ); - // The resolver receives `--base "$(Build.SourcesDirectory)"` so - // the compiler-emitted trigger-repo-relative marker path - // resolves correctly. Absolute paths in author markers are - // rejected by import.js — see its absolute-path guard. + // The resolver receives the trigger-repo checkout root as `--base` so + // the compiler-emitted trigger-repo-relative marker path resolves + // correctly. Absolute paths in author markers are rejected by + // import.js — see its absolute-path guard. + // + // The path is now a `ShellScript` binding rather than a bare + // interpolation, so assert on the prelude *and* the use. That also + // closes the quoting exposure the old inline form carried: a + // single-quoted assignment cannot break argument parsing. assert!( resolver .script - .contains("--base \"$(Build.SourcesDirectory)\""), - "resolver step must pass --base so trigger-repo-relative markers resolve correctly" + .contains("BASE='$(Build.SourcesDirectory)'") + && resolver.script.contains("--base \"$BASE\""), + "resolver step must pass --base so trigger-repo-relative markers resolve correctly: {}", + resolver.script ); assert!( !resolver.script.contains("ADO_AW_IMPORT_BASE"), diff --git a/src/compile/extensions/azure_cli.rs b/src/compile/extensions/azure_cli.rs index c04570c0..385908ea 100644 --- a/src/compile/extensions/azure_cli.rs +++ b/src/compile/extensions/azure_cli.rs @@ -6,6 +6,103 @@ use crate::compile::common::{ }; use crate::compile::ir::condition::{Condition, Expr}; use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::shell::{Binding, ShellScript}; +use crate::shell_script; + +shell_script! { + /// Detection step: probe the host for azure-cli and export + /// `AW_AZ_MOUNTS` for the later AWF invocation. + /// + /// The two `##vso[task.setvariable variable=AW_AZ_MOUNTS]` lines are + /// load-bearing: one branch populates the mount args, the other sets an + /// empty value. Leaving it undefined in the missing-az branch would let + /// bash misparse `$(AW_AZ_MOUNTS)` later as a command substitution and + /// fail the AWF step under `set -e`. + DETECT_AZURE_CLI { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [], + body: r###" +set -eo pipefail +if [ -f /usr/bin/az ] && [ -d /opt/az ]; then + echo "##vso[task.setvariable variable=AW_AZ_MOUNTS]--mount /opt/az:/opt/az:ro --mount /usr/bin/az:/usr/bin/az:ro" + echo "Azure CLI detected on host; mounting /opt/az and /usr/bin/az into AWF sandbox." +else + echo "##vso[task.setvariable variable=AW_AZ_MOUNTS]" + echo "##vso[task.logissue type=warning]Azure CLI not detected on this runner (missing /usr/bin/az or /opt/az). The az command will not be available inside the agent sandbox. Install azure-cli on the runner image to enable it." +fi +"###, + } +} + +shell_script! { + /// Install the generated `az` wrapper into `WRAPPER_PATH`. + /// + /// The wrapper text is spliced as a fragment (not a binding) because it + /// legitimately contains the string `AZURE_DEVOPS_EXT_PAT` — a sentinel + /// value the wrapper sets so the ado-proxy container can swap it for a + /// real credential *out-of-band*. `Binding::document` rejects that string + /// by design; a fragment splices verbatim shell that is not, itself, a + /// value flowing through the typed channel. + /// + /// The heredoc delimiter `ADO_AW_AZ_WRAPPER_EOF` is single-quoted so the + /// shell writing the file does no expansion; the wrapper reads `$PATH`, + /// `$AZ_REAL`, `$@` etc. only when the *installed* file is invoked later. + INSTALL_AZ_WRAPPER { + interpreter: Bash, + bindings: [WRAPPER_DIR, WRAPPER_PATH], + externals: [], + fragments: [wrapper], + body: r#" +set -eo pipefail +mkdir -p "$WRAPPER_DIR" +cat > "$WRAPPER_PATH" << 'ADO_AW_AZ_WRAPPER_EOF' +# ado-aw:fragment wrapper +ADO_AW_AZ_WRAPPER_EOF +chmod 755 "$WRAPPER_PATH" +echo "az wrapper installed at $WRAPPER_PATH" +"#, + } +} + +shell_script! { + /// Append the ado-proxy policy prompt to the agent prompt file. + /// + /// `PROMPT` is a [`Binding::document`], which routes the potentially + /// multi-kilobyte body through a quoted heredoc in the prelude rather + /// than an inline unquoted heredoc — the value is data, so it should + /// never influence the step's control flow. + APPEND_PROXY_POLICY_PROMPT { + interpreter: Bash, + bindings: [PROMPT_PATH, PROMPT], + externals: [], + fragments: [], + body: r#" +printf '%s' "$PROMPT" >> "$PROMPT_PATH" +echo "ado-proxy policy prompt appended" +"#, + } +} + +shell_script! { + /// Append the Azure-CLI advisory to the agent prompt file. Same shape + /// as [`APPEND_PROXY_POLICY_PROMPT`] but a different message, and + /// step-level `condition:` gates on the detection result. + APPEND_AZURE_CLI_PROMPT { + interpreter: Bash, + bindings: [PROMPT_PATH, PROMPT], + externals: [], + fragments: [], + body: r#" +printf '%s' "$PROMPT" >> "$PROMPT_PATH" +echo "Azure CLI prompt appended" +"#, + } +} + +/// Path of the agent prompt file every extension appends to. +const AGENT_PROMPT_PATH: &str = "/tmp/awf-tools/agent-prompt.md"; // ─── Azure CLI (permissions.read-gated, install-free) ──────────────── @@ -135,34 +232,21 @@ fn install_az_wrapper_step(capabilities: &[Capability]) -> BashStep { ADO_MCP_TOKEN_SENTINEL, capabilities, ); - // Indent the body for the heredoc without altering its content. - let script = format!( - "set -eo pipefail\n\ - mkdir -p {AZ_WRAPPER_DIR}\n\ - cat > '{AZ_WRAPPER_PATH}' << 'ADO_AW_AZ_WRAPPER_EOF'\n\ - {wrapper}\n\ - ADO_AW_AZ_WRAPPER_EOF\n\ - chmod 755 '{AZ_WRAPPER_PATH}'\n\ - echo \"az wrapper installed at {AZ_WRAPPER_PATH}\"\n" - ); - BashStep::new("Install az wrapper (ado-proxy)", script).with_condition(Condition::Ne( - Expr::Variable("AW_AZ_MOUNTS".to_string()), - Expr::Literal(String::new()), - )) + ShellScript::new(&INSTALL_AZ_WRAPPER) + .text("WRAPPER_DIR", AZ_WRAPPER_DIR) + .text("WRAPPER_PATH", AZ_WRAPPER_PATH) + .fragment("wrapper", wrapper) + .into_step("Install az wrapper (ado-proxy)") + .with_condition(Condition::Ne( + Expr::Variable("AW_AZ_MOUNTS".to_string()), + Expr::Literal(String::new()), + )) } /// Detect azure-cli on the host and set the `AW_AZ_MOUNTS` pipeline /// variable for the later AWF invocation. fn detection_bash_step() -> BashStep { - let script = "set -eo pipefail\n\ - if [ -f /usr/bin/az ] && [ -d /opt/az ]; then\n \ - echo \"##vso[task.setvariable variable=AW_AZ_MOUNTS]--mount /opt/az:/opt/az:ro --mount /usr/bin/az:/usr/bin/az:ro\"\n \ - echo \"Azure CLI detected on host; mounting /opt/az and /usr/bin/az into AWF sandbox.\"\n\ - else\n \ - echo \"##vso[task.setvariable variable=AW_AZ_MOUNTS]\"\n \ - echo \"##vso[task.logissue type=warning]Azure CLI not detected on this runner (missing /usr/bin/az or /opt/az). The az command will not be available inside the agent sandbox. Install azure-cli on the runner image to enable it.\"\n\ - fi\n"; - BashStep::new("Detect Azure CLI on host (for AWF mount)", script) + ShellScript::new(&DETECT_AZURE_CLI).into_step("Detect Azure CLI on host (for AWF mount)") } /// Explain the effective compiler-owned ADO read policy to the agent. @@ -225,7 +309,7 @@ fn proxy_policy_prompt_step( } let scope_list = scope_lines.join("\n"); - let body = format!( + let script = format!( "\n\ ---\n\ \n\ @@ -242,14 +326,10 @@ Requests outside these capabilities or scopes, all writes, and secret-bearing ro \n\ If your task requires a read outside this list, report it as missing data/tooling and name the exact organization, project, repository, and operation that the front matter would need to grant.\n" ); - let script = format!( - "cat >> \"/tmp/awf-tools/agent-prompt.md\" << 'ADO_PROXY_POLICY_PROMPT_EOF'\n\ -{body}\ -ADO_PROXY_POLICY_PROMPT_EOF\n\ -\n\ -echo \"ado-proxy policy prompt appended\"\n" - ); - BashStep::new("Append ado-proxy policy prompt", script) + ShellScript::new(&APPEND_PROXY_POLICY_PROMPT) + .text("PROMPT_PATH", AGENT_PROMPT_PATH) + .bind("PROMPT", Binding::document(script)) + .into_step("Append ado-proxy policy prompt") } /// Append an Azure CLI advisory when the detection step found `az`. @@ -266,7 +346,7 @@ fn prompt_append_bash_step(capabilities: &[Capability]) -> BashStep { .map(|g| format!("`az {g}`")) .collect::>() .join(", "); - let body = format!( + let script = format!( "\n\ ---\n\ \n\ @@ -282,17 +362,14 @@ Requests outside that boundary are refused by a policy proxy, not by a misconfig If a read you need is refused, file a `missing-tool` safe output naming `azure-cli` and the exact command, so the operator can extend the catalog rather than leaving you blocked.\n" ); - let script = format!( - "cat >> \"/tmp/awf-tools/agent-prompt.md\" << 'AZURE_CLI_PROMPT_EOF'\n\ -{body}\ -AZURE_CLI_PROMPT_EOF\n\ -\n\ -echo \"Azure CLI prompt appended\"\n" - ); - BashStep::new("Append Azure CLI prompt", script).with_condition(Condition::Ne( - Expr::Variable("AW_AZ_MOUNTS".to_string()), - Expr::Literal(String::new()), - )) + ShellScript::new(&APPEND_AZURE_CLI_PROMPT) + .text("PROMPT_PATH", AGENT_PROMPT_PATH) + .bind("PROMPT", Binding::document(script)) + .into_step("Append Azure CLI prompt") + .with_condition(Condition::Ne( + Expr::Variable("AW_AZ_MOUNTS".to_string()), + Expr::Literal(String::new()), + )) } #[cfg(test)] @@ -395,9 +472,11 @@ mod tests { #[test] fn the_installed_wrapper_is_executable_and_starts_with_a_shebang() { let step = wrapper_step(&fm_proxied()).expect("wrapper step"); - assert!(step.script.contains(&format!("chmod 755 '{AZ_WRAPPER_PATH}'"))); + assert!(step.script.contains(r#"chmod 755 "$WRAPPER_PATH""#)); // The heredoc body must not be indented: a shebang preceded by // whitespace is not a shebang, and the file would fail to exec. + // The quote is part of the match — the delimiter is quoted, so the + // opening line ends `<< 'ADO_AW_AZ_WRAPPER_EOF'`. assert!( step.script.contains("ADO_AW_AZ_WRAPPER_EOF'\n#!/bin/sh"), "the wrapper body must start at column 0: {}", @@ -700,14 +779,21 @@ repos: .map(bash_step) .find(|step| step.display_name == "Append Azure CLI prompt") .expect("Azure CLI prompt step"); + // The rendered script uses a bound $PROMPT_PATH variable; the value + // is supplied by this file's producer. assert!( append .script - .contains(r#"cat >> "/tmp/awf-tools/agent-prompt.md""#), - "prompt-append step must append to /tmp/awf-tools/agent-prompt.md \ + .contains(r#"'/tmp/awf-tools/agent-prompt.md'"#), + "prompt-append step must bind PROMPT_PATH to /tmp/awf-tools/agent-prompt.md \ (matching wrap_prompt_append). Step:\n{}", append.script ); + assert!( + append.script.contains(r#"printf '%s' "$PROMPT" >> "$PROMPT_PATH""#), + "prompt-append body must append the bound $PROMPT to $PROMPT_PATH: {}", + append.script + ); } #[test] @@ -758,8 +844,11 @@ repos: .map(bash_step) .find(|step| step.display_name == "Append Azure CLI prompt") .expect("Azure CLI prompt step"); + // The migrated form carries the body through Binding::document, whose + // canonical heredoc delimiter is ADO_AW_SHELL_DOC_EOF and is always + // single-quoted by the binding constructor. assert!( - append.script.contains("<< 'AZURE_CLI_PROMPT_EOF'"), + append.script.contains("<<'ADO_AW_SHELL_DOC_EOF'"), "prompt-append heredoc delimiter must be single-quoted to \ prevent expansion of environment references inside the prompt \ body. Step:\n{}", diff --git a/src/compile/extensions/exec_context/ci_push.rs b/src/compile/extensions/exec_context/ci_push.rs index b3d4a1de..ba739b68 100644 --- a/src/compile/extensions/exec_context/ci_push.rs +++ b/src/compile/extensions/exec_context/ci_push.rs @@ -49,11 +49,29 @@ use crate::compile::extensions::CompileContext; use crate::compile::ado_bundle::{Bundle, TokenSource, apply_bundle_auth}; use crate::compile::extensions::ado_script::EXEC_CONTEXT_CI_PUSH_PATH; use crate::compile::ir::condition::{Condition, Expr}; -use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::ir::step::Step; +use crate::compile::shell::ShellScript; use crate::compile::types::CiPushContextConfig; +use crate::shell_script; use super::contributor::{ContextContributor, succeeded_and}; +shell_script! { + /// Invoke the exec-context-ci-push node bundle. The step's own + /// `condition:` gates on `Build.Reason ∈ {IndividualCI, BatchedCI}`; + /// no bash-side guard is needed. + EXEC_CONTEXT_CI_PUSH { + interpreter: Bash, + bindings: [BUNDLE], + externals: [], + fragments: [], + body: r#" +set -euo pipefail +node "$BUNDLE" +"#, + } +} + /// CI-push-context contributor. pub(super) struct CiPushContextContributor { config: CiPushContextConfig, @@ -90,25 +108,23 @@ impl ContextContributor for CiPushContextContributor { if !self.should_activate(ctx) { return Ok(None); } - let script = format!("set -euo pipefail\nnode '{EXEC_CONTEXT_CI_PUSH_PATH}'\n"); + let script = ShellScript::new(&EXEC_CONTEXT_CI_PUSH).text("BUNDLE", EXEC_CONTEXT_CI_PUSH_PATH); // ADO auto-injects the predefined System.*/Build.* context variables // into the step env, so the bundle reads them directly; only the // non-auto-injected SYSTEM_ACCESSTOKEN bearer is projected here. let step = apply_bundle_auth( - BashStep::new( - "Stage ci-push execution context (aw-context/ci-push/*)", - script, - ) - .with_condition(succeeded_and(Condition::Or(vec![ - Condition::Eq( - Expr::Variable("Build.Reason".to_string()), - Expr::Literal("IndividualCI".to_string()), - ), - Condition::Eq( - Expr::Variable("Build.Reason".to_string()), - Expr::Literal("BatchedCI".to_string()), - ), - ]))), + script + .into_step("Stage ci-push execution context (aw-context/ci-push/*)") + .with_condition(succeeded_and(Condition::Or(vec![ + Condition::Eq( + Expr::Variable("Build.Reason".to_string()), + Expr::Literal("IndividualCI".to_string()), + ), + Condition::Eq( + Expr::Variable("Build.Reason".to_string()), + Expr::Literal("BatchedCI".to_string()), + ), + ]))), Bundle::ExecContextCiPush, TokenSource::SystemAccessToken, ); diff --git a/src/compile/extensions/exec_context/manual.rs b/src/compile/extensions/exec_context/manual.rs index aeaaebaf..de5c327b 100644 --- a/src/compile/extensions/exec_context/manual.rs +++ b/src/compile/extensions/exec_context/manual.rs @@ -50,14 +50,31 @@ use crate::compile::extensions::CompileContext; use crate::compile::extensions::ado_script::EXEC_CONTEXT_MANUAL_PATH; use crate::compile::ir::condition::{Condition, Expr}; use crate::compile::ir::env::EnvValue; -use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::ir::step::Step; +use crate::compile::shell::ShellScript; use crate::compile::types::ManualContextConfig; +use crate::shell_script; #[cfg(test)] use crate::compile::types::FrontMatter; use super::contributor::{ContextContributor, succeeded_and}; +shell_script! { + /// Invoke the exec-context-manual node bundle. The step's own + /// `condition:` gates on `Build.Reason == Manual`. + EXEC_CONTEXT_MANUAL { + interpreter: Bash, + bindings: [BUNDLE], + externals: [], + fragments: [], + body: r#" +set -euo pipefail +node "$BUNDLE" +"#, + } +} + /// Manual-context contributor. pub(super) struct ManualContextContributor { config: ManualContextConfig, @@ -142,24 +159,22 @@ impl ContextContributor for ManualContextContributor { return Ok(None); } - let script = format!("set -euo pipefail\nnode '{EXEC_CONTEXT_MANUAL_PATH}'\n"); + let script = ShellScript::new(&EXEC_CONTEXT_MANUAL).text("BUNDLE", EXEC_CONTEXT_MANUAL_PATH); // BUILD_SOURCESDIRECTORY is auto-injected by ADO, so it is not // re-projected. BUILD_REQUESTEDFOR / BUILD_REQUESTEDFOREMAIL are // retained (identity vars behind the opt-in email hygiene gate — the // projection makes the intent explicit at the call site). Manual has // no bearer (BundleAuth::None). - let mut step = BashStep::new( - "Stage manual execution context (aw-context/manual/*)", - script, - ) - .with_condition(succeeded_and(Condition::Eq( - Expr::Variable("Build.Reason".to_string()), - Expr::Literal("Manual".to_string()), - ))) - .with_env( - "BUILD_REQUESTEDFOR", - EnvValue::ado_macro("Build.RequestedFor")?, - ); + let mut step = script + .into_step("Stage manual execution context (aw-context/manual/*)") + .with_condition(succeeded_and(Condition::Eq( + Expr::Variable("Build.Reason".to_string()), + Expr::Literal("Manual".to_string()), + ))) + .with_env( + "BUILD_REQUESTEDFOR", + EnvValue::ado_macro("Build.RequestedFor")?, + ); // Email is opt-in for hygiene — see ManualContextConfig docs. if self.config.include_email_resolved() { diff --git a/src/compile/extensions/exec_context/pipeline.rs b/src/compile/extensions/exec_context/pipeline.rs index 72381968..4414ab78 100644 --- a/src/compile/extensions/exec_context/pipeline.rs +++ b/src/compile/extensions/exec_context/pipeline.rs @@ -43,11 +43,28 @@ use crate::compile::extensions::CompileContext; use crate::compile::ado_bundle::{Bundle, TokenSource, apply_bundle_auth}; use crate::compile::extensions::ado_script::EXEC_CONTEXT_PIPELINE_PATH; use crate::compile::ir::condition::{Condition, Expr}; -use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::ir::step::Step; +use crate::compile::shell::ShellScript; use crate::compile::types::PipelineContextConfig; +use crate::shell_script; use super::contributor::{ContextContributor, succeeded_and}; +shell_script! { + /// Invoke the exec-context-pipeline node bundle. The step's own + /// `condition:` gates on `Build.Reason == ResourceTrigger`. + EXEC_CONTEXT_PIPELINE { + interpreter: Bash, + bindings: [BUNDLE], + externals: [], + fragments: [], + body: r#" +set -euo pipefail +node "$BUNDLE" +"#, + } +} + /// Pipeline-context contributor. pub(super) struct PipelineContextContributor { config: PipelineContextConfig, @@ -87,21 +104,19 @@ impl ContextContributor for PipelineContextContributor { if !self.should_activate(ctx) { return Ok(None); } - let script = format!("set -euo pipefail\nnode '{EXEC_CONTEXT_PIPELINE_PATH}'\n"); + let script = ShellScript::new(&EXEC_CONTEXT_PIPELINE).text("BUNDLE", EXEC_CONTEXT_PIPELINE_PATH); // ADO auto-injects every predefined System.*/Build.* variable into the // step env (SCREAMING_SNAKE form), so the bundle reads // SYSTEM_COLLECTIONURI / BUILD_SOURCESDIRECTORY / BUILD_TRIGGEREDBY_* // directly without re-projection. Only the non-auto-injected // SYSTEM_ACCESSTOKEN bearer is projected, via the bundle-auth applier. let step = apply_bundle_auth( - BashStep::new( - "Stage pipeline execution context (aw-context/pipeline/*)", - script, - ) - .with_condition(succeeded_and(Condition::Eq( - Expr::Variable("Build.Reason".to_string()), - Expr::Literal("ResourceTrigger".to_string()), - ))), + script + .into_step("Stage pipeline execution context (aw-context/pipeline/*)") + .with_condition(succeeded_and(Condition::Eq( + Expr::Variable("Build.Reason".to_string()), + Expr::Literal("ResourceTrigger".to_string()), + ))), Bundle::ExecContextPipeline, TokenSource::SystemAccessToken, ); diff --git a/src/compile/extensions/exec_context/pr.rs b/src/compile/extensions/exec_context/pr.rs index 1117e3e1..bc3117e8 100644 --- a/src/compile/extensions/exec_context/pr.rs +++ b/src/compile/extensions/exec_context/pr.rs @@ -64,11 +64,54 @@ use crate::compile::ado_bundle::{Bundle, TokenSource, apply_bundle_auth}; use crate::compile::extensions::ado_script::EXEC_CONTEXT_PR_PATH; use crate::compile::ir::condition::{Condition, Expr}; use crate::compile::ir::env::EnvValue; -use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::ir::step::Step; +use crate::compile::shell::ShellScript; use crate::compile::types::PrContextConfig; +use crate::shell_script; use super::contributor::{ContextContributor, succeeded_and}; +shell_script! { + /// Invoke the exec-context-pr node bundle unconditionally. + /// + /// Used on the "real PR" path — the step's own `condition:` gates on + /// `Build.Reason == PullRequest`, so no bash-side guard is needed. + EXEC_CONTEXT_PR { + interpreter: Bash, + bindings: [BUNDLE], + externals: [], + fragments: [], + body: r#" +set -euo pipefail +node "$BUNDLE" +"#, + } +} + +shell_script! { + /// Invoke the exec-context-pr node bundle from a synth-PR-active + /// Agent job. + /// + /// The Agent-job condition can only depend on same-job values, so the + /// synth-active path always runs and gates in bash: an empty + /// `$AW_PR_ID` means neither a real PR build nor a synth-promoted CI + /// build, and the bundle would have nothing to describe. + EXEC_CONTEXT_PR_SYNTH { + interpreter: Bash, + bindings: [BUNDLE], + externals: [AW_PR_ID], + fragments: [], + body: r#" +set -euo pipefail +if [ -z "$AW_PR_ID" ]; then + echo "[aw-context] No PR identifier resolved (not a PR build and not synth-promoted); skipping exec-context-pr." + exit 0 +fi +node "$BUNDLE" +"#, + } +} + /// PR-context contributor. Activates when `on.pr` is configured /// (unless explicitly disabled via `execution-context.pr.enabled: false`). pub(super) struct PrContextContributor { @@ -131,28 +174,28 @@ impl ContextContributor for PrContextContributor { // which is exactly when this step should skip. // // Coexists with `prepare_step` until production callers switch. - let (prelude, condition) = if self.synthetic_pr_active { + let (script, condition) = if self.synthetic_pr_active { ( - " if [ -z \"$AW_PR_ID\" ]; then\n echo \"[aw-context] No PR identifier resolved (not a PR build and not synth-promoted); skipping exec-context-pr.\"\n exit 0\n fi\n", + ShellScript::new(&EXEC_CONTEXT_PR_SYNTH).text("BUNDLE", EXEC_CONTEXT_PR_PATH), Condition::Succeeded, ) } else { ( - "", + ShellScript::new(&EXEC_CONTEXT_PR).text("BUNDLE", EXEC_CONTEXT_PR_PATH), succeeded_and(Condition::Eq( Expr::Variable("Build.Reason".to_string()), Expr::Literal("PullRequest".to_string()), )), ) }; - let script = format!("set -euo pipefail\n{prelude}node '{EXEC_CONTEXT_PR_PATH}'\n"); // ADO auto-injects predefined System.*/Build.* context vars, so the // bundle reads SYSTEM_TEAMPROJECT / BUILD_REPOSITORY_NAME / // BUILD_SOURCESDIRECTORY (and, in real-PR mode, SYSTEM_PULLREQUEST_*) // directly. Only the non-auto-injected SYSTEM_ACCESSTOKEN bearer and, // in synth mode, the hoisted AW_PR_* overrides are projected. let mut step = apply_bundle_auth( - BashStep::new("Stage PR execution context (aw-context/pr/*)", script) + script + .into_step("Stage PR execution context (aw-context/pr/*)") .with_condition(condition), Bundle::ExecContextPr, TokenSource::SystemAccessToken, diff --git a/src/compile/extensions/exec_context/pr_checks.rs b/src/compile/extensions/exec_context/pr_checks.rs index 9a4a5d6d..41624d38 100644 --- a/src/compile/extensions/exec_context/pr_checks.rs +++ b/src/compile/extensions/exec_context/pr_checks.rs @@ -24,11 +24,53 @@ use crate::compile::ado_bundle::{Bundle, TokenSource, apply_bundle_auth}; use crate::compile::extensions::ado_script::EXEC_CONTEXT_PR_CHECKS_PATH; use crate::compile::ir::condition::{Condition, Expr}; use crate::compile::ir::env::EnvValue; -use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::ir::step::Step; +use crate::compile::shell::ShellScript; use crate::compile::types::PrChecksContextConfig; +use crate::shell_script; use super::contributor::{ContextContributor, succeeded_and}; +shell_script! { + /// Invoke the exec-context-pr-checks node bundle. Used on the + /// "real PR" path — the step's own `condition:` gates on + /// `Build.Reason == PullRequest`. + EXEC_CONTEXT_PR_CHECKS { + interpreter: Bash, + bindings: [BUNDLE], + externals: [], + fragments: [], + body: r#" +set -euo pipefail +node "$BUNDLE" +"#, + } +} + +shell_script! { + /// Invoke the exec-context-pr-checks node bundle from a synth-PR-active + /// Agent job. + /// + /// The Agent-job condition can only depend on same-job values, so the + /// synth-active path always runs and gates in bash: an empty + /// `$SYSTEM_PULLREQUEST_PULLREQUESTID` (the env value is populated from + /// the hoisted AW_PR_ID variable) means no PR to describe. + EXEC_CONTEXT_PR_CHECKS_SYNTH { + interpreter: Bash, + bindings: [BUNDLE], + externals: [SYSTEM_PULLREQUEST_PULLREQUESTID], + fragments: [], + body: r#" +set -euo pipefail +if [ -z "$SYSTEM_PULLREQUEST_PULLREQUESTID" ]; then + echo "[aw-context] No PR identifier resolved; skipping exec-context-pr-checks." + exit 0 +fi +node "$BUNDLE" +"#, + } +} + pub(super) struct PrChecksContextContributor { config: PrChecksContextConfig, /// `mode: synthetic` flag — drives env-var selection like the PR @@ -81,10 +123,11 @@ impl ContextContributor for PrChecksContextContributor { // variable and the step always runs (guarded by a runtime prelude); // in real-PR mode the auto-injected SYSTEM_PULLREQUEST_PULLREQUESTID is // read directly and the step gates on Build.Reason == PullRequest. - let (condition, prelude) = if self.synthetic_pr_active { + let (condition, script) = if self.synthetic_pr_active { ( Condition::Succeeded, - " if [ -z \"$SYSTEM_PULLREQUEST_PULLREQUESTID\" ]; then\n echo \"[aw-context] No PR identifier resolved; skipping exec-context-pr-checks.\"\n exit 0\n fi\n", + ShellScript::new(&EXEC_CONTEXT_PR_CHECKS_SYNTH) + .text("BUNDLE", EXEC_CONTEXT_PR_CHECKS_PATH), ) } else { ( @@ -92,22 +135,20 @@ impl ContextContributor for PrChecksContextContributor { Expr::Variable("Build.Reason".to_string()), Expr::Literal("PullRequest".to_string()), )), - "", + ShellScript::new(&EXEC_CONTEXT_PR_CHECKS) + .text("BUNDLE", EXEC_CONTEXT_PR_CHECKS_PATH), ) }; - let script = format!("set -euo pipefail\n{prelude}node '{EXEC_CONTEXT_PR_CHECKS_PATH}'\n"); // ADO auto-injects predefined System.*/Build.* context vars, so only // SYSTEM_ACCESSTOKEN (bearer, not auto-injected) and the mode-dependent // PR id are projected. In synth mode the id comes from the hoisted // AW_PR_ID job variable; in real-PR mode the auto-injected value is // used directly. let mut step = apply_bundle_auth( - BashStep::new( - "Stage PR-checks execution context (aw-context/pr/checks/*)", - script, - ) - .with_condition(condition), + script + .into_step("Stage PR-checks execution context (aw-context/pr/checks/*)") + .with_condition(condition), Bundle::ExecContextPrChecks, TokenSource::SystemAccessToken, ); diff --git a/src/compile/extensions/exec_context/repo.rs b/src/compile/extensions/exec_context/repo.rs index cbbcc975..a3a3318c 100644 --- a/src/compile/extensions/exec_context/repo.rs +++ b/src/compile/extensions/exec_context/repo.rs @@ -17,11 +17,28 @@ use crate::compile::extensions::CompileContext; use crate::compile::extensions::ado_script::EXEC_CONTEXT_REPO_PATH; use crate::compile::ir::condition::Condition; use crate::compile::ir::env::EnvValue; -use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::ir::step::Step; +use crate::compile::shell::ShellScript; use crate::compile::types::RepoContextConfig; +use crate::shell_script; use super::contributor::ContextContributor; +shell_script! { + /// Invoke the exec-context-repo node bundle. Repo is always-on: no + /// runtime `Build.Reason` gate, the config toggles activation. + EXEC_CONTEXT_REPO { + interpreter: Bash, + bindings: [BUNDLE], + externals: [], + fragments: [], + body: r#" +set -euo pipefail +node "$BUNDLE" +"#, + } +} + pub(super) struct RepoContextContributor { config: RepoContextConfig, } @@ -52,12 +69,13 @@ impl ContextContributor for RepoContextContributor { if !self.should_activate(ctx) { return Ok(None); } - let script = format!("set -euo pipefail\nnode '{EXEC_CONTEXT_REPO_PATH}'\n"); + let script = ShellScript::new(&EXEC_CONTEXT_REPO).text("BUNDLE", EXEC_CONTEXT_REPO_PATH); // ADO auto-injects BUILD_SOURCESDIRECTORY / BUILD_SOURCEVERSION / // BUILD_SOURCEBRANCH into the step env, so the git-only bundle reads // them directly; only the compile-time AW_REPO_CONVENTIONS toggle is a // genuine step input. Repo has no bearer (BundleAuth::None). - let step = BashStep::new("Stage repo execution context (aw-context/repo/*)", script) + let step = script + .into_step("Stage repo execution context (aw-context/repo/*)") // Always-on (no Build.Reason gate). The compile-time // activation flag is the only gate. .with_condition(Condition::Succeeded) diff --git a/src/compile/extensions/exec_context/schedule.rs b/src/compile/extensions/exec_context/schedule.rs index 3a9d67c2..a34a810f 100644 --- a/src/compile/extensions/exec_context/schedule.rs +++ b/src/compile/extensions/exec_context/schedule.rs @@ -18,11 +18,28 @@ use crate::compile::extensions::CompileContext; use crate::compile::ado_bundle::{Bundle, TokenSource, apply_bundle_auth}; use crate::compile::extensions::ado_script::EXEC_CONTEXT_SCHEDULE_PATH; use crate::compile::ir::condition::{Condition, Expr}; -use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::ir::step::Step; +use crate::compile::shell::ShellScript; use crate::compile::types::ScheduleContextConfig; +use crate::shell_script; use super::contributor::{ContextContributor, succeeded_and}; +shell_script! { + /// Invoke the exec-context-schedule node bundle. The step's own + /// `condition:` gates on `Build.Reason == Schedule`. + EXEC_CONTEXT_SCHEDULE { + interpreter: Bash, + bindings: [BUNDLE], + externals: [], + fragments: [], + body: r#" +set -euo pipefail +node "$BUNDLE" +"#, + } +} + pub(super) struct ScheduleContextContributor { config: ScheduleContextConfig, } @@ -57,19 +74,17 @@ impl ContextContributor for ScheduleContextContributor { if !self.should_activate(ctx) { return Ok(None); } - let script = format!("set -euo pipefail\nnode '{EXEC_CONTEXT_SCHEDULE_PATH}'\n"); + let script = ShellScript::new(&EXEC_CONTEXT_SCHEDULE).text("BUNDLE", EXEC_CONTEXT_SCHEDULE_PATH); // ADO auto-injects the predefined System.*/Build.* context variables // into the step env, so the bundle reads them directly; only the // non-auto-injected SYSTEM_ACCESSTOKEN bearer is projected here. let step = apply_bundle_auth( - BashStep::new( - "Stage schedule execution context (aw-context/schedule/*)", - script, - ) - .with_condition(succeeded_and(Condition::Eq( - Expr::Variable("Build.Reason".to_string()), - Expr::Literal("Schedule".to_string()), - ))), + script + .into_step("Stage schedule execution context (aw-context/schedule/*)") + .with_condition(succeeded_and(Condition::Eq( + Expr::Variable("Build.Reason".to_string()), + Expr::Literal("Schedule".to_string()), + ))), Bundle::ExecContextSchedule, TokenSource::SystemAccessToken, ); diff --git a/src/compile/extensions/exec_context/workitem.rs b/src/compile/extensions/exec_context/workitem.rs index 62396746..811f03db 100644 --- a/src/compile/extensions/exec_context/workitem.rs +++ b/src/compile/extensions/exec_context/workitem.rs @@ -60,11 +60,48 @@ use crate::compile::ado_bundle::{Bundle, TokenSource, apply_bundle_auth}; use crate::compile::extensions::ado_script::EXEC_CONTEXT_WORKITEM_PATH; use crate::compile::ir::condition::{Condition, Expr}; use crate::compile::ir::env::EnvValue; -use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::ir::step::Step; +use crate::compile::shell::ShellScript; use crate::compile::types::WorkitemContextConfig; +use crate::shell_script; use super::contributor::{ContextContributor, succeeded_and}; +shell_script! { + /// Invoke the exec-context-workitem node bundle on the "real PR" path. + /// The step's own `condition:` gates on `Build.Reason == PullRequest`. + EXEC_CONTEXT_WORKITEM { + interpreter: Bash, + bindings: [BUNDLE], + externals: [], + fragments: [], + body: r#" +set -euo pipefail +node "$BUNDLE" +"#, + } +} + +shell_script! { + /// Invoke the exec-context-workitem node bundle from a synth-PR-active + /// Agent job. Gates in bash on an empty PR id (populated from the + /// hoisted AW_PR_ID variable). + EXEC_CONTEXT_WORKITEM_SYNTH { + interpreter: Bash, + bindings: [BUNDLE], + externals: [SYSTEM_PULLREQUEST_PULLREQUESTID], + fragments: [], + body: r#" +set -euo pipefail +if [ -z "$SYSTEM_PULLREQUEST_PULLREQUESTID" ]; then + echo "[aw-context] No PR identifier resolved; skipping exec-context-workitem." + exit 0 +fi +node "$BUNDLE" +"#, + } +} + /// Workitem-context contributor (PR-linked mode only). pub(super) struct WorkitemContextContributor { config: WorkitemContextConfig, @@ -133,34 +170,33 @@ impl ContextContributor for WorkitemContextContributor { // variable and the step always runs (guarded by a runtime prelude); // in real-PR mode the auto-injected SYSTEM_PULLREQUEST_PULLREQUESTID is // read directly and the step gates on Build.Reason == PullRequest. - let condition = if self.synthetic_pr_active { - Condition::Succeeded - } else { - succeeded_and(Condition::Eq( - Expr::Variable("Build.Reason".to_string()), - Expr::Literal("PullRequest".to_string()), - )) - }; - - let prelude = if self.synthetic_pr_active { - " if [ -z \"$SYSTEM_PULLREQUEST_PULLREQUESTID\" ]; then\n echo \"[aw-context] No PR identifier resolved; skipping exec-context-workitem.\"\n exit 0\n fi\n" + let (condition, script) = if self.synthetic_pr_active { + ( + Condition::Succeeded, + ShellScript::new(&EXEC_CONTEXT_WORKITEM_SYNTH) + .text("BUNDLE", EXEC_CONTEXT_WORKITEM_PATH), + ) } else { - "" + ( + succeeded_and(Condition::Eq( + Expr::Variable("Build.Reason".to_string()), + Expr::Literal("PullRequest".to_string()), + )), + ShellScript::new(&EXEC_CONTEXT_WORKITEM) + .text("BUNDLE", EXEC_CONTEXT_WORKITEM_PATH), + ) }; let max_items = self.config.max_items_resolved(); let max_body_kb = self.config.max_body_kb_resolved(); - let script = format!("set -euo pipefail\n{prelude}node '{EXEC_CONTEXT_WORKITEM_PATH}'\n"); // ADO auto-injects predefined System.*/Build.* context vars, so only // the SYSTEM_ACCESSTOKEN bearer (not auto-injected), the mode-dependent // synth PR id, and the computed limits are projected. let mut step = apply_bundle_auth( - BashStep::new( - "Stage workitem execution context (aw-context/workitem/*)", - script, - ) - .with_condition(condition), + script + .into_step("Stage workitem execution context (aw-context/workitem/*)") + .with_condition(condition), Bundle::ExecContextWorkitem, TokenSource::SystemAccessToken, ); diff --git a/src/compile/filter_ir.rs b/src/compile/filter_ir.rs index be419400..3a3533d6 100644 --- a/src/compile/filter_ir.rs +++ b/src/compile/filter_ir.rs @@ -32,6 +32,34 @@ use std::collections::BTreeSet; use std::fmt; +use crate::shell_script; + +shell_script! { + /// Bash body of the PR / pipeline gate step. + /// + /// The step invokes the bundled `gate.js` evaluator: the evaluator reads + /// the base64-encoded `GATE_SPEC` env var and the pipeline / PR fact env + /// vars declared by [`Fact::ado_exports`] and emits + /// `##vso[task.setvariable variable=SHOULD_RUN;isOutput=true]` — the + /// downstream Agent-job condition consumes it as a typed + /// `Condition::Eq(Expr::StepOutput(..., "SHOULD_RUN"))`. + /// + /// The shell body itself is a single `node "$EVALUATOR_PATH"` — every + /// value read by the evaluator arrives through the step's typed + /// [`crate::compile::ir::env::EnvValue`] `env:` block, not through the + /// shell. That is why the body has no externals: the shell layer knows + /// nothing about `GATE_SPEC` etc, and only the JS process reads them. + GATE_EVALUATOR { + interpreter: Bash, + bindings: [EVALUATOR_PATH], + externals: [], + fragments: [], + body: r#" +node "$EVALUATOR_PATH" +"#, + } +} + // ─── Fact Sources ─────────────────────────────────────────────────────────── /// A typed runtime fact that can be acquired and referenced by predicates. @@ -1294,7 +1322,9 @@ pub fn build_gate_step_typed( let exports = collect_ado_exports(checks)?; let pr_synth_active = synthetic_pr_active && matches!(ctx, GateContext::PullRequest); - let script = format!("node '{evaluator_path}'\n"); + let script = crate::compile::shell::ShellScript::new(&GATE_EVALUATOR) + .text("EVALUATOR_PATH", evaluator_path) + .render(); let mut step = apply_bundle_auth( BashStep::new(ctx.display_name(), script) .with_id(StepId::new(ctx.step_name())?) diff --git a/src/compile/shell/bindings.rs b/src/compile/shell/bindings.rs index 4ce8e9d3..d9949cb8 100644 --- a/src/compile/shell/bindings.rs +++ b/src/compile/shell/bindings.rs @@ -39,6 +39,11 @@ pub enum BindingKind { /// An integer, emitted bare. Number, /// `true` or `false`, emitted bare. + /// + /// Retained even while no current script needs it: [`Binding`] is a closed + /// vocabulary, and without this a future author would reach for + /// `text("true")` — the untyped fallback the whole design exists to avoid. + #[allow(dead_code)] Bool, /// A whitespace-separated word list, single-quoted, intended for /// unquoted `for` expansion. @@ -100,6 +105,9 @@ impl Binding { /// A boolean, emitted bare as `true` / `false` so `[ "$V" = true ]` reads /// naturally. + /// + /// See [`BindingKind::Bool`] for why this exists ahead of a caller. + #[allow(dead_code)] pub fn boolean(value: bool) -> Self { Self { rhs: if value { "true" } else { "false" }.to_string(), @@ -184,6 +192,9 @@ impl Binding { } /// How this binding was validated. + /// + /// Used by tests asserting on producer intent rather than rendered text. + #[allow(dead_code)] pub fn kind(&self) -> BindingKind { self.kind } @@ -214,6 +225,10 @@ pub(crate) fn single_quote(value: &str) -> String { } /// A shell variable name the prelude may assign. +/// +/// Only the registry-wide lint calls this today, so it is dead in a non-test +/// build. +#[allow(dead_code)] pub(crate) fn is_shell_var_name(name: &str) -> bool { let mut chars = name.chars(); match chars.next() { diff --git a/src/compile/shell/lint.rs b/src/compile/shell/lint.rs index 3e7b9bcf..de66feca 100644 --- a/src/compile/shell/lint.rs +++ b/src/compile/shell/lint.rs @@ -22,6 +22,8 @@ use std::process::{Command, Stdio}; use serde::Deserialize; use super::registry::{ShellScriptDef, all_scripts}; +use super::FRAGMENT_MARKER; +use super::bindings::is_shell_var_name; /// One shellcheck JSON finding. #[derive(Debug, Deserialize)] @@ -108,6 +110,80 @@ fn every_registered_script_passes_shellcheck() { ); } +#[test] +fn every_declared_variable_name_is_a_valid_shell_name() { + // A binding is emitted as `NAME=value` in the prelude. A lowercase or + // hyphenated name would either produce invalid shell or silently shadow a + // convention, and the failure would surface at pipeline runtime rather + // than here. + let mut problems = String::new(); + for def in all_scripts() { + for name in def.bindings.iter().chain(def.externals.iter()) { + if !is_shell_var_name(name) { + problems.push_str(&format!( + " {} declares `{name}`, which is not a SCREAMING_SNAKE shell name ({}:{})\n", + def.name, def.file, def.line + )); + } + } + } + assert!(problems.is_empty(), "invalid shell variable names:\n{problems}"); +} + +#[test] +fn every_phase_is_also_a_declared_fragment() { + // A `phases:` entry that is not in `fragments:` would never be spliced — + // the composed lint would silently fall back to linting an outline. + let mut problems = String::new(); + for def in all_scripts() { + for (name, _) in def.phases { + if !def.fragments.contains(name) { + problems.push_str(&format!( + " {} declares phase `{name}` without declaring it as a fragment ({}:{})\n", + def.name, def.file, def.line + )); + } + } + } + assert!(problems.is_empty(), "phase declaration drift:\n{problems}"); +} + +#[test] +fn a_composed_script_is_linted_with_its_phases_spliced() { + // Guards the mechanism the SC2034-on-every-binding failure exposed: an + // outline made only of markers must be linted as the script that runs, + // not as the outline. + let composed: Vec<&'static ShellScriptDef> = all_scripts() + .into_iter() + .filter(|def| !def.phases.is_empty()) + .collect(); + assert!( + !composed.is_empty(), + "no script declares phases; if composition was removed, remove this test too" + ); + for def in composed { + let source = def.lint_source(); + for (name, phase) in def.phases { + assert!( + !source.contains(&format!("{FRAGMENT_MARKER}{name}")), + "{}: phase `{name}` was left as a marker instead of being spliced", + def.name + ); + let first = phase + .body + .lines() + .map(str::trim) + .find(|l| !l.is_empty() && !l.starts_with('#')) + .unwrap_or_else(|| panic!("{} has no runnable line", phase.name)); + assert!( + source.contains(first), + "{}: phase `{name}` body is missing from the composed lint source", + def.name + ); + } + } +} + #[test] fn every_declared_fragment_has_a_marker_and_vice_versa() { // `splice_fragments` enforces this at render time, but only for scripts a @@ -174,11 +250,25 @@ fn every_registered_script_declares_the_variables_it_reads() { /// /// Deliberately only SCREAMING_SNAKE names: lowercase locals are the body's /// own business, and shell specials (`$1`, `$@`, `$?`) are not names. +/// +/// Single-quoted spans are skipped. Nothing expands inside `'…'`, so a `$NF` +/// in an embedded awk program or a `$1` in a sed replacement is not a shell +/// variable reference. Without this the checker would demand a declaration for +/// another language's variables and push authors into renaming them, which +/// distorts the script to satisfy the tool. fn referenced_vars(body: &str) -> Vec { let mut out = Vec::new(); let bytes: Vec = body.chars().collect(); let mut i = 0; while i < bytes.len() { + if bytes[i] == '\'' { + i += 1; + while i < bytes.len() && bytes[i] != '\'' { + i += 1; + } + i += 1; + continue; + } if bytes[i] != '$' { i += 1; continue; @@ -188,7 +278,11 @@ fn referenced_vars(body: &str) -> Vec { j += 1; } let start = j; - while j < bytes.len() && (bytes[j].is_ascii_uppercase() || bytes[j] == '_' || (j > start && bytes[j].is_ascii_digit())) { + while j < bytes.len() + && (bytes[j].is_ascii_uppercase() + || bytes[j] == '_' + || (j > start && bytes[j].is_ascii_digit())) + { j += 1; } if j > start { @@ -225,6 +319,23 @@ mod tests { assert_eq!(vars, vec!["FOO", "BAR", "BAZ2"]); } + #[test] + fn referenced_vars_skips_single_quoted_spans() { + // `$NF` belongs to awk, not to the shell — nothing expands inside + // '…'. Treating it as a shell variable would force the author to + // rename another language's variable to satisfy this checker. + let vars = referenced_vars( + r#"awk -F/ '{ if (NF>1) print $NF }' <<< "$COLLECTION""#, + ); + assert_eq!(vars, vec!["COLLECTION"]); + } + + #[test] + fn referenced_vars_resumes_after_a_closing_quote() { + let vars = referenced_vars(r#"sed 's/$X/y/' "$REAL""#); + assert_eq!(vars, vec!["REAL"]); + } + #[test] fn assigned_in_body_recognises_the_common_forms() { assert!(assigned_in_body("PROXY_DIR=$(mktemp -d)", "PROXY_DIR")); diff --git a/src/compile/shell/mod.rs b/src/compile/shell/mod.rs index 67ab1347..1cc31d6f 100644 --- a/src/compile/shell/mod.rs +++ b/src/compile/shell/mod.rs @@ -84,7 +84,9 @@ use indexmap::IndexMap; use super::ir::step::BashStep; +#[allow(unused_imports)] pub use bindings::{Binding, BindingKind}; +#[allow(unused_imports)] pub use registry::{Interpreter, ShellScriptDef, all_scripts}; /// Opening marker of the generated prelude. @@ -228,9 +230,20 @@ impl ShellScript { // whatever order the producer happened to call `bind` in, so a // reordered call site produces no diff. for name in self.def.bindings { + let rhs = self.bindings[name].rhs(); + if rhs.contains('$') { + // Single-quoting a `$` is the point, not a mistake. An + // `$(Agent.TempDirectory)` is substituted by Azure DevOps + // *before* bash sees the script, and the quotes then keep + // the substituted text literal so a path containing a + // space or a metacharacter cannot alter the script. + // Expanding it in the shell instead is exactly the bug + // this design prevents. + out.push_str("# shellcheck disable=SC2016\n"); + } out.push_str(name); out.push('='); - out.push_str(self.bindings[name].rhs()); + out.push_str(rhs); out.push('\n'); } out.push_str(PRELUDE_CLOSE); @@ -266,13 +279,17 @@ impl ShellScript { } /// The registration this script was built from. + #[allow(dead_code)] pub fn def(&self) -> &'static ShellScriptDef { self.def } - /// The rendered right-hand side bound to `name`, for tests that want to - /// assert on the value a producer supplied rather than on a substring of - /// the rendered script. + /// The binding a producer supplied for `name`. + /// + /// Lets a test assert on producer intent — `binding("PROXY_CONTAINER")` is + /// the value the compiler supplied — rather than on a substring of the + /// rendered script, which would also match a comment. + #[allow(dead_code)] pub fn binding(&self, name: &str) -> Option<&Binding> { self.bindings.get(name) } @@ -294,6 +311,10 @@ fn split_shebang(body: &str) -> (Option<&str>, &str) { pub(crate) const FRAGMENT_MARKER: &str = "# ado-aw:fragment "; /// The fragment named by `line`, if it is a marker line. +/// +/// `splice_fragments` inlines its own scan, so only the registry-wide lint +/// calls this today — dead in a non-test build. +#[allow(dead_code)] pub(crate) fn fragment_marker(line: &str) -> Option<&str> { line.trim_start() .strip_prefix(FRAGMENT_MARKER) @@ -302,7 +323,8 @@ pub(crate) fn fragment_marker(line: &str) -> Option<&str> { } /// Replace each fragment marker in `body` with the text `resolve` returns, -/// re-indented to the marker's own indentation. +/// re-indented to the marker's own indentation. A marker whose fragment +/// `resolve` does not supply is left in place as the comment it already is. /// /// # Panics /// @@ -311,11 +333,11 @@ pub(crate) fn fragment_marker(line: &str) -> Option<&str> { /// was meant to run simply would not. #[track_caller] fn splice_fragments<'a>( - def: &'static ShellScriptDef, + def: &ShellScriptDef, body: &str, resolve: impl Fn(&str) -> Option<&'a str>, ) -> String { - let mut seen: Vec<&str> = Vec::new(); + let mut seen: Vec<&'static str> = Vec::new(); let mut out = String::with_capacity(body.len()); for line in body.lines() { match fragment_marker(line) { @@ -336,10 +358,15 @@ fn splice_fragments<'a>( def.name ); seen.push(declared); - let indent: String = line.chars().take_while(|c| c.is_whitespace()).collect(); let Some(shell) = resolve(name) else { + // Not resolvable in this pass (a runtime-supplied fragment + // during linting). Keep the marker: it is a comment, so it + // is inert, and it shows a reader where the splice happens. + out.push_str(line); + out.push('\n'); continue; }; + let indent: String = line.chars().take_while(|c| c.is_whitespace()).collect(); for fragment_line in dedent(shell.trim_matches('\n')).lines() { if fragment_line.is_empty() { out.push('\n'); @@ -449,6 +476,7 @@ echo hello bindings: &[], externals, fragments, + phases: &[], body, file: file!(), line: line!(), @@ -595,6 +623,46 @@ docker inspect -f 'state={{.State.Status}} exit={{.State.ExitCode}}' proxy ShellScript::new(&TEST_SCRIPT).text("CONTAINR", "typo"); } + #[test] + fn an_ado_macro_binding_carries_a_shellcheck_directive() { + // SC2016 ("expressions don't expand in single quotes") is exactly the + // behaviour an ADO macro binding wants: the macro is substituted + // before bash runs, and the quotes keep the substituted text literal. + // Without the directive every prelude would fail the bash lint. + shell_script! { + MACRO_BINDING { + interpreter: Bash, + bindings: [AGENT_TEMP, PLAIN], + externals: [], + fragments: [], + body: r#" +echo "$AGENT_TEMP $PLAIN" +"#, + } + } + let rendered = ShellScript::new(&MACRO_BINDING) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .text("PLAIN", "no-dollar-here") + .render(); + assert!( + rendered.contains( + "# shellcheck disable=SC2016\nAGENT_TEMP='$(Agent.TempDirectory)'" + ), + "an ADO macro binding needs the directive: {rendered}" + ); + // A value with no `$` gets no directive — the suppression is targeted, + // not blanket. + assert!( + rendered.contains("\nPLAIN='no-dollar-here'"), + "{rendered}" + ); + assert_eq!( + rendered.matches("shellcheck disable=SC2016").count(), + 1, + "only the binding that needs it should carry the directive: {rendered}" + ); + } + #[test] fn a_shebang_stays_first_and_the_prelude_follows_it() { shell_script! { diff --git a/src/compile/shell/registry.rs b/src/compile/shell/registry.rs index b5fcfc53..f7fff857 100644 --- a/src/compile/shell/registry.rs +++ b/src/compile/shell/registry.rs @@ -62,6 +62,20 @@ pub struct ShellScriptDef { /// Named blocks of shell composed in from elsewhere. The composition /// escape hatch — see [`super::ShellScript::fragment`]. pub fragments: &'static [&'static str], + /// Fragments whose content is *another registered script*, resolved + /// statically. + /// + /// This is what lets [`lint_source`](Self::lint_source) shellcheck the + /// **composed** script rather than an outline full of markers. That + /// matters for two reasons: an outline's bindings are consumed by its + /// phases, so linting the outline alone reports every binding as unused + /// (SC2034); and splitting a script into phases introduces exactly one + /// new risk — a variable one phase sets and another reads — which only a + /// composed lint can see. + /// + /// Every entry must also appear in [`fragments`](Self::fragments); a test + /// enforces it. + pub phases: &'static [(&'static str, &'static ShellScriptDef)], /// The script itself, verbatim, exactly as it will run. pub body: &'static str, /// Source file, for the export provenance header. @@ -78,9 +92,10 @@ impl ShellScriptDef { /// reads without declaring — which is the bug worth catching — while not /// firing for every legitimately-injected value. /// - /// Fragment markers are left in place: they are ordinary comments, and the - /// fragment's own shell is linted where it is produced. Splicing an - /// unknown block in here would only produce noise. + /// Fragments declared as [`phases`](Self::phases) are spliced, so what + /// gets linted is the **composed** script that actually runs. Fragments + /// whose text is only known at runtime keep their marker comment; their + /// own shell is linted where it is produced. pub fn lint_source(&self) -> String { let (shebang, body) = super::split_shebang(self.body); let mut out = String::with_capacity(self.body.len() + 256); @@ -90,13 +105,27 @@ impl ShellScriptDef { } out.push_str("# --- ado-aw lint stubs (not emitted) ---\n"); for name in self.bindings.iter().chain(self.externals.iter()) { + if SHELL_PROVIDED.contains(name) { + // The shell itself provides these. Stubbing them is both + // unnecessary (shellcheck already treats them as set) and + // wrong: assigning PATH trips SC2123, reporting a bug the + // real script does not have. + continue; + } // A non-empty stub: an empty one makes `[ -z "$V" ]` branches // unreachable to shellcheck's flow analysis. out.push_str(name); out.push_str("='ado-aw-lint-stub'\n"); } out.push_str("# --- end lint stubs ---\n"); - out.push_str(&super::dedent(body.trim_start_matches('\n'))); + + let body = super::dedent(body.trim_start_matches('\n')); + out.push_str(&super::splice_fragments(self, &body, |name| { + self.phases + .iter() + .find(|(fragment, _)| *fragment == name) + .map(|(_, phase)| phase.body) + })); if !out.ends_with('\n') { out.push('\n'); } @@ -111,6 +140,14 @@ impl ShellScriptDef { inventory::collect!(ShellScriptDef); +/// Variables the shell or its environment always provides. +/// +/// A script may legitimately declare one of these as an `external` to +/// document that it reads it — but the lint must not stub-assign it. +/// Shellcheck already treats them as set, and assigning some of them is +/// itself a finding (`PATH=…` trips SC2123). +const SHELL_PROVIDED: &[&str] = &["PATH", "HOME", "TMPDIR", "PWD", "IFS", "SHELL", "USER", "TERM"]; + /// Every registered script, in a stable order (sorted by [`ShellScriptDef::name`]). /// /// `inventory` iteration order is link-order and therefore not stable across @@ -143,6 +180,25 @@ pub fn all_scripts() -> Vec<&'static ShellScriptDef> { /// } /// } /// ``` +/// +/// A script assembled from registered phases adds a `phases:` clause naming +/// the script behind each fragment, so the lint sees the composed result: +/// +/// ```ignore +/// shell_script! { +/// START_ADO_PROXY { +/// interpreter: Bash, +/// bindings: [PROXY_CONTAINER], +/// externals: [], +/// fragments: [resolve_org, run_container], +/// phases: [run_container = START_ADO_PROXY_RUN_CONTAINER], +/// body: r#" +/// # ado-aw:fragment resolve_org +/// # ado-aw:fragment run_container +/// "#, +/// } +/// } +/// ``` #[macro_export] macro_rules! shell_script { ( @@ -154,6 +210,29 @@ macro_rules! shell_script { fragments: [$($fragment:ident),* $(,)?], body: $body:expr $(,)? } + ) => { + $crate::shell_script! { + $(#[$meta])* + $ident { + interpreter: $interpreter, + bindings: [$($binding),*], + externals: [$($external),*], + fragments: [$($fragment),*], + phases: [], + body: $body, + } + } + }; + ( + $(#[$meta:meta])* + $ident:ident { + interpreter: $interpreter:ident, + bindings: [$($binding:ident),* $(,)?], + externals: [$($external:ident),* $(,)?], + fragments: [$($fragment:ident),* $(,)?], + phases: [$($phase:ident = $phase_def:path),* $(,)?], + body: $body:expr $(,)? + } ) => { $(#[$meta])* #[allow(dead_code)] @@ -164,6 +243,7 @@ macro_rules! shell_script { bindings: &[$(stringify!($binding)),*], externals: &[$(stringify!($external)),*], fragments: &[$(stringify!($fragment)),*], + phases: &[$((stringify!($phase), &$phase_def)),*], body: $body, file: file!(), line: line!(), @@ -176,7 +256,6 @@ macro_rules! shell_script { #[cfg(test)] mod tests { use super::*; - use crate::shell_script; shell_script! { /// Fixture for registry behaviour. diff --git a/src/engine.rs b/src/engine.rs index fc68a679..1aa3a54f 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -3,7 +3,9 @@ use std::collections::HashMap; use anyhow::Result; use crate::compile::extensions::Declarations; +use crate::compile::shell::{Binding, ShellScript}; use crate::compile::types::{CompileTarget, EngineConfig, FrontMatter, McpConfig}; +use crate::shell_script; use crate::validate::{ contains_ado_expression, contains_ado_template_expression, contains_newline, contains_pipeline_command, is_valid_arg, is_valid_command_path, is_valid_env_var_name, @@ -1105,18 +1107,15 @@ fn copilot_install_steps( // previous local implementation stripped a literal // `https://dev.azure.com/` prefix, which is a no-op for a // `*.visualstudio.com` or on-prem collection URL. - let resolve = crate::compile::resolve_ado_organization_bash() - .lines() - .map(|line| format!(" {line}\n")) - .collect::(); + let body = ShellScript::new(&RESOLVE_ADO_ORGANIZATION_STEP) + .fragment( + "resolve_org", + crate::compile::resolve_ado_organization_bash(), + ) + .render(); + let indented = indent_bash_body(&body); let step = format!( - "\ -- bash: | - set -eo pipefail -{resolve} echo \"##vso[task.setvariable variable=AW_ADO_ORG]$ADO_PROXY_ORGANIZATION\" - displayName: \"Resolve ADO organization\" - -" + "- bash: |\n{indented} displayName: \"Resolve ADO organization\"\n\n" ); (step, "$(AW_ADO_ORG)".to_string()) } @@ -1133,21 +1132,14 @@ fn copilot_install_steps( command: 'custom' arguments: 'install Microsoft.Copilot.CLI.linux-x64 -Source \"https://pkgs.dev.azure.com/{nuget_org}/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json\" {version_arg}-OutputDirectory $(Agent.TempDirectory)/tools -ExcludeVersion -NonInteractive' -- bash: | - ls -la \"$(Agent.TempDirectory)/tools\" - echo \"##vso[task.prependpath]$(Agent.TempDirectory)/tools/Microsoft.Copilot.CLI.linux-x64\" - - # Copy copilot binary to /tmp so it's accessible inside AWF container - # (AWF auto-mounts /tmp:/tmp:rw but not Agent.TempDirectory) - mkdir -p /tmp/awf-tools - cp \"$(Agent.TempDirectory)/tools/Microsoft.Copilot.CLI.linux-x64/copilot\" /tmp/awf-tools/copilot - chmod +x /tmp/awf-tools/copilot - displayName: \"Add copilot to PATH\" - -- bash: | - copilot --version - copilot -h - displayName: \"Output copilot version\"" +{add_to_path_step} + +{version_step}", + nuget_org = nuget_org, + version_arg = version_arg, + org_resolve_step = org_resolve_step, + add_to_path_step = copilot_add_to_path_onees_step(), + version_step = copilot_version_step(), )); } @@ -1172,58 +1164,188 @@ fn normalize_version_tag(version: &str) -> String { } fn copilot_install_from_github_release(base_url: &str, display_name: &str) -> Result { + let install_body = ShellScript::new(&COPILOT_INSTALL_GITHUB_RELEASE) + .text("BASE_URL", base_url) + .bind("AGENT_TEMP_DIR", Binding::ado_macro("Agent.TempDirectory")) + .render(); + let install_indented = indent_bash_body(&install_body); + + let version_body = copilot_version_step(); Ok(format!( - "\ -- bash: | - set -euo pipefail - TARBALL_NAME=\"copilot-linux-x64.tar.gz\" - BASE_URL=\"{base_url}\" - TARBALL_URL=\"$BASE_URL/$TARBALL_NAME\" - CHECKSUMS_URL=\"$BASE_URL/SHA256SUMS.txt\" - TOOLS_DIR=\"$(Agent.TempDirectory)/tools\" - TEMP_DIR=\"$(mktemp -d)\" - trap 'rm -rf \"$TEMP_DIR\"' EXIT - mkdir -p \"$TOOLS_DIR\" /tmp/awf-tools - - curl -fsSL --retry 3 --retry-delay 5 -o \"$TEMP_DIR/SHA256SUMS.txt\" \"$CHECKSUMS_URL\" - curl -fsSL --retry 3 --retry-delay 5 -o \"$TEMP_DIR/$TARBALL_NAME\" \"$TARBALL_URL\" - - EXPECTED_CHECKSUM=$(awk -v fname=\"$TARBALL_NAME\" '$2 == fname {{print $1; exit}}' \"$TEMP_DIR/SHA256SUMS.txt\" | tr 'A-F' 'a-f') - if [ -z \"$EXPECTED_CHECKSUM\" ]; then - echo \"ERROR: failed to resolve expected checksum for $TARBALL_NAME\" - exit 1 - fi - - if command -v sha256sum > /dev/null 2>&1; then - ACTUAL_CHECKSUM=$(sha256sum \"$TEMP_DIR/$TARBALL_NAME\" | awk '{{print $1}}' | tr 'A-F' 'a-f') - elif command -v shasum > /dev/null 2>&1; then - ACTUAL_CHECKSUM=$(shasum -a 256 \"$TEMP_DIR/$TARBALL_NAME\" | awk '{{print $1}}' | tr 'A-F' 'a-f') - else - echo \"ERROR: neither sha256sum nor shasum is available\" - exit 1 - fi - - if [ \"$EXPECTED_CHECKSUM\" != \"$ACTUAL_CHECKSUM\" ]; then - echo \"ERROR: checksum verification failed\" - echo \"Expected: $EXPECTED_CHECKSUM\" - echo \"Actual: $ACTUAL_CHECKSUM\" - exit 1 - fi - - tar -xz -C \"$TOOLS_DIR\" -f \"$TEMP_DIR/$TARBALL_NAME\" - ls -la \"$TOOLS_DIR\" - echo \"##vso[task.prependpath]$TOOLS_DIR\" - cp \"$TOOLS_DIR/copilot\" /tmp/awf-tools/copilot - chmod +x /tmp/awf-tools/copilot - displayName: \"{display_name}\" - -- bash: | - copilot --version - copilot -h - displayName: \"Output copilot version\"" + "- bash: |\n{install_indented} displayName: \"{display_name}\"\n\n{version_body}" )) } +/// Indent every non-empty line of a rendered bash body by four spaces so it +/// lives inside a `- bash: |` literal block scalar. +/// +/// The YAML parser strips the common leading indentation, so the shell that +/// reaches bash is byte-identical to what [`ShellScript::render`] produced — +/// the wrap is purely a YAML layout concern. +fn indent_bash_body(body: &str) -> String { + body.lines() + .map(|line| { + if line.is_empty() { + "\n".to_string() + } else { + format!(" {line}\n") + } + }) + .collect() +} + +/// Rendered YAML for the "Output copilot version" bash step. +/// +/// Shared between the 1ES NuGet path and the GitHub-release path — both +/// install copilot to the current PATH and want a one-line smoke-check that +/// the binary responds. See [`COPILOT_VERSION`]. +fn copilot_version_step() -> String { + let body = ShellScript::new(&COPILOT_VERSION).render(); + let indented = indent_bash_body(&body); + format!("- bash: |\n{indented} displayName: \"Output copilot version\"") +} + +/// Rendered YAML for the "Add copilot to PATH" bash step on the 1ES / NuGet +/// install path. Emitted directly after `NuGetCommand@2` installs the copilot +/// tool package into `$(Agent.TempDirectory)/tools`. +fn copilot_add_to_path_onees_step() -> String { + let body = ShellScript::new(&COPILOT_ADD_TO_PATH_ONEES) + .bind("AGENT_TEMP_DIR", Binding::ado_macro("Agent.TempDirectory")) + .render(); + let indented = indent_bash_body(&body); + format!("- bash: |\n{indented} displayName: \"Add copilot to PATH\"") +} + +shell_script! { + /// Bash body of the "Resolve ADO organization" step (1ES targets with no + /// compile-time org). Splices the shared derivation from + /// [`crate::compile::resolve_ado_organization_bash`] via the + /// `resolve_org` fragment marker so both this step and the ado-proxy + /// policy step can't disagree about what the org is; then re-exports + /// the resolved value as the `AW_ADO_ORG` pipeline variable that the + /// NuGet feed URL splices in via `$(AW_ADO_ORG)`. + RESOLVE_ADO_ORGANIZATION_STEP { + interpreter: Bash, + bindings: [], + externals: [ADO_PROXY_ORGANIZATION], + fragments: [resolve_org], + body: r###" +set -eo pipefail +# ado-aw:fragment resolve_org +echo "##vso[task.setvariable variable=AW_ADO_ORG]$ADO_PROXY_ORGANIZATION" +"###, + } +} + +shell_script! { + /// Bash body of the "Add copilot to PATH" step on the 1ES NuGet path. + /// + /// After `NuGetCommand@2` unpacks the copilot tool package under + /// `$(Agent.TempDirectory)/tools/Microsoft.Copilot.CLI.linux-x64/`, this + /// step (a) prepends that directory to the outer PATH so plain `copilot` + /// works in later host steps, and (b) copies the binary into + /// `/tmp/awf-tools/` so the AWF-sandboxed agent (which auto-mounts + /// `/tmp:/tmp:rw` but not `Agent.TempDirectory`) can invoke it. + COPILOT_ADD_TO_PATH_ONEES { + interpreter: Bash, + bindings: [AGENT_TEMP_DIR], + externals: [], + fragments: [], + body: r###" +ls -la "$AGENT_TEMP_DIR/tools" +echo "##vso[task.prependpath]$AGENT_TEMP_DIR/tools/Microsoft.Copilot.CLI.linux-x64" + +# Copy copilot binary to /tmp so it's accessible inside AWF container +# (AWF auto-mounts /tmp:/tmp:rw but not Agent.TempDirectory) +mkdir -p /tmp/awf-tools +cp "$AGENT_TEMP_DIR/tools/Microsoft.Copilot.CLI.linux-x64/copilot" /tmp/awf-tools/copilot +chmod +x /tmp/awf-tools/copilot +"###, + } +} + +shell_script! { + /// Bash body of the "Output copilot version" smoke step, emitted on every + /// install path. `copilot -h` doubles as an unmasked "did the binary + /// actually parse its args" probe so a broken install fails visibly next + /// to the install step rather than deep inside the AWF invocation. + COPILOT_VERSION { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [], + body: r#" +copilot --version +copilot -h +"#, + } +} + +shell_script! { + /// Bash body of the "Install Copilot CLI (…)" step on the non-1ES / + /// GitHub-release path. Downloads the release tarball, verifies its + /// SHA-256 checksum against the published `SHA256SUMS.txt`, extracts + /// into `$AGENT_TEMP_DIR/tools`, prepends that directory to PATH and + /// stages the binary under `/tmp/awf-tools/` so the AWF-sandboxed + /// agent can invoke it. + /// + /// `BASE_URL` is either `${RELEASES}/latest/download` or + /// `${RELEASES}/download/${version_tag}`; the calling code picks the + /// shape and passes the concrete URL through. + /// + /// The checksum check is the point of this step. Fetching the tarball + /// without verification against the co-signed checksum manifest would + /// let a compromised mirror substitute a malicious binary; failing + /// closed here is the boundary. + COPILOT_INSTALL_GITHUB_RELEASE { + interpreter: Bash, + bindings: [BASE_URL, AGENT_TEMP_DIR], + externals: [], + fragments: [], + body: r###" +set -euo pipefail +TARBALL_NAME="copilot-linux-x64.tar.gz" +TARBALL_URL="$BASE_URL/$TARBALL_NAME" +CHECKSUMS_URL="$BASE_URL/SHA256SUMS.txt" +TOOLS_DIR="$AGENT_TEMP_DIR/tools" +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "$TEMP_DIR"' EXIT +mkdir -p "$TOOLS_DIR" /tmp/awf-tools + +curl -fsSL --retry 3 --retry-delay 5 -o "$TEMP_DIR/SHA256SUMS.txt" "$CHECKSUMS_URL" +curl -fsSL --retry 3 --retry-delay 5 -o "$TEMP_DIR/$TARBALL_NAME" "$TARBALL_URL" + +EXPECTED_CHECKSUM=$(awk -v fname="$TARBALL_NAME" '$2 == fname {print $1; exit}' "$TEMP_DIR/SHA256SUMS.txt" | tr 'A-F' 'a-f') +if [ -z "$EXPECTED_CHECKSUM" ]; then + echo "ERROR: failed to resolve expected checksum for $TARBALL_NAME" + exit 1 +fi + +if command -v sha256sum > /dev/null 2>&1; then + ACTUAL_CHECKSUM=$(sha256sum "$TEMP_DIR/$TARBALL_NAME" | awk '{print $1}' | tr 'A-F' 'a-f') +elif command -v shasum > /dev/null 2>&1; then + ACTUAL_CHECKSUM=$(shasum -a 256 "$TEMP_DIR/$TARBALL_NAME" | awk '{print $1}' | tr 'A-F' 'a-f') +else + echo "ERROR: neither sha256sum nor shasum is available" + exit 1 +fi + +if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]; then + echo "ERROR: checksum verification failed" + echo "Expected: $EXPECTED_CHECKSUM" + echo "Actual: $ACTUAL_CHECKSUM" + exit 1 +fi + +tar -xz -C "$TOOLS_DIR" -f "$TEMP_DIR/$TARBALL_NAME" +ls -la "$TOOLS_DIR" +echo "##vso[task.prependpath]$TOOLS_DIR" +cp "$TOOLS_DIR/copilot" /tmp/awf-tools/copilot +chmod +x /tmp/awf-tools/copilot +"###, + } +} + /// Build the full AWF `--` command string for the Copilot CLI. /// /// The returned string goes inside `-- '...'` in the pipeline YAML. diff --git a/src/runtimes/dotnet/extension.rs b/src/runtimes/dotnet/extension.rs index 7bd08f76..f36fe322 100644 --- a/src/runtimes/dotnet/extension.rs +++ b/src/runtimes/dotnet/extension.rs @@ -5,9 +5,47 @@ use crate::compile::extensions::{CompileContext, CompilerExtension, Declarations use crate::compile::ir::step::{BashStep, Step, TaskStep}; use crate::compile::ir::tasks::nuget_authenticate::NuGetAuthenticate; use crate::compile::ir::tasks::use_dotnet::UseDotNet; +use crate::compile::shell::ShellScript; +use crate::shell_script; use crate::validate; use anyhow::Result; +shell_script! { + /// Ensure a workspace-level `nuget.config` exists before + /// `NuGetAuthenticate@1` runs. + /// + /// The existence check covers the three case variations NuGet itself + /// recognises on case-sensitive filesystems (`nuget.config`, + /// `NuGet.config`, `NuGet.Config`); the file is always created with + /// the lowercase form, matching the cross-platform convention. The + /// heredoc is intentionally unquoted so `$FEED_URL` from the + /// generated prelude is substituted; the emitted XML content has no + /// other shell metacharacters. + ENSURE_NUGET_CONFIG { + interpreter: Bash, + bindings: [FEED_URL], + externals: [], + fragments: [], + body: r#" +set -eo pipefail +if [ ! -f nuget.config ] && [ ! -f NuGet.config ] && [ ! -f NuGet.Config ]; then + cat > nuget.config < + + + + + + +EOF + echo "Created nuget.config with source=$FEED_URL" +else + echo 'nuget.config already exists, skipping creation' +fi +"#, + } +} + /// .NET runtime extension. /// /// Injects: ecosystem network hosts (dotnet), bash commands (dotnet), @@ -178,24 +216,9 @@ fn ensure_nuget_config_bash_step(config: &DotnetRuntimeConfig) -> BashStep { let feed_url = config .feed_url() .unwrap_or("https://api.nuget.org/v3/index.json"); - let script = format!( - "set -eo pipefail\n\ - if [ ! -f nuget.config ] && [ ! -f NuGet.config ] && [ ! -f NuGet.Config ]; then\n \ - cat > nuget.config <<'EOF'\n\ - \n\ - \n \ - \n \ - \n \ - \n \ - \n\ - \n\ - EOF\n \ - echo 'Created nuget.config with source={feed_url}'\n\ - else\n \ - echo 'nuget.config already exists, skipping creation'\n\ - fi\n" - ); - BashStep::new("Ensure nuget.config exists", script) + ShellScript::new(&ENSURE_NUGET_CONFIG) + .text("FEED_URL", feed_url) + .into_step("Ensure nuget.config exists") } #[cfg(test)] diff --git a/src/runtimes/dotnet/mod.rs b/src/runtimes/dotnet/mod.rs index fbb96378..fe2af687 100644 --- a/src/runtimes/dotnet/mod.rs +++ b/src/runtimes/dotnet/mod.rs @@ -147,195 +147,3 @@ pub struct DotnetOptions { /// Bash commands that the .NET runtime adds to the allow-list. pub const DOTNET_BASH_COMMANDS: &[&str] = &["dotnet"]; - -/// Generate the `UseDotNet@2` pipeline step. -/// -/// Emits one of three shapes: -/// - `version: "global.json"` → `useGlobalJson: true` (discovers SDK -/// versions from `global.json` files in the workspace). -/// - explicit `version: "8.0.x"` → `version: '8.0.x'`. -/// - no version → `version: '8.0.x'` (compiler default). -pub fn generate_dotnet_install(config: &DotnetRuntimeConfig) -> String { - if config.use_global_json() { - return "\ -- task: UseDotNet@2 - inputs: - packageType: 'sdk' - useGlobalJson: true - displayName: 'Install .NET SDK (from global.json)'" - .to_string(); - } - - let version = config.version().unwrap_or("8.0.x"); - format!( - "\ -- task: UseDotNet@2 - inputs: - packageType: 'sdk' - version: '{version}' - displayName: 'Install .NET SDK {version}'" - ) -} - -/// Generate the `NuGetAuthenticate@1` pipeline step. -/// -/// Emitted when `feed-url:` or `config:` is set, authenticating the ADO -/// build service identity against any Azure Artifacts feeds referenced by -/// `nuget.config` files in the workspace. `NuGetAuthenticate@1` auto- -/// discovers `nuget.config` files — no `workingFile:` input is required, -/// unlike `npmAuthenticate@0`. -pub fn generate_nuget_authenticate() -> String { - "\ -- task: NuGetAuthenticate@1 - displayName: 'Authenticate NuGet (build service identity)'" - .to_string() -} - -/// Generate a step that ensures a `nuget.config` exists before -/// `NuGetAuthenticate@1`. -/// -/// `NuGetAuthenticate@1` is a no-op without a `nuget.config` to authenticate -/// against. This step writes a minimal `nuget.config` (with the configured -/// feed URL added as a package source) only when one doesn't already exist -/// at the repo root, preserving any repo-checked-in `nuget.config`. -/// -/// The existence check covers the three case variations NuGet itself -/// recognises on case-sensitive filesystems (`nuget.config`, `NuGet.config`, -/// `NuGet.Config`); the file is always created with the lowercase form, -/// matching the cross-platform convention. -pub fn generate_ensure_nuget_config(config: &DotnetRuntimeConfig) -> String { - let feed_url = config - .feed_url() - .unwrap_or("https://api.nuget.org/v3/index.json"); - - format!( - r#"- bash: | - set -eo pipefail - if [ ! -f nuget.config ] && [ ! -f NuGet.config ] && [ ! -f NuGet.Config ]; then - cat > nuget.config <<'EOF' - - - - - - - - EOF - echo 'Created nuget.config with source={feed_url}' - else - echo 'nuget.config already exists, skipping creation' - fi - displayName: 'Ensure nuget.config exists'"# - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - // ── generate_dotnet_install ──────────────────────────────────── - - #[test] - fn test_generate_dotnet_install_default() { - let config = DotnetRuntimeConfig::Enabled(true); - let step = generate_dotnet_install(&config); - assert!(step.contains("UseDotNet@2"), "should use UseDotNet@2 task"); - assert!( - step.contains("packageType: 'sdk'"), - "should pin packageType to 'sdk'" - ); - assert!( - step.contains("version: '8.0.x'"), - "default version should be 8.0.x: {step}" - ); - assert!( - !step.contains("useGlobalJson"), - "should not emit useGlobalJson for default" - ); - } - - #[test] - fn test_generate_dotnet_install_explicit_version() { - let config = DotnetRuntimeConfig::WithOptions(DotnetOptions { - version: Some("9.0.x".to_string()), - ..Default::default() - }); - let step = generate_dotnet_install(&config); - assert!( - step.contains("version: '9.0.x'"), - "should use specified version: {step}" - ); - assert!( - !step.contains("useGlobalJson"), - "should not emit useGlobalJson with explicit version" - ); - } - - #[test] - fn test_generate_dotnet_install_global_json() { - let config = DotnetRuntimeConfig::WithOptions(DotnetOptions { - version: Some("global.json".to_string()), - ..Default::default() - }); - let step = generate_dotnet_install(&config); - assert!( - step.contains("useGlobalJson: true"), - "should emit useGlobalJson: true: {step}" - ); - assert!( - !step.contains("version: '"), - "should not emit explicit version with useGlobalJson: {step}" - ); - } - - #[test] - fn test_generate_dotnet_install_global_json_case_insensitive() { - let config = DotnetRuntimeConfig::WithOptions(DotnetOptions { - version: Some("Global.JSON".to_string()), - ..Default::default() - }); - let step = generate_dotnet_install(&config); - assert!( - step.contains("useGlobalJson: true"), - "sentinel should be case-insensitive: {step}" - ); - } - - // ── generate_ensure_nuget_config ────────────────────────────── - - #[test] - fn test_generate_ensure_nuget_config_contains_feed_url() { - let feed = "https://pkgs.dev.azure.com/myorg/_packaging/myfeed/nuget/v3/index.json"; - let config = DotnetRuntimeConfig::WithOptions(DotnetOptions { - feed_url: Some(feed.to_string()), - ..Default::default() - }); - let step = generate_ensure_nuget_config(&config); - assert!( - step.contains(feed), - "step should interpolate the configured feed URL: {step}" - ); - assert!( - step.contains(""), - "should emit valid nuget.config XML" - ); - assert!( - step.contains("nuget.config"), - "step should reference nuget.config" - ); - assert!( - step.contains("displayName: 'Ensure nuget.config exists'"), - "step should carry the expected displayName" - ); - } - - #[test] - fn test_generate_ensure_nuget_config_default_feed() { - let config = DotnetRuntimeConfig::Enabled(true); - let step = generate_ensure_nuget_config(&config); - assert!( - step.contains("https://api.nuget.org/v3/index.json"), - "default feed should be the public nuget.org v3 index: {step}" - ); - } -} diff --git a/src/runtimes/lean/extension.rs b/src/runtimes/lean/extension.rs index d0a50551..8d54fb3c 100644 --- a/src/runtimes/lean/extension.rs +++ b/src/runtimes/lean/extension.rs @@ -5,8 +5,33 @@ use crate::compile::extensions::{ AwfMount, AwfMountMode, CompileContext, CompilerExtension, Declarations, ExtensionPhase, }; use crate::compile::ir::step::{BashStep, Step}; +use crate::compile::shell::ShellScript; +use crate::shell_script; use anyhow::Result; +shell_script! { + /// Install Lean 4 via elan into `$HOME/.elan`, and register elan's + /// `bin` directory on PATH for subsequent steps. + /// + /// The AWF chroot only sees `$HOME/.elan` because the runtime's + /// `required_awf_mounts()` mounts it read-only; the `PATH` prepend + /// makes `lean` and `lake` resolvable inside the sandbox. + INSTALL_LEAN { + interpreter: Bash, + bindings: [TOOLCHAIN], + externals: [HOME], + fragments: [], + body: r###" +set -eo pipefail +curl https://elan.lean-lang.org/elan-init.sh -sSf | sh -s -- -y --default-toolchain "$TOOLCHAIN" +echo "##vso[task.prependpath]$HOME/.elan/bin" +export PATH="$HOME/.elan/bin:$PATH" +lean --version || echo "Lean installed via elan" +lake --version || echo "Lake installed via elan" +"###, + } +} + /// Lean 4 runtime extension. /// /// Injects: network hosts (elan, lean-lang), bash commands (lean, lake, @@ -85,15 +110,9 @@ the toolchain. Lean files use the `.lean` extension.\n" /// lowers through `ir::emit` to the canonical pipeline YAML. fn lean_install_bash_step(config: &LeanRuntimeConfig) -> BashStep { let toolchain = config.toolchain().unwrap_or("stable"); - let script = format!( - "set -eo pipefail\n\ - curl https://elan.lean-lang.org/elan-init.sh -sSf | sh -s -- -y --default-toolchain {toolchain}\n\ - echo \"##vso[task.prependpath]$HOME/.elan/bin\"\n\ - export PATH=\"$HOME/.elan/bin:$PATH\"\n\ - lean --version || echo \"Lean installed via elan\"\n\ - lake --version || echo \"Lake installed via elan\"\n" - ); - BashStep::new("Install Lean 4 (elan)", script) + ShellScript::new(&INSTALL_LEAN) + .text("TOOLCHAIN", toolchain) + .into_step("Install Lean 4 (elan)") } #[cfg(test)] @@ -127,7 +146,8 @@ mod tests { Step::Bash(b) => { assert_eq!(b.display_name, "Install Lean 4 (elan)"); assert!(b.script.contains("elan-init.sh")); - assert!(b.script.contains("--default-toolchain stable")); + assert!(b.script.contains("TOOLCHAIN='stable'")); + assert!(b.script.contains(r#"--default-toolchain "$TOOLCHAIN""#)); } other => panic!("expected Step::Bash, got {other:?}"), } @@ -155,8 +175,8 @@ mod tests { match &decl.agent_prepare_steps[0] { Step::Bash(b) => assert!( b.script - .contains("--default-toolchain leanprover/lean4:v4.29.1"), - "expected pinned toolchain in script: {}", + .contains("TOOLCHAIN='leanprover/lean4:v4.29.1'"), + "expected pinned toolchain in binding prelude: {}", b.script ), other => panic!("expected Step::Bash, got {other:?}"), diff --git a/src/runtimes/lean/mod.rs b/src/runtimes/lean/mod.rs index 37728446..360caf48 100644 --- a/src/runtimes/lean/mod.rs +++ b/src/runtimes/lean/mod.rs @@ -78,48 +78,3 @@ pub struct LeanOptions { /// Bash commands that the Lean runtime adds to the allow-list. pub const LEAN_BASH_COMMANDS: &[&str] = &["lean", "lake", "elan"]; - -/// Generate the elan installation step for Lean 4. -/// -/// Installs elan (Lean toolchain manager) and the specified toolchain. -/// Defaults to "stable" if no toolchain is specified in the front matter. -/// -/// AWF chroot access is provided by the `--mount` flag declared via -/// `LeanExtension::required_awf_mounts()`, which mounts `$HOME/.elan` -/// read-only into the container. -pub fn generate_lean_install(config: &LeanRuntimeConfig) -> String { - let toolchain = config.toolchain().unwrap_or("stable"); - let script = format!( - "\ -set -eo pipefail -curl https://elan.lean-lang.org/elan-init.sh -sSf | sh -s -- -y --default-toolchain {toolchain} -echo \"##vso[task.prependpath]$HOME/.elan/bin\" -export PATH=\"$HOME/.elan/bin:$PATH\" -lean --version || echo \"Lean installed via elan\" -lake --version || echo \"Lake installed via elan\"" - ); - // Indent each line of the script body by 4 spaces for YAML block scalar - let indented: String = script - .lines() - .map(|line| format!(" {line}")) - .collect::>() - .join("\n"); - format!("- bash: |\n{indented}\n displayName: \"Install Lean 4 (elan)\"") -} - -/// Generate the prompt append step to inform the agent that Lean 4 is available. -pub fn generate_lean_prompt() -> String { - r#"- bash: | - cat >> "/tmp/awf-tools/agent-prompt.md" << 'LEAN_PROMPT_EOF' - - --- - - ## Lean 4 Formal Verification - - Lean 4 is installed and available. Use `lean` to typecheck `.lean` files, `lake build` to build Lake projects, and `lake env printPaths` to inspect the toolchain. Lean files use the `.lean` extension. - LEAN_PROMPT_EOF - - echo "Lean prompt appended" - displayName: "Append Lean 4 prompt""# - .to_string() -} diff --git a/src/runtimes/node/extension.rs b/src/runtimes/node/extension.rs index 98eeb6a3..6190e4f0 100644 --- a/src/runtimes/node/extension.rs +++ b/src/runtimes/node/extension.rs @@ -5,9 +5,37 @@ use crate::compile::extensions::{CompileContext, CompilerExtension, Declarations use crate::compile::ir::step::{BashStep, Step, TaskStep}; use crate::compile::ir::tasks::npm_authenticate::NpmAuthenticate; use crate::compile::ir::tasks::use_node::UseNode; +use crate::compile::shell::ShellScript; +use crate::shell_script; use crate::validate; use anyhow::Result; +shell_script! { + /// Ensure a workspace-level `.npmrc` exists before `npmAuthenticate@0` + /// runs. + /// + /// `npmAuthenticate@0` requires its `workingFile:` to point at an + /// existing file, so a repo without a checked-in `.npmrc` would fail + /// the auth step. This script leaves any existing `.npmrc` untouched + /// and creates a minimal one otherwise, pointing at the configured + /// registry (or public npmjs when nothing is configured). + ENSURE_NPMRC { + interpreter: Bash, + bindings: [REGISTRY], + externals: [], + fragments: [], + body: r#" +set -eo pipefail +if [ ! -f .npmrc ]; then + echo "registry=$REGISTRY" > .npmrc + echo "Created .npmrc with registry=$REGISTRY" +else + echo '.npmrc already exists, skipping creation' +fi +"#, + } +} + /// Node.js runtime extension. /// /// Injects: ecosystem network hosts (node), bash commands (node, npm, npx), @@ -143,16 +171,9 @@ fn npm_authenticate_task_step() -> TaskStep { /// configured feed (or the default npmjs registry). fn ensure_npmrc_bash_step(config: &NodeRuntimeConfig) -> BashStep { let registry = config.feed_url().unwrap_or("https://registry.npmjs.org/"); - let script = format!( - "set -eo pipefail\n\ - if [ ! -f .npmrc ]; then\n \ - echo 'registry={registry}' > .npmrc\n \ - echo 'Created .npmrc with registry={registry}'\n\ - else\n \ - echo '.npmrc already exists, skipping creation'\n\ - fi\n" - ); - BashStep::new("Ensure .npmrc exists", script) + ShellScript::new(&ENSURE_NPMRC) + .text("REGISTRY", registry) + .into_step("Ensure .npmrc exists") } #[cfg(test)] diff --git a/src/runtimes/node/mod.rs b/src/runtimes/node/mod.rs index 55e5740e..0a03ae24 100644 --- a/src/runtimes/node/mod.rs +++ b/src/runtimes/node/mod.rs @@ -113,121 +113,3 @@ pub struct NodeOptions { /// Bash commands that the Node.js runtime adds to the allow-list. pub const NODE_BASH_COMMANDS: &[&str] = &["node", "npm", "npx"]; - -/// Generate the `UseNode@1` pipeline step (inline, decoupled from ado-script). -pub fn generate_node_install(config: &NodeRuntimeConfig) -> String { - let version = config.version().unwrap_or("22.x"); - format!( - "\ -- task: UseNode@1 - inputs: - version: '{version}' - displayName: 'Install Node.js {version}'" - ) -} - -/// Generate the `npmAuthenticate@0` pipeline step. -/// -/// Emitted when `feed-url:` or `config:` is set, authenticating the ADO -/// build service identity for internal npm feeds. This runs before AWF. -/// -/// Requires a `.npmrc` file to exist; call [`generate_ensure_npmrc`] first -/// to create one if the repo doesn't already have one. -pub fn generate_npm_authenticate() -> String { - "\ -- task: npmAuthenticate@0 - inputs: - workingFile: .npmrc - displayName: 'Authenticate npm (build service identity)'" - .to_string() -} - -/// Generate a step that ensures `.npmrc` exists before `npmAuthenticate@0`. -/// -/// `npmAuthenticate@0` requires `workingFile:` to point at an existing file — -/// unlike `PipAuthenticate@1` it fails if the file is missing. This step -/// creates a minimal `.npmrc` (with the configured registry or the default -/// npmjs registry) only when one doesn't already exist, preserving any -/// repo-checked-in `.npmrc`. -pub fn generate_ensure_npmrc(config: &NodeRuntimeConfig) -> String { - let registry = config.feed_url().unwrap_or("https://registry.npmjs.org/"); - - format!( - r#"- bash: | - set -eo pipefail - if [ ! -f .npmrc ]; then - echo 'registry={registry}' > .npmrc - echo 'Created .npmrc with registry={registry}' - else - echo '.npmrc already exists, skipping creation' - fi - displayName: 'Ensure .npmrc exists'"# - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_generate_node_install_default_version() { - let config = NodeRuntimeConfig::Enabled(true); - let step = generate_node_install(&config); - assert!( - step.contains("version: '22.x'"), - "should default to 22.x, got: {step}" - ); - assert!(step.contains("UseNode@1")); - assert!(step.contains("Install Node.js 22.x")); - } - - #[test] - fn test_generate_node_install_pinned_version() { - let config = NodeRuntimeConfig::WithOptions(NodeOptions { - version: Some("20.x".into()), - ..Default::default() - }); - let step = generate_node_install(&config); - assert!( - step.contains("version: '20.x'"), - "should use pinned version, got: {step}" - ); - assert!(step.contains("Install Node.js 20.x")); - } - - #[test] - fn test_generate_npm_authenticate_emits_task() { - let step = generate_npm_authenticate(); - assert!(step.contains("npmAuthenticate@0")); - assert!(step.contains("workingFile: .npmrc")); - } - - #[test] - fn test_generate_ensure_npmrc_default_registry() { - let config = NodeRuntimeConfig::Enabled(true); - let step = generate_ensure_npmrc(&config); - assert!( - step.contains("https://registry.npmjs.org/"), - "should fallback to npm registry, got: {step}" - ); - assert!(step.contains("Ensure .npmrc exists")); - } - - #[test] - fn test_generate_ensure_npmrc_custom_feed_url() { - let custom = "https://pkgs.dev.azure.com/myorg/_packaging/myfeed/npm/registry/"; - let config = NodeRuntimeConfig::WithOptions(NodeOptions { - feed_url: Some(custom.into()), - ..Default::default() - }); - let step = generate_ensure_npmrc(&config); - assert!( - step.contains("pkgs.dev.azure.com"), - "should use custom feed URL, got: {step}" - ); - assert!( - !step.contains("https://registry.npmjs.org/"), - "should not fall back to default when custom feed is set, got: {step}" - ); - } -} diff --git a/src/runtimes/python/mod.rs b/src/runtimes/python/mod.rs index af2ba632..e74e8de2 100644 --- a/src/runtimes/python/mod.rs +++ b/src/runtimes/python/mod.rs @@ -110,79 +110,3 @@ pub struct PythonOptions { /// Bash commands that the Python runtime adds to the allow-list. pub const PYTHON_BASH_COMMANDS: &[&str] = &["python", "python3", "pip", "pip3", "uv"]; - -/// Generate the `UsePythonVersion@0` pipeline step. -pub fn generate_python_install(config: &PythonRuntimeConfig) -> String { - let version = config.version().unwrap_or("3.x"); - format!( - "\ -- task: UsePythonVersion@0 - inputs: - versionSpec: '{version}' - displayName: 'Install Python {version}'" - ) -} - -/// Generate the `PipAuthenticate@1` pipeline step. -/// -/// Emitted when `feed-url:` is set, authenticating the ADO build service -/// identity for internal package feeds. This runs before AWF, setting up -/// credentials via `##vso[task.setvariable]`. -pub fn generate_pip_authenticate() -> String { - "\ -- task: PipAuthenticate@1 - inputs: - artifactFeeds: '' - displayName: 'Authenticate pip (build service identity)'" - .to_string() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_generate_python_install_default_version() { - let config = PythonRuntimeConfig::Enabled(true); - let step = generate_python_install(&config); - assert!( - step.contains("versionSpec: '3.x'"), - "should default to 3.x, got: {step}" - ); - assert!( - step.contains("UsePythonVersion@0"), - "should use UsePythonVersion task" - ); - assert!( - step.contains("Install Python 3.x"), - "should set displayName" - ); - } - - #[test] - fn test_generate_python_install_pinned_version() { - let config = PythonRuntimeConfig::WithOptions(PythonOptions { - version: Some("3.12".into()), - ..Default::default() - }); - let step = generate_python_install(&config); - assert!( - step.contains("versionSpec: '3.12'"), - "should use pinned version, got: {step}" - ); - assert!(step.contains("Install Python 3.12")); - } - - #[test] - fn test_generate_pip_authenticate_emits_task() { - let step = generate_pip_authenticate(); - assert!( - step.contains("PipAuthenticate@1"), - "should emit PipAuthenticate task" - ); - assert!( - step.contains("artifactFeeds"), - "should include artifactFeeds input" - ); - } -} diff --git a/src/tools/cache_memory/extension.rs b/src/tools/cache_memory/extension.rs index 93296d19..06e0bb5b 100644 --- a/src/tools/cache_memory/extension.rs +++ b/src/tools/cache_memory/extension.rs @@ -4,9 +4,59 @@ use crate::compile::ir::tasks::download_pipeline_artifact::{ ArtifactSource, DownloadPipelineArtifact, RunVersion, }; use crate::compile::ir::step::{BashStep, Step, TaskStep}; +use crate::compile::shell::{Binding, ShellScript}; use crate::compile::types::CacheMemoryToolConfig; +use crate::shell_script; use anyhow::Result; +shell_script! { + /// Restore the agent memory directory from the previous run's + /// `safe_outputs` artifact, if one was downloaded. + /// + /// The artifact download step ([`DownloadPipelineArtifact@2`]) writes + /// into `$AGENT_TEMP/previous_memory/`; when it succeeded, this + /// script copies the `agent_memory` subfolder into the staging area + /// the agent reads. `cp -a` preserves modes; the `|| true` swallows + /// spurious "no such file" tails when nested optional trees are + /// missing, matching the previous behaviour. + RESTORE_AGENT_MEMORY { + interpreter: Bash, + bindings: [MEMORY_DIR, AGENT_TEMP], + externals: [], + fragments: [], + body: r#" +mkdir -p "$MEMORY_DIR" +if [ -d "$AGENT_TEMP/previous_memory/agent_memory" ]; then + cp -a "$AGENT_TEMP/previous_memory/agent_memory/." "$MEMORY_DIR/" 2>/dev/null || true + echo "Previous agent memory restored to $MEMORY_DIR" + ls -laR "$MEMORY_DIR" +else + echo "No previous agent memory found - empty memory directory created" +fi +"#, + } +} + +shell_script! { + /// Initialise an empty agent memory directory when the operator + /// forces a fresh run via `clearMemory=true`. + INIT_AGENT_MEMORY { + interpreter: Bash, + bindings: [MEMORY_DIR], + externals: [], + fragments: [], + body: r#" +mkdir -p "$MEMORY_DIR" +echo "Memory cleared by pipeline parameter - starting fresh" +"#, + } +} + +/// Absolute path to the agent's staging memory directory. Fixed by the +/// AWF-mount layout and mirrored in the prompt supplement below, so a +/// change here requires a matching change to the prompt text. +const STAGING_MEMORY_DIR: &str = "/tmp/awf-tools/staging/agent_memory"; + /// Cache memory tool extension. /// /// Injects: prepare steps (download/restore previous memory), and a @@ -100,17 +150,13 @@ fn download_previous_memory_task_step() -> TaskStep { /// previous_memory artifact into the staging directory. Runs only /// when `clearMemory=false`. fn restore_previous_memory_bash_step() -> BashStep { - let script = "mkdir -p /tmp/awf-tools/staging/agent_memory\n\ - if [ -d \"$(Agent.TempDirectory)/previous_memory/agent_memory\" ]; then\n \ - cp -a \"$(Agent.TempDirectory)/previous_memory/agent_memory/.\" /tmp/awf-tools/staging/agent_memory/ 2>/dev/null || true\n \ - echo \"Previous agent memory restored to /tmp/awf-tools/staging/agent_memory\"\n \ - ls -laR /tmp/awf-tools/staging/agent_memory\n\ - else\n \ - echo \"No previous agent memory found - empty memory directory created\"\n\ - fi\n"; - let mut b = BashStep::new("Restore previous agent memory", script).with_condition( - Condition::Custom("eq(${{ parameters.clearMemory }}, false)".to_string()), - ); + let mut b = ShellScript::new(&RESTORE_AGENT_MEMORY) + .text("MEMORY_DIR", STAGING_MEMORY_DIR) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .into_step("Restore previous agent memory") + .with_condition(Condition::Custom( + "eq(${{ parameters.clearMemory }}, false)".to_string(), + )); b.continue_on_error = true; b } @@ -118,11 +164,12 @@ fn restore_previous_memory_bash_step() -> BashStep { /// Typed bash step that initialises an empty agent_memory directory /// when the operator forces a fresh run via `clearMemory=true`. fn initialize_empty_memory_bash_step() -> BashStep { - let script = "mkdir -p /tmp/awf-tools/staging/agent_memory\n\ - echo \"Memory cleared by pipeline parameter - starting fresh\"\n"; - BashStep::new("Initialize empty agent memory (clearMemory=true)", script).with_condition( - Condition::Custom("eq(${{ parameters.clearMemory }}, true)".to_string()), - ) + ShellScript::new(&INIT_AGENT_MEMORY) + .text("MEMORY_DIR", STAGING_MEMORY_DIR) + .into_step("Initialize empty agent memory (clearMemory=true)") + .with_condition(Condition::Custom( + "eq(${{ parameters.clearMemory }}, true)".to_string(), + )) } #[cfg(test)] diff --git a/tests/compiler_tests.rs b/tests/compiler_tests.rs index 3784da1a..cf20b5e1 100644 --- a/tests/compiler_tests.rs +++ b/tests/compiler_tests.rs @@ -2114,14 +2114,17 @@ fn test_fixture_azure_devops_mcp_compiled_output() { let compiled = fs::read_to_string(&output_path).expect("Should read compiled output"); - let policy_marker = "cat > \"$PROXY_DIR/policy/policy.json\" <<'ADO_PROXY_POLICY_EOF'\n"; + // The policy document is now carried by the `POLICY` binding, which + // `Binding::document` renders as a quoted heredoc in the generated + // prelude. Nothing expands inside it, so the JSON survives verbatim. + let policy_marker = "POLICY=$(cat <<'ADO_AW_SHELL_DOC_EOF'\n"; let policy_start = compiled .find(policy_marker) .map(|index| index + policy_marker.len()) .expect("compiled pipeline must carry an ado-proxy policy document"); let policy_tail = &compiled[policy_start..]; let policy_end = policy_tail - .find("\n ADO_PROXY_POLICY_EOF") + .find("\n ADO_AW_SHELL_DOC_EOF") .expect("compiled policy heredoc must terminate"); let policy_json = policy_tail[..policy_end] .lines() @@ -2131,6 +2134,12 @@ fn test_fixture_azure_devops_mcp_compiled_output() { let policy: serde_json::Value = serde_json::from_str(&policy_json).expect("compiled policy must be valid JSON"); + // The document is written to the path the container mounts read-only. + assert!( + compiled.contains(r#"printf '%s\n' "$POLICY" > "$PROXY_DIR/policy/policy.json""#), + "the policy binding must be written to the mounted policy path" + ); + // No unreplaced template markers (except ADO ${{ }} expressions) for line in compiled.lines() { let stripped = line.replace("${{", ""); @@ -2227,7 +2236,8 @@ fn test_fixture_azure_devops_mcp_compiled_output() { "only explicitly selected capabilities plus discovery may be emitted" ); assert!( - compiled.contains("case \" devops repos rest \" in"), + compiled.contains("ALLOWED_GROUPS='devops repos rest'") + && compiled.contains(r#"case " $ALLOWED_GROUPS " in"#), "the az wrapper must narrow to the same capability set" ); assert!( @@ -4266,8 +4276,16 @@ fn assert_aw_info_step_present( compiled.contains("condition: always()"), "{fixture_name}: compiled YAML missing always() condition on aw_info step" ); + // `Agent.TempDirectory` now reaches the script as a binding rather than + // being interpolated inline, so assert on both the binding and its use. + // That is stronger: it proves the producer supplied the macro, where a + // bare substring would also match a comment. + assert!( + compiled.contains("AGENT_TEMP='$(Agent.TempDirectory)'"), + "{fixture_name}: compiled YAML missing the Agent.TempDirectory binding" + ); assert!( - compiled.contains("cat >\"$(Agent.TempDirectory)/staging/aw_info.json\" <<'AW_INFO_EOF'"), + compiled.contains("cat >\"$AGENT_TEMP/staging/aw_info.json\" <<'AW_INFO_EOF'"), "{fixture_name}: compiled YAML missing quoted heredoc aw_info write step" ); // Softer suffix check on the source path: fixtures compile under @@ -5195,8 +5213,9 @@ fn test_pr_filter_tier1_has_evaluator_gate() { "Should include base64-encoded spec" ); assert!( - compiled.contains("node '/tmp/ado-aw-scripts/ado-script/gate.js'"), - "Should invoke node gate evaluator" + compiled.contains("EVALUATOR_PATH='/tmp/ado-aw-scripts/ado-script/gate.js'") + && compiled.contains(r#"node "$EVALUATOR_PATH""#), + "Should invoke node gate evaluator via its bound path" ); assert!( compiled.contains("ado-script.zip"), @@ -5398,8 +5417,9 @@ fn test_pr_filter_tier2_has_extension_gate() { "Tier 2 should include base64-encoded spec" ); assert!( - compiled.contains("node '/tmp/ado-aw-scripts/ado-script/gate.js'"), - "Tier 2 should invoke node gate evaluator" + compiled.contains("EVALUATOR_PATH='/tmp/ado-aw-scripts/ado-script/gate.js'") + && compiled.contains(r#"node "$EVALUATOR_PATH""#), + "Tier 2 should invoke node gate evaluator via its bound path" ); assert!(compiled.contains("name: prGate"), "Should have prGate step"); } @@ -6492,8 +6512,9 @@ fn test_execution_context_pr_emits_prepare_step_and_prompt_supplement() { // `ExtensionPhase::System` and thus appears before this step // (which runs in `ExtensionPhase::Tool`). assert!( - compiled.contains("node '/tmp/ado-aw-scripts/ado-script/exec-context-pr.js'"), - "v7: prepare step must invoke the exec-context-pr.js bundle" + compiled.contains("BUNDLE='/tmp/ado-aw-scripts/ado-script/exec-context-pr.js'") + && compiled.contains(r#"node "$BUNDLE""#), + "v7: prepare step must invoke the exec-context-pr.js bundle via its bound path" ); // v7: all the bash-side specifics (GIT_CONFIG_*, regex validation, From a518a6579c2922d30f98ad74aabde45438c592ee Mon Sep 17 00:00:00 2001 From: James Devine Date: Mon, 10 Aug 2026 13:22:34 +0100 Subject: [PATCH 3/7] fix(compile): keep the shebang on line 1 of an exported shell script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provenance header was written above the shebang, so every exported `sh` script was unrunnable as a file. Shellcheck reports it as SC1128 when the exported file is checked directly, which is precisely what the export exists to enable. Found by shellchecking the exported files independently of the in-process harness — worth doing, because a harness that generates and then checks its own input can agree with itself while both are wrong. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: aa46b482-6121-45e2-8278-1a9465b4f73e --- src/compile/shell/export.rs | 58 +++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/src/compile/shell/export.rs b/src/compile/shell/export.rs index ab523ded..35f78a36 100644 --- a/src/compile/shell/export.rs +++ b/src/compile/shell/export.rs @@ -78,8 +78,7 @@ pub fn export(out_dir: &Path, format: ExportFormat) -> Result { ExportFormat::Files => { for (def, script) in scripts.iter().zip(&exported) { let path = out_dir.join(def.export_file_name()); - let contents = format!("{}{}", provenance_header(script), script.source); - std::fs::write(&path, contents) + std::fs::write(&path, with_provenance(script)) .with_context(|| format!("writing {}", path.display()))?; } } @@ -88,6 +87,22 @@ pub fn export(out_dir: &Path, format: ExportFormat) -> Result { Ok(exported.len()) } +/// Prepend the provenance header, keeping any shebang on line 1. +/// +/// A `#!` is only honoured as the first line of a file, so a header written +/// above it would leave the exported script unrunnable — and shellcheck +/// reports exactly that (SC1128) when the file is checked directly, which is +/// the whole point of exporting. +fn with_provenance(script: &ExportedScript) -> String { + let header = provenance_header(script); + match script.source.split_once('\n') { + Some((first, rest)) if first.starts_with("#!") => { + format!("{first}\n{header}{rest}") + } + _ => format!("{header}{}", script.source), + } +} + /// A header that points a reader back at the producing Rust source, so a /// finding in an exported file is actionable without a repository-wide grep. fn provenance_header(script: &ExportedScript) -> String { @@ -177,6 +192,45 @@ docker rm -f "$PROXY_CONTAINER" 2>/dev/null || true assert_eq!(entry["externals"].as_array().expect("array").len(), 0); } + #[test] + fn a_shebang_stays_on_line_one_ahead_of_the_provenance_header() { + // A `#!` is only honoured as the first line of a file. Writing the + // header above it would make every exported `sh` script unrunnable, + // and shellcheck reports it as SC1128 — which defeats the purpose of + // exporting the scripts to be checked. + let script = ExportedScript { + name: "test::WRAPPER".into(), + interpreter: "sh", + file: "src/test.rs", + line: 1, + bindings: &[], + externals: &[], + fragments: &[], + source: "#!/bin/sh\nexec az \"$@\"\n".into(), + }; + let out = with_provenance(&script); + assert!(out.starts_with("#!/bin/sh\n"), "{out}"); + assert!(out.contains("# script: test::WRAPPER")); + assert!(out.trim_end().ends_with("exec az \"$@\""), "{out}"); + } + + #[test] + fn a_script_without_a_shebang_gets_the_header_first() { + let script = ExportedScript { + name: "test::PLAIN".into(), + interpreter: "bash", + file: "src/test.rs", + line: 1, + bindings: &[], + externals: &[], + fragments: &[], + source: "echo hi\n".into(), + }; + let out = with_provenance(&script); + assert!(out.starts_with("# ado-aw generated export"), "{out}"); + assert!(out.trim_end().ends_with("echo hi"), "{out}"); + } + #[test] fn empty_declarations_render_as_none_rather_than_blank() { assert_eq!(join_or_none(&[]), "(none)"); From 34d54af7bf6dc888adfd995e9cc699fce2816692 Mon Sep 17 00:00:00 2001 From: James Devine Date: Mon, 10 Aug 2026 17:17:02 +0100 Subject: [PATCH 4/7] refactor(compile): route the remaining generated shell through ShellScript MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the migration begun in f53f6d3e. Every `bash:` step the compiler emits is now a registered, independently shellchecked script: 85 in total, up from 50. `agentic_pipeline.rs` accounts for most of it — the MCPG lifecycle, log copying, prompt preparation, threat analysis, safe-output execution and the conclusion reporter. Four call sites remain on the old `bash()` helper because their bodies come from payload helpers shared with `extensions/ado_script.rs`; migrating them means changing a helper signature across a module boundary, so `bash()` and `dedent()` stay for now. Two improvements fell out of making the shell legible enough to lint: * The AWF invocations are built as bash arrays (`AWF_ARGS=(…)` / `AWF_ARGS+=(…)`) rather than backslash-continuation chains with fragment markers interleaved. Shellcheck could not follow a continuation interrupted by a comment; the array form needs no suppression and says what it means. * `REVOKE_GITHUB_APP_TOKEN` moves from a fragment to bindings plus a `${API_URL:+--api-url "$API_URL"}` guard. The old shape left a dangling `\` at end of file when no api-url was configured — valid POSIX, but only by accident. Three suppressions are added, each targeted at one line with a stated reason: SC2207 where an ADO macro is deliberately word-split into an array, and SC2086 where `Binding::words` values expand unquoted, which is that binding's documented contract. Note on the earlier survey: counting `\n\` continuations overstated how much shell was left. Most of those lines are Rust markdown and error text — `create_pull_request.rs` has 38 of them and no shell at all, and its git calls are argv-based, so wrapping them would have *added* a shell that is not there today. Left alone deliberately. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: aa46b482-6121-45e2-8278-1a9465b4f73e --- src/compile/agentic_pipeline.rs | 2342 +++++++++++++++++--------- src/compile/extensions/ado_script.rs | 103 +- tests/compiler_tests.rs | 154 +- 3 files changed, 1732 insertions(+), 867 deletions(-) diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index 5c5feddb..432971ec 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -2042,15 +2042,72 @@ fn custom_job_condition(def: &CustomSafeOutputJobDef) -> Result { Ok(Condition::And(parts)) } +shell_script! { + /// Prepare the compiler binary at the well-known `/tmp/awf-tools/ado-aw` + /// location the custom safe-output executor picks up. + PREPARE_CUSTOM_EXECUTOR_BINARY { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [], + body: r#" +mkdir -p /tmp/awf-tools +AGENTIC_PIPELINES_PATH="$(Pipeline.Workspace)/agentic-pipeline-compiler/ado-aw" +chmod +x "$AGENTIC_PIPELINES_PATH" +cp "$AGENTIC_PIPELINES_PATH" /tmp/awf-tools/ado-aw +chmod +x /tmp/awf-tools/ado-aw +"#, + } +} + +shell_script! { + /// Add the downloaded compiler binary to PATH and mark it executable. + ADD_COMPILER_TO_PATH { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [], + body: r###" +ls -la "$(Pipeline.Workspace)/agentic-pipeline-compiler" +chmod +x "$(Pipeline.Workspace)/agentic-pipeline-compiler/ado-aw" +echo "##vso[task.prependpath]$(Pipeline.Workspace)/agentic-pipeline-compiler" +"###, + } +} + +shell_script! { + /// Create the per-job staging output directory. + PREPARE_OUTPUT_DIRECTORY { + interpreter: Bash, + bindings: [AGENT_TEMP], + externals: [], + fragments: [], + body: r#" +mkdir -p "$AGENT_TEMP/staging" +"#, + } +} + fn prepare_custom_executor_binary_step() -> BashStep { - bash( - "Prepare custom safe-output executor", - "mkdir -p /tmp/awf-tools\n\ - AGENTIC_PIPELINES_PATH=\"$(Pipeline.Workspace)/agentic-pipeline-compiler/ado-aw\"\n\ - chmod +x \"$AGENTIC_PIPELINES_PATH\"\n\ - cp \"$AGENTIC_PIPELINES_PATH\" /tmp/awf-tools/ado-aw\n\ - chmod +x /tmp/awf-tools/ado-aw\n", - ) + ShellScript::new(&PREPARE_CUSTOM_EXECUTOR_BINARY) + .into_step("Prepare custom safe-output executor") +} + +shell_script! { + /// Write the compiler-generated custom-tools runtime config to a file. + /// The payload is base64-encoded at compile time so no re-parsing is + /// needed at runtime — `base64 --decode` is the last command, so ADO's + /// fail-on-last-command default surfaces a corrupted transfer. + WRITE_CUSTOM_RUNTIME_CONFIG { + interpreter: Bash, + bindings: [ENCODED, AGENT_TEMP, CONFIG_FILENAME], + externals: [], + fragments: [], + body: r#" +mkdir -p "$AGENT_TEMP/ado-aw-custom" +printf '%s' "$ENCODED" | base64 --decode > "$AGENT_TEMP/$CONFIG_FILENAME" +"#, + } } fn write_custom_runtime_config_step( @@ -2062,31 +2119,56 @@ fn write_custom_runtime_config_step( let json = serde_json::to_string_pretty(&parsed) .context("failed to serialize custom job runtime config")?; let encoded = STANDARD.encode(json.as_bytes()); - // No runtime JSON re-validation: the payload was round-trip parsed and - // re-serialized above, so it is valid JSON by construction. `base64 - // --decode` is the last command in the script, so ADO's fail-on-last-command - // default already surfaces a corrupted transfer. Custom jobs run on - // consumer-owned pools, and this step is their only interpreter-dependent - // command, so keeping it to bash + base64 avoids a hard python3 dependency. - let script = format!( - "mkdir -p \"$(Agent.TempDirectory)/ado-aw-custom\"\n\ - printf '%s' {encoded} | base64 --decode > \"{config_path}\"\n", - encoded = shell_quote(&encoded), - ); - Ok(bash("Write custom job runtime config", script)) + let filename = agent_temp_filename(config_path); + Ok(ShellScript::new(&WRITE_CUSTOM_RUNTIME_CONFIG) + .text("ENCODED", encoded) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .text("CONFIG_FILENAME", filename) + .into_step("Write custom job runtime config")) +} + +shell_script! { + /// Materialise the aggregate `ADO_AW_AGENT_OUTPUT` payload for a + /// custom safe-output job by invoking `ado-aw execute` in + /// `--prepare-custom-agent-output` mode. + PREPARE_CUSTOM_AGENT_OUTPUT { + interpreter: Bash, + bindings: [AGENT_TEMP, PIPELINE_WORKSPACE, BUILD_ID, CONFIG_FILENAME, OUTPUT_FILENAME], + externals: [], + fragments: [], + body: r#" +/tmp/awf-tools/ado-aw execute \ + --safe-output-dir "$PIPELINE_WORKSPACE/analyzed_outputs_$BUILD_ID" \ + --resolved-config "$AGENT_TEMP/$CONFIG_FILENAME" \ + --prepare-custom-agent-output "$AGENT_TEMP/$OUTPUT_FILENAME" +"#, + } } fn prepare_custom_agent_output_step(config_path: &str, output_path: &str) -> BashStep { - let script = format!( - "# shellcheck disable=SC2016 # ADO expands path macros before bash evaluates the single-quoted arguments.\n\ - /tmp/awf-tools/ado-aw execute \ - --safe-output-dir \"$(Pipeline.Workspace)/analyzed_outputs_$(Build.BuildId)\" \ - --resolved-config {config} \ - --prepare-custom-agent-output {output}\n", - config = shell_quote(config_path), - output = shell_quote(output_path), - ); - bash("Prepare custom Agent output", script) + ShellScript::new(&PREPARE_CUSTOM_AGENT_OUTPUT) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .bind("BUILD_ID", Binding::ado_macro("Build.BuildId")) + .text("CONFIG_FILENAME", agent_temp_filename(config_path)) + .text("OUTPUT_FILENAME", agent_temp_filename(output_path)) + .into_step("Prepare custom Agent output") +} + +/// Extract the filename portion of an `$(Agent.TempDirectory)/` +/// path so it can be passed through [`Binding::text`] (which forbids `$(`) +/// while the `$(Agent.TempDirectory)` prefix is contributed separately as +/// [`Binding::ado_macro`]. +fn agent_temp_filename(path: &str) -> String { + let prefix = "$(Agent.TempDirectory)/"; + path.strip_prefix(prefix) + .unwrap_or_else(|| panic!( + "custom-tools config path {path:?} must start with {prefix:?}" + )) + .to_string() } fn component_step_with_custom_env( @@ -2267,17 +2349,15 @@ fn build_safeoutputs_job( front_matter.supply_chain(), )); // Add compiler to path - steps.push(Step::Bash(bash( - "Add agentic compiler to path", - "ls -la \"$(Pipeline.Workspace)/agentic-pipeline-compiler\"\n\ - chmod +x \"$(Pipeline.Workspace)/agentic-pipeline-compiler/ado-aw\"\n\ - echo \"##vso[task.prependpath]$(Pipeline.Workspace)/agentic-pipeline-compiler\"\n", - ))); + steps.push(Step::Bash( + ShellScript::new(&ADD_COMPILER_TO_PATH).into_step("Add agentic compiler to path"), + )); // Prepare output directory - steps.push(Step::Bash(bash( - "Prepare output directory", - "mkdir -p \"$(Agent.TempDirectory)/staging\"\n", - ))); + steps.push(Step::Bash( + ShellScript::new(&PREPARE_OUTPUT_DIRECTORY) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .into_step("Prepare output directory"), + )); // When `create-pull-request` is configured, fetch/deepen each target branch // in THIS job's checkout, immediately before the executor runs (issue // #1453). The prepare step also runs in the Agent job (for the containerized @@ -2674,15 +2754,9 @@ fn build_conclusion_job( steps.push(Step::Task(download_artifact)); let conclusion_path = super::extensions::ado_script::CONCLUSION_PATH; - let conclusion_script = format!( - "\ -if command -v node >/dev/null 2>&1 && [ -f {conclusion_path} ]; then\n \ - node {conclusion_path}\n\ -else\n \ - echo \"##vso[task.logissue type=warning]conclusion.js unavailable; skipping conclusion reporting\"\n\ -fi\n" - ); - let mut conclusion_step = bash("Report pipeline conclusion", conclusion_script); + let mut conclusion_step = ShellScript::new(&REPORT_CONCLUSION) + .text("CONCLUSION_PATH", conclusion_path) + .into_step("Report pipeline conclusion"); conclusion_step = conclusion_step.with_condition(Condition::Always); // The Conclusion job's contract is "always runs, never fails": it exists to // surface OTHER jobs' failures, so it must not turn a non-zero exit of its @@ -3414,28 +3488,100 @@ fn download_compiler_step( ]; } - let script = format!( - "set -eo pipefail\n\ - COMPILER_VERSION=\"{compiler_version}\"\n\ - DOWNLOAD_DIR=\"$(Pipeline.Workspace)/agentic-pipeline-compiler\"\n\ - DOWNLOAD_URL=\"https://github.com/githubnext/ado-aw/releases/download/v${{COMPILER_VERSION}}/ado-aw-linux-x64\"\n\ - CHECKSUM_URL=\"https://github.com/githubnext/ado-aw/releases/download/v${{COMPILER_VERSION}}/checksums.txt\"\n\ - \n\ - mkdir -p \"$DOWNLOAD_DIR\"\n\ - echo \"Downloading ado-aw v${{COMPILER_VERSION}} from GitHub Releases...\"\n\ - curl -fsSL -o \"$DOWNLOAD_DIR/ado-aw-linux-x64\" \"$DOWNLOAD_URL\"\n\ - curl -fsSL -o \"$DOWNLOAD_DIR/checksums.txt\" \"$CHECKSUM_URL\"\n\ - \n\ - echo \"Verifying checksum...\"\n\ - cd \"$DOWNLOAD_DIR\" || exit 1\n\ - grep \"ado-aw-linux-x64\" checksums.txt | sha256sum -c -\n\ - mv ado-aw-linux-x64 ado-aw\n\ - chmod +x ado-aw\n" - ); - vec![Step::Bash(bash( - format!("Download agentic pipeline compiler (v{compiler_version})"), - script, - ))] + vec![Step::Bash( + ShellScript::new(&DOWNLOAD_COMPILER_FROM_RELEASES) + .text("COMPILER_VERSION", compiler_version) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .into_step(format!( + "Download agentic pipeline compiler (v{compiler_version})" + )), + )] +} + +shell_script! { + /// Fallback download path when no supply-chain feed or pipeline artifact + /// is configured: fetch the `ado-aw` binary directly from the GitHub + /// Releases page, verify its SHA-256 against the published + /// `checksums.txt`, and stage it at `/agentic-pipeline-compiler/ado-aw`. + DOWNLOAD_COMPILER_FROM_RELEASES { + interpreter: Bash, + bindings: [COMPILER_VERSION, PIPELINE_WORKSPACE], + externals: [], + fragments: [], + body: r###" +set -eo pipefail +DOWNLOAD_DIR="$PIPELINE_WORKSPACE/agentic-pipeline-compiler" +DOWNLOAD_URL="https://github.com/githubnext/ado-aw/releases/download/v$COMPILER_VERSION/ado-aw-linux-x64" +CHECKSUM_URL="https://github.com/githubnext/ado-aw/releases/download/v$COMPILER_VERSION/checksums.txt" + +mkdir -p "$DOWNLOAD_DIR" +echo "Downloading ado-aw v$COMPILER_VERSION from GitHub Releases..." +curl -fsSL -o "$DOWNLOAD_DIR/ado-aw-linux-x64" "$DOWNLOAD_URL" +curl -fsSL -o "$DOWNLOAD_DIR/checksums.txt" "$CHECKSUM_URL" + +echo "Verifying checksum..." +cd "$DOWNLOAD_DIR" || exit 1 +grep "ado-aw-linux-x64" checksums.txt | sha256sum -c - +mv ado-aw-linux-x64 ado-aw +chmod +x ado-aw +"###, + } +} + +shell_script! { + /// Fallback download path for the AWF (Agentic Workflow Firewall) + /// binary: fetch it directly from the GitHub Releases page of + /// `github/gh-aw-firewall`, verify its SHA-256, and expose it on `PATH`. + DOWNLOAD_AWF_FROM_RELEASES { + interpreter: Bash, + bindings: [AWF_VERSION, PIPELINE_WORKSPACE], + externals: [], + fragments: [], + body: r###" +set -eo pipefail + +DOWNLOAD_DIR="$PIPELINE_WORKSPACE/awf" +DOWNLOAD_URL="https://github.com/github/gh-aw-firewall/releases/download/v$AWF_VERSION/awf-linux-x64" +CHECKSUM_URL="https://github.com/github/gh-aw-firewall/releases/download/v$AWF_VERSION/checksums.txt" + +mkdir -p "$DOWNLOAD_DIR" +echo "Downloading AWF v$AWF_VERSION from GitHub Releases..." +curl -fsSL -o "$DOWNLOAD_DIR/awf-linux-x64" "$DOWNLOAD_URL" +curl -fsSL -o "$DOWNLOAD_DIR/checksums.txt" "$CHECKSUM_URL" + +echo "Verifying checksum..." +cd "$DOWNLOAD_DIR" || exit 1 +grep "awf-linux-x64" checksums.txt | sha256sum -c - +mv awf-linux-x64 awf +chmod +x awf +echo "##vso[task.prependpath]$PIPELINE_WORKSPACE/awf" +./awf --version +"###, + } +} + +shell_script! { + /// Pre-pull every AWF container image (and optionally MCPG) so that the + /// subsequent `docker run` on the isolated agent network has all images + /// available locally. The `mcpg_pull` fragment holds an optional extra + /// `docker pull` line for the MCPG image. + PREPULL_IMAGES { + interpreter: Bash, + bindings: [SQUID_IMAGE, AGENT_IMAGE, API_PROXY_IMAGE], + externals: [], + fragments: [mcpg_pull], + body: r###" +set -eo pipefail + +docker pull "$SQUID_IMAGE" +docker pull "$AGENT_IMAGE" +docker pull "$API_PROXY_IMAGE" +# ado-aw:fragment mcpg_pull +"###, + } } fn substitute_integrity_check(yaml: &str, pipeline_path: &str, trigger_repo_dir: &str) -> String { @@ -3446,106 +3592,153 @@ fn substitute_integrity_check(yaml: &str, pipeline_path: &str, trigger_repo_dir: .replace("{{ trigger_repo_directory }}", trigger_repo_dir) } +shell_script! { + /// Stage the runtime MCPG config JSON, generate a per-run gateway API + /// key, and (optionally) stage the compiler-generated custom-tools JSON. + /// The two JSON payloads are spliced in via `mcpg_config_heredoc` and + /// `custom_tools_block` fragments so the compiler owns the heredoc + /// sentinels (each derived from the SHA of its own payload). + PREPARE_MCPG_CONFIG { + interpreter: Bash, + bindings: [AGENT_TEMP, MCPG_PORT, MCPG_DOMAIN], + externals: [], + fragments: [mcpg_config_heredoc, custom_tools_block], + body: r###" +mkdir -p "$AGENT_TEMP/staging" + +# Generate MCPG API key early so it's available as an ADO secret variable +# for both the MCPG config and the agent's mcp-config.json +MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') +echo "##vso[task.setvariable variable=MCP_GATEWAY_API_KEY;issecret=true]$MCP_GATEWAY_API_KEY" + +# Export gateway port and domain as pipeline variables (matching gh-aw pattern). +# These duplicate the compile-time values baked into the YAML, but MCPG's +# Docker container requires MCP_GATEWAY_PORT and MCP_GATEWAY_DOMAIN env vars +# to start — the ADO variable indirection satisfies that contract. +echo "##vso[task.setvariable variable=MCP_GATEWAY_PORT]$MCPG_PORT" +echo "##vso[task.setvariable variable=MCP_GATEWAY_DOMAIN]$MCPG_DOMAIN" + +# Write MCPG (MCP Gateway) configuration to a file +# ado-aw:fragment mcpg_config_heredoc + +# ado-aw:fragment custom_tools_block +echo "MCPG config:" +cat "$AGENT_TEMP/staging/mcpg-config.json" + +# Validate JSON +python3 -m json.tool "$AGENT_TEMP/staging/mcpg-config.json" > /dev/null && echo "JSON is valid" +"###, + } +} + fn prepare_mcpg_config_step( mcpg_config_json: &str, custom_tools_json: Option<&str>, ) -> Result { - // mcpg_config_json is pretty-printed JSON. We want `{` to align with - // the surrounding `cat`/`echo` lines (no extra leading indent) so the - // emitted block-scalar bash body matches base.yml. - let custom_tools_script = if let Some(custom_tools_json) = custom_tools_json { - let sentinel = super::common::heredoc_sentinel("CUSTOM_TOOLS_JSON_EOF", custom_tools_json)?; + let mcpg_sentinel = super::common::heredoc_sentinel("MCPG_CONFIG_EOF", mcpg_config_json)?; + let mcpg_config_heredoc = format!( + "cat > \"$AGENT_TEMP/staging/mcpg-config.json\" << '{mcpg_sentinel}'\n\ + {mcpg_config_json}\n\ + {mcpg_sentinel}" + ); + let custom_tools_fragment = if let Some(custom_tools_json) = custom_tools_json { + let sentinel = + super::common::heredoc_sentinel("CUSTOM_TOOLS_JSON_EOF", custom_tools_json)?; format!( "# Write compiler-generated dynamic SafeOutputs tool definitions\n\ - cat > \"$(Agent.TempDirectory)/staging/custom-tools.json\" << '{sentinel}'\n\ -{custom_tools_json}\n\ + cat > \"$AGENT_TEMP/staging/custom-tools.json\" << '{sentinel}'\n\ + {custom_tools_json}\n\ {sentinel}\n\ - python3 -m json.tool \"$(Agent.TempDirectory)/staging/custom-tools.json\" > /dev/null\n\ - \n" + python3 -m json.tool \"$AGENT_TEMP/staging/custom-tools.json\" > /dev/null" ) } else { String::new() }; - let script = format!( - "mkdir -p \"$(Agent.TempDirectory)/staging\"\n\ - \n\ - # Generate MCPG API key early so it's available as an ADO secret variable\n\ - # for both the MCPG config and the agent's mcp-config.json\n\ - MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=')\n\ - echo \"##vso[task.setvariable variable=MCP_GATEWAY_API_KEY;issecret=true]$MCP_GATEWAY_API_KEY\"\n\ - \n\ - # Export gateway port and domain as pipeline variables (matching gh-aw pattern).\n\ - # These duplicate the compile-time values baked into the YAML, but MCPG's\n\ - # Docker container requires MCP_GATEWAY_PORT and MCP_GATEWAY_DOMAIN env vars\n\ - # to start — the ADO variable indirection satisfies that contract.\n\ - echo \"##vso[task.setvariable variable=MCP_GATEWAY_PORT]{MCPG_PORT}\"\n\ - echo \"##vso[task.setvariable variable=MCP_GATEWAY_DOMAIN]{MCPG_DOMAIN}\"\n\ - \n\ - # Write MCPG (MCP Gateway) configuration to a file\n\ - cat > \"$(Agent.TempDirectory)/staging/mcpg-config.json\" << 'MCPG_CONFIG_EOF'\n\ -{mcpg_config_json}\n\ - MCPG_CONFIG_EOF\n\ - \n\ -{custom_tools_script}\ - echo \"MCPG config:\"\n\ - cat \"$(Agent.TempDirectory)/staging/mcpg-config.json\"\n\ - \n\ - # Validate JSON\n\ - python3 -m json.tool \"$(Agent.TempDirectory)/staging/mcpg-config.json\" > /dev/null && echo \"JSON is valid\"\n" - ); - Ok(bash("Prepare MCPG config", script)) + Ok(ShellScript::new(&PREPARE_MCPG_CONFIG) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .bind("MCPG_PORT", Binding::number(MCPG_PORT.into())) + .text("MCPG_DOMAIN", MCPG_DOMAIN) + .fragment("mcpg_config_heredoc", mcpg_config_heredoc) + .fragment("custom_tools_block", custom_tools_fragment) + .into_step("Prepare MCPG config")) +} + +shell_script! { + /// Prepare the AWF working directory (`/tmp/awf-tools/`) with the + /// compiler binary and MCPG staging JSON. Copies the compiler out of the + /// Pipeline.Workspace and into `/tmp/` so it is reachable inside the + /// AWF-managed container (AWF auto-mounts `/tmp:/tmp:rw`). + PREPARE_TOOLING { + interpreter: Bash, + bindings: [PIPELINE_WORKSPACE, AGENT_TEMP], + externals: [HOME], + fragments: [], + body: r###" +mkdir -p /tmp/awf-tools/staging + +echo "HOME: $HOME" + +# Use absolute path since MCP subprocess may not inherit PATH +AGENTIC_PIPELINES_PATH="$PIPELINE_WORKSPACE/agentic-pipeline-compiler/ado-aw" + +# Verify the binary exists and is executable +ls -la "$AGENTIC_PIPELINES_PATH" +chmod +x "$AGENTIC_PIPELINES_PATH" + +$AGENTIC_PIPELINES_PATH -h + +# Copy compiler binary to /tmp so it's accessible inside AWF container +cp "$AGENTIC_PIPELINES_PATH" /tmp/awf-tools/ado-aw +chmod +x /tmp/awf-tools/ado-aw + +# Copy MCPG config to /tmp +cp "$AGENT_TEMP/staging/mcpg-config.json" /tmp/awf-tools/staging/mcpg-config.json +if [ -f "$AGENT_TEMP/staging/custom-tools.json" ]; then + cp "$AGENT_TEMP/staging/custom-tools.json" /tmp/awf-tools/staging/custom-tools.json +fi +"###, + } } fn prepare_tooling_step() -> BashStep { - let script = "mkdir -p /tmp/awf-tools/staging\n\ - \n\ - echo \"HOME: $HOME\"\n\ - \n\ - # Use absolute path since MCP subprocess may not inherit PATH\n\ - AGENTIC_PIPELINES_PATH=\"$(Pipeline.Workspace)/agentic-pipeline-compiler/ado-aw\"\n\ - \n\ - # Verify the binary exists and is executable\n\ - ls -la \"$AGENTIC_PIPELINES_PATH\"\n\ - chmod +x \"$AGENTIC_PIPELINES_PATH\"\n\ - \n\ - $AGENTIC_PIPELINES_PATH -h\n\ - \n\ - # Copy compiler binary to /tmp so it's accessible inside AWF container\n\ - cp \"$AGENTIC_PIPELINES_PATH\" /tmp/awf-tools/ado-aw\n\ - chmod +x /tmp/awf-tools/ado-aw\n\ - \n\ - # Copy MCPG config to /tmp\n\ - cp \"$(Agent.TempDirectory)/staging/mcpg-config.json\" /tmp/awf-tools/staging/mcpg-config.json\n\ - if [ -f \"$(Agent.TempDirectory)/staging/custom-tools.json\" ]; then\n\ - cp \"$(Agent.TempDirectory)/staging/custom-tools.json\" /tmp/awf-tools/staging/custom-tools.json\n\ - fi\n"; - bash("Prepare tooling", script) + ShellScript::new(&PREPARE_TOOLING) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .into_step("Prepare tooling") +} + +shell_script! { + /// Write the agent-prompt markdown to `/tmp/awf-tools/agent-prompt.md` + /// so that it is reachable inside the AWF container (AWF auto-mounts + /// `/tmp:/tmp:rw`). The `heredoc` fragment carries a per-content + /// SHA-derived sentinel so a malicious agent markdown body cannot + /// terminate the heredoc early and inject shell into the Agent job. + PREPARE_AGENT_PROMPT { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [heredoc], + body: r###" +# Write agent instructions to /tmp so it's accessible inside AWF container +# ado-aw:fragment heredoc + +echo "Agent prompt:" +cat "/tmp/awf-tools/agent-prompt.md" +"###, + } } fn prepare_agent_prompt_step(agent_content: &str) -> Result { - // The agent_content lands inside a bash heredoc at the same indent as - // `cat > ...` (no extra prefix), matching base.yml's emission. - // The template uses leading-9-space `\n\` continuations; `dedent()` - // strips them uniformly so the resulting bash body has 0-indent - // surrounding lines and the interpolated content lands flush left. - // - // The sentinel is per-content SHA-derived so a malicious agent - // markdown body cannot terminate the heredoc early and inject - // shell commands into the Agent job. See - // [`crate::compile::common::heredoc_sentinel`]. let sentinel = super::common::heredoc_sentinel("AGENT_PROMPT_EOF", agent_content)?; - let template = format!( - "\ - # Write agent instructions to /tmp so it's accessible inside AWF container\n\ - cat > \"/tmp/awf-tools/agent-prompt.md\" << '{sentinel}'\n\ - {{INTERP}}\n\ - {sentinel}\n\ - \n\ - echo \"Agent prompt:\"\n\ - cat \"/tmp/awf-tools/agent-prompt.md\"\n" + let heredoc = format!( + "cat > \"/tmp/awf-tools/agent-prompt.md\" << '{sentinel}'\n{agent_content}\n{sentinel}" ); - let script = dedent(&template).replace("{INTERP}", agent_content); - Ok(bash("Prepare agent prompt", script)) + Ok(ShellScript::new(&PREPARE_AGENT_PROMPT) + .fragment("heredoc", heredoc) + .into_step("Prepare agent prompt")) } fn download_awf_step(supply_chain: Option<&SupplyChainConfig>) -> Vec { @@ -3595,31 +3788,17 @@ fn download_awf_step(supply_chain: Option<&SupplyChainConfig>) -> Vec { ]; } - let script = format!( - "set -eo pipefail\n\ - \n\ - AWF_VERSION=\"{AWF_VERSION}\"\n\ - DOWNLOAD_DIR=\"$(Pipeline.Workspace)/awf\"\n\ - DOWNLOAD_URL=\"https://github.com/github/gh-aw-firewall/releases/download/v${{AWF_VERSION}}/awf-linux-x64\"\n\ - CHECKSUM_URL=\"https://github.com/github/gh-aw-firewall/releases/download/v${{AWF_VERSION}}/checksums.txt\"\n\ - \n\ - mkdir -p \"$DOWNLOAD_DIR\"\n\ - echo \"Downloading AWF v${{AWF_VERSION}} from GitHub Releases...\"\n\ - curl -fsSL -o \"$DOWNLOAD_DIR/awf-linux-x64\" \"$DOWNLOAD_URL\"\n\ - curl -fsSL -o \"$DOWNLOAD_DIR/checksums.txt\" \"$CHECKSUM_URL\"\n\ - \n\ - echo \"Verifying checksum...\"\n\ - cd \"$DOWNLOAD_DIR\" || exit 1\n\ - grep \"awf-linux-x64\" checksums.txt | sha256sum -c -\n\ - mv awf-linux-x64 awf\n\ - chmod +x awf\n\ - echo \"##vso[task.prependpath]$(Pipeline.Workspace)/awf\"\n\ - ./awf --version\n" - ); - vec![Step::Bash(bash( - format!("Download AWF (Agentic Workflow Firewall) v{AWF_VERSION}"), - script, - ))] + vec![Step::Bash( + ShellScript::new(&DOWNLOAD_AWF_FROM_RELEASES) + .text("AWF_VERSION", AWF_VERSION) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .into_step(format!( + "Download AWF (Agentic Workflow Firewall) v{AWF_VERSION}" + )), + )] } fn prepull_images_step(include_mcpg: bool, supply_chain: Option<&SupplyChainConfig>) -> Vec { @@ -3642,19 +3821,17 @@ fn prepull_images_step(include_mcpg: bool, supply_chain: Option<&SupplyChainConf registry_base, ); - let mut script = format!( - "set -eo pipefail\n\ - \n\ - docker pull {squid}\n\ - docker pull {agent}\n\ - docker pull {api_proxy}\n" - ); - let display = if include_mcpg { + let (display, mcpg_pull) = if include_mcpg { let mcpg = image_ref(MCPG_IMAGE, &format!("v{MCPG_VERSION}"), registry_base); - script.push_str(&format!("docker pull {mcpg}\n")); - format!("Pre-pull AWF and MCPG container images (v{AWF_VERSION})") + ( + format!("Pre-pull AWF and MCPG container images (v{AWF_VERSION})"), + format!("docker pull \"{mcpg}\""), + ) } else { - format!("Pre-pull AWF container images (v{AWF_VERSION})") + ( + format!("Pre-pull AWF container images (v{AWF_VERSION})"), + String::new(), + ) }; let mut steps = Vec::new(); @@ -3667,10 +3844,151 @@ fn prepull_images_step(include_mcpg: bool, supply_chain: Option<&SupplyChainConf ) { steps.push(Step::Task(acr_login_step(base, conn))); } - steps.push(Step::Bash(bash(display, script))); + steps.push(Step::Bash( + ShellScript::new(&PREPULL_IMAGES) + .text("SQUID_IMAGE", &squid) + .text("AGENT_IMAGE", &agent) + .text("API_PROXY_IMAGE", &api_proxy) + .fragment("mcpg_pull", mcpg_pull) + .into_step(display), + )); steps } +shell_script! { + /// Start the MCP Gateway (MCPG) on the runner's Docker daemon so that AWF + /// can later attach it to its isolated internal network. This is + /// contractually a *single* Bash task — the API key never leaves the + /// process — so the block-scalar body has to carry the full multi-line + /// `docker run …` invocation. The compiler-owned `docker_env_lines` and + /// `debug_flag` fragments splice any extra `-e VAR=…` and `-e DEBUG=…` + /// continuation lines directly into the middle of that invocation. + /// + /// Uses: + /// - the pipeline variables `MCP_GATEWAY_API_KEY` (secret) and + /// `ADO_PROXY_IP` (from the optional ado-proxy startup step) as + /// externals routed via `env:`, so the secret is not baked into the + /// emitted YAML prelude + /// - `Binding::text` for the two static topology names (container + + /// image ref) and `Binding::number` for the fixed listen port + /// - Compile-time constant `MCP_GATEWAY_DOMAIN` bound as text so the + /// docker container receives the same domain constant used elsewhere + START_MCPG { + interpreter: Bash, + bindings: [MCPG_CONTAINER, MCPG_IMAGE, MCPG_PORT, MCPG_DOMAIN], + externals: [MCP_GATEWAY_API_KEY, ADO_PROXY_IP], + fragments: [debug_flag, docker_env_lines], + body: r###" +# Substitute runtime values into MCPG config +MCP_RUNNER_UID=$(id -u) +MCP_RUNNER_GID=$(id -g) +MCPG_CONFIG=$(sed \ + -e "s|\${MCP_RUNNER_UID}|$MCP_RUNNER_UID|g" \ + -e "s|\${MCP_RUNNER_GID}|$MCP_RUNNER_GID|g" \ + -e "s|\${MCP_GATEWAY_API_KEY}|$MCP_GATEWAY_API_KEY|g" \ + -e "s|\${ADO_PROXY_IP}|${ADO_PROXY_IP:-}|g" \ + /tmp/awf-tools/staging/mcpg-config.json) + +# A client redirected at an empty address would resolve the real +# Azure DevOps instead of the policy engine, quietly restoring the +# direct path this design removes. Fail loudly rather than start. +if grep -q 'ADO_PROXY_IP' /tmp/awf-tools/staging/mcpg-config.json \ + && [ -z "${ADO_PROXY_IP:-}" ]; then + echo "##vso[task.complete result=Failed]ado-proxy address is unknown; refusing to start MCP clients unredirected" + exit 1 +fi + +# Log the template config (before API key substitution) for debugging. +echo "Starting MCPG with config template:" +python3 -m json.tool < /tmp/awf-tools/staging/mcpg-config.json + +# Remove any leftover container or stale output from a previous interrupted run +# (--rm only cleans up on clean exit; OOM/SIGKILL may leave it behind) +docker rm -f "$MCPG_CONTAINER" 2>/dev/null || true +GATEWAY_OUTPUT="/tmp/gh-aw/mcp-config/gateway-output.json" +mkdir -p "$(dirname "$GATEWAY_OUTPUT")" /tmp/gh-aw/mcp-logs +rm -f "$GATEWAY_OUTPUT" + +# Start MCPG on Docker's bridge network. AWF attaches this named, +# trusted container to its internal network after creating awf-net. +# The Docker socket mount is required because MCPG spawns stdio-based MCP +# servers as sibling containers. This grants significant host access — acceptable +# here because the pipeline agent is already trusted and network-isolated by AWF. +# +# stdout → gateway-output.json (machine-readable config, read after health check) +echo "$MCPG_CONFIG" | docker run -i --rm \ + --name "$MCPG_CONTAINER" \ + --network bridge \ + -p "127.0.0.1:$MCPG_PORT:$MCPG_PORT" \ + --entrypoint /app/awmg \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e MCP_GATEWAY_PORT="$MCPG_PORT" \ + -e MCP_GATEWAY_DOMAIN="$MCPG_DOMAIN" \ + -e MCP_GATEWAY_API_KEY="$MCP_GATEWAY_API_KEY" \ + # ado-aw:fragment debug_flag + # ado-aw:fragment docker_env_lines + "$MCPG_IMAGE" \ + --routed --listen "0.0.0.0:$MCPG_PORT" --config-stdin --log-dir /tmp/gh-aw/mcp-logs \ + > "$GATEWAY_OUTPUT" 2> >(tee /tmp/gh-aw/mcp-logs/stderr.log >&2) & +MCPG_PID=$! +echo "MCPG started (PID: $MCPG_PID)" + +# Wait for MCPG to be ready +READY=false +for _i in $(seq 1 30); do + if curl -sf "http://localhost:$MCPG_PORT/health" > /dev/null 2>&1; then + echo "MCPG is ready" + READY=true + break + fi + sleep 1 +done +if [ "$READY" != "true" ]; then + echo "##vso[task.complete result=Failed]MCPG did not become ready within 30s" + exit 1 +fi + +# Wait for gateway output file to contain valid JSON with mcpServers. +# Health check passing doesn't guarantee stdout is flushed, so poll. +echo "Waiting for gateway output file..." +GATEWAY_READY=false +for _i in $(seq 1 15); do + if [ -s "$GATEWAY_OUTPUT" ] && jq -e '.mcpServers' "$GATEWAY_OUTPUT" > /dev/null 2>&1; then + echo "Gateway output is ready" + GATEWAY_READY=true + break + fi + sleep 1 +done +if [ "$GATEWAY_READY" != "true" ]; then + echo "##vso[task.complete result=Failed]Gateway output file not ready within 15s" + echo "Gateway output content:" + cat "$GATEWAY_OUTPUT" 2>/dev/null || echo "(empty or missing)" + exit 1 +fi + +echo "Gateway output:" +cat "$GATEWAY_OUTPUT" + +# Convert gateway output to Copilot CLI mcp-config.json. +# Mirrors gh-aw's convert_gateway_config_copilot.cjs: +# - Rewrite gateway URLs to the stable MCPG container name that AWF +# attaches to its internal network +# - Ensure tools: ["*"] on each server entry (Copilot CLI requirement) +# - Mark generated MCPG entries as default/trusted servers for Copilot CLI +# - Preserve all other fields (headers, type, etc.) +jq --arg prefix "http://$MCPG_DOMAIN:$MCPG_PORT" \ + '.mcpServers |= (to_entries | sort_by(.key) | map(.value.url |= sub("^http://[^/]+/"; "\($prefix)/") | .value.tools = ["*"] | .value.isDefaultServer = true) | from_entries)' \ + "$GATEWAY_OUTPUT" > /tmp/awf-tools/mcp-config.json + +chmod 600 /tmp/awf-tools/mcp-config.json + +echo "Generated MCP config at: /tmp/awf-tools/mcp-config.json" +cat /tmp/awf-tools/mcp-config.json +"###, + } +} + fn start_mcpg_step( mcpg_docker_env: &str, mcpg_step_env: &str, @@ -3681,155 +3999,48 @@ fn start_mcpg_step( .and_then(|sc| sc.registry.as_ref()) .map(|r| r.name.as_str()); let mcpg_image_v = image_ref(MCPG_IMAGE, &format!("v{MCPG_VERSION}"), registry_base); - // Build the docker-env block as additional `-e VAR=...` lines, one per - // line, joined with `\n ` (newline + 2-space continuation indent to - // match the surrounding `-e MCP_GATEWAY_*` lines). When no extensions - // contribute docker env, emit two empty `\`-continuation lines as - // placeholders for the legacy `{{ mcpg_debug_flags }}` and - // `{{ mcpg_docker_env }}` markers — bash treats them as no-op - // continuations and ignoring them keeps the lock file shape stable. - // Build the docker-env block as additional `-e VAR=...` lines, one per - // line, joined with `\n ` (newline + 2-space continuation indent to - // match the surrounding `-e MCP_GATEWAY_*` lines). When no extensions - // contribute docker env, emit two empty `\`-continuation lines as - // placeholders for the legacy `{{ mcpg_debug_flags }}` and - // `{{ mcpg_docker_env }}` markers — bash treats them as no-op - // continuations and ignoring them keeps the lock file shape stable. - // + + // Match the legacy layout of two placeholder `\`-continuation lines when + // no extensions contribute docker env — bash treats a lone `\` as a + // no-op continuation and preserving the shape keeps the docker-run + // command's argument boundaries identical to the pre-migration YAML. // `generate_mcpg_docker_env` returns a single `\` byte when no - // extensions contribute, so check for that sentinel as well as a - // literal empty string. + // extensions contribute, so match that sentinel as well as an empty + // string. let docker_env_lines: String = if mcpg_docker_env.trim().is_empty() || mcpg_docker_env.trim() == "\\" { // Two empty continuation lines mirror the legacy template's // two-marker layout. "\\\n \\".to_string() } else { - // `generate_mcpg_docker_env` already terminates every line with a - // ` \` continuation, so re-indent the lines without re-appending - // another ` \` (doing so would emit a stray `\ \` that bash reads - // as a one-character " " argument, corrupting the `docker run` - // image reference — see issue #1034). + // `generate_mcpg_docker_env` already terminates every line with + // ` \` continuation, so re-indent the lines without appending + // another ` \` (issue #1034). mcpg_docker_env.lines().collect::>().join("\n ") }; - // `--debug-pipeline` injects an extra `-e DEBUG="*" \` continuation - // line into the `docker run …` invocation so MCPG (and the stdio - // backends it spawns) emit verbose logs to the gateway stderr stream. - // Mirrors the legacy `{{ mcpg_debug_flags }}` template marker; emits - // the trailing `\n ` so the next continuation line aligns under it. + // `--debug-pipeline` injects an extra `-e DEBUG="*" \` continuation line + // into the `docker run …` invocation so MCPG (and the stdio backends it + // spawns) emit verbose logs to the gateway stderr stream. let debug_flag = if debug_pipeline { - "-e DEBUG=\"*\" \\\n ".to_string() + "-e DEBUG=\"*\" \\".to_string() } else { - String::new() + "\\".to_string() }; - let script = format!( - "# Substitute runtime values into MCPG config\n\ - MCP_RUNNER_UID=$(id -u)\n\ - MCP_RUNNER_GID=$(id -g)\n\ - MCPG_CONFIG=$(sed \\\n \ - -e \"s|\\${{MCP_RUNNER_UID}}|$MCP_RUNNER_UID|g\" \\\n \ - -e \"s|\\${{MCP_RUNNER_GID}}|$MCP_RUNNER_GID|g\" \\\n \ - -e \"s|\\${{MCP_GATEWAY_API_KEY}}|$(MCP_GATEWAY_API_KEY)|g\" \\\n \ - -e \"s|\\${{ADO_PROXY_IP}}|${{ADO_PROXY_IP:-}}|g\" \\\n \ - /tmp/awf-tools/staging/mcpg-config.json)\n\ - \n\ - # A client redirected at an empty address would resolve the real\n\ - # Azure DevOps instead of the policy engine, quietly restoring the\n\ - # direct path this design removes. Fail loudly rather than start.\n\ - if grep -q 'ADO_PROXY_IP' /tmp/awf-tools/staging/mcpg-config.json \\\n \ - && [ -z \"${{ADO_PROXY_IP:-}}\" ]; then\n \ - echo \"##vso[task.complete result=Failed]ado-proxy address is unknown; refusing to start MCP clients unredirected\"\n \ - exit 1\n\ - fi\n\ - \n\ - # Log the template config (before API key substitution) for debugging.\n\ - echo \"Starting MCPG with config template:\"\n\ - python3 -m json.tool < /tmp/awf-tools/staging/mcpg-config.json\n\ - \n\ - # Remove any leftover container or stale output from a previous interrupted run\n\ - # (--rm only cleans up on clean exit; OOM/SIGKILL may leave it behind)\n\ - docker rm -f {MCPG_CONTAINER_NAME} 2>/dev/null || true\n\ - GATEWAY_OUTPUT=\"/tmp/gh-aw/mcp-config/gateway-output.json\"\n\ - mkdir -p \"$(dirname \"$GATEWAY_OUTPUT\")\" /tmp/gh-aw/mcp-logs\n\ - rm -f \"$GATEWAY_OUTPUT\"\n\ - \n\ - # Start MCPG on Docker's bridge network. AWF attaches this named,\n\ - # trusted container to its internal network after creating awf-net.\n\ - # The Docker socket mount is required because MCPG spawns stdio-based MCP\n\ - # servers as sibling containers. This grants significant host access — acceptable\n\ - # here because the pipeline agent is already trusted and network-isolated by AWF.\n\ - #\n\ - # stdout → gateway-output.json (machine-readable config, read after health check)\n\ - echo \"$MCPG_CONFIG\" | docker run -i --rm \\\n \ - --name {MCPG_CONTAINER_NAME} \\\n \ - --network bridge \\\n \ - -p 127.0.0.1:{MCPG_PORT}:{MCPG_PORT} \\\n \ - --entrypoint /app/awmg \\\n \ - -v /var/run/docker.sock:/var/run/docker.sock \\\n \ - -e MCP_GATEWAY_PORT=\"$(MCP_GATEWAY_PORT)\" \\\n \ - -e MCP_GATEWAY_DOMAIN=\"$(MCP_GATEWAY_DOMAIN)\" \\\n \ - -e MCP_GATEWAY_API_KEY=\"$(MCP_GATEWAY_API_KEY)\" \\\n \ - {debug_flag}{docker_env_lines}\n \ - {mcpg_image_v} \\\n \ - --routed --listen 0.0.0.0:{MCPG_PORT} --config-stdin --log-dir /tmp/gh-aw/mcp-logs \\\n \ - > \"$GATEWAY_OUTPUT\" 2> >(tee /tmp/gh-aw/mcp-logs/stderr.log >&2) &\n\ - MCPG_PID=$!\n\ - echo \"MCPG started (PID: $MCPG_PID)\"\n\ - \n\ - # Wait for MCPG to be ready\n\ - READY=false\n\ - for _i in $(seq 1 30); do\n \ - if curl -sf \"http://localhost:{MCPG_PORT}/health\" > /dev/null 2>&1; then\n \ - echo \"MCPG is ready\"\n \ - READY=true\n \ - break\n \ - fi\n \ - sleep 1\n\ - done\n\ - if [ \"$READY\" != \"true\" ]; then\n \ - echo \"##vso[task.complete result=Failed]MCPG did not become ready within 30s\"\n \ - exit 1\n\ - fi\n\ - \n\ - # Wait for gateway output file to contain valid JSON with mcpServers.\n\ - # Health check passing doesn't guarantee stdout is flushed, so poll.\n\ - echo \"Waiting for gateway output file...\"\n\ - GATEWAY_READY=false\n\ - for _i in $(seq 1 15); do\n \ - if [ -s \"$GATEWAY_OUTPUT\" ] && jq -e '.mcpServers' \"$GATEWAY_OUTPUT\" > /dev/null 2>&1; then\n \ - echo \"Gateway output is ready\"\n \ - GATEWAY_READY=true\n \ - break\n \ - fi\n \ - sleep 1\n\ - done\n\ - if [ \"$GATEWAY_READY\" != \"true\" ]; then\n \ - echo \"##vso[task.complete result=Failed]Gateway output file not ready within 15s\"\n \ - echo \"Gateway output content:\"\n \ - cat \"$GATEWAY_OUTPUT\" 2>/dev/null || echo \"(empty or missing)\"\n \ - exit 1\n\ - fi\n\ - \n\ - echo \"Gateway output:\"\n\ - cat \"$GATEWAY_OUTPUT\"\n\ - \n\ - # Convert gateway output to Copilot CLI mcp-config.json.\n\ - # Mirrors gh-aw's convert_gateway_config_copilot.cjs:\n\ - # - Rewrite gateway URLs to the stable MCPG container name that AWF\n\ - # attaches to its internal network\n\ - # - Ensure tools: [\"*\"] on each server entry (Copilot CLI requirement)\n\ - # - Mark generated MCPG entries as default/trusted servers for Copilot CLI\n\ - # - Preserve all other fields (headers, type, etc.)\n\ - jq --arg prefix \"http://$(MCP_GATEWAY_DOMAIN):$(MCP_GATEWAY_PORT)\" \\\n \ - '.mcpServers |= (to_entries | sort_by(.key) | map(.value.url |= sub(\"^http://[^/]+/\"; \"\\($prefix)/\") | .value.tools = [\"*\"] | .value.isDefaultServer = true) | from_entries)' \\\n \ - \"$GATEWAY_OUTPUT\" > /tmp/awf-tools/mcp-config.json\n\ - \n\ - chmod 600 /tmp/awf-tools/mcp-config.json\n\ - \n\ - echo \"Generated MCP config at: /tmp/awf-tools/mcp-config.json\"\n\ - cat /tmp/awf-tools/mcp-config.json\n" - ); - let mut step = bash("Start MCP Gateway (MCPG)", script); + + use super::ir::env::EnvValue; + let mut step = ShellScript::new(&START_MCPG) + .text("MCPG_CONTAINER", MCPG_CONTAINER_NAME) + .text("MCPG_IMAGE", &mcpg_image_v) + .bind("MCPG_PORT", Binding::number(MCPG_PORT.into())) + .text("MCPG_DOMAIN", MCPG_DOMAIN) + .fragment("debug_flag", debug_flag) + .fragment("docker_env_lines", docker_env_lines) + .into_step("Start MCP Gateway (MCPG)") + .with_env( + "MCP_GATEWAY_API_KEY", + EnvValue::pipeline_var("MCP_GATEWAY_API_KEY"), + ) + .with_env("ADO_PROXY_IP", EnvValue::pipeline_var("ADO_PROXY_IP")); for (k, v) in parse_env_block(mcpg_step_env)? { step = step.with_env(k, v); } @@ -3879,6 +4090,81 @@ fn awf_exclude_env_flags(exclude_keys: &[String]) -> String { block } +shell_script! { + /// Invoke the AI agent inside AWF's network-isolated Docker topology. + /// + /// This is the workflow's *single* Bash task — the pre-signed engine + /// command must reach `awf` without any wrapper mutating it, so the + /// entire multi-line `awf …` invocation lives in a single block-scalar + /// body. Everything variable is spliced via fragments: + /// - `topology_attach` — one `--topology-attach` line per trusted peer + /// (MCPG always, ado-proxy when the policy engine is enabled) + /// - `image_flags` — `--image-tag` plus optional `--image-registry` + /// - `exclude_env` — one `--exclude-env ` line per BYOM/BYOK secret + /// AWF's api-proxy sidecar strips out of the agent env + /// - `awf_mounts` — the compiler-supplied chain of `--mount "…"` args + /// - `routed_engine_run` — the single-quoted `NO_PROXY` prefix + engine + /// command that AWF invokes inside the sandbox + RUN_AGENT { + interpreter: Bash, + bindings: [AGENT_TEMP, PIPELINE_WORKSPACE, ALLOWED_DOMAINS], + externals: [WORKING_DIRECTORY], + fragments: [topology_attach, image_flags, exclude_env, awf_mounts, routed_engine_run], + body: r###" +set -o pipefail + +AGENT_OUTPUT_FILE="$AGENT_TEMP/staging/logs/agent-output.txt" +mkdir -p "$AGENT_TEMP/staging/logs" +AGENT_EXIT_CODE=0 + +echo "=== Running AI agent with AWF network isolation ===" +echo "Allowed domains: $ALLOWED_DOMAINS" + +# AWF provides L7 domain whitelisting via a rootless Docker topology. +# The named MCPG container is attached to AWF's internal network as a +# trusted endpoint; the agent has no route to the host. +# AWF auto-mounts /tmp:/tmp:rw into the container, so copilot binary, +# agent prompt, and MCP config are placed under /tmp/awf-tools/. +# The argument list is assembled into an array so runtime-supplied +# fragments splice in as ordinary shell statements (`AWF_ARGS+=(...)`) +# — no `\`-continuation chain to break with fragment marker comments. +AWF_ARGS=( + --allow-domains "$ALLOWED_DOMAINS" + --network-isolation +) +# ado-aw:fragment topology_attach +# ado-aw:fragment image_flags +AWF_ARGS+=(--skip-pull --env-all) +# ado-aw:fragment exclude_env +# ado-aw:fragment awf_mounts +AWF_ARGS+=( + --container-workdir "$WORKING_DIRECTORY" + --log-level info + --proxy-logs-dir "$AGENT_TEMP/staging/logs/firewall" +) +# ado-aw:fragment routed_engine_run + +# Stream agent output in real-time while filtering VSO commands. +# sed -u = unbuffered (line-by-line) so output appears immediately. +# tee writes to both stdout (ADO pipeline log) and the artifact file. +# pipefail (set above) ensures AWF's exit code propagates through the pipe. +# shellcheck disable=SC2016 # The single-quoted engine command inside AWF_ARGS is intentionally expanded by AWF inside the sandbox +"$PIPELINE_WORKSPACE/awf/awf" "${AWF_ARGS[@]}" 2>&1 \ + | sed -u 's/##vso\[/[VSO-FILTERED] vso[/g; s/##\[/[VSO-FILTERED] [/g' \ + | tee "$AGENT_OUTPUT_FILE" \ + || AGENT_EXIT_CODE=$? + +# Print firewall summary if available +if [ -x "$PIPELINE_WORKSPACE/awf/awf" ]; then + echo "=== Firewall Summary ===" + "$PIPELINE_WORKSPACE/awf/awf" logs summary --source "$AGENT_TEMP/staging/logs/firewall" 2>/dev/null || true +fi + +exit "$AGENT_EXIT_CODE" +"###, + } +} + #[allow(clippy::too_many_arguments)] fn run_agent_step( allowed_domains: &str, @@ -3890,43 +4176,87 @@ fn run_agent_step( supply_chain: Option<&SupplyChainConfig>, ado_proxy_enabled: bool, ) -> Result { - // The awf_mounts string is a `\`-joined chain of `--mount "..."` lines. - // Render each at 2-space indent inside the bash body (the surrounding - // `--allow-domains` line is at 2-space indent too — the block-scalar - // body indent is set by the first non-empty line). + // The awf_mounts string is a `\`-joined chain of `--mount "..."` lines; + // splice it in at the fragment marker's own indent. let awf_mounts_block: String = if awf_mounts == "\\" { - " \\".to_string() + // "\\" is the sentinel for "no mounts" in the legacy string + // shape; produce an empty append so the array stays unchanged. + String::new() } else { - awf_mounts - .lines() - .map(|l| format!(" {l}")) - .collect::>() - .join("\n") + // The legacy shape is `--mount "..." \` per line. Strip the trailing + // `\`, split on whitespace-separated `--mount` occurrences, and rebuild + // as an `AWF_ARGS+=(...)` statement. + let mut lines: Vec = Vec::new(); + for line in awf_mounts.lines() { + let line = line.trim(); + let line = line.strip_suffix('\\').unwrap_or(line).trim_end(); + if line.is_empty() { + continue; + } + lines.push(line.to_string()); + } + if lines.is_empty() { + String::new() + } else { + // The ADO macro `$(AW_AZ_MOUNTS)` (contributed by the Azure CLI + // extension) is substituted at YAML load time before bash sees it. + // shellcheck cannot see the ADO substitution and mis-reads it as + // bash command substitution word-splitting into the array (SC2207), + // which is precisely what we want here because ADO expands the + // macro to zero or more `--mount ...` tokens. + format!( + "# shellcheck disable=SC2207 # $(AW_AZ_MOUNTS) is an ADO macro substituted at YAML load; word-splitting into the array is intentional.\nAWF_ARGS+=({})", + lines.join(" ") + ) + } }; let image_flags_block = awf_image_flags(supply_chain); let exclude_env_block = awf_exclude_env_flags(byom_exclude_keys); // AWF attaches externally-launched trusted containers to its internal - // network by name. The flag is repeatable (verified against the pinned - // v0.27.32 binary: "Repeatable. Example: --topology-attach mcp-gateway - // --topology-attach difc-proxy"), which is what lets the policy engine - // join alongside MCPG. - // - // Attaching also gives the agent an `/etc/hosts` entry for the container, - // so the `az` wrapper can resolve the engine by name without relying on - // Docker's embedded DNS — which AWF itself works around under gVisor and - // ARC/DinD. + // network by name. The flag is repeatable, which is what lets the policy + // engine join alongside MCPG. Attaching also gives the agent an + // `/etc/hosts` entry for the container, so the `az` wrapper can resolve + // the engine by name without relying on Docker's embedded DNS. let topology_attach_block = { - // The preceding `--network-isolation` continuation supplies the indent - // for the first line; any additional line must carry its own, matching - // `awf_image_flags`. - let mut block = format!("--topology-attach \"{MCPG_CONTAINER_NAME}\" \\\n"); + let mut parts = vec![format!("--topology-attach \"{MCPG_CONTAINER_NAME}\"")]; if ado_proxy_enabled { - block.push_str(&format!( - " --topology-attach \"{ADO_PROXY_CONTAINER_NAME}\" \\\n" - )); + parts.push(format!("--topology-attach \"{ADO_PROXY_CONTAINER_NAME}\"")); + } + format!("AWF_ARGS+=({})", parts.join(" ")) + }; + + // Convert `awf_image_flags`'s legacy ` --flag "..." \\\n` shape into + // `AWF_ARGS+=(--flag "..." ...)`. + let image_flags_line = { + let mut parts: Vec = Vec::new(); + for line in image_flags_block.lines() { + let line = line.trim(); + let line = line.strip_suffix('\\').unwrap_or(line).trim_end(); + if line.is_empty() { + continue; + } + parts.push(line.to_string()); + } + format!("AWF_ARGS+=({})", parts.join(" ")) + }; + + // Same conversion for `awf_exclude_env_flags`. + let exclude_env_line = { + let mut parts: Vec = Vec::new(); + for line in exclude_env_block.lines() { + let line = line.trim(); + let line = line.strip_suffix('\\').unwrap_or(line).trim_end(); + if line.is_empty() { + continue; + } + parts.push(line.to_string()); + } + if parts.is_empty() { + String::new() + } else { + format!("AWF_ARGS+=({})", parts.join(" ")) } - block }; // Trusted peers must bypass Squid: their names are not public DNS, and @@ -3938,55 +4268,23 @@ fn run_agent_step( MCPG_CONTAINER_NAME.to_string() }; let routed_engine_run = format!( - "export NO_PROXY=\"${{NO_PROXY:+$NO_PROXY,}}{no_proxy_peers}\"; \ - export no_proxy=\"$NO_PROXY\"; {engine_run}" + "AWF_ARGS+=(-- 'export NO_PROXY=\"${{NO_PROXY:+$NO_PROXY,}}{no_proxy_peers}\"; \ + export no_proxy=\"$NO_PROXY\"; {engine_run}')" ); - let script = format!( - "set -o pipefail\n\ - \n\ - AGENT_OUTPUT_FILE=\"$(Agent.TempDirectory)/staging/logs/agent-output.txt\"\n\ - mkdir -p \"$(Agent.TempDirectory)/staging/logs\"\n\ - \n\ - echo \"=== Running AI agent with AWF network isolation ===\"\n\ - echo \"Allowed domains: {allowed_domains}\"\n\ - \n\ - # AWF provides L7 domain whitelisting via a rootless Docker topology.\n\ - # The named MCPG container is attached to AWF's internal network as a\n\ - # trusted endpoint; the agent has no route to the host.\n\ - # AWF auto-mounts /tmp:/tmp:rw into the container, so copilot binary,\n\ - # agent prompt, and MCP config are placed under /tmp/awf-tools/.\n\ - # Stream agent output in real-time while filtering VSO commands.\n\ - # sed -u = unbuffered (line-by-line) so output appears immediately.\n\ - # tee writes to both stdout (ADO pipeline log) and the artifact file.\n\ - # pipefail (set above) ensures AWF's exit code propagates through the pipe.\n\ - # shellcheck disable=SC2046,SC2016 # ADO macros are substituted before bash; the single-quoted engine command is intentionally expanded by AWF inside the sandbox\n\ - \"$(Pipeline.Workspace)/awf/awf\" \\\n \ - --allow-domains \"{allowed_domains}\" \\\n \ - --network-isolation \\\n \ -{topology_attach_block}\ -{image_flags_block}\ - --skip-pull \\\n \ - --env-all \\\n \ -{exclude_env_block}\ -{awf_mounts_block}\n \ - --container-workdir \"{working_directory}\" \\\n \ - --log-level info \\\n \ - --proxy-logs-dir \"$(Agent.TempDirectory)/staging/logs/firewall\" \\\n \ - -- '{routed_engine_run}' \\\n \ - 2>&1 \\\n \ - | sed -u 's/##vso\\[/[VSO-FILTERED] vso[/g; s/##\\[/[VSO-FILTERED] [/g' \\\n \ - | tee \"$AGENT_OUTPUT_FILE\" \\\n \ - && AGENT_EXIT_CODE=0 || AGENT_EXIT_CODE=$?\n\ - \n\ - # Print firewall summary if available\n\ - if [ -x \"$(Pipeline.Workspace)/awf/awf\" ]; then\n \ - echo \"=== Firewall Summary ===\"\n \ - \"$(Pipeline.Workspace)/awf/awf\" logs summary --source \"$(Agent.TempDirectory)/staging/logs/firewall\" 2>/dev/null || true\n\ - fi\n\ - \n\ - exit \"$AGENT_EXIT_CODE\"\n" - ); - let mut step = bash("Run copilot (AWF network isolated)", script); + + let mut step = ShellScript::new(&RUN_AGENT) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .text("ALLOWED_DOMAINS", allowed_domains) + .fragment("topology_attach", topology_attach_block) + .fragment("image_flags", image_flags_line) + .fragment("exclude_env", exclude_env_line) + .fragment("awf_mounts", awf_mounts_block) + .fragment("routed_engine_run", routed_engine_run) + .into_step("Run copilot (AWF network isolated)"); step.working_directory = Some(working_directory.to_string()); // Engine env comes as a multi-line YAML env block — `KEY: VALUE` lines // joined by `\n`, no `env:` prefix (it's the value side of an env: mapping). @@ -3998,12 +4296,54 @@ fn run_agent_step( .collect::>() .join("\n") ); + use super::ir::env::EnvValue; + // WORKING_DIRECTORY is passed via env: so ADO substitutes any `$(...)` + // macros in the value before bash sees it. The prelude `Binding::text` + // channel deliberately refuses `$(` to prevent unreviewed macro + // substitutions. + step = step.with_env("WORKING_DIRECTORY", EnvValue::literal(working_directory)); for (k, v) in parse_env_block(&synthetic_block)? { step = step.with_env(k, v); } Ok(step) } +shell_script! { + /// Run `ado-aw execute` (Stage 3). Translates a `SucceededWithIssues` + /// exit code (2) from the executor into an ADO SucceededWithIssues + /// result rather than a hard failure. + /// + /// The path externals are supplied through the step `env:` block so ADO + /// expands their `$(…)` macros before bash sees the value; the compiler + /// itself would refuse to bake an unreviewed `$(` into a binding. + /// + /// `FILTER_ARGS` is intentionally expanded unquoted so an authored value + /// like `--only foo --exclude bar` word-splits into individual flags. + /// The producer restricts each token to the safe-output allow-list + /// vocabulary (`is_safe_tool_name`), so no shell metacharacter can appear. + EXECUTE_SAFE_OUTPUTS { + interpreter: Bash, + bindings: [FILTER_ARGS], + externals: [ + ADO_AW_SOURCE_PATH, + ADO_AW_RESOLVED_CONFIG_PATH, + ADO_AW_SAFE_OUTPUT_DIR, + ADO_AW_OUTPUT_DIR, + ], + fragments: [], + body: r###" +# shellcheck disable=SC2086 # FILTER_ARGS is a compiler-owned run of --only/--exclude flags; unquoted expansion is intentional. +ado-aw execute --source "$ADO_AW_SOURCE_PATH" --resolved-config "$ADO_AW_RESOLVED_CONFIG_PATH" --safe-output-dir "$ADO_AW_SAFE_OUTPUT_DIR" --output-dir "$ADO_AW_OUTPUT_DIR" $FILTER_ARGS +EXIT_CODE=$? +if [ $EXIT_CODE -eq 2 ]; then + echo "##vso[task.complete result=SucceededWithIssues;]Executor completed with warnings" + exit 0 +fi +exit $EXIT_CODE +"###, + } +} + fn execute_safe_outputs_step( source_path: &str, resolved_config_path: &str, @@ -4017,21 +4357,30 @@ fn execute_safe_outputs_step( ) -> Result { // `filter_args` is either empty or a leading-space-prefixed run of // `--only ` / `--exclude ` flags appended to the command. - let script = format!( - "ado-aw execute --source \"{source_path}\" --resolved-config \"{resolved_config_path}\" --safe-output-dir \"$(Pipeline.Workspace)/analyzed_outputs_$(Build.BuildId)\" --output-dir \"$(Agent.TempDirectory)/staging\"{filter_args}\n\ - EXIT_CODE=$?\n\ - if [ $EXIT_CODE -eq 2 ]; then\n \ - echo \"##vso[task.complete result=SucceededWithIssues;]Executor completed with warnings\"\n \ - exit 0\n\ - fi\n\ - exit $EXIT_CODE\n", - ); - let mut step = bash("Execute safe outputs (Stage 3)", script); - step.working_directory = Some(self_repository_directory.to_string()); + let mut script = ShellScript::new(&EXECUTE_SAFE_OUTPUTS) + .text("FILTER_ARGS", filter_args.trim()) + .into_step("Execute safe outputs (Stage 3)"); + script.working_directory = Some(self_repository_directory.to_string()); + // Path externals reach bash through ADO env expansion, which is the + // documented mechanism for values holding predefined macros. + script = script + .with_env("ADO_AW_SOURCE_PATH", EnvValue::literal(source_path)) + .with_env( + "ADO_AW_RESOLVED_CONFIG_PATH", + EnvValue::literal(resolved_config_path), + ) + .with_env( + "ADO_AW_SAFE_OUTPUT_DIR", + EnvValue::literal("$(Pipeline.Workspace)/analyzed_outputs_$(Build.BuildId)"), + ) + .with_env( + "ADO_AW_OUTPUT_DIR", + EnvValue::literal("$(Agent.TempDirectory)/staging"), + ); for (k, v) in parse_env_block(executor_ado_env)? { - step = step.with_env(k, v); + script = script.with_env(k, v); } - step = step.with_env( + script = script.with_env( "ADO_AW_SELF_REPOSITORY_DIRECTORY", // The value embeds `$(Build.SourcesDirectory)`, but it is still a // `Literal`: ADO expands `$(...)` macros in step `env:` values at agent @@ -4041,20 +4390,51 @@ fn execute_safe_outputs_step( // no part of it needs separate lowering. EnvValue::literal(self_repository_directory), ); - step = step.with_env( + script = script.with_env( "ADO_AW_SELF_REPOSITORY_NAME", self_repository_name.clone(), ); - Ok(step) + Ok(script) +} + +shell_script! { + /// Copy staged safe outputs from AWF's `/tmp` mount back into the + /// ADO staging directory for artifact publish. + COLLECT_SAFE_OUTPUTS { + interpreter: Bash, + bindings: [AGENT_TEMP], + externals: [], + fragments: [], + body: r#" +# Copy safe outputs from /tmp back to staging for artifact publish +mkdir -p "$AGENT_TEMP/staging" +cp -r /tmp/awf-tools/staging/* "$AGENT_TEMP/staging/" 2>/dev/null || true +echo "Safe outputs copied to $AGENT_TEMP/staging" +ls -la "$AGENT_TEMP/staging" 2>/dev/null || echo "No safe outputs found" +"#, + } } fn collect_safe_outputs_step() -> BashStep { - let script = "# Copy safe outputs from /tmp back to staging for artifact publish\n\ - mkdir -p \"$(Agent.TempDirectory)/staging\"\n\ - cp -r /tmp/awf-tools/staging/* \"$(Agent.TempDirectory)/staging/\" 2>/dev/null || true\n\ - echo \"Safe outputs copied to $(Agent.TempDirectory)/staging\"\n\ - ls -la \"$(Agent.TempDirectory)/staging\" 2>/dev/null || echo \"No safe outputs found\"\n"; - bash("Collect safe outputs from AWF container", script).with_condition(Condition::Always) + ShellScript::new(&COLLECT_SAFE_OUTPUTS) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .into_step("Collect safe outputs from AWF container") + .with_condition(Condition::Always) +} + +shell_script! { + /// Render the proposed safe outputs to a sanitized markdown file for the + /// build summary tab. Best-effort: a non-zero exit is downgraded to a + /// warning so the summary can never block the review gate. + SAFE_OUTPUTS_SUMMARY { + interpreter: Bash, + bindings: [APPROVAL_SUMMARY_PATH], + externals: [], + fragments: [], + body: r###" +node "$APPROVAL_SUMMARY_PATH" || echo "##vso[task.logissue type=warning]approval-summary step failed (non-fatal)" +"###, + } } /// Render the proposed safe outputs to a sanitized markdown file and attach it @@ -4085,11 +4465,9 @@ fn collect_safe_outputs_step() -> BashStep { fn safe_outputs_summary_step(reviewed: &[String]) -> BashStep { use super::ir::env::EnvValue; let approval_summary_path = super::extensions::ado_script::APPROVAL_SUMMARY_PATH; - let script = format!( - "node '{approval_summary_path}' \ - || echo \"##vso[task.logissue type=warning]approval-summary step failed (non-fatal)\"\n" - ); - bash("Render safe-outputs summary", script) + ShellScript::new(&SAFE_OUTPUTS_SUMMARY) + .text("APPROVAL_SUMMARY_PATH", approval_summary_path) + .into_step("Render safe-outputs summary") .with_env( "AW_SAFE_OUTPUTS_NDJSON", EnvValue::literal("$(Agent.TempDirectory)/staging/safe_outputs.ndjson"), @@ -4102,26 +4480,86 @@ fn safe_outputs_summary_step(reviewed: &[String]) -> BashStep { .with_condition(Condition::Always) } +shell_script! { + /// Prepare the isolated Docker network shared by the proxy and optional + /// MCP. `--internal` is load-bearing, not tidiness. + PREPARE_ADO_PROXY_NETWORK { + interpreter: Bash, + bindings: [PROXY_NETWORK], + externals: [], + fragments: [], + body: r#" +set -euo pipefail + +# Network shared by the policy engine and the Azure DevOps MCP. +# +# `--internal` is load-bearing, not tidiness. A normal user-defined +# bridge has outbound NAT, so the MCP would keep a direct route to the +# internet — including Azure DevOps hosts that are not redirected — +# and the engine would police only the one hostname we happen to +# override. Measured: a container on a normal bridge reaches the +# internet; on an internal bridge it cannot, while still reaching its +# peers. The engine keeps its own egress because AWF dual-homes it +# onto awf-net, where Squid lives. +if ! docker network inspect "$PROXY_NETWORK" >/dev/null 2>&1; then + docker network create --internal "$PROXY_NETWORK" +fi +"#, + } +} + /// Prepare the isolated Docker network shared by the proxy and optional MCP. fn prepare_ado_proxy_network_step() -> BashStep { - let script = format!( - "set -euo pipefail\n\ - \n\ - # Network shared by the policy engine and the Azure DevOps MCP.\n\ - #\n\ - # `--internal` is load-bearing, not tidiness. A normal user-defined\n\ - # bridge has outbound NAT, so the MCP would keep a direct route to the\n\ - # internet — including Azure DevOps hosts that are not redirected —\n\ - # and the engine would police only the one hostname we happen to\n\ - # override. Measured: a container on a normal bridge reaches the\n\ - # internet; on an internal bridge it cannot, while still reaching its\n\ - # peers. The engine keeps its own egress because AWF dual-homes it\n\ - # onto awf-net, where Squid lives.\n\ - if ! docker network inspect {ADO_PROXY_NETWORK_NAME} >/dev/null 2>&1; then\n \ - docker network create --internal {ADO_PROXY_NETWORK_NAME}\n\ - fi\n" - ); - bash("Prepare ado-proxy network", script) + ShellScript::new(&PREPARE_ADO_PROXY_NETWORK) + .text("PROXY_NETWORK", ADO_PROXY_NETWORK_NAME) + .into_step("Prepare ado-proxy network") +} + +shell_script! { + /// Stage the Azure DevOps MCP package on the runner. It is installed on + /// the runner (which has registry access) and mounted read-only into a + /// container that does not. The mount point is load-bearing: Node + /// resolves dependencies by walking upward from the importing file, so + /// the tree must land at `/app/node_modules`. + PREPARE_ADO_MCP { + interpreter: Bash, + bindings: [MCP_HOST_NODE_MODULES, MCP_PACKAGE, MCP_VERSION], + externals: [], + fragments: [], + body: r###" +set -euo pipefail + +# Install the MCP on the runner and stage it for mounting. The +# container it is mounted into can reach nothing but the engine, so it +# cannot fetch this itself. +MCP_STAGE="$(dirname "$MCP_HOST_NODE_MODULES")" +rm -rf "$MCP_STAGE" +mkdir -p "$MCP_STAGE" +cd "$MCP_STAGE" +npm init -y >/dev/null 2>&1 +npm install --omit=dev --no-audit --no-fund --save-exact \ + "$MCP_PACKAGE@$MCP_VERSION" + +# Verify the pin actually took. `npm install` resolves a *range* for +# anything it also has to satisfy transitively, so a matching request +# does not by itself guarantee a matching tree — and the agent's tool +# surface is defined by whatever ends up on disk here. +MCP_INSTALLED=$(node -p \ + "require('$MCP_HOST_NODE_MODULES/$MCP_PACKAGE/package.json').version") +if [ "$MCP_INSTALLED" != "$MCP_VERSION" ]; then + echo "##vso[task.complete result=Failed]Azure DevOps MCP resolved to $MCP_INSTALLED, expected $MCP_VERSION" + exit 1 +fi + +# Fail here rather than at MCP start time, where a missing entry +# script surfaces as an opaque MCPG backend error. +if [ ! -f "$MCP_HOST_NODE_MODULES/$MCP_PACKAGE/dist/index.js" ]; then + echo "##vso[task.complete result=Failed]Azure DevOps MCP package did not install" + exit 1 +fi +echo "Azure DevOps MCP $MCP_INSTALLED staged at $MCP_HOST_NODE_MODULES" +"###, + } } /// Stage the Azure DevOps MCP package only when its tool is enabled. @@ -4131,59 +4569,56 @@ fn prepare_ado_proxy_network_step() -> BashStep { /// Node resolves dependencies by walking upward from the importing file, so /// the tree must land at `/app/node_modules`. fn prepare_ado_mcp_step(version: &str) -> BashStep { - let script = format!( - "set -euo pipefail\n\ - \n\ - # Install the MCP on the runner and stage it for mounting. The\n\ - # container it is mounted into can reach nothing but the engine, so it\n\ - # cannot fetch this itself.\n\ - MCP_STAGE=\"$(dirname {ADO_MCP_HOST_NODE_MODULES})\"\n\ - rm -rf \"$MCP_STAGE\"\n\ - mkdir -p \"$MCP_STAGE\"\n\ - cd \"$MCP_STAGE\"\n\ - npm init -y >/dev/null 2>&1\n\ - npm install --omit=dev --no-audit --no-fund --save-exact \\\n \ - \"{ADO_MCP_PACKAGE}@{version}\"\n\ - \n\ - # Verify the pin actually took. `npm install` resolves a *range* for\n\ - # anything it also has to satisfy transitively, so a matching request\n\ - # does not by itself guarantee a matching tree — and the agent's tool\n\ - # surface is defined by whatever ends up on disk here.\n\ - MCP_INSTALLED=$(node -p \\\n \ - \"require('{ADO_MCP_HOST_NODE_MODULES}/{ADO_MCP_PACKAGE}/package.json').version\")\n\ - if [ \"$MCP_INSTALLED\" != \"{version}\" ]; then\n \ - echo \"##vso[task.complete result=Failed]Azure DevOps MCP resolved to $MCP_INSTALLED, expected {version}\"\n \ - exit 1\n\ - fi\n\ - \n\ - # Fail here rather than at MCP start time, where a missing entry\n\ - # script surfaces as an opaque MCPG backend error.\n\ - if [ ! -f \"{ADO_MCP_HOST_NODE_MODULES}/{ADO_MCP_PACKAGE}/dist/index.js\" ]; then\n \ - echo \"##vso[task.complete result=Failed]Azure DevOps MCP package did not install\"\n \ - exit 1\n\ - fi\n\ - echo \"Azure DevOps MCP $MCP_INSTALLED staged at {ADO_MCP_HOST_NODE_MODULES}\"\n" - ); - bash("Prepare Azure DevOps MCP", script) + ShellScript::new(&PREPARE_ADO_MCP) + .text("MCP_HOST_NODE_MODULES", ADO_MCP_HOST_NODE_MODULES) + .text("MCP_PACKAGE", ADO_MCP_PACKAGE) + .text("MCP_VERSION", version) + .into_step("Prepare Azure DevOps MCP") +} + +shell_script! { + /// Remove the network created for the policy engine and its clients. + TEARDOWN_ADO_PROXY_NETWORK { + interpreter: Bash, + bindings: [PROXY_NETWORK], + externals: [], + fragments: [], + body: r#" +# Remove the policy-engine network once its containers are gone +docker network rm "$PROXY_NETWORK" 2>/dev/null || true +"#, + } } /// Remove the network created for the policy engine and its clients. fn teardown_ado_proxy_network_step() -> BashStep { - let script = format!( - "# Remove the policy-engine network once its containers are gone\n\ - docker network rm {ADO_PROXY_NETWORK_NAME} 2>/dev/null || true\n" - ); - bash("Remove ado-proxy network", script).with_condition(Condition::Always) + ShellScript::new(&TEARDOWN_ADO_PROXY_NETWORK) + .text("PROXY_NETWORK", ADO_PROXY_NETWORK_NAME) + .into_step("Remove ado-proxy network") + .with_condition(Condition::Always) +} + +shell_script! { + /// Stop the MCPG container. + STOP_MCPG { + interpreter: Bash, + bindings: [MCPG_CONTAINER], + externals: [], + fragments: [], + body: r#" +# Stop MCPG container +echo "Stopping MCPG..." +docker stop "$MCPG_CONTAINER" 2>/dev/null || true +echo "MCPG and stdio child containers stopped" +"#, + } } fn stop_mcpg_step() -> BashStep { - let script = format!( - "# Stop MCPG container\n\ - echo \"Stopping MCPG...\"\n\ - docker stop {MCPG_CONTAINER_NAME} 2>/dev/null || true\n\ - echo \"MCPG and stdio child containers stopped\"\n" - ); - bash("Stop MCPG", script).with_condition(Condition::Always) + ShellScript::new(&STOP_MCPG) + .text("MCPG_CONTAINER", MCPG_CONTAINER_NAME) + .into_step("Stop MCPG") + .with_condition(Condition::Always) } /// Start the `ado-proxy` policy engine as a host container. @@ -4827,109 +5262,284 @@ echo "ado-proxy policy and client configuration are ready; runtime denials will } } +shell_script! { + /// Report the overall pipeline conclusion. Best-effort: falls back to a + /// warning when `conclusion.js` (delivered by the ado-script extension) + /// is unavailable, since the Conclusion job is contractually always + /// runs / never fails. + REPORT_CONCLUSION { + interpreter: Bash, + bindings: [CONCLUSION_PATH], + externals: [], + fragments: [], + body: r###" +if command -v node >/dev/null 2>&1 && [ -f "$CONCLUSION_PATH" ]; then + node "$CONCLUSION_PATH" +else + echo "##vso[task.logissue type=warning]conclusion.js unavailable; skipping conclusion reporting" +fi +"###, + } +} + +shell_script! { + /// Copy every engine + ado-aw log to the Detection job's analyzed-outputs + /// artifact directory (each source landing in its own subdirectory). + /// The `engine_log_dir` fragment assigns `ENGINE_LOG_DIR` from a + /// compile-time-constant literal so a shell like `$HOME` re-expands. + COPY_LOGS_DETECTION { + interpreter: Bash, + bindings: [AGENT_TEMP], + externals: [ADO_AW_LOG_DIR, ENGINE_LOG_DIR, HOME], + fragments: [engine_log_dir], + body: r###" +# Copy all logs to analyzed outputs for artifact upload +mkdir -p "$AGENT_TEMP/analyzed_outputs/logs" +# ado-aw:fragment engine_log_dir +if [ -d "$ENGINE_LOG_DIR" ]; then + mkdir -p "$AGENT_TEMP/analyzed_outputs/logs/copilot" + cp -r "$ENGINE_LOG_DIR"/* "$AGENT_TEMP/analyzed_outputs/logs/copilot/" 2>/dev/null || true +fi +ADO_AW_LOG_DIR="${ADO_AW_LOG_DIR:-$HOME/.ado-aw/logs}" +if [ -d "$ADO_AW_LOG_DIR" ]; then + mkdir -p "$AGENT_TEMP/analyzed_outputs/logs/ado-aw" + cp -r "$ADO_AW_LOG_DIR"/* "$AGENT_TEMP/analyzed_outputs/logs/ado-aw/" 2>/dev/null || true +fi +echo "Logs copied to $AGENT_TEMP/analyzed_outputs/logs" +ls -laR "$AGENT_TEMP/analyzed_outputs/logs" 2>/dev/null || echo "No logs found" +"###, + } +} + +shell_script! { + /// Copy every engine + ado-aw + MCPG + ado-proxy log to the Agent job's + /// staging/logs directory (each source landing in its own subdirectory + /// when it exists at runtime). + COPY_LOGS_AGENT { + interpreter: Bash, + bindings: [AGENT_TEMP], + externals: [ADO_AW_LOG_DIR, ENGINE_LOG_DIR, HOME], + fragments: [engine_log_dir], + body: r###" +# Copy all logs to output directory for artifact upload +mkdir -p "$AGENT_TEMP/staging/logs" +# ado-aw:fragment engine_log_dir +if [ -d "$ENGINE_LOG_DIR" ]; then + cp -r "$ENGINE_LOG_DIR"/* "$AGENT_TEMP/staging/logs/" 2>/dev/null || true +fi +ADO_AW_LOG_DIR="${ADO_AW_LOG_DIR:-$HOME/.ado-aw/logs}" +if [ -d "$ADO_AW_LOG_DIR" ]; then + cp -r "$ADO_AW_LOG_DIR"/* "$AGENT_TEMP/staging/logs/" 2>/dev/null || true +fi +if [ -d /tmp/gh-aw/mcp-logs ]; then + mkdir -p "$AGENT_TEMP/staging/logs/mcpg" + cp -r /tmp/gh-aw/mcp-logs/* "$AGENT_TEMP/staging/logs/mcpg/" 2>/dev/null || true +fi +if [ -d /tmp/gh-aw/ado-proxy-logs ]; then + mkdir -p "$AGENT_TEMP/staging/logs/ado-proxy" + cp -r /tmp/gh-aw/ado-proxy-logs/* "$AGENT_TEMP/staging/logs/ado-proxy/" 2>/dev/null || true +fi +echo "Logs copied to $AGENT_TEMP/staging/logs" +ls -la "$AGENT_TEMP/staging/logs" 2>/dev/null || echo "No logs found" +"###, + } +} + fn copy_logs_step(engine_log_dir: &str, is_detection: bool) -> BashStep { + // Fragment content assigns ENGINE_LOG_DIR from a double-quoted literal so + // that a shell variable such as `$HOME` re-expands at runtime — a + // `Binding::text` value is single-quoted and would leave `$HOME` literal. + // The value is a compiler-controlled constant (`Engine::log_dir`), never + // a runtime input, so quoting it verbatim carries no injection risk. + let engine_log_dir_fragment = format!("ENGINE_LOG_DIR=\"{engine_log_dir}\""); if is_detection { - // Detection job copies its logs into analyzed_outputs/logs (the - // artifact published from that job), with per-subdir nesting. - let script = format!( - "# Copy all logs to analyzed outputs for artifact upload\n\ - mkdir -p \"$(Agent.TempDirectory)/analyzed_outputs/logs\"\n\ - if [ -d \"{engine_log_dir}\" ]; then\n \ - mkdir -p \"$(Agent.TempDirectory)/analyzed_outputs/logs/copilot\"\n \ - cp -r \"{engine_log_dir}\"/* \"$(Agent.TempDirectory)/analyzed_outputs/logs/copilot/\" 2>/dev/null || true\n\ - fi\n\ - ADO_AW_LOG_DIR=\"${{ADO_AW_LOG_DIR:-$HOME/.ado-aw/logs}}\"\n\ - if [ -d \"$ADO_AW_LOG_DIR\" ]; then\n \ - mkdir -p \"$(Agent.TempDirectory)/analyzed_outputs/logs/ado-aw\"\n \ - cp -r \"$ADO_AW_LOG_DIR\"/* \"$(Agent.TempDirectory)/analyzed_outputs/logs/ado-aw/\" 2>/dev/null || true\n\ - fi\n\ - echo \"Logs copied to $(Agent.TempDirectory)/analyzed_outputs/logs\"\n\ - ls -laR \"$(Agent.TempDirectory)/analyzed_outputs/logs\" 2>/dev/null || echo \"No logs found\"\n" - ); - return bash("Copy logs to output directory", script).with_condition(Condition::Always); + return ShellScript::new(©_LOGS_DETECTION) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .fragment("engine_log_dir", engine_log_dir_fragment) + .into_step("Copy logs to output directory") + .with_condition(Condition::Always); + } + ShellScript::new(©_LOGS_AGENT) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .fragment("engine_log_dir", engine_log_dir_fragment) + .into_step("Copy logs to output directory") + .with_condition(Condition::Always) +} + +shell_script! { + /// Copy the SafeOutputs job's own logs to `staging/logs/`, plus the + /// Agent job's `agent-output.txt` and the executed-outputs NDJSON so the + /// Conclusion job can read diagnostic signals from the SafeOutputs + /// artifact. + COPY_LOGS_SAFEOUTPUTS { + interpreter: Bash, + bindings: [AGENT_TEMP, PIPELINE_WORKSPACE, BUILD_ID], + externals: [ADO_AW_LOG_DIR, ENGINE_LOG_DIR, HOME], + fragments: [engine_log_dir], + body: r###" +# Copy all logs to output directory for artifact upload +mkdir -p "$AGENT_TEMP/staging/logs" +# Copy agent output log from analyzed_outputs for optimisation use +cp "$PIPELINE_WORKSPACE/analyzed_outputs_$BUILD_ID/logs/agent-output.txt" \ + "$AGENT_TEMP/staging/logs/agent-output.txt" 2>/dev/null || true +# Copy executed NDJSON manifest so the Conclusion job can read diagnostic signals +cp "$PIPELINE_WORKSPACE/analyzed_outputs_$BUILD_ID/safe-outputs-executed.ndjson" \ + "$AGENT_TEMP/staging/safe-outputs-executed.ndjson" 2>/dev/null || true +# ado-aw:fragment engine_log_dir +if [ -d "$ENGINE_LOG_DIR" ]; then + mkdir -p "$AGENT_TEMP/staging/logs/copilot" + cp -r "$ENGINE_LOG_DIR"/* "$AGENT_TEMP/staging/logs/copilot/" 2>/dev/null || true +fi +ADO_AW_LOG_DIR="${ADO_AW_LOG_DIR:-$HOME/.ado-aw/logs}" +if [ -d "$ADO_AW_LOG_DIR" ]; then + mkdir -p "$AGENT_TEMP/staging/logs/ado-aw" + cp -r "$ADO_AW_LOG_DIR"/* "$AGENT_TEMP/staging/logs/ado-aw/" 2>/dev/null || true +fi +echo "Logs copied to $AGENT_TEMP/staging/logs" +ls -laR "$AGENT_TEMP/staging/logs" 2>/dev/null || echo "No logs found" +"###, } - let script = format!( - "# Copy all logs to output directory for artifact upload\n\ - mkdir -p \"$(Agent.TempDirectory)/staging/logs\"\n\ - if [ -d \"{engine_log_dir}\" ]; then\n \ - cp -r \"{engine_log_dir}\"/* \"$(Agent.TempDirectory)/staging/logs/\" 2>/dev/null || true\n\ - fi\n\ - ADO_AW_LOG_DIR=\"${{ADO_AW_LOG_DIR:-$HOME/.ado-aw/logs}}\"\n\ - if [ -d \"$ADO_AW_LOG_DIR\" ]; then\n \ - cp -r \"$ADO_AW_LOG_DIR\"/* \"$(Agent.TempDirectory)/staging/logs/\" 2>/dev/null || true\n\ - fi\n\ - if [ -d /tmp/gh-aw/mcp-logs ]; then\n \ - mkdir -p \"$(Agent.TempDirectory)/staging/logs/mcpg\"\n \ - cp -r /tmp/gh-aw/mcp-logs/* \"$(Agent.TempDirectory)/staging/logs/mcpg/\" 2>/dev/null || true\n\ - fi\n\ - if [ -d /tmp/gh-aw/ado-proxy-logs ]; then\n \ - mkdir -p \"$(Agent.TempDirectory)/staging/logs/ado-proxy\"\n \ - cp -r /tmp/gh-aw/ado-proxy-logs/* \"$(Agent.TempDirectory)/staging/logs/ado-proxy/\" 2>/dev/null || true\n \ - fi\n\ - echo \"Logs copied to $(Agent.TempDirectory)/staging/logs\"\n\ - ls -la \"$(Agent.TempDirectory)/staging/logs\" 2>/dev/null || echo \"No logs found\"\n" - ); - bash("Copy logs to output directory", script).with_condition(Condition::Always) } fn copy_logs_safeoutputs_step(engine_log_dir: &str) -> BashStep { - let script = format!( - "# Copy all logs to output directory for artifact upload\n\ - mkdir -p \"$(Agent.TempDirectory)/staging/logs\"\n\ - # Copy agent output log from analyzed_outputs for optimisation use\n\ - cp \"$(Pipeline.Workspace)/analyzed_outputs_$(Build.BuildId)/logs/agent-output.txt\" \\\n \ - \"$(Agent.TempDirectory)/staging/logs/agent-output.txt\" 2>/dev/null || true\n\ - # Copy executed NDJSON manifest so the Conclusion job can read diagnostic signals\n\ - cp \"$(Pipeline.Workspace)/analyzed_outputs_$(Build.BuildId)/safe-outputs-executed.ndjson\" \\\n \ - \"$(Agent.TempDirectory)/staging/safe-outputs-executed.ndjson\" 2>/dev/null || true\n\ - if [ -d \"{engine_log_dir}\" ]; then\n \ - mkdir -p \"$(Agent.TempDirectory)/staging/logs/copilot\"\n \ - cp -r \"{engine_log_dir}\"/* \"$(Agent.TempDirectory)/staging/logs/copilot/\" 2>/dev/null || true\n\ - fi\n\ - ADO_AW_LOG_DIR=\"${{ADO_AW_LOG_DIR:-$HOME/.ado-aw/logs}}\"\n\ - if [ -d \"$ADO_AW_LOG_DIR\" ]; then\n \ - mkdir -p \"$(Agent.TempDirectory)/staging/logs/ado-aw\"\n \ - cp -r \"$ADO_AW_LOG_DIR\"/* \"$(Agent.TempDirectory)/staging/logs/ado-aw/\" 2>/dev/null || true\n\ - fi\n\ - echo \"Logs copied to $(Agent.TempDirectory)/staging/logs\"\n\ - ls -laR \"$(Agent.TempDirectory)/staging/logs\" 2>/dev/null || echo \"No logs found\"\n" - ); - bash("Copy logs to output directory", script).with_condition(Condition::Always) + let engine_log_dir_fragment = format!("ENGINE_LOG_DIR=\"{engine_log_dir}\""); + ShellScript::new(©_LOGS_SAFEOUTPUTS) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .bind("BUILD_ID", Binding::ado_macro("Build.BuildId")) + .fragment("engine_log_dir", engine_log_dir_fragment) + .into_step("Copy logs to output directory") + .with_condition(Condition::Always) +} + +shell_script! { + /// Copy the Agent job's proposed safe outputs into the Detection job's + /// working directory for analysis. + PREPARE_SAFE_OUTPUTS_FOR_ANALYSIS { + interpreter: Bash, + bindings: [PIPELINE_WORKSPACE, BUILD_ID], + externals: [WORKING_DIRECTORY], + fragments: [], + body: r#" +mkdir -p "$WORKING_DIRECTORY/safe_outputs" +cp -a "$PIPELINE_WORKSPACE/agent_outputs_$BUILD_ID/." "$WORKING_DIRECTORY/safe_outputs" +"#, + } } fn prepare_safe_outputs_for_analysis(working_directory: &str) -> BashStep { - let script = format!( - "mkdir -p \"{working_directory}/safe_outputs\"\n\ - cp -a \"$(Pipeline.Workspace)/agent_outputs_$(Build.BuildId)/.\" \"{working_directory}/safe_outputs\"\n" - ); - bash("Prepare safe outputs for analysis", script) + use super::ir::env::EnvValue; + ShellScript::new(&PREPARE_SAFE_OUTPUTS_FOR_ANALYSIS) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .bind("BUILD_ID", Binding::ado_macro("Build.BuildId")) + .into_step("Prepare safe outputs for analysis") + .with_env("WORKING_DIRECTORY", EnvValue::literal(working_directory)) +} + +shell_script! { + /// Write the threat-analysis prompt to + /// `/tmp/awf-tools/threat-analysis-prompt.md`. The `heredoc` fragment + /// carries a per-content SHA-derived sentinel — the same mitigation + /// used in [`PREPARE_AGENT_PROMPT`] — so a malicious front-matter + /// `description:` (which lands inside this prompt body) cannot + /// terminate the heredoc early and inject commands into the Detection + /// job. + PREPARE_THREAT_ANALYSIS_PROMPT { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [heredoc], + body: r###" +# Write threat analysis prompt to /tmp (accessible inside AWF container) +# ado-aw:fragment heredoc + +echo "Threat analysis prompt:" +cat "/tmp/awf-tools/threat-analysis-prompt.md" +"###, + } } fn prepare_threat_analysis_prompt_step(threat_prompt: &str) -> Result { - // Same heredoc-injection mitigation as `prepare_agent_prompt_step`: - // the sentinel is SHA-derived per content so a malicious - // front-matter `description:` (which lands inside this prompt - // body) cannot terminate the heredoc early and inject commands - // into the Detection job. let sentinel = super::common::heredoc_sentinel("THREAT_ANALYSIS_EOF", threat_prompt)?; - let template = format!( - "\ - # Write threat analysis prompt to /tmp (accessible inside AWF container)\n\ - cat > \"/tmp/awf-tools/threat-analysis-prompt.md\" << '{sentinel}'\n\ - {{INTERP}}\n\ - {sentinel}\n\ - \n\ - echo \"Threat analysis prompt:\"\n\ - cat \"/tmp/awf-tools/threat-analysis-prompt.md\"\n" + let heredoc = format!( + "cat > \"/tmp/awf-tools/threat-analysis-prompt.md\" << '{sentinel}'\n\ + {threat_prompt}\n\ + {sentinel}" ); - let script = dedent(&template).replace("{INTERP}", threat_prompt); - Ok(bash("Prepare threat analysis prompt", script)) + Ok(ShellScript::new(&PREPARE_THREAT_ANALYSIS_PROMPT) + .fragment("heredoc", heredoc) + .into_step("Prepare threat analysis prompt")) +} + +shell_script! { + /// Ensure the downloaded compiler is executable at the well-known path. + SETUP_COMPILER { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [], + body: r###" +AGENTIC_PIPELINES_PATH="$(Pipeline.Workspace)/agentic-pipeline-compiler/ado-aw" +chmod +x "$AGENTIC_PIPELINES_PATH" +"###, + } } fn setup_compiler_step() -> BashStep { - let script = "AGENTIC_PIPELINES_PATH=\"$(Pipeline.Workspace)/agentic-pipeline-compiler/ado-aw\"\n\ - chmod +x \"$AGENTIC_PIPELINES_PATH\"\n"; - bash("Setup agentic pipeline compiler", script) + ShellScript::new(&SETUP_COMPILER).into_step("Setup agentic pipeline compiler") +} + +shell_script! { + /// Invoke the Detection stage's threat-analysis agent inside AWF's + /// network-isolated Docker topology. Structured like [`RUN_AGENT`] but + /// without a topology-attach block (Detection has no MCPG/ado-proxy + /// peers) and with the single-quoted engine command passed through + /// verbatim. + RUN_THREAT_ANALYSIS { + interpreter: Bash, + bindings: [AGENT_TEMP, PIPELINE_WORKSPACE, ALLOWED_DOMAINS], + externals: [WORKING_DIRECTORY], + fragments: [image_flags, exclude_env, engine_run_detection], + body: r###" +set -o pipefail + +# Run threat analysis with AWF network isolation +THREAT_OUTPUT_FILE="$AGENT_TEMP/threat-analysis-output.txt" +AGENT_EXIT_CODE=0 + +# The argument list is assembled into an array so runtime-supplied +# fragments splice in as ordinary shell statements (`AWF_ARGS+=(...)`) +# — no `\`-continuation chain to break with fragment marker comments. +AWF_ARGS=( + --allow-domains "$ALLOWED_DOMAINS" + --network-isolation +) +# ado-aw:fragment image_flags +AWF_ARGS+=(--skip-pull --env-all) +# ado-aw:fragment exclude_env +AWF_ARGS+=( + --container-workdir "$WORKING_DIRECTORY" + --log-level info + --proxy-logs-dir "$AGENT_TEMP/threat-analysis-logs/firewall" +) +# ado-aw:fragment engine_run_detection + +# Stream threat analysis output in real-time with VSO command filtering +# shellcheck disable=SC2016 # The single-quoted engine command inside AWF_ARGS is intentionally expanded by AWF inside the sandbox +"$PIPELINE_WORKSPACE/awf/awf" "${AWF_ARGS[@]}" 2>&1 \ + | sed -u 's/##vso\[/[VSO-FILTERED] vso[/g; s/##\[/[VSO-FILTERED] [/g' \ + | tee "$THREAT_OUTPUT_FILE" \ + || AGENT_EXIT_CODE=$? + +exit "$AGENT_EXIT_CODE" +"###, + } } fn run_threat_analysis_step( @@ -4943,39 +5553,56 @@ fn run_threat_analysis_step( ) -> Result { let image_flags_block = awf_image_flags(supply_chain); let exclude_env_block = awf_exclude_env_flags(byom_exclude_keys); - let script = format!( - "set -o pipefail\n\ - \n\ - # Run threat analysis with AWF network isolation\n\ - THREAT_OUTPUT_FILE=\"$(Agent.TempDirectory)/threat-analysis-output.txt\"\n\ - \n\ - # Stream threat analysis output in real-time with VSO command filtering\n\ - # shellcheck disable=SC2016 # The single-quoted engine command is intentionally expanded by AWF inside the sandbox\n\ - \"$(Pipeline.Workspace)/awf/awf\" \\\n \ - --allow-domains \"{allowed_domains}\" \\\n \ - --network-isolation \\\n\ -{image_flags_block}\ - --skip-pull \\\n \ - --env-all \\\n\ -{exclude_env_block} \ - --container-workdir \"{working_directory}\" \\\n \ - --log-level info \\\n \ - --proxy-logs-dir \"$(Agent.TempDirectory)/threat-analysis-logs/firewall\" \\\n \ - -- '{engine_run_detection}' \\\n \ - 2>&1 \\\n \ - | sed -u 's/##vso\\[/[VSO-FILTERED] vso[/g; s/##\\[/[VSO-FILTERED] [/g' \\\n \ - | tee \"$THREAT_OUTPUT_FILE\" \\\n \ - && AGENT_EXIT_CODE=0 || AGENT_EXIT_CODE=$?\n\ - \n\ - exit \"$AGENT_EXIT_CODE\"\n" - ); - let mut step = bash("Run threat analysis (AWF network isolated)", script); + let image_flags_line = { + let mut parts: Vec = Vec::new(); + for line in image_flags_block.lines() { + let line = line.trim(); + let line = line.strip_suffix('\\').unwrap_or(line).trim_end(); + if line.is_empty() { + continue; + } + parts.push(line.to_string()); + } + format!("AWF_ARGS+=({})", parts.join(" ")) + }; + let exclude_env_line = { + let mut parts: Vec = Vec::new(); + for line in exclude_env_block.lines() { + let line = line.trim(); + let line = line.strip_suffix('\\').unwrap_or(line).trim_end(); + if line.is_empty() { + continue; + } + parts.push(line.to_string()); + } + if parts.is_empty() { + String::new() + } else { + format!("AWF_ARGS+=({})", parts.join(" ")) + } + }; + let engine_run_detection_line = format!("AWF_ARGS+=(-- '{engine_run_detection}')"); + + let mut step = ShellScript::new(&RUN_THREAT_ANALYSIS) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .text("ALLOWED_DOMAINS", allowed_domains) + .fragment("image_flags", image_flags_line) + .fragment("exclude_env", exclude_env_line) + .fragment("engine_run_detection", engine_run_detection_line) + .into_step("Run threat analysis (AWF network isolated)"); step.working_directory = Some(working_directory.to_string()); // env block: GITHUB_TOKEN + GITHUB_READ_ONLY — emit the latter as // a typed YAML integer so it round-trips unquoted (matching the // legacy copilot_env output of `GITHUB_READ_ONLY: 1`, not `'1'`). + // WORKING_DIRECTORY is passed via env: so ADO substitutes any `$(...)` + // macros in the value before bash sees it. use super::ir::env::EnvValue; step = step + .with_env("WORKING_DIRECTORY", EnvValue::literal(working_directory)) .with_env("GITHUB_TOKEN", EnvValue::pipeline_var(github_token_var)) .with_env( "GITHUB_READ_ONLY", @@ -4989,53 +5616,111 @@ fn run_threat_analysis_step( Ok(step) } +shell_script! { + /// Detection job: copy the original Agent proposal payload into + /// `analyzed_outputs/`, then extract the JSON verdict from the + /// `THREAT_DETECTION_RESULT:` line printed by the threat-analysis + /// engine run. + PREPARE_ANALYZED_OUTPUTS { + interpreter: Bash, + bindings: [AGENT_TEMP, PIPELINE_WORKSPACE, BUILD_ID], + externals: [], + fragments: [], + body: r###" +# Create analyzed outputs directory with original safe outputs and analysis +mkdir -p "$AGENT_TEMP/analyzed_outputs" + +# Copy original safe outputs +cp -a "$PIPELINE_WORKSPACE/agent_outputs_$BUILD_ID/." "$AGENT_TEMP/analyzed_outputs/" + +# Copy threat analysis output +if [ -f "$AGENT_TEMP/threat-analysis-output.txt" ]; then + cp "$AGENT_TEMP/threat-analysis-output.txt" "$AGENT_TEMP/analyzed_outputs/" +fi + +# Extract JSON from THREAT_DETECTION_RESULT line in threat analysis output +if [ -f "$AGENT_TEMP/threat-analysis-output.txt" ]; then + RESULT_LINE=$(grep "THREAT_DETECTION_RESULT:" "$AGENT_TEMP/threat-analysis-output.txt" | tail -1) + if [ -n "$RESULT_LINE" ]; then + # Extract JSON after the prefix + JSON_CONTENT="${RESULT_LINE##*THREAT_DETECTION_RESULT:}" + echo "$JSON_CONTENT" > "$AGENT_TEMP/analyzed_outputs/threat-analysis.json" + echo "Extracted threat analysis JSON:" + cat "$AGENT_TEMP/analyzed_outputs/threat-analysis.json" + else + echo "Warning: No THREAT_DETECTION_RESULT found in threat analysis output" + fi +else + echo "Warning: No threat analysis output file found" +fi + +echo "Analyzed outputs directory contents:" +ls -laR "$AGENT_TEMP/analyzed_outputs" +"###, + } +} + fn prepare_analyzed_outputs_step() -> BashStep { - let script = "# Create analyzed outputs directory with original safe outputs and analysis\n\ - mkdir -p \"$(Agent.TempDirectory)/analyzed_outputs\"\n\ - \n\ - # Copy original safe outputs\n\ - cp -a \"$(Pipeline.Workspace)/agent_outputs_$(Build.BuildId)/.\" \"$(Agent.TempDirectory)/analyzed_outputs/\"\n\ - \n\ - # Copy threat analysis output\n\ - if [ -f \"$(Agent.TempDirectory)/threat-analysis-output.txt\" ]; then\n \ - cp \"$(Agent.TempDirectory)/threat-analysis-output.txt\" \"$(Agent.TempDirectory)/analyzed_outputs/\"\n\ - fi\n\ - \n\ - # Extract JSON from THREAT_DETECTION_RESULT line in threat analysis output\n\ - if [ -f \"$(Agent.TempDirectory)/threat-analysis-output.txt\" ]; then\n \ - RESULT_LINE=$(grep \"THREAT_DETECTION_RESULT:\" \"$(Agent.TempDirectory)/threat-analysis-output.txt\" | tail -1)\n \ - if [ -n \"$RESULT_LINE\" ]; then\n \ - # Extract JSON after the prefix\n \ - JSON_CONTENT=\"${RESULT_LINE##*THREAT_DETECTION_RESULT:}\"\n \ - echo \"$JSON_CONTENT\" > \"$(Agent.TempDirectory)/analyzed_outputs/threat-analysis.json\"\n \ - echo \"Extracted threat analysis JSON:\"\n \ - cat \"$(Agent.TempDirectory)/analyzed_outputs/threat-analysis.json\"\n \ - else\n \ - echo \"Warning: No THREAT_DETECTION_RESULT found in threat analysis output\"\n \ - fi\n\ - else\n \ - echo \"Warning: No threat analysis output file found\"\n\ - fi\n\ - \n\ - echo \"Analyzed outputs directory contents:\"\n\ - ls -laR \"$(Agent.TempDirectory)/analyzed_outputs\"\n"; - bash("Prepare analyzed outputs", script).with_condition(Condition::Always) + ShellScript::new(&PREPARE_ANALYZED_OUTPUTS) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .bind("BUILD_ID", Binding::ado_macro("Build.BuildId")) + .into_step("Prepare analyzed outputs") + .with_condition(Condition::Always) +} + +shell_script! { + /// Detection job (AI threat detection disabled): copy Agent proposals to + /// `analyzed_outputs/` unchanged. The Detection stage still runs as a + /// pipeline boundary even when analysis is skipped. + PREPARE_ANALYZED_OUTPUTS_PASSTHROUGH { + interpreter: Bash, + bindings: [AGENT_TEMP, PIPELINE_WORKSPACE, BUILD_ID], + externals: [], + fragments: [], + body: r###" +set -eo pipefail +mkdir -p "$AGENT_TEMP/analyzed_outputs" +cp -a "$PIPELINE_WORKSPACE/agent_outputs_$BUILD_ID/." "$AGENT_TEMP/analyzed_outputs/" +echo "AI threat detection is disabled; copied Agent outputs unchanged." +"###, + } } fn prepare_analyzed_outputs_passthrough_step() -> BashStep { - let script = "set -eo pipefail\n\ - mkdir -p \"$(Agent.TempDirectory)/analyzed_outputs\"\n\ - cp -a \"$(Pipeline.Workspace)/agent_outputs_$(Build.BuildId)/.\" \ - \"$(Agent.TempDirectory)/analyzed_outputs/\"\n\ - echo \"AI threat detection is disabled; copied Agent outputs unchanged.\"\n"; - bash("Prepare analyzed outputs (detection disabled)", script) + ShellScript::new(&PREPARE_ANALYZED_OUTPUTS_PASSTHROUGH) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .bind( + "PIPELINE_WORKSPACE", + Binding::ado_macro("Pipeline.Workspace"), + ) + .bind("BUILD_ID", Binding::ado_macro("Build.BuildId")) + .into_step("Prepare analyzed outputs (detection disabled)") +} + +shell_script! { + /// Detection-disabled short-circuit: publish `SafeToProcess=true` so + /// downstream jobs consuming this output variable behave as if analysis + /// had run and passed. + THREAT_ANALYSIS_DISABLED { + interpreter: Bash, + bindings: [], + externals: [], + fragments: [], + body: r###" +echo "AI threat detection was explicitly disabled by workflow configuration." +echo "##vso[task.setvariable variable=SafeToProcess;isOutput=true]true" +echo "SafeToProcess set to: true" +"###, + } } fn threat_analysis_disabled_step() -> BashStep { - let script = "echo \"AI threat detection was explicitly disabled by workflow configuration.\"\n\ - echo \"##vso[task.setvariable variable=SafeToProcess;isOutput=true]true\"\n\ - echo \"SafeToProcess set to: true\"\n"; - bash("Bypass AI threat analysis", script) + ShellScript::new(&THREAT_ANALYSIS_DISABLED) + .into_step("Bypass AI threat analysis") .with_id( StepId::new("threatAnalysis") .expect("threatAnalysis is a valid StepId — see StepId::new contract"), @@ -5043,32 +5728,50 @@ fn threat_analysis_disabled_step() -> BashStep { .with_output(OutputDecl::new("SafeToProcess")) } +shell_script! { + /// Detection stage: read the JSON verdict extracted by + /// [`PREPARE_ANALYZED_OUTPUTS`] and publish a `SafeToProcess` output + /// variable driving the SafeOutputs job's `condition:`. Defaults to + /// `false` (unsafe) on any parse/read failure so the pipeline + /// fails safe. + EVALUATE_THREAT_ANALYSIS { + interpreter: Bash, + bindings: [AGENT_TEMP], + externals: [], + fragments: [], + body: r###" +SAFE_TO_PROCESS="false" +JSON_FILE="$AGENT_TEMP/analyzed_outputs/threat-analysis.json" + +if [ -f "$JSON_FILE" ]; then + if jq -e . "$JSON_FILE" > /dev/null 2>&1; then + echo "JSON is valid" + + # Check if any threat field is true + if jq -e '.prompt_injection or .secret_leak or .malicious_patch' "$JSON_FILE" > /dev/null 2>&1; then + echo "##vso[task.logissue type=warning]Threats detected - safe outputs will NOT be processed" + jq -r '.reasons[]? // empty' "$JSON_FILE" | sed 's/^/ - /' + else + echo "No threats detected - safe outputs will be processed" + SAFE_TO_PROCESS="true" + fi + else + echo "##vso[task.logissue type=warning]Invalid JSON in threat analysis - defaulting to unsafe" + fi +else + echo "##vso[task.logissue type=warning]No threat analysis JSON found - defaulting to unsafe" +fi + +echo "##vso[task.setvariable variable=SafeToProcess;isOutput=true]$SAFE_TO_PROCESS" +echo "SafeToProcess set to: $SAFE_TO_PROCESS" +"###, + } +} + fn evaluate_threat_analysis_step() -> BashStep { - let script = "SAFE_TO_PROCESS=\"false\"\n\ - JSON_FILE=\"$(Agent.TempDirectory)/analyzed_outputs/threat-analysis.json\"\n\ - \n\ - if [ -f \"$JSON_FILE\" ]; then\n \ - if jq -e . \"$JSON_FILE\" > /dev/null 2>&1; then\n \ - echo \"JSON is valid\"\n \ - \n \ - # Check if any threat field is true\n \ - if jq -e '.prompt_injection or .secret_leak or .malicious_patch' \"$JSON_FILE\" > /dev/null 2>&1; then\n \ - echo \"##vso[task.logissue type=warning]Threats detected - safe outputs will NOT be processed\"\n \ - jq -r '.reasons[]? // empty' \"$JSON_FILE\" | sed 's/^/ - /'\n \ - else\n \ - echo \"No threats detected - safe outputs will be processed\"\n \ - SAFE_TO_PROCESS=\"true\"\n \ - fi\n \ - else\n \ - echo \"##vso[task.logissue type=warning]Invalid JSON in threat analysis - defaulting to unsafe\"\n \ - fi\n\ - else\n \ - echo \"##vso[task.logissue type=warning]No threat analysis JSON found - defaulting to unsafe\"\n\ - fi\n\ - \n\ - echo \"##vso[task.setvariable variable=SafeToProcess;isOutput=true]$SAFE_TO_PROCESS\"\n\ - echo \"SafeToProcess set to: $SAFE_TO_PROCESS\"\n"; - bash("Evaluate threat analysis", script) + ShellScript::new(&EVALUATE_THREAT_ANALYSIS) + .bind("AGENT_TEMP", Binding::ado_macro("Agent.TempDirectory")) + .into_step("Evaluate threat analysis") .with_id( StepId::new("threatAnalysis") .expect("threatAnalysis is a valid StepId — see StepId::new contract"), @@ -5077,76 +5780,109 @@ fn evaluate_threat_analysis_step() -> BashStep { .with_condition(Condition::Always) } -/// Scan the agent's proposed safe-output NDJSON for any approval-gated tool -/// and publish a `HasReviewedProposals` output variable. The ManualReview gate -/// is conditioned on this so a run never pauses for a human when the agent did -/// not propose anything that requires review. +shell_script! { + /// Scan the agent's proposed safe-output NDJSON for any approval-gated + /// tool and publish a `HasReviewedProposals` output variable. The + /// ManualReview gate is conditioned on this so a run never pauses for a + /// human when the agent did not propose anything that requires review. + DETECT_REVIEWED_PROPOSALS { + interpreter: Bash, + bindings: [ALTERNATION], + externals: [WORKING_DIRECTORY], + fragments: [], + body: r###" +HAS_REVIEWED="false" +NAMES="" +PROPOSALS=$(find "$WORKING_DIRECTORY/safe_outputs" -name "safe_outputs.ndjson" 2>/dev/null | head -n 1) +if [ -n "$PROPOSALS" ] && [ -f "$PROPOSALS" ]; then + if command -v jq >/dev/null 2>&1; then + # Match only the top-level "name" of each NDJSON object so a + # "name" key nested inside a tool's params can't false-positive. + if NAMES=$(jq -r 'select(type=="object") | .name // empty' "$PROPOSALS" 2>/dev/null); then + if printf '%s\n' "$NAMES" | grep -Eqx "($ALTERNATION)"; then + HAS_REVIEWED="true" + fi + else + # jq failed (e.g. corrupt/truncated proposals). Fall back to the + # broad raw scan so detection fails safe (over-match, never under- + # match) and record that detection was inconclusive. + echo "##vso[task.logissue type=warning]approval-gate: jq failed to parse $PROPOSALS; using raw scan for reviewed-proposal detection" + if grep -Eq "\"name\"[[:space:]]*:[[:space:]]*\"($ALTERNATION)\"" "$PROPOSALS"; then + HAS_REVIEWED="true" + fi + fi + elif grep -Eq "\"name\"[[:space:]]*:[[:space:]]*\"($ALTERNATION)\"" "$PROPOSALS"; then + # jq unavailable: fall back to a broad scan. May over-match (pause + # unnecessarily) but never under-matches, so the gate stays fail-safe. + HAS_REVIEWED="true" + fi +fi +echo "##vso[task.setvariable variable=HasReviewedProposals;isOutput=true]$HAS_REVIEWED" +echo "HasReviewedProposals set to: $HAS_REVIEWED" +"###, + } +} + fn detect_reviewed_proposals_step(working_directory: &str, reviewed: &[String]) -> BashStep { + use super::ir::env::EnvValue; // `reviewed` are compiler-controlled safe-output names (ASCII // alphanumeric/hyphen only — see `validate::is_safe_tool_name`), so they // are safe to embed directly in a jq/grep alternation. let alternation = reviewed.join("|"); - let script = format!( - "HAS_REVIEWED=\"false\"\n\ - PROPOSALS=$(find \"{working_directory}/safe_outputs\" -name \"safe_outputs.ndjson\" 2>/dev/null | head -n 1)\n\ - if [ -n \"$PROPOSALS\" ] && [ -f \"$PROPOSALS\" ]; then\n \ - if command -v jq >/dev/null 2>&1; then\n \ - # Match only the top-level \"name\" of each NDJSON object so a\n \ - # \"name\" key nested inside a tool's params can't false-positive.\n \ - if NAMES=$(jq -r 'select(type==\"object\") | .name // empty' \"$PROPOSALS\" 2>/dev/null); then\n \ - if printf '%s\\n' \"$NAMES\" | grep -Eqx '({alternation})'; then\n \ - HAS_REVIEWED=\"true\"\n \ - fi\n \ - else\n \ - # jq failed (e.g. corrupt/truncated proposals). Fall back to the\n \ - # broad raw scan so detection fails safe (over-match, never under-\n \ - # match) and record that detection was inconclusive.\n \ - echo \"##vso[task.logissue type=warning]approval-gate: jq failed to parse $PROPOSALS; using raw scan for reviewed-proposal detection\"\n \ - if grep -Eq '\"name\"[[:space:]]*:[[:space:]]*\"({alternation})\"' \"$PROPOSALS\"; then\n \ - HAS_REVIEWED=\"true\"\n \ - fi\n \ - fi\n \ - elif grep -Eq '\"name\"[[:space:]]*:[[:space:]]*\"({alternation})\"' \"$PROPOSALS\"; then\n \ - # jq unavailable: fall back to a broad scan. May over-match (pause\n \ - # unnecessarily) but never under-matches, so the gate stays fail-safe.\n \ - HAS_REVIEWED=\"true\"\n \ - fi\n\ - fi\n\ - echo \"##vso[task.setvariable variable=HasReviewedProposals;isOutput=true]$HAS_REVIEWED\"\n\ - echo \"HasReviewedProposals set to: $HAS_REVIEWED\"\n" - ); - bash("Detect reviewed proposals", script) + ShellScript::new(&DETECT_REVIEWED_PROPOSALS) + .text("ALTERNATION", alternation) + .into_step("Detect reviewed proposals") .with_id( StepId::new("reviewedProposals") .expect("reviewedProposals is a valid StepId — see StepId::new contract"), ) .with_output(OutputDecl::new("HasReviewedProposals")) .with_condition(Condition::Always) + .with_env("WORKING_DIRECTORY", EnvValue::literal(working_directory)) +} + +shell_script! { + /// Scan the analyzed proposal NDJSON once and publish one output variable + /// per custom tool. The `tool_checks` fragment is populated by + /// [`detect_custom_proposals_step`] with one block of shell per registered + /// custom tool. Custom executor jobs use these booleans in their job-level + /// `condition:` so an empty/no-op custom proposal set does not start a + /// job. + DETECT_CUSTOM_PROPOSALS { + interpreter: Bash, + bindings: [], + externals: [WORKING_DIRECTORY], + fragments: [tool_checks], + body: r###" +PROPOSALS=$(find "$WORKING_DIRECTORY/safe_outputs" -name "safe_outputs.ndjson" 2>/dev/null | head -n 1) +NAMES="" +RAW_SCAN="false" +if [ -n "$PROPOSALS" ] && [ -f "$PROPOSALS" ]; then + if command -v jq >/dev/null 2>&1; then + if ! NAMES=$(jq -r 'select(type=="object") | .name // empty' "$PROPOSALS" 2>/dev/null); then + echo "##vso[task.logissue type=warning]custom-proposals: jq failed to parse $PROPOSALS; using raw scan" + RAW_SCAN="true" + fi + else + RAW_SCAN="true" + fi +fi +# Fake use so shellcheck (which cannot see the compiler-spliced +# tool_checks fragment) does not flag NAMES / RAW_SCAN as SC2034 +# unused. This is a runtime no-op — `:` discards its arguments. +: "${NAMES}" "${RAW_SCAN}" +# ado-aw:fragment tool_checks +"###, + } } -/// Scan the analyzed proposal NDJSON once and publish one output variable per -/// custom tool. Custom executor jobs use these booleans in their job-level -/// `condition:` so an empty/no-op custom proposal set does not start a job. fn detect_custom_proposals_step(working_directory: &str, tools: &[String]) -> Result { - let mut script = format!( - "PROPOSALS=$(find \"{working_directory}/safe_outputs\" -name \"safe_outputs.ndjson\" 2>/dev/null | head -n 1)\n\ - NAMES=\"\"\n\ - RAW_SCAN=\"false\"\n\ - if [ -n \"$PROPOSALS\" ] && [ -f \"$PROPOSALS\" ]; then\n \ - if command -v jq >/dev/null 2>&1; then\n \ - if ! NAMES=$(jq -r 'select(type==\"object\") | .name // empty' \"$PROPOSALS\" 2>/dev/null); then\n \ - echo \"##vso[task.logissue type=warning]custom-proposals: jq failed to parse $PROPOSALS; using raw scan\"\n \ - RAW_SCAN=\"true\"\n \ - fi\n \ - else\n \ - RAW_SCAN=\"true\"\n \ - fi\n\ - fi\n" - ); - let mut step = bash("Detect custom proposals", ""); + use super::ir::env::EnvValue; + let mut tool_checks = String::new(); + let mut outputs = Vec::with_capacity(tools.len()); for tool in tools { let output = custom_tool_output_var(tool); - script.push_str(&format!( + tool_checks.push_str(&format!( "{output}=\"false\"\n\ if [ -n \"$NAMES\" ] && printf '%s\\n' \"$NAMES\" | grep -Fxq {tool_q}; then\n \ {output}=\"true\"\n\ @@ -5157,79 +5893,93 @@ fn detect_custom_proposals_step(working_directory: &str, tools: &[String]) -> Re echo \"{output} set to: ${output}\"\n", tool_q = shell_quote(tool), )); + outputs.push(output); + } + let mut step = ShellScript::new(&DETECT_CUSTOM_PROPOSALS) + .fragment("tool_checks", tool_checks) + .into_step("Detect custom proposals") + .with_env("WORKING_DIRECTORY", EnvValue::literal(working_directory)); + for output in outputs { step = step.with_output(OutputDecl::new(output)); } - step.script = dedent(&script); Ok(step .with_id(StepId::new(CUSTOM_PROPOSALS_STEP_ID)?) .with_condition(Condition::Always)) } +shell_script! { + /// Debug-only probe (emitted when `--debug-pipeline` is on). Probes every + /// MCPG backend via MCP `initialize` + `tools/list` to surface broken + /// backends early. Mirrors the legacy + /// `generate_debug_pipeline_replacements` bash body. + VERIFY_MCP_BACKENDS { + interpreter: Bash, + bindings: [MCPG_PORT], + externals: [MCPG_API_KEY], + fragments: [], + body: r###" +echo "=== Probing MCP backends ===" +PROBE_FAILED=false +for server in $(jq -r '.mcpServers | keys[]' /tmp/awf-tools/mcp-config.json); do + echo "" + echo "--- Probing: $server ---" + # MCP requires initialize handshake before tools/list. + # Send initialize first, then tools/list in a second request + # using the session ID from the initialize response. + INIT_RESPONSE=$(curl -s -D /tmp/probe-headers.txt -o /tmp/probe-init.json -w "%{http_code}" --max-time 120 -X POST \ + -H "Authorization: $MCPG_API_KEY" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json, text/event-stream" \ + -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"ado-aw-probe","version":"1.0"}}}' \ + "http://localhost:$MCPG_PORT/mcp/$server" 2>&1) + SESSION_ID=$(grep -i "mcp-session-id" /tmp/probe-headers.txt 2>/dev/null | tr -d '\r' | awk '{print $2}') + echo "Initialize: HTTP $INIT_RESPONSE, session=$SESSION_ID" + + if [ -z "$SESSION_ID" ]; then + echo "##vso[task.logissue type=warning]MCP backend '$server' did not return a session ID" + cat /tmp/probe-init.json 2>/dev/null || true + PROBE_FAILED=true + continue + fi + + # Now send tools/list with the session + HTTP_CODE=$(curl -s -o /tmp/probe-response.json -w "%{http_code}" --max-time 120 -X POST \ + -H "Authorization: $MCPG_API_KEY" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json, text/event-stream" \ + -H "Mcp-Session-Id: $SESSION_ID" \ + -d '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' \ + "http://localhost:$MCPG_PORT/mcp/$server" 2>&1) + BODY=$(cat /tmp/probe-response.json 2>/dev/null || echo "(empty)") + # Extract tool count from SSE data line + TOOL_COUNT=$(echo "$BODY" | grep '^data:' | sed 's/^data: //' | jq -r '.result.tools | length' 2>/dev/null || echo "?") + echo "tools/list: HTTP $HTTP_CODE" + if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ] && [ "$TOOL_COUNT" != "?" ]; then + echo "✓ $server: $TOOL_COUNT tools available" + else + echo "##vso[task.logissue type=warning]MCP backend '$server' tools/list returned HTTP $HTTP_CODE" + echo "Response: $BODY" + PROBE_FAILED=true + fi +done + +echo "" +echo "=== MCPG health after probes ===" +curl -sf "http://localhost:$MCPG_PORT/health" | jq . || true + +if [ "$PROBE_FAILED" = "true" ]; then + echo "##vso[task.logissue type=warning]One or more MCP backends failed to initialize — check logs above" +fi +"###, + } +} + fn verify_mcp_backends_step() -> BashStep { - // Debug-only probe (emitted when --debug-pipeline is on). Probes every - // MCPG backend via MCP initialize + tools/list to surface broken - // backends early. Mirrors the legacy `generate_debug_pipeline_replacements` - // bash body. `{{ mcpg_port }}` in the legacy template is interpolated - // here as the `MCPG_PORT` const value. - let script = format!( - "echo \"=== Probing MCP backends ===\"\n\ -PROBE_FAILED=false\n\ -for server in $(jq -r '.mcpServers | keys[]' /tmp/awf-tools/mcp-config.json); do\n \ - echo \"\"\n \ - echo \"--- Probing: $server ---\"\n \ - # MCP requires initialize handshake before tools/list.\n \ - # Send initialize first, then tools/list in a second request\n \ - # using the session ID from the initialize response.\n \ - INIT_RESPONSE=$(curl -s -D /tmp/probe-headers.txt -o /tmp/probe-init.json -w \"%{{http_code}}\" --max-time 120 -X POST \\\n \ - -H \"Authorization: $MCPG_API_KEY\" \\\n \ - -H \"Content-Type: application/json\" \\\n \ - -H \"Accept: application/json, text/event-stream\" \\\n \ - -d '{{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{{\"protocolVersion\":\"2025-03-26\",\"capabilities\":{{}},\"clientInfo\":{{\"name\":\"ado-aw-probe\",\"version\":\"1.0\"}}}}}}' \\\n \ - \"http://localhost:{MCPG_PORT}/mcp/$server\" 2>&1)\n \ - SESSION_ID=$(grep -i \"mcp-session-id\" /tmp/probe-headers.txt 2>/dev/null | tr -d '\\r' | awk '{{print $2}}')\n \ - echo \"Initialize: HTTP $INIT_RESPONSE, session=$SESSION_ID\"\n \ -\n \ - if [ -z \"$SESSION_ID\" ]; then\n \ - echo \"##vso[task.logissue type=warning]MCP backend '$server' did not return a session ID\"\n \ - cat /tmp/probe-init.json 2>/dev/null || true\n \ - PROBE_FAILED=true\n \ - continue\n \ - fi\n \ -\n \ - # Now send tools/list with the session\n \ - HTTP_CODE=$(curl -s -o /tmp/probe-response.json -w \"%{{http_code}}\" --max-time 120 -X POST \\\n \ - -H \"Authorization: $MCPG_API_KEY\" \\\n \ - -H \"Content-Type: application/json\" \\\n \ - -H \"Accept: application/json, text/event-stream\" \\\n \ - -H \"Mcp-Session-Id: $SESSION_ID\" \\\n \ - -d '{{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/list\"}}' \\\n \ - \"http://localhost:{MCPG_PORT}/mcp/$server\" 2>&1)\n \ - BODY=$(cat /tmp/probe-response.json 2>/dev/null || echo \"(empty)\")\n \ - # Extract tool count from SSE data line\n \ - TOOL_COUNT=$(echo \"$BODY\" | grep '^data:' | sed 's/^data: //' | jq -r '.result.tools | length' 2>/dev/null || echo \"?\")\n \ - echo \"tools/list: HTTP $HTTP_CODE\"\n \ - if [ \"$HTTP_CODE\" -ge 200 ] && [ \"$HTTP_CODE\" -lt 300 ] && [ \"$TOOL_COUNT\" != \"?\" ]; then\n \ - echo \"\u{2713} $server: $TOOL_COUNT tools available\"\n \ - else\n \ - echo \"##vso[task.logissue type=warning]MCP backend '$server' tools/list returned HTTP $HTTP_CODE\"\n \ - echo \"Response: $BODY\"\n \ - PROBE_FAILED=true\n \ - fi\n\ -done\n\ -\n\ -echo \"\"\n\ -echo \"=== MCPG health after probes ===\"\n\ -curl -sf \"http://localhost:{MCPG_PORT}/health\" | jq . || true\n\ -\n\ -if [ \"$PROBE_FAILED\" = \"true\" ]; then\n \ - echo \"##vso[task.logissue type=warning]One or more MCP backends failed to initialize \u{2014} check logs above\"\n\ -fi\n" - ); use super::ir::env::EnvValue; - bash("Verify MCP backends", script).with_env( - "MCPG_API_KEY", - EnvValue::pipeline_var("MCP_GATEWAY_API_KEY"), - ) + ShellScript::new(&VERIFY_MCP_BACKENDS) + .bind("MCPG_PORT", Binding::number(MCPG_PORT.into())) + .into_step("Verify MCP backends") + .with_env("MCPG_API_KEY", EnvValue::pipeline_var("MCP_GATEWAY_API_KEY")) } // ───────────────────────────────────────────────────────────────────── @@ -6118,12 +6868,17 @@ safe-outputs: // address, which does not exist until the engine is running. Starting // MCPG first would leave the redirect unresolvable. let network_script = prepare_ado_proxy_network_step().script; - assert!(network_script.contains(&format!( - "docker network create --internal {ADO_PROXY_NETWORK_NAME}" - ))); + assert!( + network_script.contains(&format!("PROXY_NETWORK='{ADO_PROXY_NETWORK_NAME}'")) + && network_script.contains(r#"docker network create --internal "$PROXY_NETWORK""#), + "the internal network must be created from the compiler-supplied \ + PROXY_NETWORK binding: {network_script}" + ); let script = prepare_ado_mcp_step(common::ADO_MCP_VERSION).script; assert!( - script.contains(&format!("{ADO_MCP_PACKAGE}@{}", common::ADO_MCP_VERSION)), + script.contains(&format!("MCP_PACKAGE='{ADO_MCP_PACKAGE}'")) + && script.contains(&format!("MCP_VERSION='{}'", common::ADO_MCP_VERSION)) + && script.contains(r#""$MCP_PACKAGE@$MCP_VERSION""#), "the MCP package must be pinned, not floating: {script}" ); assert!( @@ -6416,7 +7171,10 @@ safe-outputs: assert!(copy.script.contains("/tmp/gh-aw/ado-proxy-logs")); assert!( copy.script - .contains("$(Agent.TempDirectory)/staging/logs/ado-proxy"), + .contains("AGENT_TEMP='$(Agent.TempDirectory)'") + && copy + .script + .contains(r#""$AGENT_TEMP/staging/logs/ado-proxy""#), "proxy lifecycle and sanitized decision logs must reach the agent artifact" ); } diff --git a/src/compile/extensions/ado_script.rs b/src/compile/extensions/ado_script.rs index 161d8ae7..aa52c127 100644 --- a/src/compile/extensions/ado_script.rs +++ b/src/compile/extensions/ado_script.rs @@ -143,14 +143,17 @@ shell_script! { /// `revoke` mode always exits 0 (downgrading failures to warnings), so /// no `set -eo pipefail` — aborting the shell early on a non-zero would /// only risk turning a benign revoke hiccup into a timeline error. + /// + /// `API_URL` is bound empty when the caller has no `--api-url` override; + /// the `${API_URL:+…}` guard emits the flag only when set, so the empty + /// case never produces a trailing line-continuation dangling to EOF. REVOKE_GITHUB_APP_TOKEN { interpreter: Bash, - bindings: [GITHUB_APP_TOKEN_PATH], + bindings: [GITHUB_APP_TOKEN_PATH, API_URL], externals: [], - fragments: [args], + fragments: [], body: r#" -node "$GITHUB_APP_TOKEN_PATH" revoke \ -# ado-aw:fragment args +node "$GITHUB_APP_TOKEN_PATH" revoke ${API_URL:+--api-url "$API_URL"} "#, } } @@ -751,10 +754,10 @@ pub fn github_app_token_step_typed_for( .context("serialize GitHub App token permissions")?; args.push(format!("--permissions-json {}", sh_single_quote(&json))); } - let script = format!( - "set -eo pipefail\nnode '{GITHUB_APP_TOKEN_PATH}' {}\n", - args.join(" ") - ); + let script = ShellScript::new(&MINT_GITHUB_APP_TOKEN) + .text("GITHUB_APP_TOKEN_PATH", GITHUB_APP_TOKEN_PATH) + .fragment("args", args.join(" ")) + .render(); let step = BashStep::new(display_name, script) .with_condition(Condition::Succeeded) // Only the secret rides in env — masked, never on the command line. Its @@ -825,11 +828,11 @@ pub fn prepare_pr_base_step_typed(mode: PreparePrBaseMode, repos: &[PreparePrBas ) }) .collect(); - let script = format!( - "set -eo pipefail\nnode '{PREPARE_PR_BASE_PATH}' --mode {}{}\n", - mode.as_arg(), - repo_flags, - ); + let script = ShellScript::new(&PREPARE_PR_BASE) + .text("PREPARE_PR_BASE_PATH", PREPARE_PR_BASE_PATH) + .text("MODE", mode.as_arg()) + .fragment("repo_flags", repo_flags.trim_start().to_string()) + .render(); let step = crate::compile::ado_bundle::apply_bundle_auth( BashStep::new(mode.display_name(), script).with_condition(Condition::Succeeded), crate::compile::ado_bundle::Bundle::PreparePrBase, @@ -875,11 +878,10 @@ pub fn github_app_token_revoke_step_typed_for( // so aborting the shell early on a non-zero would neither help nor change // the outcome — it would only risk turning a benign revoke hiccup into a // timeline error. - let api_url_arg = match &cfg.api_url { - Some(api_url) => format!(" --api-url {}", sh_single_quote(api_url)), - None => String::new(), - }; - let script = format!("node '{GITHUB_APP_TOKEN_PATH}' revoke{api_url_arg}\n"); + let script = ShellScript::new(&REVOKE_GITHUB_APP_TOKEN) + .text("GITHUB_APP_TOKEN_PATH", GITHUB_APP_TOKEN_PATH) + .text("API_URL", cfg.api_url.as_deref().unwrap_or("")) + .render(); let step = BashStep::new(display_name, script) .with_condition(Condition::Always) .with_continue_on_error(true) @@ -900,10 +902,9 @@ pub fn github_app_token_revoke_step_typed_for( /// legacy emitter, and the value every consumer must use in its /// `OutputRef`. pub fn synthetic_pr_step_typed(spec_b64: &str) -> Result { - let script = format!( - "set -euo pipefail\n\ - node '{EXEC_CONTEXT_PR_SYNTH_PATH}'\n" - ); + let script = ShellScript::new(&RESOLVE_SYNTHETIC_PR) + .text("BUNDLE", EXEC_CONTEXT_PR_SYNTH_PATH) + .render(); let condition = Condition::And(vec![ Condition::Succeeded, Condition::Ne( @@ -1498,9 +1499,15 @@ mod tests { "Mint GitHub App token (Copilot engine auth)" ); assert!( + step.script.contains( + "GITHUB_APP_TOKEN_PATH='/tmp/ado-aw-scripts/ado-script/github-app-token.js'" + ), + "the bundle path must be projected through the prelude:\n{}", step.script - .contains("node '/tmp/ado-aw-scripts/ado-script/github-app-token.js'"), - "script must invoke the bundle:\n{}", + ); + assert!( + step.script.contains("node \"$GITHUB_APP_TOKEN_PATH\""), + "the body must invoke the bundle through the bound path:\n{}", step.script ); // Non-secret inputs are single-quoted argv flags (shadow-proof). The @@ -1703,9 +1710,16 @@ mod tests { let Step::Bash(step) = github_app_token_revoke_step_typed(&cfg).unwrap() else { panic!("expected a bash step"); }; + assert!( + step.script.contains( + "GITHUB_APP_TOKEN_PATH='/tmp/ado-aw-scripts/ado-script/github-app-token.js'" + ), + "the bundle path must be projected through the prelude:\n{}", + step.script + ); assert!( step.script - .contains("node '/tmp/ado-aw-scripts/ado-script/github-app-token.js' revoke"), + .contains("node \"$GITHUB_APP_TOKEN_PATH\" revoke"), "revoke step must invoke the bundle in revoke mode:\n{}", step.script ); @@ -1714,11 +1728,18 @@ mod tests { step.env.get("GH_APP_TOKEN"), Some(EnvValue::Secret(v)) if v == "GITHUB_APP_TOKEN" )); - // api-url is an argv flag (non-secret), not an env var. + // api-url is projected through the API_URL binding (non-secret prelude), + // guarded by ${API_URL:+…} so the empty case never emits the flag. + assert!( + step.script + .contains("API_URL='https://ghe.example.com/api/v3'"), + "revoke must project the api-url through the prelude:\n{}", + step.script + ); assert!( step.script - .contains("revoke --api-url 'https://ghe.example.com/api/v3'"), - "revoke must pass api-url as an argv flag:\n{}", + .contains("revoke ${API_URL:+--api-url \"$API_URL\"}"), + "revoke body must guard the --api-url flag with ${{API_URL:+…}}:\n{}", step.script ); assert!(!step.env.contains_key("GH_APP_API_URL")); @@ -1769,19 +1790,29 @@ mod tests { }; assert_eq!(step.display_name, "Prepare create-pull-request patch base"); assert!( + step.script.contains( + "PREPARE_PR_BASE_PATH='/tmp/ado-aw-scripts/ado-script/prepare-pr-base.js'" + ), + "the bundle path must be projected through the prelude:\n{}", + step.script + ); + assert!( + step.script.contains("MODE='patch-base'"), + "the mode must be projected through the prelude:\n{}", step.script - .contains("node '/tmp/ado-aw-scripts/ado-script/prepare-pr-base.js'"), - "script must invoke the bundle:\n{}", + ); + assert!( + step.script.contains("node \"$PREPARE_PR_BASE_PATH\" --mode \"$MODE\""), + "the body must invoke the bundle through the bound path and mode:\n{}", step.script ); // The repo dir (== MCP server bounding_directory) is a double-quoted argv // flag (ADO-macro path convention); its target is a single-quoted literal. assert!( step.script.contains( - "--mode patch-base --repo-dir \"$(Build.SourcesDirectory)\" \ - --target-branch 'main'" + "--repo-dir \"$(Build.SourcesDirectory)\" --target-branch 'main'" ), - "must emit typed mode/source/target flags:\n{}", + "must emit typed source/target flags:\n{}", step.script ); // The ADO bearer is projected as a masked secret (bundle uses it for the @@ -1879,7 +1910,11 @@ mod tests { step.display_name, "Prepare create-pull-request target worktree ref" ); - assert!(step.script.contains("--mode target-worktree")); + assert!( + step.script.contains("MODE='target-worktree'"), + "the mode must be projected through the prelude:\n{}", + step.script + ); assert!(!step.script.contains("--source-ref")); } diff --git a/tests/compiler_tests.rs b/tests/compiler_tests.rs index cf20b5e1..cd08451d 100644 --- a/tests/compiler_tests.rs +++ b/tests/compiler_tests.rs @@ -199,11 +199,11 @@ fn test_compiled_output_no_unreplaced_markers() { "Compiled output should reference GitHub Releases for AWF" ); assert!( - compiled.contains("AWF_VERSION=\"0.27.32\""), + compiled.contains("AWF_VERSION='0.27.32'"), "Compiled output should download the AWF version that provides strict topology isolation" ); assert!( - !compiled.contains("AWF_VERSION=\"0.27.9\""), + !compiled.contains("AWF_VERSION='0.27.9'"), "Compiled output must not fall back to the legacy host-access AWF version" ); @@ -249,9 +249,11 @@ fn test_compiled_output_no_unreplaced_markers() { "Generated AWF invocations must not use legacy host security" ); assert!( - compiled.contains("--name awmg-mcpg") + compiled.contains("MCPG_CONTAINER='awmg-mcpg'") + && compiled.contains(r#"--name "$MCPG_CONTAINER""#) && compiled.contains("--network bridge") - && compiled.contains("-p 127.0.0.1:8080:8080"), + && compiled.contains("MCPG_PORT=8080") + && compiled.contains(r#"-p "127.0.0.1:$MCPG_PORT:$MCPG_PORT""#), "MCPG must use the stable bridge-network topology" ); for line in compiled @@ -5896,9 +5898,14 @@ fn test_byom_provider_env_compiles_and_merges() { // the exact version selected by --image-tag. assert_eq!( compiled - .matches("docker pull ghcr.io/github/gh-aw-firewall/api-proxy:") + .matches("API_PROXY_IMAGE='ghcr.io/github/gh-aw-firewall/api-proxy:") .count(), 2, + "BYOK must bind the api-proxy image in both the Agent and Detection jobs: {compiled}" + ); + assert_eq!( + compiled.matches(r#"docker pull "$API_PROXY_IMAGE""#).count(), + 2, "BYOK must pre-pull the api-proxy container image in both the Agent and Detection jobs: {compiled}" ); } @@ -5980,9 +5987,14 @@ fn test_non_byom_agent_uses_always_on_api_proxy() { ); assert_eq!( compiled - .matches("docker pull ghcr.io/github/gh-aw-firewall/api-proxy:") + .matches("API_PROXY_IMAGE='ghcr.io/github/gh-aw-firewall/api-proxy:") .count(), 2, + "Agent and Detection must bind the api-proxy image for pre-pull: {compiled}" + ); + assert_eq!( + compiled.matches(r#"docker pull "$API_PROXY_IMAGE""#).count(), + 2, "Agent and Detection must pre-pull the always-on api-proxy image: {compiled}" ); assert_eq!( @@ -7843,9 +7855,16 @@ supply-chain: assert!(ok, "pipeline-artifact + registry should compile: {stderr}"); assert!(compiled.contains("source: specific")); assert!(compiled.contains("runId: '630001'")); - assert!(compiled.contains("docker pull myacr.azurecr.io/candidate/squid:")); + assert!( + compiled.contains("SQUID_IMAGE='myacr.azurecr.io/candidate/squid:") + && compiled.contains(r#"docker pull "$SQUID_IMAGE""#), + "AWF squid image must be bound from the internal registry: {compiled}" + ); assert!(compiled.contains("azureSubscription: acr-conn")); - assert!(!compiled.contains("docker pull ghcr.io")); + assert!( + !compiled.contains("_IMAGE='ghcr.io"), + "no image binding should point at ghcr.io in registry mode: {compiled}" + ); assert!(!compiled.contains("DownloadPackage@1")); assert!(!compiled.contains("NuGetAuthenticate@1")); assert!(!compiled.contains("github.com/githubnext/ado-aw/releases")); @@ -7943,12 +7962,14 @@ fn test_supply_chain_full_reroutes_all_artifacts() { "ACR login must be emitted before docker pull in registry mode" ); assert!( - compiled.contains("docker pull myacr.azurecr.io/oss-mirror/squid:"), - "AWF images must be pulled from the internal registry base path (artifact name only)" + compiled.contains("SQUID_IMAGE='myacr.azurecr.io/oss-mirror/squid:") + && compiled.contains(r#"docker pull "$SQUID_IMAGE""#), + "AWF squid image must be bound from the internal registry base path (artifact name only): {compiled}" ); assert!( - compiled.contains("docker pull myacr.azurecr.io/oss-mirror/api-proxy:"), - "the always-on api-proxy must be pulled from the internal registry" + compiled.contains("API_PROXY_IMAGE='myacr.azurecr.io/oss-mirror/api-proxy:") + && compiled.contains(r#"docker pull "$API_PROXY_IMAGE""#), + "the always-on api-proxy must be bound from the internal registry" ); assert!( compiled.contains("myacr.azurecr.io/oss-mirror/gh-aw-mcpg:"), @@ -8873,18 +8894,29 @@ engine: /// nowhere else, for a given compiled pipeline. fn assert_github_app_token_wiring(compiled: &str) { // Total bundle invocations = mint (Agent + Detection) + revoke - // (Agent + Detection) = 4. Revoke invocations carry the ` revoke` arg. + // (Agent + Detection) = 4. Both mint and revoke read the bundle path + // through the bound `$GITHUB_APP_TOKEN_PATH`; only revoke follows it + // with the literal `revoke` word. let total_bundle = compiled - .matches("node '/tmp/ado-aw-scripts/ado-script/github-app-token.js'") + .matches("node \"$GITHUB_APP_TOKEN_PATH\"") .count(); let revoke_hits = compiled - .matches("node '/tmp/ado-aw-scripts/ado-script/github-app-token.js' revoke") + .matches("node \"$GITHUB_APP_TOKEN_PATH\" revoke") .count(); let mint_hits = total_bundle - revoke_hits; assert_eq!( mint_hits, 2, "expected the mint step in exactly Agent + Detection, found {mint_hits}:\n{compiled}" ); + // The bundle path is projected through the prelude in every mint/revoke + // step (Agent + Detection = 4 preludes). + let path_bindings = compiled + .matches("GITHUB_APP_TOKEN_PATH='/tmp/ado-aw-scripts/ado-script/github-app-token.js'") + .count(); + assert_eq!( + path_bindings, 4, + "expected the bundle path bound in every mint + revoke prelude, found {path_bindings}:\n{compiled}" + ); let mint_display = compiled .matches("Mint GitHub App token (Copilot engine auth)") .count(); @@ -8993,17 +9025,17 @@ fn test_github_app_token_skip_revocation() { // Mint step still present in Agent + Detection. assert_eq!( compiled - .matches("node '/tmp/ado-aw-scripts/ado-script/github-app-token.js'") + .matches("node \"$GITHUB_APP_TOKEN_PATH\"") .count() - compiled - .matches("node '/tmp/ado-aw-scripts/ado-script/github-app-token.js' revoke") + .matches("node \"$GITHUB_APP_TOKEN_PATH\" revoke") .count(), 2, "mint step must still be present:\n{compiled}" ); // ...but no revoke step. assert!( - !compiled.contains("github-app-token.js' revoke"), + !compiled.contains("\"$GITHUB_APP_TOKEN_PATH\" revoke"), "skip-token-revocation must suppress the revoke step:\n{compiled}" ); } @@ -9048,14 +9080,21 @@ fn test_github_app_token_literal_app_id_and_api_url() { !compiled.contains("$(GITHUB_APP_PRIVATE_KEY)"), "override must replace the default private-key variable:\n{compiled}" ); - // GHES api-url flows into both mint and revoke steps as an argv flag. - // Mint: `... --api-url '...'`; revoke: `revoke --api-url '...'`. - let api_url_args = compiled + // GHES api-url flows into both mint (fragment argv) and revoke (prelude + // binding + `${API_URL:+…}` guard) steps in both jobs. + let mint_api_url = compiled .matches("--api-url 'https://ghe.example.com/api/v3'") .count(); assert_eq!( - api_url_args, 4, - "api-url must appear as an argv flag in both mint and both revoke steps (2 jobs x 2):\n{compiled}" + mint_api_url, 2, + "api-url must appear as a mint-step argv fragment in both jobs (Agent + Detection):\n{compiled}" + ); + let revoke_api_url = compiled + .matches("API_URL='https://ghe.example.com/api/v3'") + .count(); + assert_eq!( + revoke_api_url, 2, + "api-url must be projected through the revoke-step prelude in both jobs (Agent + Detection):\n{compiled}" ); assert!( !compiled.contains("GH_APP_API_URL:"), @@ -9270,7 +9309,15 @@ fn test_create_pull_request_emits_prepare_pr_base_step_in_agent() { ); let agent = job_block(&compiled, "Agent"); assert!( - agent.contains("node '/tmp/ado-aw-scripts/ado-script/prepare-pr-base.js' --mode patch-base --repo-dir \"$(Build.SourcesDirectory)\" --target-branch 'main'"), + agent.contains("PREPARE_PR_BASE_PATH='/tmp/ado-aw-scripts/ado-script/prepare-pr-base.js'"), + "Agent job must project the bundle path through the prelude:\n{agent}" + ); + assert!( + agent.contains("MODE='patch-base'"), + "Agent job must project the patch-base mode through the prelude:\n{agent}" + ); + assert!( + agent.contains("--repo-dir \"$(Build.SourcesDirectory)\" --target-branch 'main'"), "Agent job must invoke patch-base without shell-expanding the runtime self source ref:\n{agent}" ); assert!( @@ -9312,7 +9359,15 @@ fn test_create_pull_request_prepare_step_defaults_target_branch() { ); let agent = job_block(&compiled, "Agent"); assert!( - agent.contains("node '/tmp/ado-aw-scripts/ado-script/prepare-pr-base.js' --mode patch-base --repo-dir \"$(Build.SourcesDirectory)\" --target-branch 'main'"), + agent.contains("PREPARE_PR_BASE_PATH='/tmp/ado-aw-scripts/ado-script/prepare-pr-base.js'"), + "bare create-pull-request must project the bundle path through the prelude:\n{agent}" + ); + assert!( + agent.contains("MODE='patch-base'"), + "bare create-pull-request must project the patch-base mode through the prelude:\n{agent}" + ); + assert!( + agent.contains("--repo-dir \"$(Build.SourcesDirectory)\" --target-branch 'main'"), "bare create-pull-request must emit the prepare step targeting 'main':\n{agent}" ); // Single `self` checkout ⇒ exactly one --repo-dir (the working directory). @@ -9398,7 +9453,19 @@ fn test_create_pull_request_emits_prepare_pr_base_step_in_safeoutputs() { ); let safeoutputs = job_block(&compiled, "SafeOutputs"); assert!( - safeoutputs.contains("node '/tmp/ado-aw-scripts/ado-script/prepare-pr-base.js' --mode target-worktree --repo-dir \"$(Build.SourcesDirectory)\" --target-branch 'main'"), + safeoutputs.contains( + "PREPARE_PR_BASE_PATH='/tmp/ado-aw-scripts/ado-script/prepare-pr-base.js'" + ), + "SafeOutputs job must project the bundle path through the prelude:\n{safeoutputs}" + ); + assert!( + safeoutputs.contains("MODE='target-worktree'"), + "SafeOutputs job must project the target-worktree mode through the prelude:\n{safeoutputs}" + ); + assert!( + safeoutputs.contains( + "--repo-dir \"$(Build.SourcesDirectory)\" --target-branch 'main'" + ), "SafeOutputs job must invoke prepare-pr-base with the self dir/target pair:\n{safeoutputs}" ); assert!( @@ -9631,9 +9698,13 @@ fn test_issue_1731_safeoutputs_executor_source_path_uses_multi_checkout_layout() ); let safeoutputs = job_block(&compiled, "SafeOutputs"); // With additional repos, self is pinned to $(Build.SourcesDirectory)/self. + // The compiler now passes the source path to the executor through the step + // `env:` block (ADO_AW_SOURCE_PATH), which the shell reads as + // `--source "$ADO_AW_SOURCE_PATH"`. assert!( - safeoutputs - .contains("ado-aw execute --source \"$(Build.SourcesDirectory)/self/"), + safeoutputs.contains("ADO_AW_SOURCE_PATH: $(Build.SourcesDirectory)/self/") + && safeoutputs + .contains(r#"ado-aw execute --source "$ADO_AW_SOURCE_PATH""#), "SafeOutputs executor --source must use the multi-checkout layout path:\n{safeoutputs}" ); assert!( @@ -9724,10 +9795,9 @@ fn test_issue_1731_split_approval_additional_checkouts_only_in_pr_variant() { "auto SafeOutputs must NOT check out additional repos (it never runs create-pull-request):\n{auto}" ); assert!( - auto.contains("ado-aw execute --source \"$(Build.SourcesDirectory)/") - && !auto.contains( - "ado-aw execute --source \"$(Build.SourcesDirectory)/self/" - ), + auto.contains("ADO_AW_SOURCE_PATH: $(Build.SourcesDirectory)") + && !auto.contains("ADO_AW_SOURCE_PATH: $(Build.SourcesDirectory)/self/") + && auto.contains(r#"ado-aw execute --source "$ADO_AW_SOURCE_PATH""#), "self-only SafeOutputs must use its single-checkout source path:\n{auto}" ); assert!( @@ -9738,10 +9808,11 @@ fn test_issue_1731_split_approval_additional_checkouts_only_in_pr_variant() { "self-only SafeOutputs must pass its checkout root as the self repo:\n{auto}" ); assert!( - reviewed.contains("ado-aw execute --source \"$(Build.SourcesDirectory)/self/") + reviewed.contains("ADO_AW_SOURCE_PATH: $(Build.SourcesDirectory)/self/") && reviewed.contains( "ADO_AW_SELF_REPOSITORY_DIRECTORY: $(Build.SourcesDirectory)/self" - ), + ) + && reviewed.contains(r#"ado-aw execute --source "$ADO_AW_SOURCE_PATH""#), "PR-capable reviewed job must use its multi-checkout self path:\n{reviewed}" ); } @@ -9771,17 +9842,17 @@ fn test_issue_1731_split_approval_additional_checkouts_in_auto_when_sibling_gate "SafeOutputs_Reviewed must NOT check out additional repos when it doesn't run create-pull-request:\n{reviewed}" ); assert!( - auto.contains("ado-aw execute --source \"$(Build.SourcesDirectory)/self/") + auto.contains("ADO_AW_SOURCE_PATH: $(Build.SourcesDirectory)/self/") + && auto.contains(r#"ado-aw execute --source "$ADO_AW_SOURCE_PATH""#) && auto.contains( "ADO_AW_SELF_REPOSITORY_DIRECTORY: $(Build.SourcesDirectory)/self" ), "PR-capable automatic job must use its multi-checkout self path:\n{auto}" ); assert!( - reviewed.contains("ado-aw execute --source \"$(Build.SourcesDirectory)/") - && !reviewed.contains( - "ado-aw execute --source \"$(Build.SourcesDirectory)/self/" - ), + reviewed.contains("ADO_AW_SOURCE_PATH: $(Build.SourcesDirectory)") + && !reviewed.contains("ADO_AW_SOURCE_PATH: $(Build.SourcesDirectory)/self/") + && reviewed.contains(r#"ado-aw execute --source "$ADO_AW_SOURCE_PATH""#), "self-only reviewed job must use its single-checkout source path:\n{reviewed}" ); assert!( @@ -9809,7 +9880,8 @@ fn test_issue_1731_split_checkout_layout_compiles_for_every_target() { "{target}: tools must be checked out in Agent and the PR-capable Stage 3 job only:\n{compiled}" ); assert!( - compiled.contains("ado-aw execute --source \"$(Build.SourcesDirectory)/self/"), + compiled.contains("ADO_AW_SOURCE_PATH: $(Build.SourcesDirectory)/self/") + && compiled.contains(r#"ado-aw execute --source "$ADO_AW_SOURCE_PATH""#), "{target}: PR-capable Stage 3 source must use multi-checkout layout:\n{compiled}" ); assert!( @@ -9819,7 +9891,7 @@ fn test_issue_1731_split_checkout_layout_compiles_for_every_target() { "{target}: Stage 3 self identity must be compile-time resolved:\n{compiled}" ); assert!( - compiled.contains("ado-aw execute --source \"$(Build.SourcesDirectory)/") + compiled.contains("ADO_AW_SOURCE_PATH: $(Build.SourcesDirectory)") && compiled.contains( "ADO_AW_SELF_REPOSITORY_DIRECTORY: $(Build.SourcesDirectory)" ), From dc9cfa091d1c865bc45273dc1e2480d77a3d2782 Mon Sep 17 00:00:00 2001 From: James Devine Date: Mon, 10 Aug 2026 18:05:45 +0100 Subject: [PATCH 5/7] refactor(compile): retire the untyped bash() and dedent() helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last four `bash()` call sites went through two payload helpers, `stage_candidate_artifact_payload_bash` and `extract_package_payload_bash`, each carrying a SAFETY comment warning that every parameter is interpolated into a shell body with no escaping and that callers must pass only compiler-owned constants. That is precisely the hazard `ShellScript` removes, so both are now registered scripts with typed bindings and a `tail` fragment. The comments are gone because the property is now enforced rather than requested. Both keep their `-> String` signature: `extensions/ado_script.rs` shares `stage_candidate_artifact_payload_bash` and wraps it differently. Adds `Binding::ado_path` for the shape these need — a path built around an ADO variable, e.g. `$(Pipeline.Workspace)/agentic-pipeline-compiler`. `ado_macro` takes a bare name and rightly refused it. Rather than widening `text` to allow `$(`, which would have let an arbitrary command substitution through, `ado_path` validates that every embedded `$(…)` is a well-formed predefined-variable name. The value can therefore only expand to something Azure DevOps substitutes before bash runs, never to a command the runner executes. `bash()` and `dedent()` are deleted. Every shell body the compiler emits is now a registered script: 87 in total, all passing shellcheck both in-process and as exported standalone files. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: aa46b482-6121-45e2-8278-1a9465b4f73e --- docs/extending.md | 6 +- src/compile/agentic_pipeline.rs | 261 +++++++++++++++----------------- src/compile/shell/bindings.rs | 71 +++++++++ 3 files changed, 200 insertions(+), 138 deletions(-) diff --git a/docs/extending.md b/docs/extending.md index ea068cc2..93347b0a 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -338,11 +338,15 @@ contains. Pick the constructor that matches the shape: | `Binding::boolean(b)` | `true` / `false` | flags read as `[ "$V" = true ]` | | `Binding::words([…])` | `'a b c'` | a list the body expands unquoted in `for` | | `Binding::ado_macro("Agent.TempDirectory")` | `'$(Agent.TempDirectory)'` | ADO predefined variables | +| `Binding::ado_path(p)` | `'$(Pipeline.Workspace)/x'` | a path built around one | | `Binding::document(text)` | quoted heredoc | JSON, prompts, certificates | Each validates its own shape: `words` rejects an entry containing whitespace or a glob (the consumer expands it unquoted, so that would silently change the -list), and `ado_macro` accepts only a well-formed dotted name. +list), `ado_macro` accepts only a well-formed dotted name, and `ado_path` +checks every embedded `$(…)` is such a name — so the value can only ever +expand to a variable Azure DevOps substitutes, never to a command the runner +executes. ### Declaring the variable surface diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index 432971ec..f998ec49 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -3322,16 +3322,63 @@ print("Validated candidate provenance:") print(json.dumps(diagnostic, indent=2, sort_keys=True)) "#; -/// Build the staging script for one payload from a pinned candidate artifact. +shell_script! { + /// Stage a payload out of a provenance-checked candidate pipeline + /// artifact: locate exactly one payload, checksum manifest and provenance + /// document, copy them into place, verify an *exact* filename checksum + /// entry, then validate producer identity before the caller's tail runs. + /// + /// The exactly-one requirement is load-bearing. A `find` that matched two + /// files would otherwise let an attacker who can add a file to the + /// artifact decide which one is staged. + STAGE_CANDIDATE_ARTIFACT_PAYLOAD { + interpreter: Bash, + bindings: [STAGING, DEST, PAYLOAD_NAME, PROVENANCE_VALIDATOR, DEFINITION_ID, RUN_ID], + externals: [], + fragments: [tail], + body: r###" +set -eo pipefail +mkdir -p "$DEST" + +locate_one() { + local name="$1" + mapfile -d '' -t matches < <(find "$STAGING" -type f -name "$name" -print0) + if [ "${#matches[@]}" -ne 1 ]; then + echo "##vso[task.complete result=Failed]Expected exactly one $name in candidate artifact, found ${#matches[@]}" >&2 + exit 1 + fi + printf '%s' "${matches[0]}" +} + +PAYLOAD="$(locate_one "$PAYLOAD_NAME")" +CHK="$(locate_one checksums.txt)" +PROVENANCE="$(locate_one provenance.json)" +cp "$PAYLOAD" "$DEST/$PAYLOAD_NAME" +cp "$CHK" "$DEST/checksums.txt" +cp "$PROVENANCE" "$DEST/provenance.json" + +echo "Verifying exact checksum entry for $PAYLOAD_NAME..." +cd "$DEST" || exit 1 +awk -v name="$PAYLOAD_NAME" ' + { candidate=$2; sub(/^\*/, "", candidate); if (candidate == name) { count++; line=$0 } } + END { if (count != 1) exit 1; print line } +' checksums.txt | sha256sum -c - + +python3 -c "$PROVENANCE_VALIDATOR" provenance.json "$DEFINITION_ID" "$RUN_ID" +# ado-aw:fragment tail +"###, + } +} + +/// Bash body that stages a payload out of a provenance-checked candidate +/// pipeline artifact, then runs the caller-supplied verify/relocate tail. /// /// The producer contract is `schema: ado-aw/candidate-artifact/1` with numeric -/// `producer_definition_id` and `producer_build_id` fields. The script requires -/// exactly one payload, checksum manifest, and provenance document; verifies an -/// exact filename checksum entry; and validates producer identity before -/// running `tail`. +/// `producer_definition_id` and `producer_build_id` fields. /// -/// SAFETY: shell-interpolated path, payload, and tail arguments must be -/// compiler-owned constants. Producer IDs are validated positive `u64`s. +/// Returns the rendered script rather than a step because two callers wrap it +/// differently — see [`download_compiler_step`] here and +/// `install_and_download_steps_typed` in `extensions/ado_script.rs`. pub(crate) fn stage_candidate_artifact_payload_bash( config: &PipelineArtifactConfig, staging: &str, @@ -3339,92 +3386,78 @@ pub(crate) fn stage_candidate_artifact_payload_bash( payload: &str, tail: &str, ) -> String { - format!( - "set -eo pipefail\n\ - STAGING=\"{staging}\"\n\ - DEST=\"{dest_dir}\"\n\ - PAYLOAD_NAME='{payload}'\n\ - mkdir -p \"$DEST\"\n\ - \n\ - locate_one() {{\n \ - local name=\"$1\"\n \ - mapfile -d '' -t matches < <(find \"$STAGING\" -type f -name \"$name\" -print0)\n \ - if [ \"${{#matches[@]}}\" -ne 1 ]; then\n \ - echo \"##vso[task.complete result=Failed]Expected exactly one $name in candidate artifact, found ${{#matches[@]}}\" >&2\n \ - exit 1\n \ - fi\n \ - printf '%s' \"${{matches[0]}}\"\n\ - }}\n\ - \n\ - PAYLOAD=\"$(locate_one \"$PAYLOAD_NAME\")\"\n\ - CHK=\"$(locate_one checksums.txt)\"\n\ - PROVENANCE=\"$(locate_one provenance.json)\"\n\ - cp \"$PAYLOAD\" \"$DEST/$PAYLOAD_NAME\"\n\ - cp \"$CHK\" \"$DEST/checksums.txt\"\n\ - cp \"$PROVENANCE\" \"$DEST/provenance.json\"\n\ - \n\ - echo \"Verifying exact checksum entry for $PAYLOAD_NAME...\"\n\ - cd \"$DEST\" || exit 1\n\ - awk -v name=\"$PAYLOAD_NAME\" '\n \ - {{ candidate=$2; sub(/^\\*/, \"\", candidate); if (candidate == name) {{ count++; line=$0 }} }}\n \ - END {{ if (count != 1) exit 1; print line }}\n\ - ' checksums.txt | sha256sum -c -\n\ - \n\ - python3 -c '{provenance_validator}' provenance.json {definition_id} {run_id}\n\ - {tail}", - provenance_validator = CANDIDATE_PROVENANCE_VALIDATOR_PY, - definition_id = config.definition_id, - run_id = config.run_id, - ) + ShellScript::new(&STAGE_CANDIDATE_ARTIFACT_PAYLOAD) + .bind("STAGING", Binding::ado_path(staging)) + .bind("DEST", Binding::ado_path(dest_dir)) + .text("PAYLOAD_NAME", payload) + .bind( + "PROVENANCE_VALIDATOR", + Binding::document(CANDIDATE_PROVENANCE_VALIDATOR_PY), + ) + .bind("DEFINITION_ID", Binding::number(config.definition_id)) + .bind("RUN_ID", Binding::number(config.run_id)) + .fragment("tail", tail) + .render() +} + +shell_script! { + /// Locate a payload inside a `DownloadPackage@1` staging directory — + /// handling both the extracted-tree and raw-`.nupkg` delivery shapes — + /// copy it plus `checksums.txt` into place, verify the checksum, then run + /// the caller's tail with `DEST` as the working directory. + EXTRACT_PACKAGE_PAYLOAD { + interpreter: Bash, + bindings: [STAGING, DEST, PAYLOAD_NAME], + externals: [], + fragments: [tail], + body: r###" +set -eo pipefail +mkdir -p "$DEST" + +# DownloadPackage@1 may deliver an extracted tree or a raw .nupkg; +# handle both by unzipping any .nupkg when the payload is absent. +if [ -z "$(find "$STAGING" -name "$PAYLOAD_NAME" -print -quit)" ]; then + NUPKG="$(find "$STAGING" -name '*.nupkg' -print -quit)" + if [ -n "$NUPKG" ]; then + unzip -o "$NUPKG" -d "$STAGING" >/dev/null + fi +fi + +BIN="$(find "$STAGING" -name "$PAYLOAD_NAME" -print -quit)" +CHK="$(find "$STAGING" -name 'checksums.txt' -print -quit)" +if [ -z "$BIN" ] || [ -z "$CHK" ]; then + echo "##vso[task.complete result=Failed]$PAYLOAD_NAME or checksums.txt not found in package" + exit 1 +fi +cp "$BIN" "$DEST/$PAYLOAD_NAME" +cp "$CHK" "$DEST/checksums.txt" + +echo "Verifying checksum..." +cd "$DEST" || exit 1 +grep "$PAYLOAD_NAME" checksums.txt | sha256sum -c - +# ado-aw:fragment tail +"###, + } } -/// Bash body that locates a payload file inside a `DownloadPackage@1` staging -/// directory — handling both the extracted-tree and raw-`.nupkg` delivery -/// shapes — copies it (plus `checksums.txt`) into `dest_dir`, then runs the -/// caller-supplied verify/relocate tail. `payload` is the artifact file name -/// (e.g. `ado-aw-linux-x64`); `tail` is appended after the files are staged in -/// `dest_dir` (the working directory is `dest_dir`). +/// Bash body that stages a payload out of a `DownloadPackage@1` staging +/// directory and runs the caller-supplied verify/relocate tail. /// -/// SAFETY: every parameter is interpolated verbatim into a `format!()` shell -/// body with no escaping. All callers MUST pass compile-time-constant, -/// trusted strings only (today: hardcoded ADO macro paths and literal payload -/// names). Never pass user/front-matter-controlled data here — doing so would -/// introduce shell-command injection into the generated pipeline. +/// `payload` is the artifact file name (e.g. `ado-aw-linux-x64`); `tail` is +/// appended once the files are staged in `dest_dir`, which is also the working +/// directory by then. fn extract_package_payload_bash( staging: &str, dest_dir: &str, payload: &str, tail: &str, ) -> String { - format!( - "set -eo pipefail\n\ - STAGING=\"{staging}\"\n\ - DEST=\"{dest_dir}\"\n\ - mkdir -p \"$DEST\"\n\ - \n\ - # DownloadPackage@1 may deliver an extracted tree or a raw .nupkg;\n\ - # handle both by unzipping any .nupkg when the payload is absent.\n\ - if [ -z \"$(find \"$STAGING\" -name '{payload}' -print -quit)\" ]; then\n \ - NUPKG=\"$(find \"$STAGING\" -name '*.nupkg' -print -quit)\"\n \ - if [ -n \"$NUPKG\" ]; then\n \ - unzip -o \"$NUPKG\" -d \"$STAGING\" >/dev/null\n \ - fi\n\ - fi\n\ - \n\ - BIN=\"$(find \"$STAGING\" -name '{payload}' -print -quit)\"\n\ - CHK=\"$(find \"$STAGING\" -name 'checksums.txt' -print -quit)\"\n\ - if [ -z \"$BIN\" ] || [ -z \"$CHK\" ]; then\n \ - echo \"##vso[task.complete result=Failed]{payload} or checksums.txt not found in package\"\n \ - exit 1\n\ - fi\n\ - cp \"$BIN\" \"$DEST/{payload}\"\n\ - cp \"$CHK\" \"$DEST/checksums.txt\"\n\ - \n\ - echo \"Verifying checksum...\"\n\ - cd \"$DEST\" || exit 1\n\ - grep \"{payload}\" checksums.txt | sha256sum -c -\n\ - {tail}" - ) + ShellScript::new(&EXTRACT_PACKAGE_PAYLOAD) + .bind("STAGING", Binding::ado_path(staging)) + .bind("DEST", Binding::ado_path(dest_dir)) + .text("PAYLOAD_NAME", payload) + .fragment("tail", tail) + .render() } /// `NuGetAuthenticate@1` step to emit **once per job** when the feed mirror is @@ -3461,7 +3494,10 @@ fn download_compiler_step( "Download candidate artifact for agentic pipeline compiler", staging, )), - Step::Bash(bash("Stage candidate agentic pipeline compiler", body)), + Step::Bash(BashStep::new( + "Stage candidate agentic pipeline compiler", + body, + )), ]; } @@ -3481,7 +3517,7 @@ fn download_compiler_step( compiler_version, staging, )), - Step::Bash(bash( + Step::Bash(BashStep::new( format!("Stage agentic pipeline compiler (v{compiler_version})"), body, )), @@ -3757,7 +3793,7 @@ fn download_awf_step(supply_chain: Option<&SupplyChainConfig>) -> Vec { "Download candidate artifact for AWF", staging, )), - Step::Bash(bash( + Step::Bash(BashStep::new( "Stage candidate AWF (Agentic Workflow Firewall)", body, )), @@ -3781,7 +3817,7 @@ fn download_awf_step(supply_chain: Option<&SupplyChainConfig>) -> Vec { AWF_VERSION, staging, )), - Step::Bash(bash( + Step::Bash(BashStep::new( format!("Stage AWF (Agentic Workflow Firewall) v{AWF_VERSION}"), body, )), @@ -5986,55 +6022,6 @@ fn verify_mcp_backends_step() -> BashStep { // Helpers // ───────────────────────────────────────────────────────────────────── -/// Construct a [`BashStep`] with its script body run through -/// [`dedent`]. Every compiler-generated bash body in this module is -/// built by `format!()` with `\n\` continuations whose source -/// indentation leaks into the emitted YAML; `dedent()` strips it. -fn bash(name: impl Into, script: impl Into) -> BashStep { - BashStep::new(name, dedent(&script.into())) -} - -/// Strip the common leading whitespace from every non-empty line of -/// `s`, **and** strip trailing whitespace from every line. The -/// trailing-whitespace strip is critical for block-scalar emission: -/// serde_yaml falls back to the double-quoted form when a block -/// scalar contains lines with trailing spaces (because the scalar's -/// re-parse would lose them), which produces hard-to-read YAML. -/// -/// Used to clean Rust source-string indentation out of the bash -/// bodies we hand to [`BashStep::new`]. Without this, the -/// `\n\`-continuation indent in Rust source ends up inside the -/// emitted YAML block scalar. -fn dedent(s: &str) -> String { - let min = s - .lines() - .filter(|l| !l.trim().is_empty()) - .map(|l| l.chars().take_while(|c| *c == ' ').count()) - .min() - .unwrap_or(0); - let mut out = String::with_capacity(s.len()); - let mut first = true; - for line in s.lines() { - if !first { - out.push('\n'); - } - first = false; - // Only strip the leading `min` chars when the line actually - // has that many leading spaces; otherwise leave it alone - // (this avoids mangling interpolated content whose indent is - // intentionally lower than the surrounding template indent). - let leading_spaces = line.chars().take_while(|c| *c == ' ').count(); - let strip = leading_spaces.min(min); - let stripped_leading = &line[strip..]; - let stripped = stripped_leading.trim_end_matches([' ', '\t']); - out.push_str(stripped); - } - if s.ends_with('\n') { - out.push('\n'); - } - out -} - /// Classify a single raw env-var value string into a typed [`EnvValue`]. /// /// An ADO **macro** `$(NAME)` (with no nested `$` or `(`) becomes an diff --git a/src/compile/shell/bindings.rs b/src/compile/shell/bindings.rs index d9949cb8..8c772405 100644 --- a/src/compile/shell/bindings.rs +++ b/src/compile/shell/bindings.rs @@ -167,6 +167,48 @@ impl Binding { } } + /// A path that embeds one or more Azure DevOps predefined variables, e.g. + /// `$(Pipeline.Workspace)/agentic-pipeline-compiler`. + /// + /// [`Binding::ado_macro`] takes a bare variable name; this takes a path + /// built around one. Every `$(…)` occurrence is validated as a well-formed + /// predefined-variable name, so the only thing the value can expand to is + /// a variable Azure DevOps substitutes before bash runs — not a command + /// substitution, and not a shell metacharacter. The result is + /// single-quoted, so the substituted text stays literal. + #[track_caller] + pub fn ado_path(value: impl AsRef) -> Self { + let value = value.as_ref(); + assert!( + !value.contains('\n') && !value.contains('\r'), + "shell binding value must be a single line, got {value:?}" + ); + assert!( + !value.contains('`') && !value.contains("${"), + "an ADO path must not contain a backtick or `${{`, got {value:?}" + ); + let mut rest = value; + while let Some(open) = rest.find("$(") { + let after = &rest[open + 2..]; + let close = after.find(')').unwrap_or_else(|| { + panic!("unterminated `$(` in ADO path {value:?}") + }); + let name = &after[..close]; + assert!( + is_ado_macro_name(name), + "ADO path {value:?} embeds {name:?}, which is not a dotted \ + alphanumeric predefined-variable name; a command substitution \ + is not permitted here" + ); + rest = &after[close + 1..]; + } + assert_not_secret(value); + Self { + rhs: single_quote(value), + kind: BindingKind::AdoMacro, + } + } + /// Bulk text — a JSON document, a prompt, a certificate — assigned through /// a quoted heredoc so no expansion occurs and no escaping is needed. /// @@ -339,6 +381,35 @@ mod tests { Binding::ado_macro("System.AccessToken"); } + #[test] + fn ado_path_accepts_a_macro_with_a_compiler_owned_suffix() { + let binding = Binding::ado_path("$(Pipeline.Workspace)/compiler/_pkg"); + assert_eq!(binding.rhs(), "'$(Pipeline.Workspace)/compiler/_pkg'"); + assert_eq!(binding.kind(), BindingKind::AdoMacro); + // A plain path with no macro is fine too. + assert_eq!(Binding::ado_path("/tmp/scripts").rhs(), "'/tmp/scripts'"); + } + + #[test] + #[should_panic(expected = "is not a dotted alphanumeric predefined-variable name")] + fn ado_path_refuses_a_command_substitution() { + // This is the whole point: `$(…)` in a path must be an ADO variable + // Azure DevOps substitutes, never a shell command the runner executes. + Binding::ado_path("/tmp/$(rm -rf /)/x"); + } + + #[test] + #[should_panic(expected = "unterminated")] + fn ado_path_refuses_an_unterminated_macro() { + Binding::ado_path("/tmp/$(Pipeline.Workspace/x"); + } + + #[test] + #[should_panic(expected = "backtick")] + fn ado_path_refuses_a_backtick() { + Binding::ado_path("/tmp/`id`"); + } + #[test] fn document_uses_a_quoted_heredoc_so_nothing_expands() { let binding = Binding::document("{\"a\": \"$NOT_EXPANDED\"}\n"); From f8ee73b9a53c47ec949ac59b6a2294ab702063b5 Mon Sep 17 00:00:00 2001 From: James Devine Date: Mon, 10 Aug 2026 18:30:27 +0100 Subject: [PATCH 6/7] test(compile): guard generated shell against regressing to format! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `src/compile/shell/` makes generated shell reviewable and lintable, but nothing stopped a future change from writing `BashStep::new("X", format!("set -eu\n\ …"))` again. That shape is invisible to both linters: the registry lint only sees registered scripts, and the compiled-YAML lint would report a finding but never the shape. The guard checks three things: the script argument of `BashStep::new` is never built inline, a `shell_script!` body contains no escaped continuation, and neither retired helper comes back. It deliberately does **not** grep for `\n\` across the codebase. The earlier survey in this work used exactly that as a proxy for "how much shell is left" and was badly wrong — most such lines are Rust markdown and error text, and `safe_outputs/create_pull_request.rs` has 38 of them and no shell at all. Acting on that count would have meant wrapping argv-based `git` calls in a shell that is not there today. Parsing the call by balancing parentheses matters for the same reason: the first draft scanned for a terminator, overran the call, and reported a `format!` in the next call's *display name* as a shell body. A test exercises the discriminator on both shapes, since a guard that only ever passes is indistinguishable from one that does nothing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: aa46b482-6121-45e2-8278-1a9465b4f73e --- AGENTS.md | 5 + docs/extending.md | 13 +++ src/compile/agentic_pipeline.rs | 20 +++- tests/generated_shell_guard.rs | 194 ++++++++++++++++++++++++++++++++ 4 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 tests/generated_shell_guard.rs diff --git a/AGENTS.md b/AGENTS.md index bcc0a05b..18aca608 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -561,6 +561,11 @@ stays on `.with_env(…, EnvValue::secret(…))`. Every variable a body reads must be declared as a `binding` or an `external`. Both the render path and a registry-wide test enforce it. +`tests/generated_shell_guard.rs` fails the build if shell regresses to the old +shape — a `BashStep::new` whose script argument is built with `format!`, an +escaped continuation inside a `shell_script!` body, or a reintroduced +`bash()` / `dedent()` helper. + ### Bash step lint Shell is linted at two levels. diff --git a/docs/extending.md b/docs/extending.md index 93347b0a..32220772 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -406,6 +406,19 @@ cargo run -- export-bash-scripts --out /tmp/ado-aw-shell --format json Writes one `.sh` per registered script with a provenance header naming the producing Rust source, for review with ordinary shell tooling. +### The guard + +`tests/generated_shell_guard.rs` fails the build if generated shell regresses +to the old shape: a `BashStep::new` whose *script* argument is built with +`format!`, an escaped `\n\` continuation inside a `shell_script!` body, or a +reintroduced `bash()` / `dedent()` helper. + +It deliberately does not grep for `\n\` across the codebase. Most such lines +are Rust markdown and error text — `safe_outputs/create_pull_request.rs` has 38 +of them and no shell at all — so counting them says nothing about how much +shell is left. The guard checks one unambiguous thing instead, and a test +proves it distinguishes an inline body from a `format!` display name. + ## Filter IR (`src/compile/filter_ir.rs`) Trigger filter expressions still use the separate filter IR. It lowers `PrFilters` / `PipelineFilters` into typed checks, validates conflicts, and emits bash consumed by `AdoScriptExtension` declarations. The generated gate steps are now returned as typed IR steps instead of being spliced into YAML templates. diff --git a/src/compile/agentic_pipeline.rs b/src/compile/agentic_pipeline.rs index f998ec49..29199b45 100644 --- a/src/compile/agentic_pipeline.rs +++ b/src/compile/agentic_pipeline.rs @@ -6877,11 +6877,25 @@ safe-outputs: "the resolved version must be verified, not just requested: {script}" ); + // A caller-supplied version must reach the script, and the compiled-in + // default must not survive alongside it. The version is a binding now, + // so assert on the prelude — that proves the producer supplied it, + // where a bare substring would also match the verification message. let override_script = prepare_ado_mcp_step("2.9.0").script; - assert!(override_script.contains(&format!("{ADO_MCP_PACKAGE}@2.9.0"))); - assert!(override_script.contains("expected 2.9.0")); assert!( - !override_script.contains(&format!("{ADO_MCP_PACKAGE}@{}", common::ADO_MCP_VERSION)) + override_script.contains("MCP_VERSION='2.9.0'"), + "the override must be bound: {override_script}" + ); + assert!( + !override_script.contains(&format!("MCP_VERSION='{}'", common::ADO_MCP_VERSION)), + "the default version must not survive an override: {override_script}" + ); + // The pin and its verification both read the binding, so an override + // cannot be applied to one and not the other. + assert!( + override_script.contains(r#""$MCP_PACKAGE@$MCP_VERSION""#) + && override_script.contains("expected $MCP_VERSION"), + "install and verification must share one version: {override_script}" ); } diff --git a/tests/generated_shell_guard.rs b/tests/generated_shell_guard.rs new file mode 100644 index 00000000..56533906 --- /dev/null +++ b/tests/generated_shell_guard.rs @@ -0,0 +1,194 @@ +//! Guard against generated shell regressing to unstructured `format!` bodies. +//! +//! # Why a source-level guard +//! +//! `src/compile/shell/` makes generated shell reviewable and lintable, but +//! nothing stops a future change from going back to +//! `BashStep::new("X", format!("set -eu\n\ …"))`. That would be invisible to +//! both linters: the registry lint only sees registered scripts, and the +//! compiled-YAML lint would flag a *finding* but never the *shape*. +//! +//! The check is deliberately narrow. An earlier survey used the count of +//! `\n\` continuations per file as a proxy for "how much shell is left" and +//! was badly wrong — most of those lines are Rust markdown and error text +//! (`safe_outputs/create_pull_request.rs` has 38 of them and no shell at +//! all). So this does not grep for continuations. It checks one thing that is +//! unambiguous: the script argument of `BashStep::new` must not be built +//! inline. + +use std::path::{Path, PathBuf}; + +fn src_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src") +} + +fn rust_sources(dir: &Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).expect("read src dir") { + let path = entry.expect("dir entry").path(); + if path.is_dir() { + rust_sources(&path, out); + } else if path.extension().is_some_and(|e| e == "rs") { + out.push(path); + } + } +} + +/// Byte offset at which a file's `#[cfg(test)]` module begins, if any. +/// +/// Test code legitimately builds throwaway steps inline — the rule is about +/// what the compiler *emits*, not about test fixtures. +fn test_module_start(source: &str) -> usize { + source + .find("#[cfg(test)]") + .unwrap_or(source.len()) +} + +/// The script argument of a `BashStep::new(name, script)` call starting at +/// `open` (the index of the `(`), or `None` if the call is malformed. +/// +/// Balances parentheses and splits on the top-level comma rather than +/// scanning for a terminator: a naive scan overruns the call and picks up +/// unrelated code, and — as the first draft of this guard proved — reports a +/// `format!` in the *display name* of the next call as if it were a shell +/// body. +fn script_argument(source: &str, open: usize) -> Option<&str> { + let bytes = source.as_bytes(); + let mut depth = 0usize; + let mut split = None; + for i in open..bytes.len() { + match bytes[i] { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + let start = split? + 1; + return Some(&source[start..i]); + } + } + b',' if depth == 1 && split.is_none() => split = Some(i), + _ => {} + } + } + None +} + +#[test] +fn generated_bash_steps_are_never_built_from_an_inline_format() { + let mut files = Vec::new(); + rust_sources(&src_dir(), &mut files); + assert!(!files.is_empty(), "no Rust sources found under src/"); + + let mut problems = Vec::new(); + for file in &files { + // `shell/mod.rs` owns `into_step`, which is the one sanctioned + // `BashStep::new` in the codebase. + if file.ends_with(Path::new("compile/shell/mod.rs")) + || file.ends_with(Path::new("compile\\shell\\mod.rs")) + { + continue; + } + let source = std::fs::read_to_string(file).expect("read source"); + let production = &source[..test_module_start(&source)]; + + const CALL: &str = "BashStep::new"; + for (index, _) in production.match_indices(CALL) { + let Some(script) = script_argument(production, index + CALL.len()) else { + continue; + }; + // A `format!` or an escaped continuation in the *script* argument + // means the shell was assembled at the call site rather than + // declared as a registered script. A `format!` in the display + // name is fine and common. + if script.contains("format!(") || script.contains("\\n\\") { + let line = production[..index].lines().count() + 1; + problems.push(format!( + " {}:{line} builds a bash body inline", + file.display() + )); + } + } + } + + assert!( + problems.is_empty(), + "generated shell must be declared with `shell_script!` and rendered \ + through `ShellScript`, not assembled inline. See the \"Generated \ + shell scripts\" section of docs/extending.md.\n{}", + problems.join("\n") + ); +} + +#[test] +fn registered_script_bodies_are_written_verbatim() { + // A `shell_script!` body is a raw string containing the shell exactly as + // it runs. A `\n\` continuation inside one means somebody pasted an old + // `format!` body in without unescaping it, which defeats the point: the + // body would no longer read as the script that runs. + let mut files = Vec::new(); + rust_sources(&src_dir(), &mut files); + + let mut problems = Vec::new(); + for file in &files { + let source = std::fs::read_to_string(file).expect("read source"); + for (index, _) in source.match_indices("shell_script! {") { + let tail = &source[index..]; + // A body runs to the closing raw-string delimiter. + let end = tail.find("\"#").map(|e| e + 2).unwrap_or(tail.len()); + if tail[..end].contains("\\n\\") { + let line = source[..index].lines().count() + 1; + problems.push(format!( + " {}:{line} has an escaped continuation in a script body", + file.display() + )); + } + } + } + + assert!( + problems.is_empty(), + "a `shell_script!` body must be verbatim shell — no `\\n\\` \ + continuations, no escaped quotes:\n{}", + problems.join("\n") + ); +} + +#[test] +fn the_guard_catches_an_inline_body_but_not_a_formatted_display_name() { + // A guard that only ever passes is indistinguishable from one that does + // nothing. Exercise the discriminator directly on both shapes. + const CALL: &str = "BashStep::new"; + + let offending = r#"Step::Bash(BashStep::new("Do a thing", format!("set -eu\nrm {p}\n")))"#; + let index = offending.find(CALL).expect("call present"); + let script = script_argument(offending, index + CALL.len()).expect("argument parsed"); + assert!( + script.contains("format!("), + "an inline format! body must be caught, got {script:?}" + ); + + // A `format!` display name with a rendered script is the normal shape for + // a step that needs extra configuration, and must not be flagged. + let allowed = r#"BashStep::new(format!("Stage compiler (v{v})"), body)"#; + let index = allowed.find(CALL).expect("call present"); + let script = script_argument(allowed, index + CALL.len()).expect("argument parsed"); + assert_eq!(script.trim(), "body"); + assert!(!script.contains("format!(")); +} + +#[test] +fn the_retired_helpers_are_not_reintroduced() { + // `bash()` and `dedent()` in `agentic_pipeline.rs` existed only to paper + // over `format!`-built bodies. Re-adding either would signal the pattern + // is back. + let path = src_dir().join("compile").join("agentic_pipeline.rs"); + let source = std::fs::read_to_string(&path).expect("read agentic_pipeline.rs"); + for retired in ["\nfn bash(", "\nfn dedent("] { + assert!( + !source.contains(retired), + "`{}` was reintroduced in {}; generated shell should go through \ + `ShellScript` instead", + retired.trim(), + path.display() + ); + } +} From b355c2009fd6de6ff9fedc66e290d83445b33341 Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 11 Aug 2026 11:09:04 +0100 Subject: [PATCH 7/7] test(compile): update gate and MCP assertions for the binding form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught three assertions that still expected shell values interpolated inline. The PR moves them into the generated binding prelude, so the emitted text legitimately changed: * `tests/compiler_tests.rs` — the Azure DevOps MCP version (from #1843) now renders as `MCP_VERSION='2.9.0'` with `"$MCP_PACKAGE@$MCP_VERSION"` rather than `"@azure-devops/mcp@2.9.0"`. Both the override and default cases now assert on the prelude *and* the use, which also proves the install and its verification read one version rather than two. * `tests/gate_e2e.rs` — `find_gate_spec` located the gate step by the literal `node ''`. With the path bound, that matched nothing and the test reported the gate as absent rather than as changed. Also passes `SYSTEMROOT` through the gate harness's `env_clear` on Windows. Node aborts during initialisation without it — its CSPRNG seeding resolves the OS crypto provider relative to that variable — and the failure surfaces as an assertion with empty stdout and a native stack trace, which reads like a gate-logic bug. Linux CI never hit this; it made the test unrunnable locally, which is how the stale locator survived review in the first place. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: aa46b482-6121-45e2-8278-1a9465b4f73e --- tests/compiler_tests.rs | 28 +++++++++++++++++++++++----- tests/gate_e2e.rs | 30 +++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/tests/compiler_tests.rs b/tests/compiler_tests.rs index cd08451d..ea705807 100644 --- a/tests/compiler_tests.rs +++ b/tests/compiler_tests.rs @@ -2078,9 +2078,26 @@ Test. "#, ); - assert!(compiled.contains("\"@azure-devops/mcp@2.9.0\"")); - assert!(compiled.contains("expected 2.9.0")); - assert!(!compiled.contains("\"@azure-devops/mcp@2.8.1\"")); + // The version is supplied as a `ShellScript` binding rather than + // interpolated into the install command, so assert on the generated + // prelude — that proves the override reached the producer, where a bare + // substring would also match the verification message. + assert!( + compiled.contains("MCP_PACKAGE='@azure-devops/mcp'") + && compiled.contains("MCP_VERSION='2.9.0'"), + "the front-matter version override must reach the install step" + ); + // Install and verification read the same binding, so an override cannot + // be applied to one and not the other. + assert!( + compiled.contains("\"$MCP_PACKAGE@$MCP_VERSION\"") + && compiled.contains("expected $MCP_VERSION"), + "the install and its verification must share one version" + ); + assert!( + !compiled.contains("MCP_VERSION='2.8.1'"), + "the compiler default must not survive an explicit override" + ); } /// Test that the Azure DevOps MCP fixture compiles successfully with no unreplaced markers @@ -2168,8 +2185,9 @@ fn test_fixture_azure_devops_mcp_compiled_output() { "MCPG config should contain the container image" ); assert!( - compiled.contains("\"@azure-devops/mcp@2.8.1\"") - && compiled.contains("expected 2.8.1"), + compiled.contains("MCP_PACKAGE='@azure-devops/mcp'") + && compiled.contains("MCP_VERSION='2.8.1'") + && compiled.contains("expected $MCP_VERSION"), "the unversioned frontmatter form must use and verify the compiler default" ); assert!( diff --git a/tests/gate_e2e.rs b/tests/gate_e2e.rs index 054d2eff..88b395cb 100644 --- a/tests/gate_e2e.rs +++ b/tests/gate_e2e.rs @@ -18,12 +18,21 @@ fn value_field<'a>(mapping: &'a Mapping, key: &str) -> Option<&'a Value> { mapping.get(&key) } +/// Locate the compiled gate step and return its `GATE_SPEC` env value. +/// +/// The step is identified by the evaluator path plus the `node` invocation +/// that reads it. The path arrives as a `ShellScript` binding in the generated +/// prelude rather than inline in the command, so matching the bare +/// `node ''` form would silently find nothing and report the gate as +/// missing. fn find_gate_spec(value: &Value) -> Option { + const EVALUATOR_PATH: &str = "/tmp/ado-aw-scripts/ado-script/gate.js"; match value { Value::Mapping(mapping) => { let script = string_field(mapping, "bash").or_else(|| string_field(mapping, "script")); if script.is_some_and(|script| { - script.contains("node '/tmp/ado-aw-scripts/ado-script/gate.js'") + script.contains(&format!("EVALUATOR_PATH='{EVALUATOR_PATH}'")) + && script.contains(r#"node "$EVALUATOR_PATH""#) }) { let env = value_field(mapping, "env")?.as_mapping()?; return string_field(env, "GATE_SPEC").map(str::to_owned); @@ -39,10 +48,25 @@ fn find_gate_spec(value: &Value) -> Option { fn run_gate(gate_js: &Path, gate_spec: &str, pr_title: &str) -> Output { let path = std::env::var_os("PATH").unwrap_or_default(); - Command::new("node") + let mut command = Command::new("node"); + command .arg(gate_js) .env_clear() - .env("PATH", path) + .env("PATH", path); + + // Node aborts during initialisation on Windows without `SYSTEMROOT`: its + // CSPRNG seeding calls into the OS crypto provider, which is resolved + // relative to that variable. The failure is an assertion with no stdout + // and a native stack trace, which reads like a gate-logic bug rather than + // a missing environment variable. Passing it through keeps `env_clear`'s + // intent — the gate must see only the variables set below — while letting + // the process start at all. + #[cfg(windows)] + if let Some(system_root) = std::env::var_os("SYSTEMROOT") { + command.env("SYSTEMROOT", system_root); + } + + command .env("GATE_SPEC", gate_spec) .env("ADO_BUILD_REASON", "PullRequest") .env("ADO_PR_TITLE", pr_title)