diff --git a/README.md b/README.md index dd26d04..6383758 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,9 @@ it: `image`/`build`, `command`, `environment`/`env_file`, short-form bind `volumes`, `healthcheck` (CMD/CMD-SHELL), `depends_on` (all conditions), and network `aliases`. Compose extension fields (any `x-`-prefixed key) and YAML anchors are accepted as-is, so a top-level `x-*` anchor block for shared -config just works. See `architecture/supported-subset.md` for the full -accept/ignore/reject matrix. +config just works. `${VAR}`-style variable interpolation is resolved against +the process environment (no `.env` file support). See +`architecture/supported-subset.md` for the full accept/ignore/reject matrix. ## Status diff --git a/architecture/supported-subset.md b/architecture/supported-subset.md index 11ded2b..e27aeaa 100644 --- a/architecture/supported-subset.md +++ b/architecture/supported-subset.md @@ -78,6 +78,21 @@ common way to shadow a subdirectory of a bind mount). No host-path translation is applied, since the entry names a container path, not a host source. A colon-less entry that is not absolute (e.g. `./cache`) is malformed and raises. +## Variable interpolation + +`interpolate()` (`compose2pod/interpolate.py`) resolves Compose Spec +`${VAR}` references against the process environment (`os.environ` in the +CLI) before `validate()` sees the document, in every string leaf of the +compose document — dict keys are left untouched. Supported forms: +`$VAR`, `${VAR}`, `${VAR:-default}`, `${VAR-default}`, `${VAR:?msg}`, +`${VAR?msg}`, `${VAR:+alt}`, `${VAR+alt}`, and `$$` for a literal `$`. An +unset `$VAR`/`${VAR}` with no default resolves to an empty string and +emits a warning rather than failing; `${VAR:?msg}`/`${VAR?msg}` raises +`UnsupportedComposeError(msg)` instead. There is no `.env` file support — +only the environment the caller already has is consulted; export the +values first (`set -a; . .env; set +a`) if a project relies on a `.env` +file. + ## depends_on All three conditions are honored: `service_started`, `service_healthy`, diff --git a/compose2pod/__init__.py b/compose2pod/__init__.py index 456f871..6fd5f41 100644 --- a/compose2pod/__init__.py +++ b/compose2pod/__init__.py @@ -2,6 +2,7 @@ from compose2pod.emit import EmitOptions, emit_script from compose2pod.exceptions import UnsupportedComposeError +from compose2pod.interpolate import interpolate from compose2pod.parsing import validate @@ -9,5 +10,6 @@ "EmitOptions", "UnsupportedComposeError", "emit_script", + "interpolate", "validate", ] diff --git a/compose2pod/cli.py b/compose2pod/cli.py index 0c6e356..56d6d1b 100644 --- a/compose2pod/cli.py +++ b/compose2pod/cli.py @@ -3,6 +3,7 @@ import argparse import contextlib import json +import os import re import sys from pathlib import Path @@ -11,6 +12,7 @@ from compose2pod.emit import EmitOptions, emit_script from compose2pod.exceptions import UnsupportedComposeError +from compose2pod.interpolate import interpolate from compose2pod.parsing import validate @@ -88,7 +90,8 @@ def main(argv: list[str] | None = None) -> int: sys.stderr.write(f"compose2pod: error: could not parse compose input: {error}\n") return 2 try: - warnings = validate(compose) + compose, interpolation_warnings = interpolate(compose, os.environ) + warnings = [*interpolation_warnings, *validate(compose)] script = emit_script( compose=compose, options=EmitOptions( diff --git a/compose2pod/interpolate.py b/compose2pod/interpolate.py new file mode 100644 index 0000000..5619d08 --- /dev/null +++ b/compose2pod/interpolate.py @@ -0,0 +1,72 @@ +"""Resolve Compose-spec `${VAR}` interpolation against a given environment.""" + +import re +from collections.abc import Mapping +from typing import Any + +from compose2pod.exceptions import UnsupportedComposeError + + +_PATTERN = re.compile( + r""" + \$(?: + (?P\$) + |(?P[a-zA-Z_][a-zA-Z0-9_]*) + |\{(?P[a-zA-Z_][a-zA-Z0-9_]*)(?P:-|-|:\?|\?|:\+|\+)?(?P[^}]*)\} + ) + """, + re.VERBOSE, +) + + +def _resolve(name: str, op: str | None, arg: str, environ: Mapping[str, str], warnings: list[str]) -> str: + is_set = name in environ + is_set_and_nonempty = is_set and environ[name] != "" + if op is None: + if is_set: + return environ[name] + warnings.append(f"variable '{name}' is not set, defaulting to blank string") + return "" + if op in {"-", ":-"}: + condition = is_set_and_nonempty if op == ":-" else is_set + return environ[name] if condition else arg + if op in {"?", ":?"}: + condition = is_set_and_nonempty if op == ":?" else is_set + if condition: + return environ[name] + raise UnsupportedComposeError(arg) + condition = is_set_and_nonempty if op == ":+" else is_set + return arg if condition else "" + + +def _interpolate_string(text: str, environ: Mapping[str, str], warnings: list[str]) -> str: + def substitute(match: re.Match[str]) -> str: + if match.group("escaped"): + return "$" + if match.group("named") is not None: + return _resolve(match.group("named"), None, "", environ, warnings) + return _resolve(match.group("braced"), match.group("op"), match.group("arg"), environ, warnings) + + return _PATTERN.sub(substitute, text) + + +def _walk(node: Any, environ: Mapping[str, str], warnings: list[str]) -> Any: # noqa: ANN401 - arbitrary compose value + if isinstance(node, str): + return _interpolate_string(node, environ, warnings) + if isinstance(node, dict): + return {key: _walk(value, environ, warnings) for key, value in node.items()} + if isinstance(node, list): + return [_walk(item, environ, warnings) for item in node] + return node + + +def interpolate(document: Any, environ: Mapping[str, str]) -> tuple[Any, list[str]]: # noqa: ANN401 - arbitrary compose value + """Resolve `${VAR}`-style references in every string leaf of `document`. + + Dict keys are left untouched. Returns the resolved document and any + "variable not set" warnings; raises `UnsupportedComposeError` for a + `${VAR:?msg}`/`${VAR?msg}` reference whose condition is not met. + """ + warnings: list[str] = [] + resolved = _walk(document, environ, warnings) + return resolved, warnings diff --git a/planning/changes/2026-07-08.06-variable-interpolation.md b/planning/changes/2026-07-08.06-variable-interpolation.md new file mode 100644 index 0000000..8b1c36c --- /dev/null +++ b/planning/changes/2026-07-08.06-variable-interpolation.md @@ -0,0 +1,125 @@ +--- +summary: Added `compose2pod/interpolate.py`, resolving Compose-spec `${VAR}` interpolation against the process environment before validate/emit, so placeholders never reach a container literally. +--- + +# Design: Support Compose variable interpolation + +## Summary + +compose2pod passes every compose string value through to `podman run` +verbatim. A compose file that uses `${VAR}` — idiomatic Compose syntax for +injecting CI/host environment values — currently reaches the container as +the literal text `${VAR}` instead of the value of `VAR`, because nothing in +the pipeline performs the substitution real Compose engines apply at +parse time. This change adds an `interpolate()` step, run right after +parsing and before `validate()`, that resolves the Compose Spec +interpolation syntax against `os.environ`. + +## Motivation + +```yaml +services: + app: + environment: + LLM_MODEL: ${LLM_MODEL} +``` + +With `LLM_MODEL=gpt-4` set in the CI environment invoking compose2pod, a real +`docker compose` / `podman-compose` run would substitute the value at parse +time, so the container never sees a placeholder. compose2pod instead emits +`-e 'LLM_MODEL=${LLM_MODEL}'` — single-quoted by `shlex.quote` so the shell +never expands it either — and the container receives the literal string +`${LLM_MODEL}`. Downstream app code parsing that value (e.g. pydantic +settings) fails with a confusing type error instead of getting the resolved +config. This is a silent-wrongness bug, not a missing feature notice: +nothing in compose2pod warns that interpolation isn't happening. + +## Design + +### 1. New module `compose2pod/interpolate.py` + +`interpolate(document: Any, environ: Mapping[str, str]) -> tuple[Any, list[str]]` +recursively walks the parsed compose document (dict values and list items; +**keys are never interpolated** — Compose doesn't template mapping keys +either) and applies Compose Spec substitution to every string leaf: + +- `$$` → literal `$`. +- `$VAR` / `${VAR}` → `environ[VAR]`, or `""` with a warning + (`"variable 'VAR' is not set, defaulting to blank string"`) if unset. +- `${VAR:-default}` → `VAR` if set **and non-empty**, else `default`. +- `${VAR-default}` → `VAR` if set (even empty), else `default`. +- `${VAR:?msg}` / `${VAR?msg}` → same set/unset rule as above, but raise + `UnsupportedComposeError(msg)` instead of substituting a default. +- `${VAR:+alt}` / `${VAR+alt}` → `alt` if the set/unset rule is true, else + `""`. + +`default`/`alt`/`msg` text is taken literally — no recursive interpolation +inside a default's own text. This mirrors the common case (`${FOO:-bar}`) +and keeps the regex and semantics simple; nested `${...}` inside a default +is not real-world compose usage. + +`environ` is passed in by the caller rather than read internally, so tests +don't need to mutate `os.environ`. + +### 2. Wire into `cli.py` + +```python +compose = _read_compose(text, args.format) +compose, interpolation_warnings = interpolate(compose, os.environ) +... +warnings = [*interpolation_warnings, *validate(compose)] +``` + +Interpolation runs before `validate()` so structural checks always see +resolved values (not load-bearing today, but keeps the pipeline order +matching real Compose: substitute, then validate, then convert). + +### 3. Export from the public API + +Add `interpolate` to `compose2pod/__init__.py::__all__`, alongside +`validate` and `emit_script` — library consumers who call the pipeline +functions directly (bypassing `cli.main`) need the same fix. + +### 4. Promote into `architecture/supported-subset.md` + +New "Variable interpolation" section documenting the supported syntax and +that it resolves against the process environment only (not `.env` files — +see Non-goals). + +## Non-goals + +- **No `.env` file loading.** Real `docker compose` merges a project-root + `.env` file into the interpolation environment. compose2pod has no notion + of "the project's `.env`" and reads compose input from stdin/a single + file; adding implicit file discovery is a separate, larger decision. Users + needing this can `set -a; . .env; set +a` (or `export $(cat .env)`) before + invoking compose2pod — same environment compose2pod already reads from. +- **No interpolation of `env_file` contents.** Values inside a file passed + via `env_file:` are handled by podman's own `--env-file`, not by + compose2pod, and Compose itself does not interpolate those either. +- **No nested interpolation inside default/error text** (see Design §1). +- **No interpolation of `--command`.** That's a CLI argument, not compose + YAML; the invoking shell already expands it before compose2pod sees it. + +## Testing + +TDD, `just test-ci` at 100% line coverage. + +- Unit (`test_interpolate.py`, new): one case per syntax form (`$VAR`, + `${VAR}`, `${VAR:-d}`, `${VAR-d}`, `${VAR:?m}` unset/empty/set, + `${VAR?m}`, `${VAR:+a}`, `${VAR+a}`, `$$`), unset-var warning, recursion + through nested dict/list, keys left untouched. +- Integration (`test_cli.py`): a compose document with `${VAR}` in an + `environment` value, run through `main()` with the var set in + `monkeypatch.setenv`, asserts the resolved value (not the placeholder) + appears in the emitted script's `-e` flag. +- `test_exports` updated for the new `interpolate` export. + +## Risk + +Low. The new module is a pure function with no side effects, sitting before +existing validated code paths; worst case is a regex edge case in an +unusual `${...}` form, caught by the syntax-form test matrix. The main +judgment call — resolving against `os.environ` only, no `.env` file — is +called out explicitly in Non-goals so it's a visible, revisitable decision +rather than a silent gap. diff --git a/tests/test_cli.py b/tests/test_cli.py index 6cf4c38..f66ec53 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -21,7 +21,13 @@ def run_main(compose_text: str, argv: list[str], monkeypatch: pytest.MonkeyPatch class TestPublicApi: def test_exports(self) -> None: - assert set(compose2pod.__all__) == {"EmitOptions", "UnsupportedComposeError", "emit_script", "validate"} + assert set(compose2pod.__all__) == { + "EmitOptions", + "UnsupportedComposeError", + "emit_script", + "interpolate", + "validate", + } assert compose2pod.validate({"services": {"a": {"image": "x"}}}) == [] @@ -128,6 +134,38 @@ def test_missing_file_returns_2(self, capsys: pytest.CaptureFixture[str]) -> Non assert rc == EXIT_USAGE_ERROR assert "compose2pod: error:" in capsys.readouterr().err + def test_environment_variable_reference_is_resolved( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LLM_MODEL", "gpt-4") + yaml_text = "services:\n app:\n image: x\n environment:\n MODEL: ${LLM_MODEL}\n" + rc = run_main(yaml_text, ["--target", "app", "--image", "i", "--format", "yaml"], monkeypatch) + out = capsys.readouterr() + assert rc == 0 + assert "MODEL=gpt-4" in out.out + assert "LLM_MODEL" not in out.out + + def test_unset_environment_variable_reference_warns( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("UNSET_VAR", raising=False) + yaml_text = "services:\n app:\n image: x\n environment:\n MODEL: ${UNSET_VAR}\n" + rc = run_main(yaml_text, ["--target", "app", "--image", "i", "--format", "yaml"], monkeypatch) + out = capsys.readouterr() + assert rc == 0 + assert "MODEL=" in out.out + assert "UNSET_VAR" in out.err + assert "not set" in out.err + + def test_required_environment_variable_missing_returns_2( + self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("REQUIRED_VAR", raising=False) + yaml_text = "services:\n app:\n image: x\n environment:\n MODEL: ${REQUIRED_VAR:?must be set}\n" + rc = run_main(yaml_text, ["--target", "app", "--image", "i", "--format", "yaml"], monkeypatch) + assert rc == EXIT_USAGE_ERROR + assert "must be set" in capsys.readouterr().err + def test_yaml_anchor_extension_fields_convert( self, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/test_interpolate.py b/tests/test_interpolate.py new file mode 100644 index 0000000..c85f1f7 --- /dev/null +++ b/tests/test_interpolate.py @@ -0,0 +1,102 @@ +import pytest + +from compose2pod.exceptions import UnsupportedComposeError +from compose2pod.interpolate import interpolate + + +class TestInterpolate: + def test_bare_dollar_var_substitutes(self) -> None: + document, warnings = interpolate({"k": "$FOO"}, {"FOO": "bar"}) + assert document == {"k": "bar"} + assert warnings == [] + + def test_braced_var_substitutes(self) -> None: + document, warnings = interpolate({"k": "${FOO}"}, {"FOO": "bar"}) + assert document == {"k": "bar"} + assert warnings == [] + + def test_unset_var_defaults_to_blank_and_warns(self) -> None: + document, warnings = interpolate({"k": "${FOO}"}, {}) + assert document == {"k": ""} + assert "FOO" in warnings[0] + assert "not set" in warnings[0] + + def test_dollar_dollar_is_literal_dollar(self) -> None: + document, warnings = interpolate({"k": "$$FOO"}, {"FOO": "bar"}) + assert document == {"k": "$FOO"} + assert warnings == [] + + def test_colon_dash_uses_default_when_unset(self) -> None: + document, _ = interpolate({"k": "${FOO:-fallback}"}, {}) + assert document == {"k": "fallback"} + + def test_colon_dash_uses_default_when_empty(self) -> None: + document, _ = interpolate({"k": "${FOO:-fallback}"}, {"FOO": ""}) + assert document == {"k": "fallback"} + + def test_colon_dash_uses_value_when_set_and_nonempty(self) -> None: + document, _ = interpolate({"k": "${FOO:-fallback}"}, {"FOO": "bar"}) + assert document == {"k": "bar"} + + def test_dash_uses_default_only_when_unset(self) -> None: + document, _ = interpolate({"k": "${FOO-fallback}"}, {}) + assert document == {"k": "fallback"} + + def test_dash_keeps_empty_value_when_set_empty(self) -> None: + document, _ = interpolate({"k": "${FOO-fallback}"}, {"FOO": ""}) + assert document == {"k": ""} + + def test_colon_question_raises_when_unset(self) -> None: + with pytest.raises(UnsupportedComposeError, match="FOO is required"): + interpolate({"k": "${FOO:?FOO is required}"}, {}) + + def test_colon_question_raises_when_empty(self) -> None: + with pytest.raises(UnsupportedComposeError, match="FOO is required"): + interpolate({"k": "${FOO:?FOO is required}"}, {"FOO": ""}) + + def test_colon_question_passes_when_set_and_nonempty(self) -> None: + document, _ = interpolate({"k": "${FOO:?FOO is required}"}, {"FOO": "bar"}) + assert document == {"k": "bar"} + + def test_question_raises_only_when_unset(self) -> None: + with pytest.raises(UnsupportedComposeError, match="FOO is required"): + interpolate({"k": "${FOO?FOO is required}"}, {}) + + def test_question_passes_when_set_empty(self) -> None: + document, _ = interpolate({"k": "${FOO?FOO is required}"}, {"FOO": ""}) + assert document == {"k": ""} + + def test_colon_plus_substitutes_alt_when_set_and_nonempty(self) -> None: + document, _ = interpolate({"k": "${FOO:+alt}"}, {"FOO": "bar"}) + assert document == {"k": "alt"} + + def test_colon_plus_blank_when_unset_or_empty(self) -> None: + assert interpolate({"k": "${FOO:+alt}"}, {})[0] == {"k": ""} + assert interpolate({"k": "${FOO:+alt}"}, {"FOO": ""})[0] == {"k": ""} + + def test_plus_substitutes_alt_when_set_even_empty(self) -> None: + document, _ = interpolate({"k": "${FOO+alt}"}, {"FOO": ""}) + assert document == {"k": "alt"} + + def test_plus_blank_when_unset(self) -> None: + document, _ = interpolate({"k": "${FOO+alt}"}, {}) + assert document == {"k": ""} + + def test_recurses_through_nested_dict_and_list(self) -> None: + document, _ = interpolate( + {"services": {"app": {"environment": ["A=$FOO", "B=${BAR}"]}}}, + {"FOO": "1", "BAR": "2"}, + ) + assert document == {"services": {"app": {"environment": ["A=1", "B=2"]}}} + + def test_keys_are_left_untouched(self) -> None: + document, _ = interpolate({"$FOO": "literal"}, {"FOO": "bar"}) + assert document == {"$FOO": "literal"} + + def test_non_string_leaves_pass_through(self) -> None: + document, _ = interpolate({"k": 1, "n": None, "b": True}, {}) + assert document == {"k": 1, "n": None, "b": True} + + def test_multiple_variables_in_one_string(self) -> None: + document, _ = interpolate({"k": "$FOO-${BAR}"}, {"FOO": "a", "BAR": "b"}) + assert document == {"k": "a-b"}