From 363fd984f235976a4dbd7d22577a65b6358fe11d Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Fri, 2 Oct 2026 14:40:40 -0400 Subject: [PATCH 1/3] feat(eventing): make signing mountable in a cluster, and generate the keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #879 wired signing in both directions but left it unreachable outside a laptop: the ConfigMap shipped `*_VERIFY_KEYSET_PATH` and `*_SIGNING_KID` as empty strings and said seed paths "belong in the Deployment beside the volume" — and no Deployment had one. The only volume in either was `data: emptyDir`, so there was no way to get a key into a pod. That blocks S6/S8/S9, each of which is defined as "reproducible on a local Kind cluster". * `scripts/gen_signing_keys.py` — no key generation existed anywhere, so enabling signing meant hand-rolling `os.urandom(32).hex()` and deriving public keys in a REPL. Seeds are written 0600 under a 0700 directory and **never printed**; only kids and public keys go to stdout, so a pasted terminal transcript cannot leak a private key. Public keys are derived from the seeds rather than typed, so the keyset cannot disagree with the keys it authorizes. Refuses to overwrite an existing seed without `--force`, since that key may already be deployed and approved. * Both Deployments mount a seed Secret at `/etc/ce-signing` and the keyset ConfigMap at `/etc/ce-keyset`, both `optional: true` — without that, every signing-less overlay would block in ContainerCreating waiting for a Secret that will never exist. No `defaultMode`: 0400 would be root-only and there is no `runAsUser`/`fsGroup` (OpenShift assigns the UID), so the default 0644 is what makes the file readable. * `k8s/base/keyset-configmap.yaml` ships **empty** and a test will keep it that way. "There is never a key in this file" is checkable; "only public keys in this file" is not, and a 32-byte seed is indistinguishable from a public key by length. * `k8s/overlays/kind-signed/` flips enforcement on. A separate overlay rather than patching `kind`, because S3/S5 are explicitly unsigned milestones and the before/after contrast is the demo — and because the `test` overlay's "mounts no credential" assertion should keep meaning something. **A per-service Secret, not one shared.** Caught while rendering the overlay: both Deployments initially mounted the same `event-signing-key`, which would have given the two services one seed. That collapses the asymmetry the design rests on — EventBridge would hold the key it verifies runner responses with and could forge them, which is the exact property that made HMAC unacceptable (DESIGN_PHASE2.md §4.3). Now `eventbridge-signing-key` and `eventrunner-signing-key`. This deviates from the single `event-signing-key` name in README_PHASE1.md:704-722; that section is rewritten in a later commit, as it also still describes a symmetric setup. Verified: `data` remains volume index 0 in all three existing overlays, so the demo overlay's `/spec/template/spec/volumes/0` PVC patch still targets the right volume; no overlay renders a `kind: Secret`; `kind` stays fully off while `kind-signed` renders enforcement plus both key paths; the rendered env maps onto the real config fields; and keys from the generator verify through `verify_with_keyset`. 551 passed, 5 skipped — including the three manifest tests this could have broken (`test_no_credential_is_ever_rendered_into_a_manifest`, `test_the_test_overlay_mounts_no_credential_so_mock_mode_is_automatic`, `test_no_manifest_sets_run_as_user`). ruff check and format clean. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/k8s/base/eventbridge-deployment.yaml | 35 ++++ eventing/k8s/base/eventrunner-deployment.yaml | 34 ++++ eventing/k8s/base/keyset-configmap.yaml | 28 ++++ eventing/k8s/base/kustomization.yaml | 1 + .../overlays/kind-signed/kustomization.yaml | 63 +++++++ eventing/scripts/gen_signing_keys.py | 156 ++++++++++++++++++ 6 files changed, 317 insertions(+) create mode 100644 eventing/k8s/base/keyset-configmap.yaml create mode 100644 eventing/k8s/overlays/kind-signed/kustomization.yaml create mode 100644 eventing/scripts/gen_signing_keys.py diff --git a/eventing/k8s/base/eventbridge-deployment.yaml b/eventing/k8s/base/eventbridge-deployment.yaml index 3a26e2ee..d06a2a2d 100644 --- a/eventing/k8s/base/eventbridge-deployment.yaml +++ b/eventing/k8s/base/eventbridge-deployment.yaml @@ -67,6 +67,15 @@ spec: volumeMounts: - name: data mountPath: /data + # §11. Appended, not inserted: the demo overlay JSON-patches + # `/spec/template/spec/volumes/0` to swap `data` for a PVC, so anything + # placed before `data` would be replaced instead of it. + - name: signing-key + mountPath: /etc/ce-signing + readOnly: true + - name: keyset + mountPath: /etc/ce-keyset + readOnly: true readinessProbe: httpGet: path: /healthz @@ -91,3 +100,29 @@ spec: volumes: - name: data emptyDir: {} + # §11 — the seed EventBridge signs requests and group lifecycle events with. + # NOT declared in any manifest: `k8s_deploy.py --signing-keys` applies the + # Secret via stdin, because a kustomize secretGenerator would inline the seed + # into `kubectl kustomize` output and `--from-literal` would leak it to argv. + # + # `optional: true` keeps signing-less overlays working — without it the pod + # blocks in ContainerCreating waiting for a Secret that will never exist. No + # `defaultMode`: 0400 would be root-only, and with no `runAsUser`/`fsGroup` + # (OpenShift assigns the UID) the default 0644 is what makes it readable. + # A Secret PER SERVICE, not one shared between them. A single seed would mean + # EventBridge holds the key it verifies runner responses with, so it could + # forge any agent's answer and an agent could forge another's — which is the + # exact property that made HMAC unacceptable (DESIGN_PHASE2.md §4.3). Two + # Secrets is what keeps "only this agent could have produced this" true. + - name: signing-key + secret: + secretName: eventbridge-signing-key + optional: true + # The approved-key set — `kid` -> public key. A ConfigMap, never a Secret: a + # 32-byte seed is indistinguishable from a public key by length, so keeping + # the set somewhere a Secret is not is what makes a pasted private key + # obvious in review (shared/keyset.py). + - name: keyset + configMap: + name: eventing-keyset + optional: true diff --git a/eventing/k8s/base/eventrunner-deployment.yaml b/eventing/k8s/base/eventrunner-deployment.yaml index 3f19a282..9bf482bd 100644 --- a/eventing/k8s/base/eventrunner-deployment.yaml +++ b/eventing/k8s/base/eventrunner-deployment.yaml @@ -48,6 +48,15 @@ spec: volumeMounts: - name: data mountPath: /data + # §11. Appended, not inserted: the demo overlay JSON-patches + # `/spec/template/spec/volumes/0` to swap `data` for a PVC, so anything + # placed before `data` would be replaced instead of it. + - name: signing-key + mountPath: /etc/ce-signing + readOnly: true + - name: keyset + mountPath: /etc/ce-keyset + readOnly: true # §8.4 — the failure this catches is a LIVE process with a DEAD thread. # If the Kafka consumer thread raises, the main thread keeps running, so # the pod stays Running and reports healthy while consuming nothing. A @@ -73,3 +82,28 @@ spec: volumes: - name: data emptyDir: {} + # §11 — the Ed25519 seed this runner signs terminal responses with. The + # Secret is NOT declared in any manifest: `k8s_deploy.py --signing-keys` + # applies it via stdin, because a kustomize secretGenerator would inline the + # seed into `kubectl kustomize` output (which the §14.1 digest hashes and + # people paste into terminals) and `--from-literal` would put it in argv. + # + # `optional: true` is what keeps every signing-less overlay working: without + # it the pod blocks in ContainerCreating until a Secret that will never exist + # appears. No `defaultMode` on purpose — 0400 would be root-only, and there is + # no `runAsUser`/`fsGroup` here (OpenShift assigns the UID), so the default + # 0644 is what makes the file readable by whatever UID the SCC picks. + # A Secret PER SERVICE: see the matching note in eventbridge-deployment.yaml. + # If both mounted one seed, EventBridge would hold the key it verifies this + # runner's responses with and could forge them — the HMAC property §4.3 rejected. + - name: signing-key + secret: + secretName: eventrunner-signing-key + optional: true + # The approved-key set this runner verifies incoming requests against. A + # ConfigMap, never a Secret — see shared/keyset.py on why the location is + # what keeps a pasted private key visible in review. + - name: keyset + configMap: + name: eventing-keyset + optional: true diff --git a/eventing/k8s/base/keyset-configmap.yaml b/eventing/k8s/base/keyset-configmap.yaml new file mode 100644 index 00000000..f035fbb8 --- /dev/null +++ b/eventing/k8s/base/keyset-configmap.yaml @@ -0,0 +1,28 @@ +# §11 — the approved-agent key set. THIS FILE IS THE AUTHORIZATION LIST: an event +# whose `kid` is not in it is refused, and so is one carrying no signature at all. +# Approving an agent means adding its public key; revoking it means removing the entry. +# +# Committed EMPTY, and a test asserts it stays that way. Two reasons: +# +# * A non-empty committed keyset would mean someone pasted a key into the repository. +# Public keys are not secrets, but a 32-byte seed is indistinguishable from a public +# key by length (shared/keyset.py), so "there is never a key in this file" is a rule +# that can be checked, where "only public keys in this file" is not. +# * `{}` is a valid keyset that approves nobody, and `keyset.load_if_set` treats an +# empty PATH as "not enabled" — so an operator who mounts this without populating it +# gets a clean "no approved kids" at startup rather than a confusing partial state. +# +# Populated at deploy time by `k8s_deploy.py --signing-keys DIR`, which replaces the +# whole ConfigMap from the `agents.json` that `gen_signing_keys.py` derived from the +# seeds. Unlike the seeds this may safely appear in `kubectl kustomize` output. +# +# Rotation is a restart, not a reload: `shared/keyset.py` deliberately has no live +# reload, because a verifier that silently widens the set it trusts is harder to reason +# about than one that restarts. +apiVersion: v1 +kind: ConfigMap +metadata: + name: eventing-keyset +data: + agents.json: | + {} diff --git a/eventing/k8s/base/kustomization.yaml b/eventing/k8s/base/kustomization.yaml index d6be0800..22ef7bb9 100644 --- a/eventing/k8s/base/kustomization.yaml +++ b/eventing/k8s/base/kustomization.yaml @@ -5,6 +5,7 @@ kind: Kustomization # it targets ns kafka and cannot participate in an overlay's namespace rewrite. resources: - configmap.yaml + - keyset-configmap.yaml - eventbridge-deployment.yaml - eventbridge-service.yaml - eventbridge-route.yaml diff --git a/eventing/k8s/overlays/kind-signed/kustomization.yaml b/eventing/k8s/overlays/kind-signed/kustomization.yaml new file mode 100644 index 00000000..6f2ade54 --- /dev/null +++ b/eventing/k8s/overlays/kind-signed/kustomization.yaml @@ -0,0 +1,63 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kev1 + +# §11 — the `kind` overlay with signing ENFORCED on both directions. Everything that +# makes it work (the volumes, the keyset ConfigMap, the config keys) is in the base and +# defaults to off; this overlay only flips the switches. +# +# A separate overlay rather than a patch on `kind`, for two reasons: +# +# * S3 and S5 (#2593/#2595) are explicitly **unsigned** milestones, so the plain +# `kind` path has to keep demonstrating the system without signatures — the +# before/after contrast is the demo. +# * `tests/test_manifests.py` asserts the `test` overlay mounts no credential at all +# so mock mode is auto-detected. Keeping enforcement out of the shared overlays +# means that assertion stays true and keeps meaning something. +# +# The Secret holding the seeds is NOT here and never will be: +# python3 scripts/gen_signing_keys.py --out ~/.eventing-keys +# python3 scripts/k8s_deploy.py --yes --overlay kind-signed --signing-keys ~/.eventing-keys +resources: + - ../kind + +patches: + # Enforcement is a ConfigMap merge, not a `kubectl set env`: the values belong in the + # manifest so the digest in §14.1 covers them, and so `kubectl kustomize` shows an + # auditor exactly what is enforced. Only paths and kids live here — never a seed. + - target: + kind: ConfigMap + name: eventing-config + patch: |- + - op: replace + path: /data/ER_REQUIRE_SIGNATURE + value: "true" + - op: replace + path: /data/EB_REQUIRE_RESPONSE_SIGNATURE + value: "true" + # Both services verify against the SAME approved set, mounted from the + # `eventing-keyset` ConfigMap. One list, so approving an agent is one edit. + - op: replace + path: /data/ER_VERIFY_KEYSET_PATH + value: /etc/ce-keyset/agents.json + - op: replace + path: /data/EB_VERIFY_KEYSET_PATH + value: /etc/ce-keyset/agents.json + # The kid each service names in what it signs. EB_SIGNING_KID is also the only + # kid accepted on group lifecycle events, so an approved runner cannot forge a + # `group.completed` and end a batch early. + - op: replace + path: /data/ER_SIGNING_KID + value: runner-01 + - op: replace + path: /data/EB_SIGNING_KID + value: eb-01 + # Seed paths point at the Secret mount. `add`, not `replace`: these keys are + # deliberately absent from the base ConfigMap (configmap.yaml says seed paths + # "belong in the Deployment beside the volume"), so there is nothing to replace. + - op: add + path: /data/ER_SIGNING_KEY_PATH + value: /etc/ce-signing/seed.hex + - op: add + path: /data/EB_SIGNING_KEY_PATH + value: /etc/ce-signing/seed.hex diff --git a/eventing/scripts/gen_signing_keys.py b/eventing/scripts/gen_signing_keys.py new file mode 100644 index 00000000..b6173013 --- /dev/null +++ b/eventing/scripts/gen_signing_keys.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Generate the Ed25519 seeds and the approved-key set that §11 signing needs. + +Signing was wired in both directions (EventBridge signs requests and group events, +EventRunner signs terminal responses) but there was no way to produce keys for it, so +turning it on meant hand-rolling `os.urandom(32).hex()` and deriving public keys in a +REPL. This is that, with the parts that are easy to get wrong done once: + + * **A seed is never printed.** Only kids and public keys go to stdout, so pasting a + terminal transcript into an issue cannot leak a private key. The same reason + `k8s_deploy.py` logs a 4-char prefix and a length instead of a token. + * **Seeds land at mode 0600, in a directory created 0700.** A seed written + world-readable is the whole control gone, and it is not obvious from the outside. + * **Public keys are derived, never typed.** `agents.json` is produced from the seeds + by `shared.signing.public_key`, so the keyset cannot disagree with the keys it is + supposed to authorize — the failure mode that otherwise shows up as "signing is + broken" with nothing pointing at the key set. + +What this does NOT do is attest anything. The set records which keys an operator +approved, not which workloads a platform vouched for; `shared/keyset.py` says the same +thing at more length, and it is worth repeating out loud when demoing. + +Asymmetry is the point: EventBridge gets its own seed and EventRunner gets another, so +neither can produce the other's signatures. A single shared seed would be HMAC with +extra steps, which `DESIGN_PHASE2.md` §4.3 rejected for exactly that reason. + + python3 scripts/gen_signing_keys.py --out ~/.eventing-keys + python3 scripts/gen_signing_keys.py --out ~/.eventing-keys --force + python3 scripts/gen_signing_keys.py --print-keyset # JSON only, no files + python3 scripts/gen_signing_keys.py --out d --runners 3 # runner-01..runner-03 +""" +from __future__ import annotations + +import argparse +import json +import pathlib +import secrets +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent)) + +from shared import signing # noqa: E402 + +PREFIX = "keygen" +BRIDGE_KID = "eb-01" +SEED_FILE = "seed.hex" # the key inside the Secret, per README_PHASE1.md + + +def _log(msg: str) -> None: + print(f"[{PREFIX}] {msg}") + + +def generate(n_runners: int = 1, bridge_kid: str = BRIDGE_KID) -> dict[str, bytes]: + """`kid` -> 32-byte seed. One for the bridge, `n_runners` for runners. + + `secrets.token_bytes`, not `random`: the latter is seeded predictably enough that a + key from it is not a key. + """ + keys = {bridge_kid: secrets.token_bytes(32)} + for i in range(1, n_runners + 1): + keys[f"runner-{i:02d}"] = secrets.token_bytes(32) + return keys + + +def keyset_json(seeds: dict[str, bytes]) -> str: + """The approved-key set: `kid` -> public key hex, derived from each seed. + + Public keys only. A 32-byte seed and a 32-byte public key are indistinguishable by + length, so `keyset.load()` cannot catch a pasted private key — deriving them here is + what makes that mistake impossible rather than merely discouraged. + """ + return json.dumps({kid: signing.public_key(seed).hex() + for kid, seed in sorted(seeds.items())}, indent=2) + "\n" + + +def write_seeds(out: pathlib.Path, seeds: dict[str, bytes], *, + force: bool = False) -> list[pathlib.Path]: + """One `/seed.hex` per key, 0600, under a 0700 directory. + + A directory per kid rather than `.hex` files side by side: that is the shape + `kubectl create secret generic --from-file=` wants, and it keeps the Secret's + data key `seed.hex` — the name the Deployment's mount path and + `ER_SIGNING_KEY_PATH` both already assume. + """ + out.mkdir(parents=True, exist_ok=True) + out.chmod(0o700) + written = [] + for kid, seed in sorted(seeds.items()): + d = out / kid + d.mkdir(exist_ok=True) + d.chmod(0o700) + p = d / SEED_FILE + if p.exists() and not force: + raise SystemExit( + f"[{PREFIX}] {p} exists — refusing to overwrite a key that may already " + f"be deployed and approved. Pass --force if you mean to rotate, and " + f"remember rotation means redeploying the keyset and restarting both " + f"services (shared/keyset.py: no live reload, on purpose).") + # Create at 0600 before writing, not after: a chmod afterwards leaves a window + # in which the seed exists world-readable. + p.touch(mode=0o600, exist_ok=True) + p.chmod(0o600) + p.write_text(seed.hex() + "\n") + written.append(p) + return written + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description="Generate Ed25519 seeds and the approved-key set for §11 signing.") + ap.add_argument("--out", type=pathlib.Path, + help="directory to write /seed.hex into (0600). " + "Omit with --print-keyset to generate nothing on disk.") + ap.add_argument("--runners", type=int, default=1, + help="how many runner keys to generate (default 1)") + ap.add_argument("--bridge-kid", default=BRIDGE_KID, + help=f"kid for EventBridge's own key (default {BRIDGE_KID})") + ap.add_argument("--force", action="store_true", + help="overwrite existing seeds (rotation)") + ap.add_argument("--print-keyset", action="store_true", + help="print agents.json to stdout and write no seed files") + args = ap.parse_args(argv) + + if args.runners < 1: + ap.error("--runners must be at least 1") + if not args.out and not args.print_keyset: + ap.error("give --out DIR, or --print-keyset to see a keyset without writing keys") + + seeds = generate(args.runners, args.bridge_kid) + ks = keyset_json(seeds) + + if args.print_keyset and not args.out: + # Seeds are discarded unread: this mode exists to show the shape of a keyset, + # not to produce one whose private keys were thrown away. + print(ks, end="") + print(f"[{PREFIX}] NOTE: no seeds written, so these public keys are useless — " + f"rerun with --out to keep the matching private keys.", file=sys.stderr) + return 0 + + paths = write_seeds(args.out, seeds, force=args.force) + (args.out / "agents.json").write_text(ks) + + _log(f"wrote {len(paths)} seed(s) at mode 0600 under {args.out}") + for kid, seed in sorted(seeds.items()): + # Public key only. Never the seed, not even truncated. + _log(f" {kid:<12} pub={signing.public_key(seed).hex()}") + _log(f"approved-key set -> {args.out / 'agents.json'}") + _log("next: python3 scripts/k8s_deploy.py --yes --overlay kind-signed " + f"--signing-keys {args.out}") + if args.print_keyset: + print(ks, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 21d49c9ff85167ee2316d7c822f5854f36532924 Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Fri, 2 Oct 2026 14:49:30 -0400 Subject: [PATCH 2/3] feat(eventing): apply signing key material at deploy time, and purge it on teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `k8s_deploy.py --signing-keys DIR` applies one seed Secret per service plus the approved-key ConfigMap, modelled on `ensure_ntfy_secret`: via stdin so the seed never reaches `ps` through argv, and a no-op when the flag is absent so a redeploy cannot silently switch enforcement off and strand a signed topic. It validates before applying, because every one of these mistakes produces a cluster that refuses all traffic with no local symptom — the service starts, signs under a kid nobody approved, and the other side rejects everything: * the key directory and each named seed file must exist, * `agents.json` must parse as a keyset and approve at least one kid, * the kid each service will sign under must be IN that keyset. Each failure names the file and the command that fixes it. Seeds are logged as a length only — not even a prefix. 4 hex chars of a 32-byte seed is 16 bits of a private key, and unlike an API token there is nothing to gain from identifying it. The keyset gets the opposite treatment on purpose: kids are logged in full, because "the operator deployed yesterday's key set" is otherwise invisible until it surfaces as an unexplained rejection. `--overlay kind-signed` without `--signing-keys` now aborts unless both Secrets are already deployed. Both volumes are `optional: true`, so without this the pods would start happily with enforcement on and no key, and refuse every event. Teardown: `--purge` now deletes both signing Secrets and the keyset ConfigMap. It also deletes `eventbridge-ntfy`, which was listed in neither the `doomed` preview nor the delete tuple — so a "purged" namespace kept a capability that lets anyone who learns it read and publish your notifications. Leaving private keys behind is worse: a later deploy silently inherits an identity the operator thought they had destroyed. 17 new tests covering generation and application, including that the two services are never given the same seed (a shared one would let EventBridge forge any agent's response — the HMAC property §4.3 rejected), that seeds land 0600 under a 0700 directory, that the CLI prints public keys but never a seed on either stream, and that each malformed-key-directory case is refused before it reaches a cluster. 568 passed, 5 skipped. ruff check and format clean. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/scripts/k8s_deploy.py | 138 ++++++++++++++- eventing/scripts/k8s_teardown.py | 13 ++ eventing/tests/test_signing_deploy.py | 231 ++++++++++++++++++++++++++ 3 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 eventing/tests/test_signing_deploy.py diff --git a/eventing/scripts/k8s_deploy.py b/eventing/scripts/k8s_deploy.py index f4859fcb..f8e4c7a6 100755 --- a/eventing/scripts/k8s_deploy.py +++ b/eventing/scripts/k8s_deploy.py @@ -49,6 +49,9 @@ from k8slib import Kubectl, condition_is, condition_reason, dig, wait_for # noqa: E402 from proclib import Checks, die # noqa: E402 +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent)) +from shared import keyset # noqa: E402 + PREFIX = "deploy" HERE = pathlib.Path(__file__).resolve().parent K8S = HERE.parent / "k8s" @@ -58,6 +61,13 @@ DIGEST_ANNOTATION = "rossoctl.dev/manifest-sha256" SECRET_NAME = "anthropic-credentials" NTFY_SECRET_NAME = "eventbridge-ntfy" +# §11 — one seed Secret per service, never shared: a single seed would let either side +# forge the other's signatures (DESIGN_PHASE2.md §4.3). The keyset is a ConfigMap +# because it holds only public keys, and that location is what keeps the distinction +# reviewable (shared/keyset.py). +EB_SIGNING_SECRET = "eventbridge-signing-key" +ER_SIGNING_SECRET = "eventrunner-signing-key" +KEYSET_CONFIGMAP = "eventing-keyset" NTFY_HELP = f"""\ Phone notifications are off unless NTFY_TOPIC is exported: @@ -218,6 +228,105 @@ def ensure_ntfy_secret(k: Kubectl, c: Checks, ns: str, *, dry_run: bool) -> None f"Secret/{NTFY_SECRET_NAME} applied from the environment") +def ensure_signing_material(k: Kubectl, c: Checks, ns: str, keydir: str | None, *, + dry_run: bool) -> None: + """Apply the per-service seed Secrets and the approved-key ConfigMap from a key + directory produced by `gen_signing_keys.py`. + + **Two Secrets, one per service.** A single shared seed would mean EventBridge holds + the key it verifies runner responses with, so it could forge any agent's answer and + an agent could forge another's — the property that ruled out HMAC entirely + (DESIGN_PHASE2.md §4.3). Mounting separate Secrets is what keeps "only this agent + could have produced this" true. + + A seed is a private key, so it gets the same handling as the API token: applied via + stdin so it never reaches `ps` through argv, never written into a manifest (a + kustomize secretGenerator would inline it into `kubectl kustomize` output, which the + §14.1 digest hashes and people paste into terminals), and logged only as a length. + + The keyset is the opposite and is treated so on purpose: public keys only, applied as + a ConfigMap, kids logged in full. `KeySet.kids` is documented safe to log, and + printing them is what makes "the operator deployed yesterday's key set" visible + instead of surfacing later as an unexplained rejection. + + Absent `--signing-keys` this is a no-op that leaves whatever is deployed alone, so a + redeploy cannot silently switch enforcement off and strand a signed topic. + """ + if not keydir: + for name in (EB_SIGNING_SECRET, ER_SIGNING_SECRET): + if k.get("secret", name, namespace=ns, missing_ok=True) is not None: + c.log(f"--signing-keys not given; leaving the existing " + f"Secret/{name} in place") + return + + d = pathlib.Path(keydir).expanduser() + keyset_path = d / "agents.json" + if not keyset_path.is_file(): + c.fail("signing key directory is usable", + f"{keyset_path} not found — run " + f"`python3 scripts/gen_signing_keys.py --out {d}` first") + return + + # Validate before applying: a malformed keyset deployed into a cluster fails closed + # for every legitimate agent, which looks like a broken deployment rather than a + # security control. Better to refuse here, where the error names the file. + try: + ks = keyset.load(str(keyset_path)) + except Exception as e: # noqa: BLE001 - any parse/decode problem is the same answer + c.fail("the approved-key set parses", f"{keyset_path}: {e}") + return + if len(ks) == 0: + c.fail("the approved-key set approves at least one kid", + f"{keyset_path} is empty — enforcement would reject every event") + return + + for secret_name, kid_env, kid_default in ( + (EB_SIGNING_SECRET, "EB_SIGNING_KID", "eb-01"), + (ER_SIGNING_SECRET, "ER_SIGNING_KID", "runner-01")): + kid = os.environ.get(kid_env, "").strip() or kid_default + seed_file = d / kid / "seed.hex" + if not seed_file.is_file(): + c.fail(f"seed for kid {kid!r} exists", + f"{seed_file} not found. The overlay names this kid, so the service " + f"would start, sign with nothing, and have every event refused.") + return + if kid not in ks: + c.fail(f"kid {kid!r} is in the approved set", + f"{keyset_path} holds {list(ks.kids)} — a service signing under an " + f"unapproved kid has every event refused by the other side") + return + seed_hex = seed_file.read_text().strip() + manifest = { + "apiVersion": "v1", "kind": "Secret", "type": "Opaque", + "metadata": {"name": secret_name, "namespace": ns, + "labels": {"app.kubernetes.io/part-of": "rossoctl-eventing"}}, + # The mount's file name, and what ER_/EB_SIGNING_KEY_PATH points at. + "stringData": {"seed.hex": seed_hex}, + } + # Length only. Not even a prefix: 4 hex chars of a 32-byte seed is 16 bits of a + # private key, and unlike an API token there is no benefit to identifying it. + c.log(f"Secret/{secret_name}: kid={kid} seed=({len(seed_hex)} hex chars)") + if dry_run: + c.log(f"--dry-run: Secret/{secret_name} was not applied") + continue + c.expect_run(k.apply_stdin(json.dumps(manifest), namespace=ns), + f"Secret/{secret_name} applied for kid {kid}") + + cm = { + "apiVersion": "v1", "kind": "ConfigMap", + "metadata": {"name": KEYSET_CONFIGMAP, "namespace": ns, + "labels": {"app.kubernetes.io/part-of": "rossoctl-eventing"}}, + "data": {"agents.json": keyset_path.read_text()}, + } + c.log(f"ConfigMap/{KEYSET_CONFIGMAP}: {len(ks)} approved kid(s): " + f"{', '.join(ks.kids)}") + if dry_run: + c.log(f"--dry-run: ConfigMap/{KEYSET_CONFIGMAP} was not applied") + return + c.expect_run(k.apply_stdin(json.dumps(cm), namespace=ns), + f"ConfigMap/{KEYSET_CONFIGMAP} applied with {len(ks)} kid(s)") + + # ---- the steps -------------------------------------------------------------- class Deployer: @@ -285,6 +394,14 @@ def step3_apply_overlay(self) -> None: # Any overlay may want phone notifications; the Secret is optional in the # manifest so this is safe when NTFY_TOPIC is unset. ensure_ntfy_secret(self.k, self.c, self.ns, dry_run=self.args.dry_run) + # §11 — before the apply, so the Secrets and keyset exist by the time the + # Deployments that mount them are created. `optional: true` on both volumes + # means a pod would still start without them, but it would start with + # enforcement on and no key, and refuse every event. + ensure_signing_material(self.k, self.c, self.ns, self.args.signing_keys, + dry_run=self.args.dry_run) + if self.c.failed: + return # a bad key directory must not reach a signed overlay res = self.k.apply_kustomize(str(self.overlay), server_dry_run=self.args.dry_run) if not self.c.expect_run(res, f"overlay {self.args.overlay} applied"): self.c.log(res.tail(15)) @@ -381,7 +498,13 @@ def build_args(argv=None): ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--namespace", "-n", default=os.environ.get("NS", "kev1")) - ap.add_argument("--overlay", default="test", choices=("test", "kind", "demo")) + ap.add_argument("--overlay", default="test", + choices=("test", "kind", "demo", "kind-signed")) + ap.add_argument("--signing-keys", default=None, metavar="DIR", + help="key directory from scripts/gen_signing_keys.py. Applies one " + "seed Secret per service plus the approved-key ConfigMap. " + "Required by the kind-signed overlay; omitting it leaves any " + "already-deployed key material untouched.") ap.add_argument("--kubeconfig", default=None, help="kubeconfig file (e.g. .kube/config-kind for a Kind cluster)") ap.add_argument("--context", default=None) @@ -419,6 +542,19 @@ def main(argv=None) -> int: if not overlay_path(args.overlay).is_dir(): die(f"no overlay at {overlay_path(args.overlay)}", prefix=PREFIX) + # The signed overlay turns enforcement on in the ConfigMap. Deploying it with no key + # material produces a cluster that refuses every event with no obvious cause — the + # pods start (both volumes are `optional: true`) and then reject everything. Refuse + # up front unless key material is already deployed. + if args.overlay == "kind-signed" and not args.signing_keys: + have = all(k.get("secret", n, namespace=args.namespace, missing_ok=True) + is not None for n in (EB_SIGNING_SECRET, ER_SIGNING_SECRET)) + if not have: + die("--overlay kind-signed enforces signatures but no key material is " + "deployed. Run `python3 scripts/gen_signing_keys.py --out DIR` then " + "pass --signing-keys DIR.", prefix=PREFIX) + c.log("no --signing-keys given, but both seed Secrets are already deployed") + needed, reason = deploy_needed(k, c, d.overlay, args.namespace) c.log(f"change detection: deploy {'NEEDED' if needed else 'not needed'} — {reason}") diff --git a/eventing/scripts/k8s_teardown.py b/eventing/scripts/k8s_teardown.py index a94d389e..45a29a2d 100755 --- a/eventing/scripts/k8s_teardown.py +++ b/eventing/scripts/k8s_teardown.py @@ -125,6 +125,15 @@ def purge(k: Kubectl, c: Checks, ns: str, *, assume_yes: bool) -> None: f"ns/{ns}: deploy/eventbridge, deploy/eventrunner, svc/eventbridge, " f"route/eventbridge, scaledobject/eventrunner, configmap/eventing-config", f"ns/{ns}: secret/anthropic-credentials (if present)", + # eventbridge-ntfy was listed nowhere and so survived --purge, which means a + # "purged" namespace kept a capability that lets anyone who learns it read and + # publish your notifications. + f"ns/{ns}: secret/eventbridge-ntfy (if present)", + # §11 — the private keys. Leaving these behind means a later deploy silently + # inherits an identity the operator thought they had destroyed. + f"ns/{ns}: secret/eventbridge-signing-key, secret/eventrunner-signing-key " + f"(PRIVATE KEYS, if present)", + f"ns/{ns}: configmap/eventing-keyset (the approved-agent list, if present)", f"ns/{ns}: pvc/eventbridge-data AND ITS CONTENTS (if present)", f"ns/{KAFKA_NS}: kafkatopic/{ns}-requests AND ALL ITS MESSAGES", f"ns/{KAFKA_NS}: kafkatopic/{ns}-responses AND ALL ITS MESSAGES", @@ -145,7 +154,11 @@ def purge(k: Kubectl, c: Checks, ns: str, *, assume_yes: bool) -> None: ("deploy", "eventrunner"), ("deploy", "eventbridge"), ("svc", "eventbridge"), ("route", "eventbridge"), ("configmap", "eventing-config"), + ("configmap", "eventing-keyset"), ("secret", "anthropic-credentials"), + ("secret", "eventbridge-ntfy"), + ("secret", "eventbridge-signing-key"), + ("secret", "eventrunner-signing-key"), ("pvc", "eventbridge-data")): c.expect_run(k.delete(kind, name, namespace=ns), f"deleted {kind}/{name} from {ns}") for topic in (f"{ns}-requests", f"{ns}-responses"): diff --git a/eventing/tests/test_signing_deploy.py b/eventing/tests/test_signing_deploy.py new file mode 100644 index 00000000..84a2b7e9 --- /dev/null +++ b/eventing/tests/test_signing_deploy.py @@ -0,0 +1,231 @@ +"""Key generation and the deploy-time key material. §11. + +Signing was wired in #879 but could not be switched on in a cluster: nothing generated +keys and no Deployment mounted any. These cover the two pieces that close that, and the +properties worth pinning are mostly about what must NOT happen — a seed must not be +printed, must not be world-readable, must not be shared between the two services, and a +half-configured key directory must not reach a cluster that is enforcing. +""" +from __future__ import annotations + +import json +import pathlib +import stat +import sys + +import pytest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "scripts")) + +import gen_signing_keys as G # noqa: E402 +from k8s_deploy import ensure_signing_material # noqa: E402 +from proclib import Checks, Result # noqa: E402 + +from shared import ce, keyset # noqa: E402 +from shared import signing as S # noqa: E402 + +# ---- generation ------------------------------------------------------------- + +def test_generate_makes_one_bridge_key_and_the_asked_for_runners(): + keys = G.generate(3) + assert sorted(keys) == ["eb-01", "runner-01", "runner-02", "runner-03"] + assert all(len(s) == 32 for s in keys.values()), "an Ed25519 seed is 32 bytes" + assert len(set(keys.values())) == 4, "every seed must be distinct" + + +def test_the_bridge_and_runner_never_share_a_seed(): + """The property the whole design rests on. One shared seed would mean EventBridge + holds the key it verifies runner responses with, so it could forge any agent's + answer — which is exactly why HMAC was rejected (DESIGN_PHASE2.md §4.3).""" + keys = G.generate(1) + assert keys["eb-01"] != keys["runner-01"] + + +def test_the_keyset_holds_public_keys_derived_from_the_seeds(tmp_path): + """Derived, never typed: a keyset that disagrees with the keys it authorizes shows + up as "signing is broken" with nothing pointing at the key set.""" + keys = G.generate(2) + ks = json.loads(G.keyset_json(keys)) + assert sorted(ks) == sorted(keys) + for kid, seed in keys.items(): + assert ks[kid] == S.public_key(seed).hex() + assert ks[kid] != seed.hex(), "a seed must never land in the keyset" + + +def test_the_generated_keyset_loads_and_authorizes_its_own_signatures(tmp_path): + """End to end through the real verification path, not just shape checks.""" + keys = G.generate(1) + G.write_seeds(tmp_path, keys) + (tmp_path / "agents.json").write_text(G.keyset_json(keys)) + ks = keyset.load(str(tmp_path / "agents.json")) + + seed = S.load_seed(str(tmp_path / "runner-01" / "seed.hex")) + assert seed == keys["runner-01"], "load_seed must read back what was written" + evt = ce.new_event(type=ce.TYPE_RESPONSE, source="rossoctl://er/test", + datacontenttype="application/json", correlationid="c", + sessionuuid="s", sequence=1, phase="result", final="true", + data={"text": "hi"}) + S.sign_into(evt, seed, "runner-01") + back = ce.from_kafka_binary(*ce.to_kafka_binary(evt)) + ok, why = S.verify_with_keyset(back, ks) + assert ok, why + + +def test_seeds_are_written_unreadable_by_anyone_else(tmp_path): + """A seed written world-readable is the control gone, and it is invisible from the + outside — nothing fails, the key is just no longer private.""" + keys = G.generate(1) + paths = G.write_seeds(tmp_path, keys) + assert (stat.S_IMODE(tmp_path.stat().st_mode) & 0o077) == 0, \ + "the key directory must not be group- or world-accessible" + for p in paths: + mode = stat.S_IMODE(p.stat().st_mode) + assert mode == 0o600, f"{p} is {oct(mode)}, must be 0600" + + +def test_regenerating_refuses_to_clobber_a_deployed_key(tmp_path): + """An existing seed may already be deployed and in an approved keyset. Silently + replacing it revokes a working identity with no indication why.""" + keys = G.generate(1) + G.write_seeds(tmp_path, keys) + with pytest.raises(SystemExit, match="refusing to overwrite"): + G.write_seeds(tmp_path, G.generate(1)) + # --force is the deliberate path. + rotated = G.generate(1) + G.write_seeds(tmp_path, rotated, force=True) + assert S.load_seed(str(tmp_path / "eb-01" / "seed.hex")) == rotated["eb-01"] + + +def test_the_cli_never_prints_a_seed(tmp_path, capsys): + """stdout and stderr both. A terminal transcript pasted into an issue is the most + likely way a demo key escapes.""" + rc = G.main(["--out", str(tmp_path), "--runners", "2"]) + assert rc == 0 + out = capsys.readouterr() + printed = out.out + out.err + for kid in ("eb-01", "runner-01", "runner-02"): + seed_hex = S.load_seed(str(tmp_path / kid / "seed.hex")).hex() + assert seed_hex not in printed, f"{kid}'s seed was printed" + assert S.public_key(bytes.fromhex(seed_hex)).hex() in printed, \ + f"{kid}'s public key should be shown" + + +def test_print_keyset_alone_writes_nothing(tmp_path, capsys): + rc = G.main(["--print-keyset"]) + assert rc == 0 + ks = json.loads(capsys.readouterr().out) + assert sorted(ks) == ["eb-01", "runner-01"] + assert list(tmp_path.iterdir()) == [], "no files should be written" + + +# ---- deploy-time application ------------------------------------------------ + +class _FakeKubectl: + """Records what would be applied. `get` returns None — nothing pre-exists.""" + + def __init__(self): + self.applied: list[dict] = [] + + def get(self, kind, name, namespace=None, missing_ok=False): + return None + + def apply_stdin(self, body, namespace=None): + self.applied.append(json.loads(body)) + return Result(argv=["kubectl", "apply"], rc=0, out="configured", err="", + duration_s=0.01, launched=True, timed_out=False) + + +def _keydir(tmp_path, *, runners=1): + keys = G.generate(runners) + G.write_seeds(tmp_path, keys) + (tmp_path / "agents.json").write_text(G.keyset_json(keys)) + return keys + + +def _apply(keydir, **kw): + k, c = _FakeKubectl(), Checks(prefix="test") + ensure_signing_material(k, c, "kev1", keydir, dry_run=kw.get("dry_run", False)) + return k, c + + +def test_a_valid_key_directory_applies_two_secrets_and_one_configmap(tmp_path): + keys = _keydir(tmp_path) + k, c = _apply(str(tmp_path)) + assert c.failed == 0 + names = [(m["kind"], m["metadata"]["name"]) for m in k.applied] + assert names == [("Secret", "eventbridge-signing-key"), + ("Secret", "eventrunner-signing-key"), + ("ConfigMap", "eventing-keyset")] + secrets = [m for m in k.applied if m["kind"] == "Secret"] + # The data key is the file name the mount and *_SIGNING_KEY_PATH both assume. + assert all(list(m["stringData"]) == ["seed.hex"] for m in secrets) + assert (secrets[0]["stringData"]["seed.hex"] + != secrets[1]["stringData"]["seed.hex"]), \ + "the two services must be given different seeds" + assert secrets[0]["stringData"]["seed.hex"].strip() == keys["eb-01"].hex() + + +def test_the_applied_configmap_holds_only_public_keys(tmp_path): + keys = _keydir(tmp_path) + k, _ = _apply(str(tmp_path)) + cm = [m for m in k.applied if m["kind"] == "ConfigMap"][0] + ks = json.loads(cm["data"]["agents.json"]) + assert sorted(ks) == sorted(keys) + for kid, seed in keys.items(): + assert ks[kid] == S.public_key(seed).hex() + assert seed.hex() not in json.dumps(cm), "a seed leaked into the ConfigMap" + + +def test_without_a_key_directory_nothing_is_applied(tmp_path): + """A redeploy must not silently switch enforcement off and strand a signed topic.""" + k, c = _apply(None) + assert c.failed == 0 and k.applied == [] + + +def test_a_missing_key_directory_fails_with_the_command_to_run(tmp_path, capsys): + """`Checks.failures` keeps only the description; the actionable detail is printed. + Assert on the output, since that is what the operator actually sees.""" + k, c = _apply(str(tmp_path / "absent")) + assert c.failed == 1 and k.applied == [] + assert "gen_signing_keys.py" in capsys.readouterr().out, \ + "the failure should name the command that fixes it" + + +def test_an_empty_keyset_is_refused(tmp_path): + """`{}` is a valid keyset that approves nobody — deployed under enforcement it + rejects every event, which reads as a broken system rather than a config error.""" + (tmp_path / "agents.json").write_text("{}") + k, c = _apply(str(tmp_path)) + assert c.failed == 1 and k.applied == [] + + +def test_a_malformed_keyset_is_refused_before_it_reaches_the_cluster(tmp_path): + (tmp_path / "agents.json").write_text('{"runner-01": "not-a-key"}') + k, c = _apply(str(tmp_path)) + assert c.failed == 1 and k.applied == [] + + +def test_a_kid_missing_from_the_approved_set_is_refused(tmp_path): + """The service would start, sign under that kid, and have every event refused by + the other side — a failure with no local symptom.""" + _keydir(tmp_path) + (tmp_path / "agents.json").write_text(json.dumps({"someone-else": "ab" * 32})) + k, c = _apply(str(tmp_path)) + assert c.failed == 1 and k.applied == [] + + +def test_a_missing_seed_file_is_refused(tmp_path): + keys = _keydir(tmp_path) + (tmp_path / "runner-01" / "seed.hex").unlink() + k, c = _apply(str(tmp_path)) + assert c.failed == 1 + # The bridge's Secret may already have been applied; the runner's must not be. + assert not any(m["metadata"]["name"] == "eventrunner-signing-key" + for m in k.applied) + assert keys # keys were generated; the point is the deploy stopped + + +def test_dry_run_applies_nothing(tmp_path): + _keydir(tmp_path) + k, c = _apply(str(tmp_path), dry_run=True) + assert c.failed == 0 and k.applied == [] From 803ccd906465ad07392fd5683e1b53569c76c449 Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Mon, 5 Oct 2026 15:54:58 -0400 Subject: [PATCH 3/3] =?UTF-8?q?fix(eventing):=20Phase=202=20=E2=80=94=20ve?= =?UTF-8?q?rify=20the=20seed=20derives=20the=20approved=20key,=20and=20thr?= =?UTF-8?q?ee=20more?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the #884 review. All findings reproduced against `21d49c9` before fixing, and each fix verified by reverting it. **The must-fix: `ensure_signing_material` validated everything except the one thing that mattered.** It checked the directory, that `agents.json` parses, that it approves a kid, that the seed file exists, and that the kid is *in* the keyset — never that the seed **derives** the public key the keyset holds for it. Reproduced: kid in keyset : True seed file exists : True keyset parses : True pub(seed) : 29acbae141bccaf0… keyset's pub for eb-01 : 03a107bff3ce10be… MATCH : False A divergent pair passed every check and produced exactly the cluster the function exists to prevent: pods start (both volumes are `optional: true`), enforcement is on, the service signs with a key nobody approved, and every event is refused with nothing local to look at. Now refused before anything is applied, with both fingerprints and the two remedies named. A corrupt seed is a named check failure rather than a traceback. **`write_seeds` is all-or-nothing.** It wrote kid-by-kid and raised mid-loop, while `agents.json` is rewritten by the caller afterwards — so if the FIRST kid in sorted order was missing and a later one present, the first got a brand-new seed and the keyset kept the old public key for it. That is the one state the module docstring promises cannot happen ("the keyset cannot disagree with the keys it is supposed to authorize"), and it is reachable from a partial rotation or a restored backup. The pre-check now runs over the whole set before any file is touched, and the refusal says "Nothing was written". Both of these are the same hazard from opposite ends, which is why the deploy-time check matters even with the generator fixed: the likeliest real case is neither, but an `agents.json` copied from a teammate while the seeds on disk are local. **The seed paths move out of the ConfigMap and into each Deployment's `env`.** The overlay quoted `configmap.yaml:54-55` — "they name Secret mounts and belong in the Deployment beside the volume" — immediately before doing the other thing. No leak, since the value is a path, but a rule the repo states and the patch cites should not be a rule the patch breaks. Followed the rule rather than rewriting it, which also keeps `test_manifests.py`'s Secret-vs-ConfigMap assertions aligned and leaves #883 free to extend its `SECRET_PATH_VARS` tuple to these two variables, as §8.3 implies it should. `env` does not exist on either base container (only `envFrom`), so the patch adds the whole list; an `env/-` append would fail at render time. All four overlays still build, `kind` stays fully unsigned, and `kind-signed` renders both paths on the Deployments with enforcement still in the ConfigMap where the §14.1 digest covers it. **`test_the_test_overlay_mounts_no_credential_so_mock_mode_is_automatic` passed for a narrower reason than it documented.** Its docstring says "no Secret is referenced", and the overlay now renders `secretName: eventrunner-signing-key` — only the literal `secretRef` spelling kept it green. It now checks Secret references by name against an allowlist of the two signing Secrets and rejects anything `anthropic`-shaped by either spelling. Verified by injecting a credential Secret into the test overlay: the test goes red, where before it would not have. **§12 documented 12 preflight checks; the script has 13.** `check_claude_cli_image` was added with the demo overlay and never got a table row, while `IMPLEMENTATION_REPORT1.md` and `README_PHASE1.md` both reference checks by number. Added row 13 with its reasoning (it runs on a cluster node because the CLI's bundled `bun` aborts under QEMU user-mode emulation), and corrected "checks 1-8, 10-12" to "10-13" in §13 step 0 and both places `k8s_deploy.py` repeats it. Pre-existing drift, folded in here because check 14 is on this branch's own list. Tests: 575 passing (+7), ruff clean, all four overlays build. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/agentdocs/DESIGN_PHASE1.md | 10 ++- .../overlays/kind-signed/kustomization.yaml | 41 +++++++-- eventing/scripts/gen_signing_keys.py | 27 ++++-- eventing/scripts/k8s_deploy.py | 33 ++++++- eventing/tests/test_manifests.py | 27 ++++-- eventing/tests/test_signing_deploy.py | 90 +++++++++++++++++++ 6 files changed, 205 insertions(+), 23 deletions(-) diff --git a/eventing/agentdocs/DESIGN_PHASE1.md b/eventing/agentdocs/DESIGN_PHASE1.md index e5abbe1f..82f7a434 100644 --- a/eventing/agentdocs/DESIGN_PHASE1.md +++ b/eventing/agentdocs/DESIGN_PHASE1.md @@ -691,9 +691,15 @@ checks 8–9 exist before anything is deployed. | 10 | **Topic name collisions**: no pre-existing `KafkaTopic` or broker topic named `-requests` / `-responses` that this deploy does not own | another user took the name | resolves RQ-5; prevents consuming someone else's data | | 11 | Namespace exists or is creatable; quota admits the pods | quota exhausted | otherwise pods sit `Pending` with an unhelpful message | | 12 | Default StorageClass exists, if the overlay requests a PVC | no provisioner | PVC stays `Pending` forever | +| 13 | **The claude CLI image is anonymously pullable and carries the CLI** (demo overlay only; skipped otherwise) | image missing, or built where the bundled `bun` cannot execute | T2.4's gate. Without it every real run fails with `claude binary not found` — and mock mode is exactly what stops the e2e from catching that. Runs on a cluster node rather than at build time because the CLI's `bun` aborts under QEMU user-mode emulation, so a cross-built image cannot execute it on an arm64 builder | + +Check 13 was added with the demo overlay and this table documented 12 for a while, which +`IMPLEMENTATION_REPORT1.md` and `README_PHASE1.md` both reference by number — so the +count is worth keeping right rather than discovering later that the design documents 12 +of 13. Check 9 necessarily runs *after* EventBridge is deployed, so preflight has two -phases: checks 1–8 and 10–12 run before anything is applied, and check 9 runs +phases: checks 1–8 and 10–13 run before anything is applied, and check 9 runs as the gate between deployment step 4 and step 5 (§13). **On a Kind cluster (§3.2)** the same checks apply, with three differences that are @@ -714,7 +720,7 @@ implementation. these in order. The manual equivalents are shown because they are what you will type when debugging. -**0. Preflight** — §12 checks 1–8, 10–12. Abort on any failure. +**0. Preflight** — §12 checks 1–8, 10–13. Abort on any failure. **1. Namespace.** diff --git a/eventing/k8s/overlays/kind-signed/kustomization.yaml b/eventing/k8s/overlays/kind-signed/kustomization.yaml index 6f2ade54..e0a805d9 100644 --- a/eventing/k8s/overlays/kind-signed/kustomization.yaml +++ b/eventing/k8s/overlays/kind-signed/kustomization.yaml @@ -52,12 +52,39 @@ patches: - op: replace path: /data/EB_SIGNING_KID value: eb-01 - # Seed paths point at the Secret mount. `add`, not `replace`: these keys are - # deliberately absent from the base ConfigMap (configmap.yaml says seed paths - # "belong in the Deployment beside the volume"), so there is nothing to replace. + # Seed paths are NOT here. `configmap.yaml` says they "name Secret mounts and + # belong in the Deployment beside the volume", so they are set as container `env` + # entries on each Deployment below. An earlier revision of this overlay put them + # here and quoted that rule immediately before breaking it — no leak, since the + # value is a path, but a rule the repo states and a patch cites should not be a + # rule the patch breaks. The §14.1 digest covers the Deployments too, so nothing + # is lost by keeping them there. + + # The seed paths, beside the volume that supplies the seed — which is the whole point + # of the location rule. `env` rather than the ConfigMap also keeps + # `test_manifests.py`'s Secret-vs-ConfigMap assertions able to mean what they say, + # including the Phase 3 tuple in #883 that is scoped to seed-path variables. + - target: + kind: Deployment + name: eventbridge + patch: |- + # `env` does not exist on the base container (only `envFrom`), so this adds the + # whole list rather than appending to it. A trailing-slash `env/-` op would fail + # with "path does not exist" at render time. - op: add - path: /data/ER_SIGNING_KEY_PATH - value: /etc/ce-signing/seed.hex + path: /spec/template/spec/containers/0/env + value: + - name: EB_SIGNING_KEY_PATH + value: /etc/ce-signing/seed.hex + - target: + kind: Deployment + name: eventrunner + patch: |- + # `env` does not exist on the base container (only `envFrom`), so this adds the + # whole list rather than appending to it. A trailing-slash `env/-` op would fail + # with "path does not exist" at render time. - op: add - path: /data/EB_SIGNING_KEY_PATH - value: /etc/ce-signing/seed.hex + path: /spec/template/spec/containers/0/env + value: + - name: ER_SIGNING_KEY_PATH + value: /etc/ce-signing/seed.hex diff --git a/eventing/scripts/gen_signing_keys.py b/eventing/scripts/gen_signing_keys.py index b6173013..1afac34a 100644 --- a/eventing/scripts/gen_signing_keys.py +++ b/eventing/scripts/gen_signing_keys.py @@ -84,18 +84,33 @@ def write_seeds(out: pathlib.Path, seeds: dict[str, bytes], *, """ out.mkdir(parents=True, exist_ok=True) out.chmod(0o700) + + # Check EVERY target before writing ANY of them. Raising mid-loop would leave a new + # seed on disk for one kid while `agents.json` — written by the caller, after this + # returns — still holds the old public key for it. That is precisely the state this + # module's docstring promises cannot happen ("the keyset cannot disagree with the + # keys it is supposed to authorize"), and until k8s_deploy.py grew its + # seed-derives-the-approved-key check, nothing downstream could detect it. + # + # Reachable whenever the FIRST kid in sorted order is missing and a later one is + # present — a partial rotation, or a restored backup. + if not force: + clashes = [out / kid / SEED_FILE for kid in sorted(seeds) + if (out / kid / SEED_FILE).exists()] + if clashes: + raise SystemExit( + f"[{PREFIX}] {', '.join(str(p) for p in clashes)} already exist — " + f"refusing to overwrite keys that may already be deployed and approved. " + f"Nothing was written. Pass --force if you mean to rotate, and remember " + f"rotation means redeploying the keyset and restarting both services " + f"(shared/keyset.py: no live reload, on purpose).") + written = [] for kid, seed in sorted(seeds.items()): d = out / kid d.mkdir(exist_ok=True) d.chmod(0o700) p = d / SEED_FILE - if p.exists() and not force: - raise SystemExit( - f"[{PREFIX}] {p} exists — refusing to overwrite a key that may already " - f"be deployed and approved. Pass --force if you mean to rotate, and " - f"remember rotation means redeploying the keyset and restarting both " - f"services (shared/keyset.py: no live reload, on purpose).") # Create at 0600 before writing, not after: a chmod afterwards leaves a window # in which the seed exists world-readable. p.touch(mode=0o600, exist_ok=True) diff --git a/eventing/scripts/k8s_deploy.py b/eventing/scripts/k8s_deploy.py index f8e4c7a6..9ca98104 100755 --- a/eventing/scripts/k8s_deploy.py +++ b/eventing/scripts/k8s_deploy.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Deploy in the §13 order, with the public URL as a hard gate. Tasks T4.2, T4.3. - 0. Preflight (§12 checks 1-8, 10-12). Abort on any failure. + 0. Preflight (§12 checks 1-8, 10-13). Abort on any failure. 1. Namespace. 2. Topics — in ns `kafka`, NOT the target namespace (Finding 1). 3. The overlay: EventBridge + Service + Route + EventRunner + ScaledObject. @@ -50,7 +50,7 @@ from proclib import Checks, die # noqa: E402 sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent)) -from shared import keyset # noqa: E402 +from shared import keyset, signing # noqa: E402 PREFIX = "deploy" HERE = pathlib.Path(__file__).resolve().parent @@ -296,6 +296,33 @@ def ensure_signing_material(k: Kubectl, c: Checks, ns: str, keydir: str | None, f"unapproved kid has every event refused by the other side") return seed_hex = seed_file.read_text().strip() + # The check the other four cannot make: that this seed DERIVES the public key the + # keyset approves for its kid. A kid present in both files with a mismatched pair + # passes every check above and produces exactly the cluster this function exists + # to prevent — pods start (both volumes are `optional: true`), enforcement is on, + # the service signs with a key nobody approved, and every event is refused with + # no local symptom. + # + # Reachable without hand-editing: an `agents.json` copied from a teammate or a + # previous cluster while the seeds on disk are local, a restored backup, or a + # partial regenerate. + try: + seed = signing.load_seed(str(seed_file)) + except Exception as e: # noqa: BLE001 - any decode problem is the same answer + c.fail(f"seed for kid {kid!r} is a usable Ed25519 seed", f"{seed_file}: {e}") + return + derived = signing.public_key(seed) + approved = ks.select(kid) + if derived != approved: + c.fail( + f"the seed for kid {kid!r} matches the approved public key", + f"{seed_file} derives {derived.hex()[:16]}… but {keyset_path} approves " + f"{approved.hex()[:16]}… for {kid!r}. Deploying this pair starts both " + f"services, signs with an unapproved key and has every event refused, " + f"with nothing local to look at. Regenerate the pair with " + f"`python3 scripts/gen_signing_keys.py --out {d} --force`, or restore " + f"the agents.json that goes with these seeds.") + return manifest = { "apiVersion": "v1", "kind": "Secret", "type": "Opaque", "metadata": {"name": secret_name, "namespace": ns, @@ -573,7 +600,7 @@ def main(argv=None) -> int: return c.summary() if not args.skip_preflight: - c.section("step 0: preflight (§12 checks 1-8, 10-12)") + c.section("step 0: preflight (§12 checks 1-8, 10-13)") skip = {int(s) for s in args.skip.split(",") if s.strip()} k8s_preflight.run_pre(k8s_preflight.Preflight(k, c, args), skip) if c.failed: diff --git a/eventing/tests/test_manifests.py b/eventing/tests/test_manifests.py index 2da561af..8695a829 100644 --- a/eventing/tests/test_manifests.py +++ b/eventing/tests/test_manifests.py @@ -111,13 +111,30 @@ def test_the_demo_overlay_puts_home_on_the_volume(): @needs_kubectl def test_the_test_overlay_mounts_no_credential_so_mock_mode_is_automatic(): - """What makes the e2e test free and deterministic is that NOTHING can inject a - credential into these pods: no Secret is referenced, so config.py's - auto-detection selects mock mode itself. §14 assertion 7 then checks the pod - log says `auto:` — which also proves the detection path works in-cluster, - rather than bypassing it with an explicit override.""" + """What makes the e2e test free and deterministic is that NOTHING can inject an + **Anthropic credential** into these pods, so config.py's auto-detection selects mock + mode itself. §14 assertion 7 then checks the pod log says `auto:` — which also proves + the detection path works in-cluster rather than bypassing it with an override. + + **The assertion is about credentials, not about Secrets in general**, and that + distinction is now load-bearing. The base mounts the per-service *signing* Secrets as + `optional: true` volumes, so `secretName:` does appear in this overlay — a bare + `"secretRef" not in doc` kept passing only because a volume uses the other spelling, + which would have made this test green for a reason it does not document. It is + checked by name here: any Secret reference other than the two signing ones has to be + justified, and an `ANTHROPIC`-shaped one fails outright. + """ doc = find_doc(render("test"), "Deployment", "eventrunner") + # `secretName` only — the volume is also called `signing-key`, which is a volume + # name rather than a Secret reference and must not be mistaken for one. + allowed_secrets = {"eventbridge-signing-key", "eventrunner-signing-key"} + referenced = set(re.findall(r"secretName:\s*(\S+)", doc)) + assert referenced <= allowed_secrets, \ + f"unexpected Secret reference: {sorted(referenced - allowed_secrets)}" + # No credential Secret, by either spelling, and nothing credential-shaped. assert "secretRef" not in doc, "the test overlay must mount no credential" + assert "anthropic" not in doc.lower(), \ + "the test overlay must reference no Anthropic credential Secret" assert "ANTHROPIC" not in doc assert "ER_MOCK_CLAUDE" not in doc, \ "leave the mode to auto-detection so the e2e exercises it" diff --git a/eventing/tests/test_signing_deploy.py b/eventing/tests/test_signing_deploy.py index 84a2b7e9..67441698 100644 --- a/eventing/tests/test_signing_deploy.py +++ b/eventing/tests/test_signing_deploy.py @@ -229,3 +229,93 @@ def test_dry_run_applies_nothing(tmp_path): _keydir(tmp_path) k, c = _apply(str(tmp_path), dry_run=True) assert c.failed == 0 and k.applied == [] + + +# ---- the seed must DERIVE the approved public key ------------------------ +# +# Review finding on #884: every other check passes on a pair that does not agree, which +# is the one divergence producing the cluster this validation exists to prevent — pods +# start (both volumes are `optional: true`), enforcement is on, the service signs with a +# key nobody approved, and every event is refused with nothing local to look at. + + +@pytest.mark.parametrize("kid", ["eb-01", "runner-01"]) +def test_a_seed_that_does_not_match_the_approved_key_is_refused(tmp_path, kid): + """The likeliest real case is not hand-editing: an `agents.json` copied from a + teammate or a previous cluster while the seeds on disk are local.""" + _keydir(tmp_path) + # Replace one seed, leaving agents.json alone — so the kid is still in the keyset + # and the file still exists, which is all the other checks look at. + other = G.generate(1)["eb-01"] + (tmp_path / kid / "seed.hex").write_text(other.hex() + "\n") + + k, c = _apply(str(tmp_path)) + assert c.failed == 1, "a mismatched seed/keyset pair was accepted" + assert not any(m["kind"] == "ConfigMap" for m in k.applied), \ + "the keyset was applied despite a mismatched pair" + + +def test_the_mismatch_refusal_names_both_fingerprints(tmp_path, capsys): + """An operator has to be able to tell which half is wrong, so the message carries a + prefix of each key and names the two remedies.""" + _keydir(tmp_path) + (tmp_path / "eb-01" / "seed.hex").write_text(G.generate(1)["eb-01"].hex() + "\n") + _apply(str(tmp_path)) + out = capsys.readouterr().out + assert "derives" in out and "approves" in out + assert "gen_signing_keys.py" in out, "the message should name the fix" + + +def test_a_corrupt_seed_file_is_refused_rather_than_crashing(tmp_path): + """`load_seed` raises on a non-hex or wrong-length file; that must become a named + check failure, not a traceback out of the deploy script.""" + _keydir(tmp_path) + (tmp_path / "eb-01" / "seed.hex").write_text("not-a-seed\n") + k, c = _apply(str(tmp_path)) + assert c.failed == 1 and k.applied == [] + + +def test_a_matching_directory_is_still_accepted(tmp_path): + """The gate must not reject what it should allow — the check is an equality, and an + equality test that always fails is as useless as one that never does.""" + _keydir(tmp_path, runners=2) + k, c = _apply(str(tmp_path)) + assert c.failed == 0 + assert len(k.applied) == 3 # two Secrets + the keyset ConfigMap + + +# ---- write_seeds is all-or-nothing -------------------------------------- + + +def test_a_partial_rotation_writes_nothing_at_all(tmp_path): + """Review finding on #884: `write_seeds` wrote kid-by-kid and raised mid-loop, so if + the FIRST kid in sorted order was missing and a later one present, the first got a + brand-new seed while `agents.json` — rewritten by the caller, afterwards — kept the + old public key for it. That is the state the module docstring promises cannot + happen: "the keyset cannot disagree with the keys it is supposed to authorize".""" + keys = _keydir(tmp_path, runners=1) + before = (tmp_path / "agents.json").read_text() + eb_seed = tmp_path / "eb-01" / "seed.hex" + original_eb = eb_seed.read_text() + eb_seed.unlink() # first in sorted order; runner-01 survives + + with pytest.raises(SystemExit) as e: + G.write_seeds(tmp_path, keys) + assert "already exist" in str(e.value) + assert "Nothing was written" in str(e.value) + + # The refusal left the directory exactly as it was: no new seed for eb-01, so + # agents.json cannot have drifted away from what is on disk. + assert not eb_seed.exists(), "a new seed was written despite the refusal" + assert (tmp_path / "agents.json").read_text() == before + assert original_eb # the original was readable before we removed it + + +def test_force_still_rotates_every_key(tmp_path): + """The pre-check must not make `--force` a no-op.""" + keys = _keydir(tmp_path, runners=1) + first = (tmp_path / "eb-01" / "seed.hex").read_text() + fresh = G.generate(1) + G.write_seeds(tmp_path, fresh, force=True) + assert (tmp_path / "eb-01" / "seed.hex").read_text() != first + assert keys # the originals existed; the point is force overwrote them