From a00bcea2ed5c7cb8e814bc6430a6298c66d812dc Mon Sep 17 00:00:00 2001 From: Praveen T Date: Fri, 28 Aug 2026 19:29:49 +0530 Subject: [PATCH 1/6] Add external minion onboarding script and runbook for salt-minion-vcf - scripts/onboarding/vcf-ops-onboard.py: interactive CLI that resolves a VCF Operations-managed Salt master, starts a salt-minion-vcf instance (Docker or Kubernetes/Helm), and trusts its key against the master via the GET /api/salt/master and POST /api/salt/minions/{id}/trusted-keys VCF Operations APIs - with audit logging, spinners, and a multi-minion loop, while staying fully non-interactive when driven by CLI flags. - docs/runbook.md: operational runbook covering end-to-end minion onboarding and how to supply saltext.vcf pillar data (local and master-dispatched paths) for each supported VCF component. - Ignore __pycache__/*.pyc alongside the existing ignore rules. --- salt-minion-vcf/.gitignore | 2 + salt-minion-vcf/docs/runbook.md | 228 +++++ salt-minion-vcf/scripts/onboarding/README.md | 115 +++ .../scripts/onboarding/vcf-ops-onboard.py | 886 ++++++++++++++++++ 4 files changed, 1231 insertions(+) create mode 100644 salt-minion-vcf/docs/runbook.md create mode 100644 salt-minion-vcf/scripts/onboarding/README.md create mode 100755 salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py diff --git a/salt-minion-vcf/.gitignore b/salt-minion-vcf/.gitignore index 54b0918..f68b782 100644 --- a/salt-minion-vcf/.gitignore +++ b/salt-minion-vcf/.gitignore @@ -3,6 +3,8 @@ .DS_Store dist/ build/ +__pycache__/ +*.pyc # Real pillar data (credentials) - only *.sls.example templates are tracked. pillar/*.sls diff --git a/salt-minion-vcf/docs/runbook.md b/salt-minion-vcf/docs/runbook.md new file mode 100644 index 0000000..6d20925 --- /dev/null +++ b/salt-minion-vcf/docs/runbook.md @@ -0,0 +1,228 @@ +# Runbook: Onboarding an External Minion and Connecting It to VCF Infrastructure + +This runbook covers two separate procedures: + +1. **Bring up a `salt-minion-vcf` instance and trust it against a VCF + Operations-managed Salt master** (Part 1) - using + [`scripts/onboarding/vcf-ops-onboard.py`](../scripts/onboarding/vcf-ops-onboard.py). +2. **Give that minion the credentials it needs to actually operate against + VCF components** (vCenter, NSX, SDDC Manager, ESXi, VCFA, VCF Installer, + VCF Operations) via Salt Pillar (Part 2). + +These are independent: a minion can be connected to the master (Part 1) +before it has any pillar data configured (Part 2) - it just can't run any +`saltext.vcf` operations against a real target until Part 2 is done. + +--- + +## Part 1 - Bring up the minion and connect it to the Salt master + +### Prerequisites + +- The `salt-minion-vcf` image built locally or available in a registry you + can pull from (`docker build -t salt-minion-vcf:0.1.0 .` from the repo + root - see the top-level [`README.md`](../README.md#quick-start) if this + hasn't been done yet). +- `docker` on PATH (Docker mode), or `helm` + `kubectl` on PATH (Kubernetes mode). +- Network access from wherever you run the script to your VCF Operations + instance's Suite API, and from the minion's host/cluster to the Salt + master (`SALT_MASTER_PORT`/`4506`, `SALT_PUBLISH_PORT`/`4505`). +- Credentials for a VCF Operations user with the Salt Management view/manage + privileges, and the resource UUID of the VCF instance whose master you + want to attach to. + +### Procedure + +Run the onboarding script: + +```bash +python3 scripts/onboarding/vcf-ops-onboard.py \ + --ops-host vcfops.example.com \ + --ops-user admin \ + --vcf-instance-id \ + --deployment docker # or: kubernetes +``` + +Everything not passed as a flag is prompted for interactively, with a +review/confirm summary shown before anything is actually started. The script +handles, in order: + +1. Logs in to VCF Operations. +2. Resolves the Salt master governing the given VCF instance. +3. Computes the master's identity fingerprint (`master_finger`). +4. Starts the minion (`docker run`, or `helm upgrade --install`), passing it + the master FQDN, `master_finger`, and a freshly generated minion ID. The + minion generates its own RSA keypair locally on first start - the + private key never leaves it, and VCF Operations credentials never reach it. +5. Reads back the minion's public key. +6. Registers that key as trusted with the master. +7. Waits until the master has actually accepted the connection. + +Use `--dry-run` first if you want to preview every command and API call +without executing anything. See `--help` for the full flag list, or +[`scripts/onboarding/README.md`](../scripts/onboarding/README.md) for a +complete walkthrough of every option. + +### Verification + +From the Salt master: + +```bash +salt-key -L # minion should be under "Accepted Keys" +salt '' test.ping # should return True +``` + +From the minion side (Docker): + +```bash +docker exec salt-minion-vcf salt-call --local test.version +docker logs salt-minion-vcf | grep "Minion is ready to receive requests" +``` + +(Kubernetes: substitute `kubectl exec -n --` / +`kubectl logs -n `.) + +### Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| `CERTIFICATE_VERIFY_FAILED: self signed certificate` on login | VCF Operations uses a self-signed/internal CA cert | Pass `--insecure` | +| `pull access denied for salt-minion-vcf` | Image not built locally yet - Docker tried to pull it from Docker Hub | `docker build -t salt-minion-vcf:0.1.0 .` from the repo root first, or point `--image` at wherever you built/pushed it | +| `container name already in use` on retry | A previous failed attempt left a stopped container behind | The script now detects this and offers to remove it automatically | +| `[CRITICAL] Unable to securely set the permissions of "/etc/salt/pki/minion"` / `PermissionError: Permission denied: '/etc/salt/pki/minion/tmp...'` | The PKI volume value was a host path (bind mount), not a named Docker volume - the container runs as non-root uid `10000`, and a bind-mounted host directory doesn't inherit the image's baked-in ownership | Use a plain volume name (e.g. `salt-minion-vcf-pki`, the default) instead of an absolute path. If you specifically need a host path, `chown -R 10000:10000` it first | +| Minion key is accepted on the master (`salt-key -L` shows it), but the onboarding script (or the image's own `HEALTHCHECK`/`readinessProbe`) never reports it connected | `status.master`'s answer depends on `master_alive_interval` being configured on the minion, which the entrypoint doesn't set by default - it can under-report even once genuinely connected | The onboarding script also checks the minion's logs for `Minion is ready to receive requests` as a fallback, which doesn't have this gap. If you're checking manually, use that log line or `salt '' test.ping` from the master instead of relying on `status.master` alone | + +--- + +## Part 2 - Pillar data for connecting to VCF components + +`saltext.vcf` reads all target credentials from Salt Pillar under +`saltext.vcf.`. There is **no way to pass these credentials through +the onboarding script or through `SALT_MASTER`/`SALT_MINION_ID`-style +environment variables** - they must be supplied as pillar data, by design +(see [`docs/security.md`](security.md)). + +### Supported targets + +| Target key | Component | Example file | +|---|---|---| +| `vcenter` | vCenter Server (REST + SOAP/pyVmomi) | [`pillar/vcenter.sls.example`](../pillar/vcenter.sls.example) | +| `nsx` | NSX Manager (Policy API) | [`pillar/nsx.sls.example`](../pillar/nsx.sls.example) | +| `sddc_manager` | SDDC Manager | [`pillar/sddc_manager.sls.example`](../pillar/sddc_manager.sls.example) | +| `esxi` | Standalone/unmanaged ESXi hosts only - a host already joined to vCenter uses the `vcenter` block instead (its REST session API is blocked once managed) | [`pillar/esxi.sls.example`](../pillar/esxi.sls.example) | +| `vcfa` | VCF Automation (Aria Automation) | [`pillar/vcfa.sls.example`](../pillar/vcfa.sls.example) | +| `vcf_installer` | VCF Installer (Day-0 bringup, formerly Cloud Builder) | [`pillar/vcf_installer.sls.example`](../pillar/vcf_installer.sls.example) | +| `vcf_ops` | VCF Operations (Suite API) | [`pillar/vcf_ops.sls.example`](../pillar/vcf_ops.sls.example) | + +Each file follows the same shape - copy it, rename it (drop `.example`), and +fill in real values: + +```yaml +saltext.vcf: + vcenter: + host: mgmt-vc.example.test + username: administrator@vsphere.local + password: secret + verify_ssl: false +``` + +**Never commit the real `*.sls` files** - only `*.sls.example` is tracked; +the rest are gitignored. + +### Which path applies depends on how you'll run VCF operations + +This is the detail most likely to cause confusion - pick the path that +matches how you intend to trigger `saltext.vcf` calls against this minion. + +#### Path 1 - Locally inside the container (`salt-call --local`) + +Use this if scripts inside the container/Pod call `saltext.vcf` directly, or +for ad-hoc testing. The minion always has `pillar_roots` pointed at its own +local pillar directory; `top.sls` is auto-generated to match `'*'` against +whatever `*.sls` files are present. + +**Docker** - bind-mount the directory at container start: + +```bash +docker run -d --name salt-minion-vcf \ + -e SALT_MASTER= \ + -v salt-minion-vcf-pki:/etc/salt/pki/minion \ + -v "$(pwd)/pillar:/etc/salt/pillar" \ + salt-minion-vcf:0.1.0 +``` + +Or push files into an already-running container (no restart needed - +`salt-call --local` recompiles pillar from disk on every call): + +```bash +./scripts/pillar-push.sh salt-minion-vcf pillar/vcenter.sls +``` + +**Kubernetes** - create a Secret containing your `*.sls` files plus a +`top.sls` matching `'*'` (a Pod only ever runs one minion ID, so a wildcard +is always sufficient here): + +```bash +cat > top.sls <<'EOF' +base: + '*': + - vcenter +EOF +kubectl create secret generic salt-minion-vcf-pillar \ + --from-file=top.sls \ + --from-file=vcenter.sls=pillar/vcenter.sls +helm upgrade --install vcf-executor ./helm/salt-minion-vcf \ + --set salt.master= \ + --set pillar.secretName=salt-minion-vcf-pillar +``` + +To update without restarting the Pod, update the Secret object itself - +kubelet re-syncs the mounted volume automatically (typically within ~60-90s). + +**Verify:** + +```bash +docker exec salt-minion-vcf salt-call --local pillar.items +docker exec salt-minion-vcf salt-call --local vcf_vcenter_vm.list_ +``` + +#### Path 2 - Dispatched from the Salt Master (`salt '' ...`) + +This is the intended production model: VCF Operations/RaaS dispatches jobs +to the minion from the master. **Jobs run this way are compiled using the +Master's own `pillar_roots` - anything mounted into this container (Path 1) +is invisible to them.** The customer's Salt master admin needs pillar data +on the master side (e.g. `/srv/pillar`), targeted by this minion's ID - see +[`pillar/master-top.sls.example`](../pillar/master-top.sls.example): + +```yaml +# /srv/pillar/top.sls on the customer's Salt Master +base: + '': + - vcenter +``` + +using the identical `saltext.vcf.` structure as the `pillar/*.sls.example` +files in this repo. This is outside this repo's control (it's the master +admin's own `pillar_roots`); for production, prefer an `ext_pillar` backed by +a secrets manager (e.g. Vault) over plain files in `/srv/pillar`. + +**Verify (run from the master, not `salt-call --local`):** + +```bash +salt '' pillar.items +salt '' test.ping +``` + +If you need both models at once (local ad-hoc testing *and* master-dispatched +production jobs), configure Path 1 and Path 2 independently with the same +values - they don't conflict, since each is scoped to a different pillar_roots. + +### Security reminders + +See [`docs/security.md`](security.md) for the full list. The two most +relevant here: + +- Never put VCF credentials in a Kubernetes ConfigMap - use a Secret. +- Only `*.sls.example` files are tracked in git; never force-add or commit + a real `*.sls` file. diff --git a/salt-minion-vcf/scripts/onboarding/README.md b/salt-minion-vcf/scripts/onboarding/README.md new file mode 100644 index 0000000..1910df3 --- /dev/null +++ b/salt-minion-vcf/scripts/onboarding/README.md @@ -0,0 +1,115 @@ +# VCF Operations Onboarding Script + +`vcf-ops-onboard.py` is an interactive tool that brings up a `salt-minion-vcf` +instance (Docker **or** Kubernetes/Helm) and registers it as a trusted minion +against a Salt master managed by VMware VCF Operations - without the +minion's private key ever leaving the minion, and without VCF Operations +credentials ever reaching the minion itself. + +## What it does + +```text +1. Log in to VCF Operations +2. Resolve the Salt master for a given VCF instance +3. Compute the master's identity fingerprint (master_finger) +4. Start the minion (docker run, or helm upgrade --install), with a freshly + generated minion ID - the minion generates its own RSA keypair locally +5. Read back the minion's public key (never the private key) +6. Trust that key with the master +7. Poll until the master accepts the connection +``` + +This mirrors the manual flow documented in the top-level +[`README.md`](../../README.md#salt-master-registration), just automated and +without a human needing to run `salt-key -a` by hand - trust is established +via the VCF Operations API instead. + +Steps 4-7 can be repeated for multiple minions in one session without +re-entering VCF Operations credentials or re-resolving the master. + +## Interactive features + +- **Input validation**: the VCF instance ID is checked against a UUID format + and re-prompted if invalid; deployment type is a numbered menu, not free text. +- **Review before acting**: a summary of every setting (minion ID, image, + container/release name, target master, ...) is shown before the minion is + started, and again before its key is trusted - nothing consequential runs + without an explicit confirmation. +- **Live progress**: waiting for the minion to generate its keypair and for + the master to accept the connection shows an animated spinner with a + countdown (falls back to periodic plain-text lines if output isn't a TTY, + e.g. when redirected to a file). +- **Onboard multiple minions in one session**: after each successful + onboarding you're asked whether to onboard another against the same + master - container/volume/release names are auto-suggested with a `-2`, + `-3`, ... suffix so they don't collide with the previous minion. +- **`-y`/`--yes`** skips all confirmations for scripted/CI use, and + **`--dry-run`** previews every command and API call without executing + anything. + +## Logging + +Every run writes a full step-by-step audit log to +`vcf-ops-onboard-.log` in the current directory (override the path +with `--log-file`). It captures every prompt, shell command, and API call/ +response - passwords and auth tokens are never written to it. Pass +`-v`/`--verbose` to also mirror that detail live on the console. + +## Requirements + +- Python 3.8+ (standard library only - no `pip install` needed) +- `docker` on PATH (Docker mode), or `helm` + `kubectl` on PATH (Kubernetes mode) +- Network access from wherever you run this script to your VCF Operations + instance's Suite API + +## Usage + +Fully interactive - just run it and answer the prompts: + +```bash +python3 scripts/onboarding/vcf-ops-onboard.py +``` + +Or supply anything up front via flags (anything omitted is still prompted for): + +```bash +# Docker +python3 scripts/onboarding/vcf-ops-onboard.py \ + --ops-host vcfops.example.com \ + --ops-user admin \ + --vcf-instance-id \ + --deployment docker \ + --image salt-minion-vcf:0.1.0 + +# Kubernetes / Helm (run from the salt-minion-vcf repo root, so +# --chart-path's default of ./helm/salt-minion-vcf resolves correctly) +python3 scripts/onboarding/vcf-ops-onboard.py \ + --ops-host vcfops.example.com \ + --ops-user admin \ + --vcf-instance-id \ + --deployment kubernetes \ + --namespace vcf-salt \ + --release-name vcf-executor +``` + +See `--help` for the full flag list (container/release naming, image +repository/tag, connect timeout, `--log-file`/`-v` for audit logging, +`--dry-run` to preview every command and API call without executing +anything, `-y` to skip confirmation prompts). + +## Things to validate in your own environment + +- **VCF Operations auth flow**: the script logs in via + `POST /suite-api/api/auth/token/acquire` and sends + `Authorization: OpsToken ` on subsequent calls - the same pattern + used by other existing tooling against this backend. If your deployment + fronts VCF Operations with SSO/CSP instead, adjust `OpsClient.login()`. +- **`master_finger` algorithm**: defaults to `sha256` (matches the Salt + version this image bundles). Override with `--master-finger-algo md5` if + your Salt master needs the legacy default. + +## Known limitation + +There is currently no API to *revoke* a trusted key (deregistration), so this +script only covers onboarding. To remove a minion, use your master's own +key-management tooling directly for now. diff --git a/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py b/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py new file mode 100755 index 0000000..5c68154 --- /dev/null +++ b/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py @@ -0,0 +1,886 @@ +#!/usr/bin/env python3 +""" +vcf-ops-onboard.py + +Interactive onboarding tool that brings up a salt-minion-vcf instance (Docker +or Kubernetes/Helm) and registers it as a trusted minion against a VCF +Operations-managed Salt master - with no private key ever leaving the minion, +and no VCF Operations credentials ever reaching the minion itself. + +Flow: + 1. Prompt for VCF Operations (Suite API) connection details and log in. + 2. Resolve the Salt master governing a given VCF instance + (GET /suite-api/api/salt/master?resourceId=). + 3. Compute the Salt-compatible master_finger from the returned master + public key, so the minion can verify the master's identity on connect. + 4. Start the minion (docker run, or helm install/upgrade), pointed at the + master and given a freshly generated minion ID. The minion generates + its own RSA keypair locally on first start - this script never sees it. + 5. Read back the minion's public key (never the private key) and its ID. + 6. Trust that key against the master + (POST /suite-api/api/salt/minions/{minionId}/trusted-keys). + 7. Poll the minion (already retrying in the background) until the master + accepts it, using the exact check the image's own healthcheck/readiness + probe uses: `salt-call status.master`. + +Steps 4-7 can be repeated for multiple minions in one session without +re-entering VCF Operations credentials. + +Every step is written to a timestamped log file (default: +vcf-ops-onboard-.log) in addition to the interactive console +output, for audit/troubleshooting. Passwords and auth tokens are never +logged. + +Only two dependencies: Python 3.8+, and whichever of `docker`/`helm`+`kubectl` +you're deploying with. No third-party pip packages required. + +Reference: https://github.com/saltstack/salt-helm/tree/main/salt-minion-vcf +""" + +from __future__ import annotations + +import argparse +import base64 +import getpass +import hashlib +import json +import logging +import re +import shlex +import ssl +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from dataclasses import dataclass +from datetime import datetime +from typing import Callable, Optional + + +LOG = logging.getLogger("vcf_onboard") + +UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + + +# -------------------------------------------------------------------------- +# Logging +# -------------------------------------------------------------------------- + +def setup_logging(log_file: str, verbose: bool) -> None: + LOG.setLevel(logging.DEBUG) + + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(logging.Formatter( + "%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S")) + LOG.addHandler(file_handler) + + if verbose: + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.DEBUG) + console_handler.setFormatter(logging.Formatter(" . %(message)s")) + LOG.addHandler(console_handler) + + +# -------------------------------------------------------------------------- +# Console helpers (each also writes to the log file for audit purposes) +# -------------------------------------------------------------------------- + +def _supports_color() -> bool: + return sys.stdout.isatty() + + +class _C: + RESET = "\033[0m" if _supports_color() else "" + BOLD = "\033[1m" if _supports_color() else "" + GREEN = "\033[32m" if _supports_color() else "" + RED = "\033[31m" if _supports_color() else "" + YELLOW = "\033[33m" if _supports_color() else "" + CYAN = "\033[36m" if _supports_color() else "" + DIM = "\033[2m" if _supports_color() else "" + + +def step(n: int, total: int, title: str) -> None: + bar = "=" * 70 + print(f"\n{_C.BOLD}{_C.CYAN}{bar}\n STEP {n}/{total}: {title}\n{bar}{_C.RESET}") + LOG.info(f"==== STEP {n}/{total}: {title} ====") + + +def info(msg: str) -> None: + print(f" {msg}") + LOG.info(msg) + + +def ok(msg: str) -> None: + print(f"{_C.GREEN}[OK]{_C.RESET} {msg}") + LOG.info(f"OK: {msg}") + + +def warn(msg: str) -> None: + print(f"{_C.YELLOW}[WARN]{_C.RESET} {msg}") + LOG.warning(msg) + + +def fail(msg: str) -> None: + print(f"{_C.RED}[FAIL]{_C.RESET} {msg}") + LOG.error(msg) + + +def die(msg: str, code: int = 1) -> None: + fail(msg) + sys.exit(code) + + +def prompt(text: str, default: Optional[str] = None, secret: bool = False, + validate: Optional[Callable[[str], bool]] = None, + validate_hint: str = "") -> str: + suffix = f" [{default}]" if default else "" + reader = getpass.getpass if secret else input + while True: + value = reader(f"{text}{suffix}: ").strip() + if not value and default is not None: + value = default + if not value: + print(" (this value is required)") + continue + if validate and not validate(value): + print(f" Invalid value.{(' ' + validate_hint) if validate_hint else ''}") + continue + LOG.debug(f"prompt '{text}' -> {'' if secret else value}") + return value + + +def prompt_uuid(text: str, default: Optional[str] = None) -> str: + return prompt(text, default=default, validate=lambda v: bool(UUID_RE.match(v)), + validate_hint="Expected a UUID, e.g. 3fa85f64-5717-4562-b3fc-2c963f66afa6") + + +def choose(text: str, options: list, default: Optional[str] = None) -> str: + print(f"{text}") + for i, opt in enumerate(options, 1): + marker = " (default)" if opt == default else "" + print(f" {i}. {opt}{marker}") + default_idx = str(options.index(default) + 1) if default in options else None + while True: + raw = prompt("Enter choice number", default=default_idx) + if raw.isdigit() and 1 <= int(raw) <= len(options): + choice = options[int(raw) - 1] + LOG.debug(f"choice '{text}' -> {choice}") + return choice + print(f" Please enter a number between 1 and {len(options)}") + + +def confirm(text: str, default: bool = True, assume_yes: bool = False) -> bool: + if assume_yes: + LOG.debug(f"confirm '{text}' -> yes (--yes)") + return True + suffix = "[Y/n]" if default else "[y/N]" + while True: + raw = input(f"{text} {suffix} ").strip().lower() + if not raw: + result = default + elif raw in ("y", "yes"): + result = True + elif raw in ("n", "no"): + result = False + else: + continue + LOG.debug(f"confirm '{text}' -> {result}") + return result + + +def print_summary(title: str, pairs: list) -> None: + width = max([len(k) for k, _ in pairs] + [len(title)]) + 2 + print(f"\n{_C.BOLD}{title}{_C.RESET}") + print(f"{_C.DIM}{'-' * 70}{_C.RESET}") + for key, value in pairs: + print(f" {key:<{width}} {value}") + print(f"{_C.DIM}{'-' * 70}{_C.RESET}") + LOG.info(f"{title}: " + ", ".join(f"{k}={v}" for k, v in pairs)) + + +class Spinner: + """Animated progress indicator for interactive terminals; falls back to + periodic plain-text lines when output isn't a TTY (e.g. redirected to a + file), so progress is still visible either way.""" + + FRAMES = "|/-\\" + + def __init__(self, message: str): + self.message = message + self._i = 0 + self.active = sys.stdout.isatty() + + def spin(self, extra: str = "") -> None: + if not self.active: + return + frame = self.FRAMES[self._i % len(self.FRAMES)] + self._i += 1 + suffix = f" - {extra}" if extra else "" + sys.stdout.write(f"\r {frame} {self.message}{suffix}" + " " * 10) + sys.stdout.flush() + + def stop(self, final: Optional[str] = None) -> None: + if self.active: + sys.stdout.write("\r" + " " * 100 + "\r") + sys.stdout.flush() + if final: + print(final) + + +def wait_until(predicate: Callable[[], bool], timeout: int, check_interval: float, + message: str, dry_run: bool = False) -> bool: + """Poll `predicate` at most once per check_interval until it returns True + or timeout elapses. Animates a spinner (or prints periodically) while + waiting; every check is logged to the audit log regardless.""" + if dry_run: + LOG.info(f"(dry-run) skipping wait: {message}") + return True + + spinner = Spinner(message) + deadline = time.time() + timeout + next_check = 0.0 + last_plain_print = 0.0 + + while time.time() < deadline: + now = time.time() + if now >= next_check: + result = predicate() + LOG.debug(f"check '{message}' -> {result}") + if result: + spinner.stop() + return True + next_check = now + check_interval + + remaining = int(deadline - now) + if spinner.active: + spinner.spin(f"{remaining}s remaining") + time.sleep(0.15) + else: + if now - last_plain_print >= check_interval: + print(f" {message}... ({remaining}s remaining)") + last_plain_print = now + time.sleep(check_interval) + + spinner.stop() + return False + + +# -------------------------------------------------------------------------- +# Shell command execution +# -------------------------------------------------------------------------- + +def run(cmd: list, dry_run: bool = False, capture: bool = False, check: bool = True) -> str: + printable = " ".join(shlex.quote(c) for c in cmd) + print(f" $ {printable}") + LOG.debug(f"$ {printable}") + if dry_run: + return "" + try: + result = subprocess.run( + cmd, + check=check, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None, + text=True, + ) + except FileNotFoundError: + die(f"Command not found: {cmd[0]}. Is it installed and on PATH?") + except subprocess.CalledProcessError as e: + LOG.error(f"command failed (exit {e.returncode}): {printable}") + raise + if capture: + output = (result.stdout or "").strip() + LOG.debug(f"output: {output}") + return output + return "" + + +# -------------------------------------------------------------------------- +# VCF Operations (Suite API) client +# -------------------------------------------------------------------------- + +class OpsApiError(Exception): + pass + + +class OpsClient: + """ + Thin client for the two VCF Operations Salt trust-management endpoints. + + Auth flow (matches the one already used by other internal tooling + against this same backend): + POST {base}/api/auth/token/acquire {username, password} -> {token} + Authorization: OpsToken (on every subsequent call) + + If your environment's login flow differs (e.g. CSP/SSO-fronted), adjust + `login()` accordingly - everything else in this class is unaffected. + """ + + def __init__(self, host: str, username: str, password: str, + base_path: str = "/suite-api", verify_tls: bool = True, timeout: int = 30): + self.base_url = f"https://{host}{base_path}" + self.username = username + self.password = password + self.verify_tls = verify_tls + self.timeout = timeout + self._token: Optional[str] = None + self._ssl_context = ssl.create_default_context() + if not verify_tls: + self._ssl_context.check_hostname = False + self._ssl_context.verify_mode = ssl.CERT_NONE + + def _request(self, method: str, path: str, params: Optional[dict] = None, + json_body: Optional[dict] = None, authed: bool = True) -> dict: + url = f"{self.base_url}{path}" + if params: + from urllib.parse import urlencode + url = f"{url}?{urlencode(params)}" + + LOG.debug(f"{method} {url}") + + headers = {"Content-Type": "application/json", "Accept": "application/json"} + if authed: + if not self._token: + raise OpsApiError("Not authenticated - call login() first") + headers["Authorization"] = f"OpsToken {self._token}" + + data = json.dumps(json_body).encode("utf-8") if json_body is not None else None + req = urllib.request.Request(url, data=data, headers=headers, method=method) + + try: + with urllib.request.urlopen(req, timeout=self.timeout, context=self._ssl_context) as resp: + LOG.debug(f"{method} {path} -> HTTP {resp.status}") + body = resp.read().decode("utf-8") + return json.loads(body) if body else {} + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + LOG.error(f"{method} {path} -> HTTP {e.code}: {body}") + raise OpsApiError(f"{method} {path} -> HTTP {e.code}: {body}") from None + except urllib.error.URLError as e: + LOG.error(f"{method} {path} -> connection error: {e.reason}") + raise OpsApiError(f"{method} {path} -> connection error: {e.reason}") from None + + def login(self) -> None: + data = self._request( + "POST", "/api/auth/token/acquire", + json_body={"username": self.username, "password": self.password}, + authed=False, + ) + token = data.get("token") + if not token: + raise OpsApiError("Login succeeded but no token was returned") + self._token = token + LOG.info(f"Authenticated as {self.username} (token acquired, not logged)") + + def get_master_details(self, vcf_instance_id: str) -> dict: + """GET /api/salt/master?resourceId= -> {resourceId, masterId, masterFqdn, + masterPublicKey (base64 of the PEM text), masterKeyState, presenceStatus}.""" + return self._request("GET", "/api/salt/master", params={"resourceId": vcf_instance_id}) + + def add_trusted_key(self, minion_id: str, master_id: str, minion_public_key_pem: str) -> dict: + """POST /api/salt/minions/{minionId}/trusted-keys + Body: {masterId, minionPublicKey} - minionPublicKey is RAW PEM text here + (NOT base64-encoded - only the master pubkey in GET responses is).""" + return self._request( + "POST", f"/api/salt/minions/{minion_id}/trusted-keys", + json_body={"masterId": master_id, "minionPublicKey": minion_public_key_pem}, + ) + + +# -------------------------------------------------------------------------- +# Salt master_finger computation +# -------------------------------------------------------------------------- + +def pem_finger(pem_text: str, sum_type: str = "sha256") -> str: + """ + Reproduces Salt's own salt.utils.crypt.pem_finger(): strip the PEM + header/footer lines, base64-decode the body to raw DER bytes, hash them, + and format as colon-separated hex pairs - the exact string Salt expects + for `master_finger` / SALT_MASTER_FINGER. + """ + lines = [l for l in pem_text.strip().splitlines() if l.strip()] + if len(lines) < 3: + raise ValueError("Master public key does not look like a PEM block") + body = "".join(lines[1:-1]) + der = base64.b64decode(body) + digest = hashlib.new(sum_type, der).hexdigest() + return ":".join(digest[i:i + 2] for i in range(0, len(digest), 2)) + + +# -------------------------------------------------------------------------- +# Docker deployment +# -------------------------------------------------------------------------- + +@dataclass +class DockerConfig: + image: str + container_name: str + volume: str + master_fqdn: str + master_finger: str + minion_id: str + + +def docker_container_exists(name: str, dry_run: bool) -> bool: + if dry_run: + return False + out = run(["docker", "ps", "-a", "--filter", f"name=^{name}$", "--format", "{{.Names}}"], + capture=True, check=False) + return out.strip() == name + + +def docker_start(cfg: DockerConfig, dry_run: bool, assume_yes: bool = False) -> None: + if docker_container_exists(cfg.container_name, dry_run): + warn(f"A container named '{cfg.container_name}' already exists " + f"(likely left over from a previous attempt).") + if confirm(f"Remove it and continue?", default=True, assume_yes=assume_yes): + run(["docker", "rm", "-f", cfg.container_name], dry_run=dry_run) + else: + die(f"Container '{cfg.container_name}' already exists. " + f"Choose a different --container-name or remove it manually with " + f"`docker rm -f {cfg.container_name}`.") + + cmd = [ + "docker", "run", "-d", + "--name", cfg.container_name, + "-e", f"SALT_MASTER={cfg.master_fqdn}", + "-e", f"SALT_MASTER_FINGER={cfg.master_finger}", + "-e", f"SALT_MINION_ID={cfg.minion_id}", + "-v", f"{cfg.volume}:/etc/salt/pki/minion", + cfg.image, + ] + run(cmd, dry_run=dry_run) + + +def docker_exec(container: str, args: list, dry_run: bool = False, check: bool = True) -> str: + return run(["docker", "exec", container] + args, dry_run=dry_run, capture=True, check=check) + + +def docker_read_minion_pubkey(container: str, timeout: int, dry_run: bool) -> str: + if dry_run: + return "-----BEGIN PUBLIC KEY-----\nAAAAAAAAAAAAAAAAAAAAAAAA\n-----END PUBLIC KEY-----" + + result = {} + + def _check() -> bool: + pubkey = docker_exec(container, ["cat", "/etc/salt/pki/minion/minion.pub"], check=False) + if pubkey.startswith("-----BEGIN PUBLIC KEY-----"): + result["pubkey"] = pubkey + return True + return False + + if not wait_until(_check, timeout=timeout, check_interval=2, + message="Waiting for minion to generate its keypair", dry_run=dry_run): + die(f"Timed out waiting for {container} to generate its minion keypair. " + f"Check `docker logs {container}`.") + return result["pubkey"] + + +MINION_READY_LOG_MARKER = "Minion is ready to receive requests" + + +def docker_is_connected(container: str, dry_run: bool) -> bool: + if dry_run: + return True + # status.master's answer depends on master_alive_interval being configured on the + # minion, which this image's entrypoint does not set - it can under-report even + # once actually connected. The log line below is emitted once, event-driven, the + # moment the pub/req channels with the master are established, so it doesn't have + # that gap; treat either signal as sufficient. + out = docker_exec( + container, + ["salt-call", "--out=newline_values_only", "--retcode-passthrough", "status.master"], + check=False, + ) + if out.strip().lower() == "true": + return True + logs = run(["docker", "logs", container], dry_run=dry_run, capture=True, check=False) + return MINION_READY_LOG_MARKER in logs + + +# -------------------------------------------------------------------------- +# Kubernetes / Helm deployment +# -------------------------------------------------------------------------- + +@dataclass +class HelmConfig: + chart_path: str + release_name: str + namespace: str + image_repository: str + image_tag: str + master_fqdn: str + master_finger: str + minion_id: str + + +def helm_start(cfg: HelmConfig, dry_run: bool) -> None: + cmd = [ + "helm", "upgrade", "--install", cfg.release_name, cfg.chart_path, + "--namespace", cfg.namespace, "--create-namespace", + "--set", f"salt.master={cfg.master_fqdn}", + "--set", f"salt.masterFinger={cfg.master_finger}", + "--set", f"salt.minionId={cfg.minion_id}", + "--set", f"image.repository={cfg.image_repository}", + "--set", f"image.tag={cfg.image_tag}", + ] + run(cmd, dry_run=dry_run) + + +def kubectl_get_pod_name(namespace: str, release_name: str, dry_run: bool, timeout: int = 60) -> str: + if dry_run: + return f"{release_name}-salt-minion-vcf-0" + + selector = f"app.kubernetes.io/name=salt-minion-vcf,app.kubernetes.io/instance={release_name}" + result = {} + + def _check() -> bool: + name = run( + ["kubectl", "get", "pods", "-n", namespace, "-l", selector, + "-o", "jsonpath={.items[0].metadata.name}"], + capture=True, check=False, + ) + if name: + result["name"] = name + return True + return False + + if not wait_until(_check, timeout=timeout, check_interval=2, + message="Waiting for the Pod to be scheduled", dry_run=dry_run): + die(f"Timed out waiting for a Pod matching '{selector}' in namespace {namespace}.") + return result["name"] + + +def kubectl_exec(namespace: str, pod: str, args: list, dry_run: bool = False, check: bool = True) -> str: + return run(["kubectl", "exec", "-n", namespace, pod, "--"] + args, + dry_run=dry_run, capture=True, check=check) + + +def kubectl_read_minion_pubkey(namespace: str, pod: str, timeout: int, dry_run: bool) -> str: + if dry_run: + return "-----BEGIN PUBLIC KEY-----\nAAAAAAAAAAAAAAAAAAAAAAAA\n-----END PUBLIC KEY-----" + + result = {} + + def _check() -> bool: + pubkey = kubectl_exec(namespace, pod, ["cat", "/etc/salt/pki/minion/minion.pub"], check=False) + if pubkey.startswith("-----BEGIN PUBLIC KEY-----"): + result["pubkey"] = pubkey + return True + return False + + if not wait_until(_check, timeout=timeout, check_interval=2, + message="Waiting for minion to generate its keypair", dry_run=dry_run): + die(f"Timed out waiting for {pod} to generate its minion keypair. " + f"Check `kubectl logs -n {namespace} {pod}`.") + return result["pubkey"] + + +def kubectl_is_connected(namespace: str, pod: str, dry_run: bool) -> bool: + if dry_run: + return True + # See the comment on docker_is_connected() - status.master alone can under-report; + # the log marker is an event-driven signal emitted only after successful auth. + out = kubectl_exec( + namespace, pod, + ["salt-call", "--out=newline_values_only", "--retcode-passthrough", "status.master"], + check=False, + ) + if out.strip().lower() == "true": + return True + logs = run(["kubectl", "logs", "-n", namespace, pod], dry_run=dry_run, capture=True, check=False) + return MINION_READY_LOG_MARKER in logs + + +# -------------------------------------------------------------------------- +# Main orchestration +# -------------------------------------------------------------------------- + +TOTAL_STEPS = 7 + + +def build_arg_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + description="Onboard a salt-minion-vcf instance against a VCF Operations-managed Salt master.", + ) + p.add_argument("--ops-host", help="VCF Operations FQDN or IP") + p.add_argument("--ops-user", help="VCF Operations username") + p.add_argument("--ops-base-path", default="/suite-api", + help="Suite API base path (default: /suite-api)") + p.add_argument("--insecure", action="store_true", + help="Skip TLS certificate verification against VCF Operations") + p.add_argument("--vcf-instance-id", help="VCF instance resource UUID whose master to use") + + p.add_argument("--deployment", choices=["docker", "kubernetes"], + help="Where to run the minion") + + p.add_argument("--minion-id", help="Explicit minion ID (default: auto-generated)") + p.add_argument("--minion-id-prefix", default="ext-minion", + help="Prefix for the auto-generated minion ID (default: ext-minion)") + + # Docker options. Defaults are intentionally None (not the literal + # default value) so the script can tell "explicitly passed on the CLI" + # apart from "use the built-in default" - only the first minion in a + # session honors these directly; see onboard_one_minion(). + p.add_argument("--image", help="[docker] image:tag to run (default: salt-minion-vcf:0.1.0)") + p.add_argument("--container-name", help="[docker] container name (default: salt-minion-vcf)") + p.add_argument("--volume", help="[docker] PKI volume name (default: salt-minion-vcf-pki)") + + # Kubernetes/Helm options + p.add_argument("--chart-path", default="./helm/salt-minion-vcf", help="[k8s] path to the Helm chart") + p.add_argument("--release-name", help="[k8s] Helm release name (default: vcf-executor)") + p.add_argument("--namespace", help="[k8s] target namespace (default: vcf-salt)") + p.add_argument("--image-repository", help="[k8s] image repository (default: salt-minion-vcf)") + p.add_argument("--image-tag", help="[k8s] image tag (default: 0.1.0)") + + p.add_argument("--master-finger-algo", default="sha256", choices=["sha256", "md5"], + help="Hash algorithm for master_finger (default: sha256, matches modern Salt)") + p.add_argument("--connect-timeout", type=int, default=300, + help="Seconds to wait for the minion to connect (default: 300)") + p.add_argument("--poll-interval", type=int, default=5, + help="Seconds between connection status checks (default: 5)") + p.add_argument("-y", "--yes", action="store_true", help="Assume yes on all confirmations") + p.add_argument("--dry-run", action="store_true", + help="Print every command/API call without executing anything") + p.add_argument("--log-file", help="Path to the audit log file " + "(default: vcf-ops-onboard-.log)") + p.add_argument("-v", "--verbose", action="store_true", + help="Also print detailed debug logging to the console") + return p + + +DEFAULT_IMAGE = "salt-minion-vcf:0.1.0" +DEFAULT_CONTAINER_NAME = "salt-minion-vcf" +DEFAULT_VOLUME = "salt-minion-vcf-pki" +DEFAULT_RELEASE_NAME = "vcf-executor" +DEFAULT_NAMESPACE = "vcf-salt" +DEFAULT_IMAGE_REPOSITORY = "salt-minion-vcf" +DEFAULT_IMAGE_TAG = "0.1.0" + + +def onboard_one_minion(client: OpsClient, args: argparse.Namespace, + master_id: str, master_fqdn: str, master_finger: str, + deployment: str, defaults: dict, index: int) -> dict: + """ + Runs steps 4-7 for a single minion and returns a summary dict. + + `index` counts minions onboarded in this session (starting at 1). CLI + flags for identity-bearing settings (minion ID, container/volume/release + name) are only honored on the first minion - a container name, PKI + volume, or Helm release can't be reused for a second minion without + colliding, so from the second minion onward this always prompts, with an + auto-suffixed suggestion ("-2", "-3", ...) to avoid that collision. + """ + + # ---------------------------------------------------------------- Step 4 + step(4, TOTAL_STEPS, "Start the minion") + suffix = "" if index == 1 else f"-{index}" + + minion_id = (args.minion_id if index == 1 else None) or prompt( + "Minion ID", default=f"{args.minion_id_prefix}-{uuid.uuid4()}") + + if deployment == "docker": + container_name = (args.container_name if index == 1 else None) or prompt( + "Container name", default=f"{DEFAULT_CONTAINER_NAME}{suffix}") + image = args.image or defaults.get("image") or prompt("Image", default=DEFAULT_IMAGE) + volume = (args.volume if index == 1 else None) or prompt( + "PKI volume name", default=f"{DEFAULT_VOLUME}{suffix}") + if volume.startswith("/"): + warn(f"'{volume}' looks like a host path, not a named Docker volume - " + f"it will be bind-mounted as-is. The container runs as non-root uid 10000, " + f"so that host directory must already exist and be writable by uid 10000 " + f"(e.g. `mkdir -p {volume} && chown 10000:10000 {volume}`), or the minion " + f"will fail to write its keys there.") + + print_summary("Review before starting the minion", [ + ("Deployment", "docker"), + ("Minion ID", minion_id), + ("Image", image), + ("Container name", container_name), + ("PKI volume", volume), + ("Salt master", f"{master_fqdn} (master_finger computed)"), + ]) + if not confirm("Proceed with these settings?", assume_yes=args.yes): + die("Aborted by user.", code=0) + + docker_cfg = DockerConfig( + image=image, container_name=container_name, volume=volume, + master_fqdn=master_fqdn, master_finger=master_finger, minion_id=minion_id, + ) + docker_start(docker_cfg, dry_run=args.dry_run, assume_yes=args.yes) + ok(f"Container '{container_name}' started") + defaults["image"] = image + pod_name = None + else: + release_name = (args.release_name if index == 1 else None) or prompt( + "Helm release name", default=f"{DEFAULT_RELEASE_NAME}{suffix}") + namespace = args.namespace or defaults.get("namespace") or prompt( + "Namespace", default=DEFAULT_NAMESPACE) + image_repository = args.image_repository or defaults.get("image_repository") or prompt( + "Image repository", default=DEFAULT_IMAGE_REPOSITORY) + image_tag = args.image_tag or defaults.get("image_tag") or prompt( + "Image tag", default=DEFAULT_IMAGE_TAG) + + print_summary("Review before starting the minion", [ + ("Deployment", "kubernetes"), + ("Minion ID", minion_id), + ("Release name", release_name), + ("Namespace", namespace), + ("Image", f"{image_repository}:{image_tag}"), + ("Salt master", f"{master_fqdn} (master_finger computed)"), + ]) + if not confirm("Proceed with these settings?", assume_yes=args.yes): + die("Aborted by user.", code=0) + + helm_cfg = HelmConfig( + chart_path=args.chart_path, release_name=release_name, namespace=namespace, + image_repository=image_repository, image_tag=image_tag, + master_fqdn=master_fqdn, master_finger=master_finger, minion_id=minion_id, + ) + helm_start(helm_cfg, dry_run=args.dry_run) + ok(f"Helm release '{release_name}' installed/upgraded in namespace {namespace}") + defaults["namespace"] = namespace + defaults["image_repository"] = image_repository + defaults["image_tag"] = image_tag + pod_name = kubectl_get_pod_name(namespace, release_name, dry_run=args.dry_run) + ok(f"Pod: {pod_name}") + + # ---------------------------------------------------------------- Step 5 + step(5, TOTAL_STEPS, "Read the minion's public key") + if deployment == "docker": + minion_pubkey_pem = docker_read_minion_pubkey(container_name, timeout=60, dry_run=args.dry_run) + else: + minion_pubkey_pem = kubectl_read_minion_pubkey(namespace, pod_name, timeout=60, dry_run=args.dry_run) + ok("Minion public key retrieved (private key never left the minion)") + + # ---------------------------------------------------------------- Step 6 + step(6, TOTAL_STEPS, "Trust the minion's key against the master") + if not confirm(f"Register minion '{minion_id}' as trusted against master '{master_id}'?", + assume_yes=args.yes): + die("Aborted by user.", code=0) + if args.dry_run: + info(f"(dry-run) POST /api/salt/minions/{minion_id}/trusted-keys " + f"{{masterId: {master_id}, minionPublicKey: }}") + else: + try: + trust_result = client.add_trusted_key(minion_id, master_id, minion_pubkey_pem) + except OpsApiError as e: + die(f"Failed to register the trusted key: {e}") + if trust_result.get("status", "").upper() not in ("SUCCESS", ""): + die(f"Trust registration reported failure: {trust_result}") + ok("Minion key trusted with the Salt master") + + # ---------------------------------------------------------------- Step 7 + step(7, TOTAL_STEPS, "Wait for the minion to connect") + + def _connected() -> bool: + if deployment == "docker": + return docker_is_connected(container_name, dry_run=args.dry_run) + return kubectl_is_connected(namespace, pod_name, dry_run=args.dry_run) + + connected = wait_until(_connected, timeout=args.connect_timeout, + check_interval=args.poll_interval, + message="Waiting for the master to accept the minion", dry_run=args.dry_run) + if not connected: + die(f"Minion did not connect within {args.connect_timeout}s. " + f"Check the master's `salt-key -L` and the minion's logs.") + ok("Minion connected to the Salt master") + + return {"minion_id": minion_id, "deployment": deployment} + + +def main() -> None: + args = build_arg_parser().parse_args() + log_file = args.log_file or f"vcf-ops-onboard-{datetime.now():%Y%m%d-%H%M%S}.log" + setup_logging(log_file, verbose=args.verbose) + LOG.info(f"vcf-ops-onboard started, args={vars(args)}") + + print(f"{_C.BOLD}VCF Operations - External Minion Onboarding{_C.RESET}") + info(f"Logging full step-by-step detail to: {log_file}") + if args.dry_run: + warn("Running in --dry-run mode: nothing will actually be executed.") + + # ---------------------------------------------------------------- Step 1 + step(1, TOTAL_STEPS, "Connect to VCF Operations") + ops_host = args.ops_host or prompt("VCF Operations FQDN or IP") + ops_user = args.ops_user or prompt("Username") + ops_password = prompt("Password", secret=True) + verify_tls = not args.insecure + if not verify_tls: + warn("TLS certificate verification is disabled for this session.") + + client = OpsClient(ops_host, ops_user, ops_password, + base_path=args.ops_base_path, verify_tls=verify_tls) + if not args.dry_run: + try: + client.login() + except OpsApiError as e: + die(f"Login failed: {e}") + ok(f"Authenticated to {ops_host}") + + # ---------------------------------------------------------------- Step 2 + step(2, TOTAL_STEPS, "Resolve the Salt master for your VCF instance") + vcf_instance_id = args.vcf_instance_id or prompt_uuid("VCF instance resource ID (UUID)") + + if args.dry_run: + master = {"masterId": "salt-master-", "masterFqdn": "salt-master.example.com", + "masterPublicKey": base64.b64encode( + b"-----BEGIN PUBLIC KEY-----\nAAAAAAAAAAAAAAAAAAAAAAAA\n-----END PUBLIC KEY-----").decode()} + else: + try: + master = client.get_master_details(vcf_instance_id) + except OpsApiError as e: + die(f"Could not resolve master details: {e}") + + master_id = master["masterId"] + master_fqdn = master["masterFqdn"] + master_pubkey_pem = base64.b64decode(master["masterPublicKey"]).decode("utf-8") + ok(f"Master resolved: {master_id} @ {master_fqdn}") + + # ---------------------------------------------------------------- Step 3 + step(3, TOTAL_STEPS, "Compute master identity fingerprint") + master_finger = pem_finger(master_pubkey_pem, sum_type=args.master_finger_algo) + ok(f"master_finger ({args.master_finger_algo}): {master_finger}") + + # ------------------------------------------------- Steps 4-7 (repeatable) + deployment = args.deployment or choose( + "\nWhere should this minion run?", ["docker", "kubernetes"], default="docker") + + onboarded = [] + defaults: dict = {} + index = 1 + while True: + result = onboard_one_minion(client, args, master_id, master_fqdn, master_finger, + deployment, defaults, index) + onboarded.append(result) + + print(f"\n{_C.BOLD}{_C.GREEN}Minion onboarded{_C.RESET}") + print(f" Minion ID : {result['minion_id']}") + print(f" Master : {master_id} @ {master_fqdn}") + print(f" Deployment: {result['deployment']}") + print(f"\nVerify from the Salt master:\n salt '{result['minion_id']}' test.ping") + + if args.dry_run or not confirm( + "\nOnboard another minion against this same master?", default=False, assume_yes=False): + break + index += 1 + + print(f"\n{_C.BOLD}Session summary{_C.RESET} ({len(onboarded)} minion(s) onboarded)") + for r in onboarded: + print(f" - {r['minion_id']} ({r['deployment']})") + print(f"\nFull audit log: {log_file}") + LOG.info(f"Session complete: {len(onboarded)} minion(s) onboarded: " + f"{[r['minion_id'] for r in onboarded]}") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print() + die("Interrupted by user.", code=130) From 251c74c17f7de5c860c41183fbe9879f894d392b Mon Sep 17 00:00:00 2001 From: Praveen T Date: Mon, 31 Aug 2026 00:23:37 +0530 Subject: [PATCH 2/6] Fix external minion connectivity: FIPS crypto + direct master pubkey trust Root-caused via live testing against a VCF-managed Salt master: - FIPS-validated masters don't implement SHA-1 for RSA OAEP/PKCS1v15 at all - a minion defaulting to SHA-1 doesn't get a clean rejection, it crashes the master's payload handler on every auth attempt. FIPS mode (fips_mode: True, OAEP-SHA224/PKCS1v15-SHA224) is now on by default in docker-entrypoint.sh, matching real VCF-managed minion config. - master_finger-based identity verification was unreliable in this environment even once FIPS was fixed. VCF's own internal component minions never do fingerprint verification at all - they're handed the master's public key directly and trust it. The onboarding script now pre-seeds SALT_MASTER_PUBKEY_B64 (written to minion_master.pub) for Docker minions instead, matching that pattern. master_finger is kept for the Kubernetes/Helm path, which doesn't yet support direct pubkey seeding. - Logged full response bodies for get_master_details/add_trusted_key (no secrets in either) to make this class of issue diagnosable from the audit log directly next time. Updated docs/external-minion-configuration.md and scripts/onboarding/README.md to reflect the real fixes in place of earlier troubleshooting guesses, and added the esxi-cluster-patching, usb-controller-removal, and vc-patch runbooks. --- ...ok.md => external-minion-configuration.md} | 0 .../docs/runbook-esxi-cluster-patching.md | 253 ++++++++++++++++++ .../docs/runbook-usb-controller-removal.md | 189 +++++++++++++ salt-minion-vcf/docs/runbook-vc-patch.md | 183 +++++++++++++ 4 files changed, 625 insertions(+) rename salt-minion-vcf/docs/{runbook.md => external-minion-configuration.md} (100%) create mode 100644 salt-minion-vcf/docs/runbook-esxi-cluster-patching.md create mode 100644 salt-minion-vcf/docs/runbook-usb-controller-removal.md create mode 100644 salt-minion-vcf/docs/runbook-vc-patch.md diff --git a/salt-minion-vcf/docs/runbook.md b/salt-minion-vcf/docs/external-minion-configuration.md similarity index 100% rename from salt-minion-vcf/docs/runbook.md rename to salt-minion-vcf/docs/external-minion-configuration.md diff --git a/salt-minion-vcf/docs/runbook-esxi-cluster-patching.md b/salt-minion-vcf/docs/runbook-esxi-cluster-patching.md new file mode 100644 index 0000000..3b3ccbd --- /dev/null +++ b/salt-minion-vcf/docs/runbook-esxi-cluster-patching.md @@ -0,0 +1,253 @@ +# Runbook: Patch an ESXi Cluster via vSphere Lifecycle Manager (vLCM) + +This runbook walks through patching every ESXi host in a vSphere cluster +using the desired-image vLCM workflow (configure a depot, define/commit a +desired image, set the apply policy, then check/precheck/stage/remediate), +using the `salt-minion-vcf` container/Pod. `saltext-vcf` is already +embedded in the image, so the `vcf_esxi_vlcm` module used below is +available as soon as the minion starts. + +**Prerequisite:** complete [`runbook.md`](external-minion-configuration) first. The minion +container/Pod needs a Salt master to start against (`SALT_MASTER`) even if +every command below is run locally with `salt-call --local` - there is no +masterless mode for this image. + +This runbook shows Docker commands throughout. Everything works +identically from a Kubernetes Pod - swap `docker exec salt-minion-vcf ...` +for `kubectl exec -n -- ...`, and see `runbook.md` Part 2 +for the Kubernetes Secret equivalent of the pillar pushes below. + +--- + +## Step 1 - Point the minion at your vCenter + +Same vCenter pillar block every other runbook in this series uses - skip +if already done: + +```bash +cp pillar/vcenter.sls.example pillar/vcenter.sls +``` + +```yaml +# pillar/vcenter.sls +saltext.vcf: + vcenter: + host: mgmt-vc.example.test + username: administrator@vsphere.local + password: secret + verify_ssl: false +``` + +```bash +./scripts/pillar-push.sh salt-minion-vcf pillar/vcenter.sls +docker exec salt-minion-vcf salt-call --local pillar.get saltext.vcf:vcenter +``` + +`vcf_esxi_vlcm` reuses this same vCenter session - no separate connection +config for this domain. + +## Step 2 - Find your cluster's ID + +vLCM addresses clusters by their vCenter managed-object ID (e.g. +`domain-c9`), not by display name: + +```bash +docker exec salt-minion-vcf salt-call --local vcf_vcenter_cluster.list_ +``` + +Note the `domain-c...` id for the cluster you're patching - it's used as +`cluster_id` (or as the state's `name`) in every step below. + +## Step 3 - Set the patch target (pillar) + +Everything specific to this cluster's patch run goes under +`saltext.vcf.esxi_vlcm`, as a peer of `vcenter`: + +```bash +cat > pillar/esxi_vlcm.sls <<'EOF' +saltext.vcf: + esxi_vlcm: + offline_depot: + location: http://repo.example.com/VMware-ESXi-9.2.0.0.25504872-depot.zip + image: + spec: + base_image: + version: "9.2.0.0.25504872" + policy: + enable_quick_boot: true + task: + timeout: 14400 # 4h - bump for large clusters/slow links + poll_interval: 30 +EOF +./scripts/pillar-push.sh salt-minion-vcf pillar/esxi_vlcm.sls +``` + +Every command below falls back to these pillar values for any argument you +don't pass explicitly. Nothing here is a credential, so this file doesn't +need the same secrecy as `vcenter.sls` - but keep it out of git anyway +(image version/URLs are still environment-specific). + +## Step 4 - Configure the depot + +Registers where ESXi update payloads come from. Idempotent - a no-op if a +depot at this location already exists: + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.depot_configured name=patch-depot test=True + +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.depot_configured name=patch-depot +``` + +Using an online (vendor update repository) depot instead of an offline +ZIP? Set `saltext.vcf.esxi_vlcm.online_depot.location` in Step 3 and pass +`depot_type=online` on this command instead. + +## Step 5 - Set the cluster's desired image + +Replace `` with the id from Step 2 in every command from here +on: + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.image_configured name= test=True + +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.image_configured name= +``` + +Idempotent on the committed version - a no-op if the cluster is already at +Step 3's target version. If the cluster already has an uncommitted draft, +the default behavior (`existing_draft_action=delete`) discards it and +proceeds - pass `existing_draft_action=reuse` or `=fail` instead if that's +not what you want. + +## Step 6 - Set the apply policy + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.policy_configured name= test=True + +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.policy_configured name= +``` + +Idempotent on the keys you set in Step 3's `policy` block (e.g. +`enable_quick_boot`) - other fields vCenter fills in on its own don't +trigger a spurious change. + +## Step 7 - Compliance scan + +Checks which hosts are out of compliance with the desired image. Always +runs (no cheap "already scanned" check) - inexpensive and non-disruptive: + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.compliance_checked name= test=True + +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.compliance_checked name= +``` + +## Step 8 - Precheck + +Runs vCenter's own remediation prechecks (capacity, DRS/HA constraints, +hardware compatibility) **without changing anything**. Always run this and +review the result before Step 10: + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.prechecked name= +``` + +## Step 9 - Stage + +Pre-downloads the image to each host, without applying it yet - shortens +the maintenance window in Step 10: + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.staged name= test=True + +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.staged name= +``` + +## Step 10 - Remediate + +**This is the disruptive step** - see [Risk summary](#risk-summary) first. +Applies the desired image to every host in the cluster: hosts enter +maintenance mode, install the image, and reboot, one at a time (DRS/vMotion +evacuates VMs off each host first, if enabled and there's spare capacity). + +```bash +# Dry run - only reports what would happen, no remediation call is made +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.remediated name= test=True + +# Real remediation +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.remediated name= +``` + +This calls vCenter with `accept_eula=True` by default - confirm your +organization is fine with the image's EULA being auto-accepted before +running this for real. This is also the longest step; the default task +timeout (`saltext.vcf.esxi_vlcm.task.timeout`, 4 hours) is a floor for a +multi-host cluster, not a ceiling - increase it in Step 3 for larger +clusters. + +## Step 11 - Verify + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_esxi_vlcm.reported name= +``` + +Always a read-only no-op; the comment summarizes whether a last-check, +apply-impact, and last-apply report are present. Follow up with the +execution-module equivalents for the full payload if you need the details: + +```bash +docker exec salt-minion-vcf salt-call --local vcf_esxi_vlcm.compliance_scan +``` + +Then re-run Step 2's cluster list / your own host inventory check to +confirm every host is now on the target build. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Step 5 fails: "cluster already has draft ..." | A previous partial run left an uncommitted draft, and you passed `existing_draft_action=fail` | Re-run with the default (`delete`) to discard it, or `existing_draft_action=reuse` if that draft is already at your target version | +| Step 5 fails: "commit reported success but cluster version is ..." | vCenter's commit API returned success without actually moving the version | Re-run Step 5 - if it repeats, check vCenter's own recent tasks/events for the cluster before retrying again | +| Step 10 blocks for a very long time / times out | Default `task.timeout` (4h) is too short for this cluster's host count, or DRS can't evacuate VMs fast enough | Raise `saltext.vcf.esxi_vlcm.task.timeout`/`task.poll_interval` in Step 3's pillar and re-run; also check cluster capacity/HA admission control if evacuation itself is slow | +| Step 8's precheck reports failures | Real compatibility/capacity/HA issues on specific hosts | Resolve the specific host issue vCenter reports before proceeding to Step 10 - do not skip a failing precheck | +| `depot_configured` (Step 4) fails: "requires 'location'" | Pillar not pushed, or `depot_type` doesn't match which section you filled in (`offline_depot` vs `online_depot`) | Re-check Step 3's pillar push and that `depot_type` (default `offline`) matches the section you populated | + +--- + +## Risk summary + +- **Host reboots, cluster-wide.** Every host in the cluster is patched by + Step 10 unless you scope it with `hosts=` on the compliance/stage steps + first (`remediated` itself always targets the whole cluster - there is + no host filter on that step). +- **VM impact depends on DRS/HA headroom.** If the cluster can't fully + evacuate a host being patched (insufficient spare capacity, DRS + disabled, affinity rules), VMs on that host may experience downtime + instead of a live migration. +- **No automated rollback.** Reverting means re-running this workflow + against a prior image version, not an undo button. +- **Always run Step 8 (precheck) and read the result before Step 10.** A + passing precheck is the closest thing to a safety gate this workflow + has. +- Step 10 accepts the image's EULA on your behalf by default + (`accept_eula=True`). + +See [`docs/security.md`](security.md) for general credential-handling +reminders - this use case doesn't require any credentials beyond the +vCenter pillar block set up in Step 1. diff --git a/salt-minion-vcf/docs/runbook-usb-controller-removal.md b/salt-minion-vcf/docs/runbook-usb-controller-removal.md new file mode 100644 index 0000000..63d3c3f --- /dev/null +++ b/salt-minion-vcf/docs/runbook-usb-controller-removal.md @@ -0,0 +1,189 @@ +# Runbook: Remove Unauthorized/Unused USB Controllers from VMs (KB-316384) + +This runbook walks through removing USB 2.0 (EHCI+UHCI) / USB 3.x (xHCI) +controllers from VMs managed by vCenter, using the `salt-minion-vcf` +container/Pod. `saltext-vcf` is already embedded in the `salt-minion-vcf` +image, so the `vcf_vim_vm_devices` module used below is available as soon +as the minion starts - no extra install step. + +**Prerequisite:** complete [`runbook.md`](external-minion-configuration) first. The minion +container/Pod needs a Salt master to start against (`SALT_MASTER`) even if +every command below is run locally with `salt-call --local` - there is no +masterless mode for this image. `runbook.md` Part 1 brings the minion up and +connects it to a master; Part 2 is the general pillar pattern this runbook +reuses in Step 1 below. + +This runbook shows Docker commands throughout. Everything here works +identically from a Kubernetes Pod - swap `docker exec salt-minion-vcf ...` +for `kubectl exec -n -- ...`, and see `runbook.md` Part 2 +for the Kubernetes Secret equivalent of the pillar push in Step 1. + +--- + +## Step 1 - Point the minion at your vCenter (pillar data) + +`saltext.vcf` reads the vCenter to scan from Salt Pillar under +`saltext.vcf.vcenter`. Copy the example and fill in your vCenter's details: + +```bash +cp pillar/vcenter.sls.example pillar/vcenter.sls +``` + +```yaml +# pillar/vcenter.sls +saltext.vcf: + vcenter: + host: mgmt-vc.example.test # your vCenter FQDN/IP + username: administrator@vsphere.local + password: secret + verify_ssl: false +``` + +Push it into the running container: + +```bash +./scripts/pillar-push.sh salt-minion-vcf pillar/vcenter.sls +``` + +(Running on Kubernetes instead: update the pillar Secret in place - see +`runbook.md` Part 2, Path 1, for the exact `kubectl create secret ... +--dry-run=client -o yaml | kubectl apply -f -` command. Kubelet re-syncs the +mounted Secret automatically, no Pod restart needed.) + +Confirm the minion can see it: + +```bash +docker exec salt-minion-vcf salt-call --local pillar.get saltext.vcf:vcenter +``` + +**Have more than one vCenter to target?** Add extra targets under a +`profiles` key in the same file, then pass `profile=` on any command +below to point it at that one instead of the default: + +```yaml +saltext.vcf: + vcenter: # default target + host: mgmt-vc.example.test + ... + profiles: + dr-site: + vcenter: + host: dr-vc.example.test + username: administrator@vsphere.local + password: secret + verify_ssl: false +``` + +## Step 2 - (Optional) Set the removal behavior + +By default, VMs that aren't in the vSphere "connected" state are reported +but left alone (their hardware can't be reconfigured anyway). This is +controlled by one pillar key, `usb-controller-removal.connected_only` +(default `true`). Only change it if you've confirmed disconnected VMs in +your environment are safe to reconfigure: + +```bash +cat > pillar/usb-controller-removal.sls <<'EOF' +usb-controller-removal: + connected_only: false +EOF +./scripts/pillar-push.sh salt-minion-vcf pillar/usb-controller-removal.sls +``` + +Skip this step to keep the safe default. + +## Step 3 - Audit: see what would be affected + +Read-only - lists every VM that currently has a USB controller, with no +changes made: + +```bash +docker exec salt-minion-vcf salt-call --local vcf_vim_vm_devices.list_vms_with_usb_controllers +``` + +Review this list before continuing. + +## Step 4 - Dry run + +Confirms exactly what the real run will do, without touching anything +(`test=True`): + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_vim_vm_devices.usb_controllers_absent \ + name=usb-controllers-absent \ + connected_only=True \ + test=True +``` + +Check `changes.would_remove` in the output - it lists each affected VM and +the exact device(s) that would be removed. **Removing a USB controller +disconnects any USB device currently passed through to that VM** (license +dongles, smartcard readers, USB storage) - review this list carefully +before Step 5. + +## Step 5 - Apply + +Once you've reviewed the dry-run list, run the same command without +`test=True`: + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_vim_vm_devices.usb_controllers_absent \ + name=usb-controllers-absent \ + connected_only=True +``` + +`changes.removed` lists each VM the controller was actually removed from. +If `changes.errors` appears, those specific VMs failed (e.g. permissions, +VM mid-migration) and were not touched - see Troubleshooting below. + +## Step 6 - Verify + +```bash +docker exec salt-minion-vcf salt-call --local vcf_vim_vm_devices.list_vms_with_usb_controllers +``` + +Should now be empty, or only list VMs intentionally skipped in Step 1/2 +(disconnected, with `connected_only: true`). + +--- + +## Optional - Act on a single VM + +If Step 3's audit flags one specific VM you'd rather handle by itself +instead of the fleet-wide sweep in Steps 4-5: + +```bash +docker exec salt-minion-vcf salt-call --local vcf_vim_vm_devices.usb_controllers_list +docker exec salt-minion-vcf salt-call --local vcf_vim_vm_devices.usb_controllers_remove +``` + +These are direct calls, not state functions - there is no `test=True` +dry-run gate here, so always check with `usb_controllers_list` first. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| `pillar.get saltext.vcf:vcenter` returns empty | Pillar not pushed yet, or `top.sls` doesn't reference it | Re-run Step 1's push command; see `runbook.md` Part 2 if still empty | +| Dry run shows VMs, but apply's `changes.removed` is shorter | A VM's USB controller changed between the two runs | Re-run Step 3 immediately before Step 5 | +| `changes.errors` lists a VM after apply | `ReconfigVM` failed for that VM specifically | Fix the underlying issue (permissions, VM state) then re-run `usb_controllers_remove` for just that VM | +| Command reports the wrong VMs / wrong vCenter | Targeting the default vCenter instead of a `profiles` entry | Add `profile=` to the command, matching Step 1 | + +--- + +## Risk summary + +- Fleet-wide by default: every VM visible to the targeted vCenter is + scanned and, on apply, has its USB controller removed if present and + connected. +- No rollback - if a VM needs its USB controller back, it must be re-added + manually. +- Always run Step 4 (dry run) and review the list before Step 5 (apply). + +See [`docs/security.md`](security.md) for general credential-handling +reminders - this use case doesn't require any credentials beyond the +vCenter pillar block set up in Step 1. diff --git a/salt-minion-vcf/docs/runbook-vc-patch.md b/salt-minion-vcf/docs/runbook-vc-patch.md new file mode 100644 index 0000000..411cc02 --- /dev/null +++ b/salt-minion-vcf/docs/runbook-vc-patch.md @@ -0,0 +1,183 @@ +# Runbook: Patch the vCenter Server Appliance (VCSA Self-Update) + +This runbook walks through patching the vCenter Server Appliance itself +(VAMI's `/rest/appliance/update/...` self-update workflow: configure a +repository, stage a build, precheck, install), using the `salt-minion-vcf` +container/Pod. `saltext-vcf` is already embedded in the image, so the +`vcf_vc_patch` module used below is available as soon as the minion starts. + +**Prerequisite:** complete [`runbook.md`](external-minion-configuration) first. The minion +container/Pod needs a Salt master to start against (`SALT_MASTER`) even if +every command below is run locally with `salt-call --local` - there is no +masterless mode for this image. + +This runbook shows Docker commands throughout. Everything works +identically from a Kubernetes Pod - swap `docker exec salt-minion-vcf ...` +for `kubectl exec -n -- ...`, and see `runbook.md` Part 2 +for the Kubernetes Secret equivalent of the pillar pushes below. + +--- + +## Step 1 - Point the minion at your vCenter + +Same vCenter pillar block every other runbook in this series uses - skip +if already done: + +```bash +cp pillar/vcenter.sls.example pillar/vcenter.sls +``` + +```yaml +# pillar/vcenter.sls +saltext.vcf: + vcenter: + host: mgmt-vc.example.test + username: administrator@vsphere.local + password: secret + verify_ssl: false +``` + +```bash +./scripts/pillar-push.sh salt-minion-vcf pillar/vcenter.sls +docker exec salt-minion-vcf salt-call --local pillar.get saltext.vcf:vcenter +``` + +`vcf_vc_patch` reuses this same vCenter session for its `/rest/...` calls - +no separate login step. + +## Step 2 - Set the patch target (pillar) + +Everything specific to this patch run - which build to install, the +repository it comes from, and the SSO admin password required to actually +install - goes under `saltext.vcf.vc_patch`, as a peer of `vcenter` in the +same pillar tree: + +```bash +cat > pillar/vc_patch.sls <<'EOF' +saltext.vcf: + vc_patch: + repository_url: http://repo.example.com/vcsa/ + version: "9.0.1.0.12345" + sso_password: secret # the vCenter SSO admin password - VAMI + # requires re-confirming it for install, + # even though the session above is + # already authenticated + auto_stage: false + certificate_check: true +EOF +./scripts/pillar-push.sh salt-minion-vcf pillar/vc_patch.sls +``` + +`sso_password` is as sensitive as the `vcenter.password` above - never +commit `vc_patch.sls`, same as `vcenter.sls`. + +Every command below falls back to these pillar values for any argument you +don't pass explicitly, so once this is set you generally don't need to +repeat `version=`/`repository_url=` on the command line. + +## Step 3 - Check current state before touching anything + +Read-only: + +```bash +docker exec salt-minion-vcf salt-call --local vcf_vc_patch.get_update_policy +docker exec salt-minion-vcf salt-call --local vcf_vc_patch.list_pending_updates +docker exec salt-minion-vcf salt-call --local vcf_vc_patch.get_update_status +``` + +Confirm the version you set in Step 2 actually shows up as a pending +update before continuing. + +## Step 4 - Configure the update repository + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_vc_patch.repository_configured name=vc-repo test=True + +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_vc_patch.repository_configured name=vc-repo +``` + +This always re-applies (VAMI's policy-set replaces the whole policy each +time), but re-running with the same inputs is a safe no-op in effect. + +## Step 5 - Stage the update + +Downloads and stages the resolved build, then runs a precheck. Idempotent - +a no-op if this version is already staged: + +```bash +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_vc_patch.update_prepared name=vc-staged test=True + +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_vc_patch.update_prepared name=vc-staged +``` + +This step can legitimately take a while (default timeout: 1 hour, via +`stage_timeout_seconds`). Check `changes.precheck` in the output for +warnings/errors before proceeding - a failed precheck here (disk space, +compatibility) means Step 6 will fail too. + +## Step 6 - Install + +**This is the disruptive step** - see [Risk summary](#risk-summary) before +running it for real. Take a vCenter backup/snapshot first; there is no +automated rollback. + +```bash +# Dry run - only reports what would happen, no install call is made +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_vc_patch.update_installed name=vc-installed test=True + +# Real install +docker exec salt-minion-vcf salt-call --local state.single \ + vcf_vc_patch.update_installed name=vc-installed +``` + +The appliance reboots as part of this. The command will block (or the +minion's own connection to the master may briefly appear to drop, if the +Salt master's network path routes through the same vCenter environment) +until the install/monitor cycle completes or `install_timeout_seconds` +(default: 2 hours) is hit. + +## Step 7 - Verify + +```bash +docker exec salt-minion-vcf salt-call --local vcf_vc_patch.get_update_status +docker exec salt-minion-vcf salt-call --local vcf_vc_patch.get_update_history +``` + +Confirm the installed version matches Step 2's `version`, and the history +entry for this install shows success. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| `precheck.not_allowed_error` during Step 5 | Staging is still in progress; VAMI refuses a precheck concurrently | The state already retries this internally while polling stage progress - if it still fails, staging likely didn't complete; check `changes.stage`/`changes.monitor_stage` in the Step 5 output | +| Step 5 reports a client-side timeout but the update later shows staged anyway | A slow link can time out the stage call itself even though VAMI's job kept running server-side | The state already falls back to polling `get_staged_update` after a stage-timeout error - re-run Step 5 once if it still reports failure, it should now see the completed stage | +| `401`/`403` from any `vcf_vc_patch.*` call | This vCenter build doesn't accept the `/api/session` token on the legacy `/rest/...` namespace | Verify with a read-only call (`get_update_policy`) first; if it fails, this vCenter build isn't supported for self-patching over this API | +| Step 6 fails with an authentication/password error | `sso_password` in `pillar/vc_patch.sls` is wrong or wasn't pushed | Re-check Step 2's pillar push, re-verify with `pillar.get saltext.vcf:vc_patch` (redact before sharing output - this echoes the password back) | +| Step 4/5/6 pick up the wrong `version`/`repository_url` | An explicit CLI argument or a stale pillar push is overriding what you expect | Explicit command-line args always win over pillar - drop them from the command to use Step 2's pillar values, and re-push if the pillar itself is stale | + +--- + +## Risk summary + +- **Appliance downtime.** The vCenter Server Appliance restarts its + services (and the appliance OS itself, for many updates) during install. + Plan a maintenance window - vCenter-dependent operations (this minion's + own vCenter-backed states included) are unavailable for the duration. +- **No automated rollback.** If the install fails partway or the result is + unacceptable, recovery is via your own pre-patch backup/snapshot, not + anything this module provides. +- **Always run Step 5 and Step 6 with `test=True` first**, and read + `changes.precheck` before the real install. +- `sso_password` is a credential with the same sensitivity as the vCenter + admin password - never commit it, same handling as `pillar/vcenter.sls`. + +See [`docs/security.md`](security.md) for general credential-handling +reminders. From 46374c2c312e18ad5941461bed9c35654447efd6 Mon Sep 17 00:00:00 2001 From: Praveen T Date: Mon, 31 Aug 2026 00:24:54 +0530 Subject: [PATCH 3/6] Fix external minion connectivity: FIPS crypto + direct master pubkey trust Root-caused via live testing against a VCF-managed Salt master: - FIPS-validated masters don't implement SHA-1 for RSA OAEP/PKCS1v15 at all - a minion defaulting to SHA-1 doesn't get a clean rejection, it crashes the master's payload handler on every auth attempt. FIPS mode (fips_mode: True, OAEP-SHA224/PKCS1v15-SHA224) is now on by default in docker-entrypoint.sh, matching real VCF-managed minion config. - master_finger-based identity verification was unreliable in this environment even once FIPS was fixed. VCF's own internal component minions never do fingerprint verification at all - they're handed the master's public key directly and trust it. The onboarding script now pre-seeds SALT_MASTER_PUBKEY_B64 (written to minion_master.pub) for Docker minions instead, matching that pattern. master_finger is kept for the Kubernetes/Helm path, which doesn't yet support direct pubkey seeding. - Logged full response bodies for get_master_details/add_trusted_key (no secrets in either) to make this class of issue diagnosable from the audit log directly next time. Also updates docs/external-minion-configuration.md and scripts/onboarding/README.md to reflect these fixes in place of the earlier troubleshooting guesses (this is the commit the previous one on this branch should have included, but a bad git-add pathspec silently dropped these files from it). --- .../docs/external-minion-configuration.md | 19 ++- salt-minion-vcf/scripts/docker-entrypoint.sh | 31 ++++- salt-minion-vcf/scripts/onboarding/README.md | 18 ++- .../scripts/onboarding/vcf-ops-onboard.py | 114 ++++++++++++------ 4 files changed, 136 insertions(+), 46 deletions(-) diff --git a/salt-minion-vcf/docs/external-minion-configuration.md b/salt-minion-vcf/docs/external-minion-configuration.md index 6d20925..74d1f95 100644 --- a/salt-minion-vcf/docs/external-minion-configuration.md +++ b/salt-minion-vcf/docs/external-minion-configuration.md @@ -49,11 +49,18 @@ handles, in order: 1. Logs in to VCF Operations. 2. Resolves the Salt master governing the given VCF instance. -3. Computes the master's identity fingerprint (`master_finger`). +3. Computes the master's identity fingerprint (`master_finger`) - used for + the Kubernetes/Helm path and for your own reference/audit trail. 4. Starts the minion (`docker run`, or `helm upgrade --install`), passing it - the master FQDN, `master_finger`, and a freshly generated minion ID. The - minion generates its own RSA keypair locally on first start - the - private key never leaves it, and VCF Operations credentials never reach it. + the master FQDN and a freshly generated minion ID. The minion generates + its own RSA keypair locally on first start - the private key never + leaves it, and VCF Operations credentials never reach it. Docker minions + are pre-seeded with the master's actual public key + (`SALT_MASTER_PUBKEY_B64`, written to `minion_master.pub`) rather than + just a fingerprint, so they trust it directly on first connect - the + same approach VCF's own internal component minions use. FIPS-compliant + crypto (`OAEP-SHA224`/`PKCS1v15-SHA224`) is on by default, matching what + VCF-managed Salt masters require - see Troubleshooting below. 5. Reads back the minion's public key. 6. Registers that key as trusted with the master. 7. Waits until the master has actually accepted the connection. @@ -90,7 +97,9 @@ docker logs salt-minion-vcf | grep "Minion is ready to receive requests" | `pull access denied for salt-minion-vcf` | Image not built locally yet - Docker tried to pull it from Docker Hub | `docker build -t salt-minion-vcf:0.1.0 .` from the repo root first, or point `--image` at wherever you built/pushed it | | `container name already in use` on retry | A previous failed attempt left a stopped container behind | The script now detects this and offers to remove it automatically | | `[CRITICAL] Unable to securely set the permissions of "/etc/salt/pki/minion"` / `PermissionError: Permission denied: '/etc/salt/pki/minion/tmp...'` | The PKI volume value was a host path (bind mount), not a named Docker volume - the container runs as non-root uid `10000`, and a bind-mounted host directory doesn't inherit the image's baked-in ownership | Use a plain volume name (e.g. `salt-minion-vcf-pki`, the default) instead of an absolute path. If you specifically need a host path, `chown -R 10000:10000` it first | -| Minion key is accepted on the master (`salt-key -L` shows it), but the onboarding script (or the image's own `HEALTHCHECK`/`readinessProbe`) never reports it connected | `status.master`'s answer depends on `master_alive_interval` being configured on the minion, which the entrypoint doesn't set by default - it can under-report even once genuinely connected | The onboarding script also checks the minion's logs for `Minion is ready to receive requests` as a fallback, which doesn't have this gap. If you're checking manually, use that log line or `salt '' test.ping` from the master instead of relying on `status.master` alone | +| Minion key is accepted on the master (`salt-key -L` shows it), but the onboarding script (or the image's own `HEALTHCHECK`/`readinessProbe`) never reports it connected, and can even appear to hang indefinitely | `status.master`'s answer depends on `master_alive_interval` being configured on the minion, which the entrypoint doesn't set by default - it can under-report even once genuinely connected. Worse, `salt-call status.master` (without `--local`) tries to compile pillar from the master before running the check at all, which can block for a long time (or indefinitely) while the minion is still mid-handshake | The onboarding script now checks the minion's logs for the event-driven `Minion is ready to receive requests` line *first* (a plain `docker logs`/`kubectl logs` call that can't itself hang), and only falls back to a time-boxed (8s) `salt-call --local status.master` if that line hasn't appeared yet. If you're checking manually, prefer that log line or `salt '' test.ping` from the master over `salt-call status.master` | +| Minion loops forever on `[ERROR] Sign-in attempt failed: Some exception handling minion payload` (sometimes preceded by `{'ret': 'bad sig algo'}`), even though the key is accepted on the master and both ports (4505/4506) are reachable | The master runs FIPS-validated crypto and doesn't implement SHA-1 for RSA OAEP/PKCS1v15 at all - a minion defaulting to SHA-1 doesn't get a clean rejection, it crashes the master's payload handler on every single auth attempt (visible on the master's own log as `salt.channel.server: Some exception handling a payload from minion`) | FIPS mode (`fips_mode: True`, `encryption_algorithm: OAEP-SHA224`, `signing_algorithm: PKCS1v15-SHA224`) is on by default as of this image - see `docker-entrypoint.sh`. If you're running an older image or need to override it, set `SALT_FIPS_MODE=false` only if you've confirmed your master is *not* FIPS-enforced | +| Minion key is accepted, FIPS is enabled, `_auth` completes without crashing, but the minion still never connects, with `[CRITICAL] The specified fingerprint in the master configuration file ... Does not match the authenticating master's key` | `master_finger`, computed by the onboarding script from the same `masterPublicKey` VCF Operations returns, did not match what the live master actually presented on the wire in this deployment - the underlying cause wasn't pinned down (possibly a RaaS/SSEAPI-key-vs-Salt-PKI-key distinction specific to this environment), but VCF's own internal component minions never do this fingerprint check at all - they're handed the master's public key directly and trust it | The onboarding script now pre-seeds the master's actual public key directly (`SALT_MASTER_PUBKEY_B64`, written to `/etc/salt/pki/minion/minion_master.pub`) instead of computing/checking a fingerprint, matching how internal component minions are bootstrapped. This is the default behavior as of this image/script version - `master_finger`/`SALT_MASTER_FINGER` is only used by the Kubernetes/Helm path today | --- diff --git a/salt-minion-vcf/scripts/docker-entrypoint.sh b/salt-minion-vcf/scripts/docker-entrypoint.sh index cc762a0..361d7e7 100755 --- a/salt-minion-vcf/scripts/docker-entrypoint.sh +++ b/salt-minion-vcf/scripts/docker-entrypoint.sh @@ -37,7 +37,18 @@ master_tries: -1 retry_dns: 30 EOF - if [ -n "${SALT_MASTER_FINGER:-}" ]; then + # Preferred: pre-seed the master's actual public key so the minion trusts + # it directly on first connect, instead of independently re-deriving and + # comparing a fingerprint (master_finger) against whatever key is presented + # live - the two can disagree for reasons outside this image's control + # (e.g. a management-plane key registry vs. what the wire protocol + # presents), and this is also how VCF's own internal component minions are + # bootstrapped - handed the master's public key directly, no fingerprint + # verification. SALT_MASTER_PUBKEY_B64 takes precedence over the legacy + # SALT_MASTER_FINGER when both are set. + if [ -n "${SALT_MASTER_PUBKEY_B64:-}" ]; then + echo "${SALT_MASTER_PUBKEY_B64}" | base64 -d > /etc/salt/pki/minion/minion_master.pub + elif [ -n "${SALT_MASTER_FINGER:-}" ]; then cat >> "$MASTER_CONFIG" < "$FIPS_CONFIG" < "$RUNTIME_CONFIG" <). 3. Compute the Salt-compatible master_finger from the returned master - public key, so the minion can verify the master's identity on connect. + public key (used for the Kubernetes/Helm path, and for your own + reference/audit trail either way). 4. Start the minion (docker run, or helm install/upgrade), pointed at the - master and given a freshly generated minion ID. The minion generates - its own RSA keypair locally on first start - this script never sees it. + master and given a freshly generated minion ID. Docker minions are + pre-seeded with the master's actual public key (not just its + fingerprint) so they trust it directly on first connect - the same + approach VCF's own internal component minions use. The minion + generates its own RSA keypair locally on first start - this script + never sees it. 5. Read back the minion's public key (never the private key) and its ID. 6. Trust that key against the master (POST /suite-api/api/salt/minions/{minionId}/trusted-keys). - 7. Poll the minion (already retrying in the background) until the master - accepts it, using the exact check the image's own healthcheck/readiness - probe uses: `salt-call status.master`. + 7. Poll the minion (already retrying in the background) until it connects, + primarily by watching its logs for the event-driven "Minion is ready to + receive requests" line, falling back to a time-boxed `salt-call + status.master` (the same check the image's own healthcheck/readiness + probe uses, but that check alone can hang or under-report - see the + comments on docker_is_connected()/kubectl_is_connected()). Steps 4-7 can be repeated for multiple minions in one session without re-entering VCF Operations credentials. @@ -274,7 +282,8 @@ def wait_until(predicate: Callable[[], bool], timeout: int, check_interval: floa # Shell command execution # -------------------------------------------------------------------------- -def run(cmd: list, dry_run: bool = False, capture: bool = False, check: bool = True) -> str: +def run(cmd: list, dry_run: bool = False, capture: bool = False, check: bool = True, + timeout: float = None) -> str: printable = " ".join(shlex.quote(c) for c in cmd) print(f" $ {printable}") LOG.debug(f"$ {printable}") @@ -287,9 +296,15 @@ def run(cmd: list, dry_run: bool = False, capture: bool = False, check: bool = T stdout=subprocess.PIPE if capture else None, stderr=subprocess.STDOUT if capture else None, text=True, + timeout=timeout, ) except FileNotFoundError: die(f"Command not found: {cmd[0]}. Is it installed and on PATH?") + except subprocess.TimeoutExpired: + LOG.warning(f"command timed out after {timeout}s: {printable}") + if check: + raise + return "" except subprocess.CalledProcessError as e: LOG.error(f"command failed (exit {e.returncode}): {printable}") raise @@ -380,16 +395,20 @@ def login(self) -> None: def get_master_details(self, vcf_instance_id: str) -> dict: """GET /api/salt/master?resourceId= -> {resourceId, masterId, masterFqdn, masterPublicKey (base64 of the PEM text), masterKeyState, presenceStatus}.""" - return self._request("GET", "/api/salt/master", params={"resourceId": vcf_instance_id}) + result = self._request("GET", "/api/salt/master", params={"resourceId": vcf_instance_id}) + LOG.debug(f"GET /api/salt/master response body: {result}") + return result def add_trusted_key(self, minion_id: str, master_id: str, minion_public_key_pem: str) -> dict: """POST /api/salt/minions/{minionId}/trusted-keys Body: {masterId, minionPublicKey} - minionPublicKey is RAW PEM text here (NOT base64-encoded - only the master pubkey in GET responses is).""" - return self._request( + result = self._request( "POST", f"/api/salt/minions/{minion_id}/trusted-keys", json_body={"masterId": master_id, "minionPublicKey": minion_public_key_pem}, ) + LOG.debug(f"POST /api/salt/minions/{minion_id}/trusted-keys response body: {result}") + return result # -------------------------------------------------------------------------- @@ -422,7 +441,7 @@ class DockerConfig: container_name: str volume: str master_fqdn: str - master_finger: str + master_pubkey_b64: str minion_id: str @@ -449,7 +468,7 @@ def docker_start(cfg: DockerConfig, dry_run: bool, assume_yes: bool = False) -> "docker", "run", "-d", "--name", cfg.container_name, "-e", f"SALT_MASTER={cfg.master_fqdn}", - "-e", f"SALT_MASTER_FINGER={cfg.master_finger}", + "-e", f"SALT_MASTER_PUBKEY_B64={cfg.master_pubkey_b64}", "-e", f"SALT_MINION_ID={cfg.minion_id}", "-v", f"{cfg.volume}:/etc/salt/pki/minion", cfg.image, @@ -457,8 +476,10 @@ def docker_start(cfg: DockerConfig, dry_run: bool, assume_yes: bool = False) -> run(cmd, dry_run=dry_run) -def docker_exec(container: str, args: list, dry_run: bool = False, check: bool = True) -> str: - return run(["docker", "exec", container] + args, dry_run=dry_run, capture=True, check=check) +def docker_exec(container: str, args: list, dry_run: bool = False, check: bool = True, + timeout: float = None) -> str: + return run(["docker", "exec", container] + args, dry_run=dry_run, capture=True, check=check, + timeout=timeout) def docker_read_minion_pubkey(container: str, timeout: int, dry_run: bool) -> str: @@ -484,23 +505,32 @@ def _check() -> bool: MINION_READY_LOG_MARKER = "Minion is ready to receive requests" +STATUS_MASTER_CHECK_TIMEOUT = 8 # seconds + + def docker_is_connected(container: str, dry_run: bool) -> bool: if dry_run: return True - # status.master's answer depends on master_alive_interval being configured on the - # minion, which this image's entrypoint does not set - it can under-report even - # once actually connected. The log line below is emitted once, event-driven, the - # moment the pub/req channels with the master are established, so it doesn't have - # that gap; treat either signal as sufficient. + # The log line below is emitted once, event-driven, the moment the pub/req + # channels with the master are established - check it first since it's a + # plain local `docker logs` call that cannot itself hang. + logs = run(["docker", "logs", container], dry_run=dry_run, capture=True, check=False) + if MINION_READY_LOG_MARKER in logs: + return True + # `salt-call status.master` is a weaker, secondary signal: its answer depends + # on master_alive_interval being configured on the minion (this image's + # entrypoint does not set it, so it can under-report even once connected), + # and - without --local - salt-call itself tries to compile pillar from the + # master first, which can hang for a long time (or indefinitely) while the + # minion is still mid-handshake. Run it with a hard timeout so a hang here + # can never block the overall connect-timeout/poll loop. out = docker_exec( container, - ["salt-call", "--out=newline_values_only", "--retcode-passthrough", "status.master"], + ["salt-call", "--local", "--out=newline_values_only", "--retcode-passthrough", "status.master"], check=False, + timeout=STATUS_MASTER_CHECK_TIMEOUT, ) - if out.strip().lower() == "true": - return True - logs = run(["docker", "logs", container], dry_run=dry_run, capture=True, check=False) - return MINION_READY_LOG_MARKER in logs + return out.strip().lower() == "true" # -------------------------------------------------------------------------- @@ -556,9 +586,10 @@ def _check() -> bool: return result["name"] -def kubectl_exec(namespace: str, pod: str, args: list, dry_run: bool = False, check: bool = True) -> str: +def kubectl_exec(namespace: str, pod: str, args: list, dry_run: bool = False, check: bool = True, + timeout: float = None) -> str: return run(["kubectl", "exec", "-n", namespace, pod, "--"] + args, - dry_run=dry_run, capture=True, check=check) + dry_run=dry_run, capture=True, check=check, timeout=timeout) def kubectl_read_minion_pubkey(namespace: str, pod: str, timeout: int, dry_run: bool) -> str: @@ -584,17 +615,20 @@ def _check() -> bool: def kubectl_is_connected(namespace: str, pod: str, dry_run: bool) -> bool: if dry_run: return True - # See the comment on docker_is_connected() - status.master alone can under-report; - # the log marker is an event-driven signal emitted only after successful auth. + # See the comments on docker_is_connected() - check the event-driven log + # marker first (a plain `kubectl logs` call that cannot itself hang), and + # only fall back to the weaker, hang-prone `status.master` check, bounded + # by a hard timeout, if the marker hasn't shown up yet. + logs = run(["kubectl", "logs", "-n", namespace, pod], dry_run=dry_run, capture=True, check=False) + if MINION_READY_LOG_MARKER in logs: + return True out = kubectl_exec( namespace, pod, - ["salt-call", "--out=newline_values_only", "--retcode-passthrough", "status.master"], + ["salt-call", "--local", "--out=newline_values_only", "--retcode-passthrough", "status.master"], check=False, + timeout=STATUS_MASTER_CHECK_TIMEOUT, ) - if out.strip().lower() == "true": - return True - logs = run(["kubectl", "logs", "-n", namespace, pod], dry_run=dry_run, capture=True, check=False) - return MINION_READY_LOG_MARKER in logs + return out.strip().lower() == "true" # -------------------------------------------------------------------------- @@ -664,7 +698,7 @@ def build_arg_parser() -> argparse.ArgumentParser: def onboard_one_minion(client: OpsClient, args: argparse.Namespace, - master_id: str, master_fqdn: str, master_finger: str, + master_id: str, master_fqdn: str, master_pubkey_b64: str, master_finger: str, deployment: str, defaults: dict, index: int) -> dict: """ Runs steps 4-7 for a single minion and returns a summary dict. @@ -703,14 +737,14 @@ def onboard_one_minion(client: OpsClient, args: argparse.Namespace, ("Image", image), ("Container name", container_name), ("PKI volume", volume), - ("Salt master", f"{master_fqdn} (master_finger computed)"), + ("Salt master", f"{master_fqdn} (master pubkey pre-seeded)"), ]) if not confirm("Proceed with these settings?", assume_yes=args.yes): die("Aborted by user.", code=0) docker_cfg = DockerConfig( image=image, container_name=container_name, volume=volume, - master_fqdn=master_fqdn, master_finger=master_finger, minion_id=minion_id, + master_fqdn=master_fqdn, master_pubkey_b64=master_pubkey_b64, minion_id=minion_id, ) docker_start(docker_cfg, dry_run=args.dry_run, assume_yes=args.yes) ok(f"Container '{container_name}' started") @@ -839,10 +873,16 @@ def main() -> None: master_id = master["masterId"] master_fqdn = master["masterFqdn"] - master_pubkey_pem = base64.b64decode(master["masterPublicKey"]).decode("utf-8") + master_pubkey_b64 = master["masterPublicKey"] + master_pubkey_pem = base64.b64decode(master_pubkey_b64).decode("utf-8") ok(f"Master resolved: {master_id} @ {master_fqdn}") # ---------------------------------------------------------------- Step 3 + # Docker minions are pre-seeded with the master's actual public key + # (SALT_MASTER_PUBKEY_B64) rather than a fingerprint - see + # docker-entrypoint.sh for why. The fingerprint below is still computed + # for the Kubernetes/Helm path (which only supports master_finger today) + # and for your own reference/audit trail. step(3, TOTAL_STEPS, "Compute master identity fingerprint") master_finger = pem_finger(master_pubkey_pem, sum_type=args.master_finger_algo) ok(f"master_finger ({args.master_finger_algo}): {master_finger}") @@ -855,7 +895,7 @@ def main() -> None: defaults: dict = {} index = 1 while True: - result = onboard_one_minion(client, args, master_id, master_fqdn, master_finger, + result = onboard_one_minion(client, args, master_id, master_fqdn, master_pubkey_b64, master_finger, deployment, defaults, index) onboarded.append(result) From 036c08687f77d483e5e6deffa11a856fd65e8293 Mon Sep 17 00:00:00 2001 From: Praveen T Date: Wed, 2 Sep 2026 01:42:57 +0530 Subject: [PATCH 4/6] Minion Key states --- salt-minion-vcf/Dockerfile | 3 + .../docs/external-minion-configuration.md | 43 +- .../salt-minion-vcf/templates/deployment.yaml | 12 + .../templates/statefulset.yaml | 12 + .../helm/salt-minion-vcf/values.schema.json | 1 + .../helm/salt-minion-vcf/values.yaml | 17 + salt-minion-vcf/scripts/docker-entrypoint.sh | 21 + salt-minion-vcf/scripts/onboarding/README.md | 72 ++- .../scripts/onboarding/vcf-ops-onboard.py | 517 +++++++++++------- 9 files changed, 459 insertions(+), 239 deletions(-) diff --git a/salt-minion-vcf/Dockerfile b/salt-minion-vcf/Dockerfile index f3bc915..d48063c 100644 --- a/salt-minion-vcf/Dockerfile +++ b/salt-minion-vcf/Dockerfile @@ -90,8 +90,11 @@ RUN chmod 0755 /usr/local/bin/docker-entrypoint.sh /usr/local/bin/healthcheck.sh ENV SALT_MASTER="" \ SALT_MASTER_PORT="4506" \ SALT_PUBLISH_PORT="4505" \ + SALT_MASTER_PUBKEY_B64="" \ SALT_MINION_ID="" \ SALT_MINION_ID_PREFIX="vcf-salt-executor" \ + SALT_MINION_PRIVATE_KEY_B64="" \ + SALT_MINION_PUBLIC_KEY_B64="" \ SALT_MASTER_FINGER="" \ SALT_LOG_LEVEL="info" \ SALT_FILE_CLIENT_LOCAL="false" \ diff --git a/salt-minion-vcf/docs/external-minion-configuration.md b/salt-minion-vcf/docs/external-minion-configuration.md index 74d1f95..c66fb1f 100644 --- a/salt-minion-vcf/docs/external-minion-configuration.md +++ b/salt-minion-vcf/docs/external-minion-configuration.md @@ -28,8 +28,9 @@ before it has any pillar data configured (Part 2) - it just can't run any instance's Suite API, and from the minion's host/cluster to the Salt master (`SALT_MASTER_PORT`/`4506`, `SALT_PUBLISH_PORT`/`4505`). - Credentials for a VCF Operations user with the Salt Management view/manage - privileges, and the resource UUID of the VCF instance whose master you - want to attach to. + privileges. The Salt master itself is chosen interactively from the list + VCF Operations returns - no VCF instance/resource ID is needed up front. +- `openssl` on PATH (used to generate the minion's own RSA keypair). ### Procedure @@ -39,7 +40,6 @@ Run the onboarding script: python3 scripts/onboarding/vcf-ops-onboard.py \ --ops-host vcfops.example.com \ --ops-user admin \ - --vcf-instance-id \ --deployment docker # or: kubernetes ``` @@ -48,22 +48,27 @@ review/confirm summary shown before anything is actually started. The script handles, in order: 1. Logs in to VCF Operations. -2. Resolves the Salt master governing the given VCF instance. -3. Computes the master's identity fingerprint (`master_finger`) - used for - the Kubernetes/Helm path and for your own reference/audit trail. -4. Starts the minion (`docker run`, or `helm upgrade --install`), passing it - the master FQDN and a freshly generated minion ID. The minion generates - its own RSA keypair locally on first start - the private key never - leaves it, and VCF Operations credentials never reach it. Docker minions - are pre-seeded with the master's actual public key - (`SALT_MASTER_PUBKEY_B64`, written to `minion_master.pub`) rather than - just a fingerprint, so they trust it directly on first connect - the - same approach VCF's own internal component minions use. FIPS-compliant - crypto (`OAEP-SHA224`/`PKCS1v15-SHA224`) is on by default, matching what - VCF-managed Salt masters require - see Troubleshooting below. -5. Reads back the minion's public key. -6. Registers that key as trusted with the master. -7. Waits until the master has actually accepted the connection. +2. Lists every Salt master VCF Operations knows about and prompts you to + pick one (by FQDN), flagging any that aren't `ACCEPTED`/`PRESENT` so you + don't pick one that won't actually work. +3. Generates a fresh RSA keypair for the minion locally, via `openssl` - the + private key never leaves this process except to be handed directly to + the minion's own runtime, and VCF Operations credentials never reach the + minion itself. +4. Registers the minion's public key as trusted against the selected master + - the minion ID is assigned by VCF Operations at this point, not chosen + by you or the script. +5. Starts the minion (`docker run`, or `helm upgrade --install`), pre-seeded + with that exact keypair and minion ID. Docker minions are also pre-seeded + with the master's actual public key (`SALT_MASTER_PUBKEY_B64`, written to + `minion_master.pub`) rather than just a fingerprint, so they trust the + master directly on first connect - the same approach VCF's own internal + component minions use. FIPS-compliant crypto (`OAEP-SHA224`/ + `PKCS1v15-SHA224`) is on by default, matching what VCF-managed Salt + masters require - see Troubleshooting below. Because trust was already + established in step 4 *before* the minion starts, there's no + waiting-for-acceptance window and no manual `salt-key -a`. +6. Waits until the master has actually accepted the connection. Use `--dry-run` first if you want to preview every command and API call without executing anything. See `--help` for the full flag list, or diff --git a/salt-minion-vcf/helm/salt-minion-vcf/templates/deployment.yaml b/salt-minion-vcf/helm/salt-minion-vcf/templates/deployment.yaml index dcc5127..2aafbe1 100644 --- a/salt-minion-vcf/helm/salt-minion-vcf/templates/deployment.yaml +++ b/salt-minion-vcf/helm/salt-minion-vcf/templates/deployment.yaml @@ -44,6 +44,18 @@ spec: value: kubernetes - name: SALT_MINION_ID value: {{ default (include "salt-minion-vcf.fullname" .) .Values.salt.minionId | quote }} + {{- if .Values.salt.minionKeySecretName }} + - name: SALT_MINION_PRIVATE_KEY_B64 + valueFrom: + secretKeyRef: + name: {{ .Values.salt.minionKeySecretName }} + key: private-key-b64 + - name: SALT_MINION_PUBLIC_KEY_B64 + valueFrom: + secretKeyRef: + name: {{ .Values.salt.minionKeySecretName }} + key: public-key-b64 + {{- end }} - name: SALT_LOG_LEVEL value: {{ .Values.salt.logLevel | quote }} - name: SALT_FILE_CLIENT_LOCAL diff --git a/salt-minion-vcf/helm/salt-minion-vcf/templates/statefulset.yaml b/salt-minion-vcf/helm/salt-minion-vcf/templates/statefulset.yaml index 94c6ad9..bfc0eb4 100644 --- a/salt-minion-vcf/helm/salt-minion-vcf/templates/statefulset.yaml +++ b/salt-minion-vcf/helm/salt-minion-vcf/templates/statefulset.yaml @@ -53,6 +53,18 @@ spec: - name: SALT_MINION_ID value: {{ .Values.salt.minionId | quote }} {{- end }} + {{- if .Values.salt.minionKeySecretName }} + - name: SALT_MINION_PRIVATE_KEY_B64 + valueFrom: + secretKeyRef: + name: {{ .Values.salt.minionKeySecretName }} + key: private-key-b64 + - name: SALT_MINION_PUBLIC_KEY_B64 + valueFrom: + secretKeyRef: + name: {{ .Values.salt.minionKeySecretName }} + key: public-key-b64 + {{- end }} - name: SALT_FILE_CLIENT_LOCAL value: {{ .Values.salt.fileClientLocal | quote }} {{- if .Values.vault.addr }} diff --git a/salt-minion-vcf/helm/salt-minion-vcf/values.schema.json b/salt-minion-vcf/helm/salt-minion-vcf/values.schema.json index 5cddd59..1a1f2e2 100644 --- a/salt-minion-vcf/helm/salt-minion-vcf/values.schema.json +++ b/salt-minion-vcf/helm/salt-minion-vcf/values.schema.json @@ -52,6 +52,7 @@ "enum": ["error", "warning", "info", "debug", "trace"] }, "minionId": { "type": "string" }, + "minionKeySecretName": { "type": "string" }, "masterTries": { "type": "integer" }, "retryDns": { "type": "integer", "minimum": 0 }, "extraMinionConfig": { "type": "object" }, diff --git a/salt-minion-vcf/helm/salt-minion-vcf/values.yaml b/salt-minion-vcf/helm/salt-minion-vcf/values.yaml index 8d22240..7285ae7 100644 --- a/salt-minion-vcf/helm/salt-minion-vcf/values.yaml +++ b/salt-minion-vcf/helm/salt-minion-vcf/values.yaml @@ -22,8 +22,25 @@ salt: # # Deployment: # Leave empty to use the stable Helm resource name. + # + # When minionKeySecretName is set (see below), this MUST match the minion + # ID that was already registered as trusted for that keypair (e.g. the + # minionId vcf-ops-onboard.py received back from POST /api/salt/minions) - + # VCF Operations, not this chart, is the source of truth for that ID. minionId: "" + # Name of an EXISTING Kubernetes Secret (created out-of-band, e.g. by + # vcf-ops-onboard.py, never via Helm values) with keys 'private-key-b64' + # and 'public-key-b64' - a base64-encoded PEM RSA keypair for the minion + # itself. Required by the trust-then-start flow this image now expects: + # the minion's public key must already be registered as trusted with VCF + # Operations *before* the minion starts, so the minion must be handed that + # exact keypair rather than generating its own. Leave empty to fall back + # to the minion generating its own keypair on first start (legacy + # behavior - incompatible with pre-registered trust, since VCF Operations + # would not know that self-generated key). + minionKeySecretName: "" + masterTries: -1 retryDns: 30 diff --git a/salt-minion-vcf/scripts/docker-entrypoint.sh b/salt-minion-vcf/scripts/docker-entrypoint.sh index 361d7e7..31db7e7 100755 --- a/salt-minion-vcf/scripts/docker-entrypoint.sh +++ b/salt-minion-vcf/scripts/docker-entrypoint.sh @@ -18,6 +18,26 @@ if [ -z "${SALT_MINION_ID:-}" ]; then fi fi +# Pre-seed the minion's own RSA keypair when supplied. vcf-ops-onboard.py now +# generates this keypair and registers its public half as trusted via the +# VCF Operations API *before* the minion ever starts - so the minion must be +# handed that exact keypair rather than generating its own (which would not +# match whatever key was already registered as trusted, and would just sit +# untrusted). Independent of the master-side config model below (Docker/env +# or Kubernetes/ConfigMap+Secret) - either can supply these two variables. +# +# Only applied when minion.pem doesn't already exist, so a restarted/ +# rescheduled container (persistent PKI volume) keeps its established +# identity rather than re-seeding on every start. +if [ ! -s /etc/salt/pki/minion/minion.pem ] \ + && [ -n "${SALT_MINION_PRIVATE_KEY_B64:-}" ] && [ -n "${SALT_MINION_PUBLIC_KEY_B64:-}" ]; then + echo "${SALT_MINION_PRIVATE_KEY_B64}" | base64 -d > /etc/salt/pki/minion/minion.pem + chmod 0400 /etc/salt/pki/minion/minion.pem + echo "${SALT_MINION_PUBLIC_KEY_B64}" | base64 -d > /etc/salt/pki/minion/minion.pub + chmod 0644 /etc/salt/pki/minion/minion.pub + echo "Pre-seeded minion keypair (already registered as trusted)" +fi + # There are two supported configuration models: # 1. Docker/env: SALT_MASTER is supplied and this script writes master config. # 2. Kubernetes/ConfigMap: 10-master.conf is mounted by Kubernetes/Helm. @@ -180,6 +200,7 @@ echo "====================================================" echo " Salt Minion VCF" echo "====================================================" echo "Minion ID : ${SALT_MINION_ID}" +echo "Minion Keypair : $([ -n "${SALT_MINION_PRIVATE_KEY_B64:-}" ] && echo "pre-seeded (pre-registered as trusted)" || echo "self-generated on first start")" echo "Deployment Type : ${DEPLOYMENT_TYPE:-docker}" if [ -n "${SALT_MASTER:-}" ]; then echo "Salt Master : ${SALT_MASTER}" diff --git a/salt-minion-vcf/scripts/onboarding/README.md b/salt-minion-vcf/scripts/onboarding/README.md index 0f27a77..e7e98f3 100644 --- a/salt-minion-vcf/scripts/onboarding/README.md +++ b/salt-minion-vcf/scripts/onboarding/README.md @@ -10,39 +10,41 @@ credentials ever reaching the minion itself. ```text 1. Log in to VCF Operations -2. Resolve the Salt master for a given VCF instance -3. Compute the master's identity fingerprint (master_finger) - used for the - Kubernetes/Helm path and for your own reference/audit trail -4. Start the minion (docker run, or helm upgrade --install), with a freshly - generated minion ID - the minion generates its own RSA keypair locally. - Docker minions are pre-seeded with the master's actual public key - (not just its fingerprint) so they trust it directly on first connect - - the same approach VCF's own internal component minions use -5. Read back the minion's public key (never the private key) -6. Trust that key with the master -7. Poll until the master accepts the connection +2. List every Salt master VCF Operations knows about and pick one, by FQDN - + masters that aren't ACCEPTED/PRESENT are flagged so you don't pick one + that won't actually work +3. Generate a fresh RSA keypair for the minion locally, via `openssl` - the + private key never leaves this process +4. Register the minion's public key as trusted against the selected master. + The minion ID is assigned by VCF Operations here, not chosen up front - + this is the first point the ID is known +5. Start the minion (docker run, or helm upgrade --install), pre-seeded with + that exact keypair and minion ID. Docker minions are also pre-seeded with + the master's actual public key (not just its fingerprint) so they trust + it directly on first connect - the same approach VCF's own internal + component minions use +6. Poll until the master accepts the connection ``` -This mirrors the manual flow documented in the top-level -[`README.md`](../../README.md#salt-master-registration), just automated and -without a human needing to run `salt-key -a` by hand - trust is established -via the VCF Operations API instead. +Because trust is established in step 4 *before* the minion ever starts, +there's no waiting-for-acceptance window and no human needing to run +`salt-key -a` by hand. -Steps 4-7 can be repeated for multiple minions in one session without -re-entering VCF Operations credentials or re-resolving the master. +Steps 3-6 can be repeated for multiple minions in one session without +re-entering VCF Operations credentials or re-listing masters. ## Interactive features -- **Input validation**: the VCF instance ID is checked against a UUID format - and re-prompted if invalid; deployment type is a numbered menu, not free text. +- **Master picker**: every Salt master VCF Operations knows about is listed + with its key/presence state; deployment type is also a numbered menu, not + free text. - **Review before acting**: a summary of every setting (minion ID, image, - container/release name, target master, ...) is shown before the minion is - started, and again before its key is trusted - nothing consequential runs - without an explicit confirmation. -- **Live progress**: waiting for the minion to generate its keypair and for - the master to accept the connection shows an animated spinner with a - countdown (falls back to periodic plain-text lines if output isn't a TTY, - e.g. when redirected to a file). + container/release name, target master, ...) is shown before the minion's + key is registered as trusted, and again before the minion is started - + nothing consequential runs without an explicit confirmation. +- **Live progress**: waiting for the master to accept the connection shows + an animated spinner with a countdown (falls back to periodic plain-text + lines if output isn't a TTY, e.g. when redirected to a file). - **Onboard multiple minions in one session**: after each successful onboarding you're asked whether to onboard another against the same master - container/volume/release names are auto-suggested with a `-2`, @@ -62,6 +64,7 @@ response - passwords and auth tokens are never written to it. Pass ## Requirements - Python 3.8+ (standard library only - no `pip install` needed) +- `openssl` on PATH (generates the minion's own RSA keypair) - `docker` on PATH (Docker mode), or `helm` + `kubectl` on PATH (Kubernetes mode) - Network access from wherever you run this script to your VCF Operations instance's Suite API @@ -81,7 +84,6 @@ Or supply anything up front via flags (anything omitted is still prompted for): python3 scripts/onboarding/vcf-ops-onboard.py \ --ops-host vcfops.example.com \ --ops-user admin \ - --vcf-instance-id \ --deployment docker \ --image salt-minion-vcf:0.1.0 @@ -90,10 +92,16 @@ python3 scripts/onboarding/vcf-ops-onboard.py \ python3 scripts/onboarding/vcf-ops-onboard.py \ --ops-host vcfops.example.com \ --ops-user admin \ - --vcf-instance-id \ --deployment kubernetes \ --namespace vcf-salt \ --release-name vcf-executor + +# Skip the interactive master picker if you already know the master ID +python3 scripts/onboarding/vcf-ops-onboard.py \ + --ops-host vcfops.example.com \ + --ops-user admin \ + --master-id salt-master-7a1b2c3d-4e5f-6789-abcd-ef0123456789 \ + --deployment docker ``` See `--help` for the full flag list (container/release naming, image @@ -113,6 +121,14 @@ anything, `-y` to skip confirmation prompts). your Salt master needs the legacy default. Only used by the Kubernetes/Helm path today - Docker minions are pre-seeded with the master's actual public key instead (see "What it does" above). +- **Minion keypair Secret (Kubernetes)**: the generated keypair is written + to a Kubernetes Secret named `-minion-key` + (`kubectl apply`'d directly, not through Helm values, so it never lands + in `helm get values`/release history) and referenced by the chart's new + `salt.minionKeySecretName` value. Delete it yourself if you remove the + release and don't intend to reuse that identity. +- **RSA key size**: defaults to 2048 bits (Salt's own default). Override + with `--key-size 4096` if your security policy requires it. - **FIPS mode**: enabled by default (`fips_mode: True`, `encryption_algorithm: OAEP-SHA224`, `signing_algorithm: PKCS1v15-SHA224`) since VCF-managed Salt masters are typically FIPS-enforced and reject diff --git a/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py b/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py index 3e77800..09ab316 100755 --- a/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py +++ b/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py @@ -2,45 +2,51 @@ """ vcf-ops-onboard.py -Interactive onboarding tool that brings up a salt-minion-vcf instance (Docker -or Kubernetes/Helm) and registers it as a trusted minion against a VCF -Operations-managed Salt master - with no private key ever leaving the minion, -and no VCF Operations credentials ever reaching the minion itself. +Interactive onboarding tool that registers a new externally managed Salt +minion against a VCF Operations-managed Salt master, then brings up a +salt-minion-vcf instance (Docker or Kubernetes/Helm) that is already +trusted on first connect. Flow: 1. Prompt for VCF Operations (Suite API) connection details and log in. - 2. Resolve the Salt master governing a given VCF instance - (GET /suite-api/api/salt/master?resourceId=). - 3. Compute the Salt-compatible master_finger from the returned master - public key (used for the Kubernetes/Helm path, and for your own - reference/audit trail either way). - 4. Start the minion (docker run, or helm install/upgrade), pointed at the - master and given a freshly generated minion ID. Docker minions are - pre-seeded with the master's actual public key (not just its - fingerprint) so they trust it directly on first connect - the same - approach VCF's own internal component minions use. The minion - generates its own RSA keypair locally on first start - this script - never sees it. - 5. Read back the minion's public key (never the private key) and its ID. - 6. Trust that key against the master - (POST /suite-api/api/salt/minions/{minionId}/trusted-keys). - 7. Poll the minion (already retrying in the background) until it connects, - primarily by watching its logs for the event-driven "Minion is ready to - receive requests" line, falling back to a time-boxed `salt-call - status.master` (the same check the image's own healthcheck/readiness - probe uses, but that check alone can hang or under-report - see the - comments on docker_is_connected()/kubectl_is_connected()). - -Steps 4-7 can be repeated for multiple minions in one session without -re-entering VCF Operations credentials. + 2. List every Salt master known to VCF Operations + (GET /api/salt/masters) and interactively select one - by FQDN, with + its key/presence state shown so you don't pick a master that isn't + actually usable. + 3. Generate a fresh RSA keypair for the minion, locally, via `openssl`. + The private key never leaves this process except to be handed + directly to the minion's own runtime (an env var for Docker, or a + Kubernetes Secret this script creates for you) - it is never sent to + VCF Operations, and never written to the console or the audit log. + 4. Register the minion's public key as trusted against the selected + master (POST /api/salt/minions). The minion ID is always assigned by + VCF Operations, not chosen here - the response is the first time this + script (or you) learns what it is. + 5. Start the minion (docker run, or helm install/upgrade), pre-seeded + with that exact keypair and minion ID, and with the master's actual + public key (not just its fingerprint) so it trusts the master + directly on first connect - the same approach VCF's own internal + component minions use. Because the trust relationship was already + established in step 4 *before* the minion ever starts, there is no + manual `salt-key -a` step and no waiting-for-acceptance window. + 6. Poll the minion (already retrying in the background) until it + connects, primarily by watching its logs for the event-driven "Minion + is ready to receive requests" line, falling back to a time-boxed + `salt-call status.master` (the same check the image's own healthcheck/ + readiness probe uses, but that check alone can hang or under-report - + see the comments on docker_is_connected()/kubectl_is_connected()). + +Steps 3-6 can be repeated for multiple minions in one session without +re-entering VCF Operations credentials or re-listing masters. Every step is written to a timestamped log file (default: vcf-ops-onboard-.log) in addition to the interactive console -output, for audit/troubleshooting. Passwords and auth tokens are never -logged. +output, for audit/troubleshooting. Passwords, auth tokens, and private key +material are never logged. -Only two dependencies: Python 3.8+, and whichever of `docker`/`helm`+`kubectl` -you're deploying with. No third-party pip packages required. +Only three dependencies: Python 3.8+, `openssl` (minion keypair generation), +and whichever of `docker`/`helm`+`kubectl` you're deploying with. No +third-party pip packages required. Reference: https://github.com/saltstack/salt-helm/tree/main/salt-minion-vcf """ @@ -50,10 +56,8 @@ import argparse import base64 import getpass -import hashlib import json import logging -import re import shlex import ssl import subprocess @@ -61,7 +65,6 @@ import time import urllib.error import urllib.request -import uuid from dataclasses import dataclass from datetime import datetime from typing import Callable, Optional @@ -69,9 +72,8 @@ LOG = logging.getLogger("vcf_onboard") -UUID_RE = re.compile( - r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" -) +HEALTHY_KEY_STATE = "ACCEPTED" +HEALTHY_PRESENCE = "PRESENT" # -------------------------------------------------------------------------- @@ -162,11 +164,6 @@ def prompt(text: str, default: Optional[str] = None, secret: bool = False, return value -def prompt_uuid(text: str, default: Optional[str] = None) -> str: - return prompt(text, default=default, validate=lambda v: bool(UUID_RE.match(v)), - validate_hint="Expected a UUID, e.g. 3fa85f64-5717-4562-b3fc-2c963f66afa6") - - def choose(text: str, options: list, default: Optional[str] = None) -> str: print(f"{text}") for i, opt in enumerate(options, 1): @@ -283,15 +280,18 @@ def wait_until(predicate: Callable[[], bool], timeout: int, check_interval: floa # -------------------------------------------------------------------------- def run(cmd: list, dry_run: bool = False, capture: bool = False, check: bool = True, - timeout: float = None) -> str: + timeout: float = None, input_data: Optional[str] = None, + redact_input_in_log: bool = False) -> str: printable = " ".join(shlex.quote(c) for c in cmd) - print(f" $ {printable}") - LOG.debug(f"$ {printable}") + stdin_note = " < " if (input_data and redact_input_in_log) else (" < -" if input_data else "") + print(f" $ {printable}{stdin_note}") + LOG.debug(f"$ {printable}{stdin_note}") if dry_run: return "" try: result = subprocess.run( cmd, + input=input_data, check=check, stdout=subprocess.PIPE if capture else None, stderr=subprocess.STDOUT if capture else None, @@ -325,7 +325,8 @@ class OpsApiError(Exception): class OpsClient: """ - Thin client for the two VCF Operations Salt trust-management endpoints. + Thin client for the two VCF Operations Salt trust-management endpoints + this script needs: listing masters, and registering a minion's key. Auth flow (matches the one already used by other internal tooling against this same backend): @@ -392,43 +393,155 @@ def login(self) -> None: self._token = token LOG.info(f"Authenticated as {self.username} (token acquired, not logged)") - def get_master_details(self, vcf_instance_id: str) -> dict: - """GET /api/salt/master?resourceId= -> {resourceId, masterId, masterFqdn, - masterPublicKey (base64 of the PEM text), masterKeyState, presenceStatus}.""" - result = self._request("GET", "/api/salt/master", params={"resourceId": vcf_instance_id}) - LOG.debug(f"GET /api/salt/master response body: {result}") - return result - - def add_trusted_key(self, minion_id: str, master_id: str, minion_public_key_pem: str) -> dict: - """POST /api/salt/minions/{minionId}/trusted-keys - Body: {masterId, minionPublicKey} - minionPublicKey is RAW PEM text here - (NOT base64-encoded - only the master pubkey in GET responses is).""" + def list_masters(self, page_size: int = 1000) -> list: + """GET /api/salt/masters -> a page of + {masterId, masterFqdn, masterPublicKey (base64), masterKeyState, + presenceStatus}. Unscoped - no VCF instance/resource ID needed. + + Fetches a single large page, which is fine for interactive use; + if your environment has more masters than page_size, pass a larger + value or add real pagination here.""" + result = self._request("GET", "/api/salt/masters", params={"page": 0, "pageSize": page_size}) + masters = result.get("masters", []) + page_info = result.get("pageInfo") or {} + total = page_info.get("totalCount") + if isinstance(total, int) and total > len(masters): + warn(f"VCF Operations reports {total} master(s) total, but only {len(masters)} were " + f"fetched (page_size={page_size}). Increase --master-page-size to see the rest.") + LOG.debug(f"GET /api/salt/masters response body: {result}") + return masters + + def create_minion(self, master_id: str, minion_public_key_pem: str) -> dict: + """POST /api/salt/minions + Body: {masterId, minionPublicKey}. The minion ID is always assigned + by VCF Operations - it is not accepted as request input. Response: + {minionId, masterId, minionPublicKey, masterPublicKey (base64), + masterFqdn, keyState}.""" result = self._request( - "POST", f"/api/salt/minions/{minion_id}/trusted-keys", + "POST", "/api/salt/minions", json_body={"masterId": master_id, "minionPublicKey": minion_public_key_pem}, ) - LOG.debug(f"POST /api/salt/minions/{minion_id}/trusted-keys response body: {result}") + LOG.debug(f"POST /api/salt/minions response body: {result}") return result # -------------------------------------------------------------------------- -# Salt master_finger computation +# Master selection # -------------------------------------------------------------------------- -def pem_finger(pem_text: str, sum_type: str = "sha256") -> str: +def _is_healthy_master(master: dict) -> bool: + return (master.get("masterKeyState") or "").upper() == HEALTHY_KEY_STATE \ + and (master.get("presenceStatus") or "").upper() == HEALTHY_PRESENCE + + +def select_master(client: OpsClient, args: argparse.Namespace) -> dict: + """Lists every Salt master known to VCF Operations and either honors + --master-id (still validated against the live list) or prompts the user + to choose one interactively, by FQDN, with key/presence state shown so + an unusable master isn't picked by accident.""" + if args.dry_run: + masters = [{ + "masterId": "salt-master-", + "masterFqdn": "salt-master.example.com", + "masterKeyState": HEALTHY_KEY_STATE, + "presenceStatus": HEALTHY_PRESENCE, + "masterPublicKey": base64.b64encode( + b"-----BEGIN PUBLIC KEY-----\nAAAAAAAAAAAAAAAAAAAAAAAA\n-----END PUBLIC KEY-----").decode(), + }] + else: + try: + masters = client.list_masters(page_size=args.master_page_size) + except OpsApiError as e: + die(f"Could not list Salt masters: {e}") + + if not masters: + die("No Salt masters are known to VCF Operations. Nothing to onboard against.") + + print(f"\n{_C.BOLD}Available Salt masters{_C.RESET}") + print(f"{_C.DIM}{'-' * 92}{_C.RESET}") + print(f" {'#':<3} {'FQDN':<38} {'Master ID':<28} {'Key State':<10} Presence") + for i, m in enumerate(masters, 1): + healthy = _is_healthy_master(m) + flag = "" if healthy else f" {_C.YELLOW}<- not {HEALTHY_KEY_STATE}/{HEALTHY_PRESENCE}{_C.RESET}" + print(f" {i:<3} {str(m.get('masterFqdn', '')):<38} {str(m.get('masterId', '')):<28} " + f"{str(m.get('masterKeyState', '')):<10} {str(m.get('presenceStatus', ''))}{flag}") + print(f"{_C.DIM}{'-' * 92}{_C.RESET}") + + unhealthy_count = sum(1 for m in masters if not _is_healthy_master(m)) + if unhealthy_count: + warn(f"{unhealthy_count} master(s) above are not {HEALTHY_KEY_STATE}/{HEALTHY_PRESENCE} - " + f"onboarding against one of them will likely fail, or leave the minion unable to " + f"connect even after trust is registered.") + + chosen = None + if args.master_id: + chosen = next((m for m in masters if m.get("masterId") == args.master_id), None) + if chosen is None: + die(f"--master-id '{args.master_id}' was not found in the list above.") + LOG.debug(f"--master-id matched: {chosen}") + else: + while True: + raw = prompt("Select a master by number") + if raw.isdigit() and 1 <= int(raw) <= len(masters): + chosen = masters[int(raw) - 1] + break + print(f" Please enter a number between 1 and {len(masters)}") + + if not _is_healthy_master(chosen): + if not confirm( + f"'{chosen.get('masterFqdn')}' is not {HEALTHY_KEY_STATE}/{HEALTHY_PRESENCE} " + f"(state={chosen.get('masterKeyState')}, presence={chosen.get('presenceStatus')}). " + f"Proceed anyway?", default=False, assume_yes=args.yes): + die("Aborted by user.", code=0) + + ok(f"Selected master: {chosen.get('masterId')} @ {chosen.get('masterFqdn')}") + return chosen + + +# -------------------------------------------------------------------------- +# Minion RSA keypair generation +# -------------------------------------------------------------------------- + +def generate_minion_keypair(key_size: int, dry_run: bool) -> "tuple[str, str]": """ - Reproduces Salt's own salt.utils.crypt.pem_finger(): strip the PEM - header/footer lines, base64-decode the body to raw DER bytes, hash them, - and format as colon-separated hex pairs - the exact string Salt expects - for `master_finger` / SALT_MASTER_FINGER. + Generates a fresh RSA keypair for the minion via `openssl` - the only + external tool this needs beyond docker/helm/kubectl, so no third-party + pip dependency is required. Returns (private_key_pem, public_key_pem). + + The private key is generated here, in this process, and handed directly + to the minion's own runtime (an env var for Docker, a Kubernetes Secret + this script creates for you) - it is never sent to VCF Operations, and + never written to the console or the audit log. """ - lines = [l for l in pem_text.strip().splitlines() if l.strip()] - if len(lines) < 3: - raise ValueError("Master public key does not look like a PEM block") - body = "".join(lines[1:-1]) - der = base64.b64decode(body) - digest = hashlib.new(sum_type, der).hexdigest() - return ":".join(digest[i:i + 2] for i in range(0, len(digest), 2)) + LOG.info(f"$ openssl genrsa {key_size} (output not logged: private key material)") + if dry_run: + return ( + "-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n", + "-----BEGIN PUBLIC KEY-----\n\n-----END PUBLIC KEY-----\n", + ) + try: + priv = subprocess.run( + ["openssl", "genrsa", str(key_size)], + check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ).stdout + except FileNotFoundError: + die("Command not found: openssl. Install it and ensure it's on PATH.") + raise # unreachable, keeps type-checkers happy + except subprocess.CalledProcessError as e: + die(f"Failed to generate the minion's RSA keypair: {e.stderr.strip()}") + raise # unreachable + + LOG.debug("$ openssl rsa -pubout") + try: + pub = subprocess.run( + ["openssl", "rsa", "-pubout"], + input=priv, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ).stdout + except subprocess.CalledProcessError as e: + die(f"Failed to derive the minion's public key: {e.stderr.strip()}") + raise # unreachable + + return priv, pub # -------------------------------------------------------------------------- @@ -443,6 +556,8 @@ class DockerConfig: master_fqdn: str master_pubkey_b64: str minion_id: str + minion_private_key_b64: str + minion_public_key_b64: str def docker_container_exists(name: str, dry_run: bool) -> bool: @@ -470,10 +585,25 @@ def docker_start(cfg: DockerConfig, dry_run: bool, assume_yes: bool = False) -> "-e", f"SALT_MASTER={cfg.master_fqdn}", "-e", f"SALT_MASTER_PUBKEY_B64={cfg.master_pubkey_b64}", "-e", f"SALT_MINION_ID={cfg.minion_id}", + "-e", f"SALT_MINION_PRIVATE_KEY_B64={cfg.minion_private_key_b64}", + "-e", f"SALT_MINION_PUBLIC_KEY_B64={cfg.minion_public_key_b64}", "-v", f"{cfg.volume}:/etc/salt/pki/minion", cfg.image, ] - run(cmd, dry_run=dry_run) + printable = " ".join( + shlex.quote(c) if "PRIVATE_KEY_B64" not in c else "SALT_MINION_PRIVATE_KEY_B64=" + for c in cmd + ) + print(f" $ {printable}") + LOG.debug(f"$ {printable}") + if dry_run: + return + try: + subprocess.run(cmd, check=True) + except FileNotFoundError: + die("Command not found: docker. Is it installed and on PATH?") + except subprocess.CalledProcessError as e: + die(f"Failed to start the minion container (exit {e.returncode}).") def docker_exec(container: str, args: list, dry_run: bool = False, check: bool = True, @@ -482,26 +612,6 @@ def docker_exec(container: str, args: list, dry_run: bool = False, check: bool = timeout=timeout) -def docker_read_minion_pubkey(container: str, timeout: int, dry_run: bool) -> str: - if dry_run: - return "-----BEGIN PUBLIC KEY-----\nAAAAAAAAAAAAAAAAAAAAAAAA\n-----END PUBLIC KEY-----" - - result = {} - - def _check() -> bool: - pubkey = docker_exec(container, ["cat", "/etc/salt/pki/minion/minion.pub"], check=False) - if pubkey.startswith("-----BEGIN PUBLIC KEY-----"): - result["pubkey"] = pubkey - return True - return False - - if not wait_until(_check, timeout=timeout, check_interval=2, - message="Waiting for minion to generate its keypair", dry_run=dry_run): - die(f"Timed out waiting for {container} to generate its minion keypair. " - f"Check `docker logs {container}`.") - return result["pubkey"] - - MINION_READY_LOG_MARKER = "Minion is ready to receive requests" @@ -547,6 +657,32 @@ class HelmConfig: master_fqdn: str master_finger: str minion_id: str + minion_key_secret_name: str + + +def kubectl_upsert_minion_key_secret(namespace: str, secret_name: str, + minion_private_key_b64: str, minion_public_key_b64: str, + dry_run: bool) -> None: + """ + Creates or updates a Kubernetes Secret holding the minion's keypair - + the same "existing Secret, created out-of-band, never through Helm + values" pattern this chart already uses for pillar/vault secrets (Helm + values end up readable in `helm get values`/release history/Secrets; + a plain Secret object does not get any less secret for it, but at least + keeps this key out of the *release* history specifically). + """ + manifest = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": secret_name, "namespace": namespace}, + "type": "Opaque", + "stringData": { + "private-key-b64": minion_private_key_b64, + "public-key-b64": minion_public_key_b64, + }, + } + run(["kubectl", "apply", "-f", "-"], dry_run=dry_run, capture=False, check=True, + input_data=json.dumps(manifest), redact_input_in_log=True) def helm_start(cfg: HelmConfig, dry_run: bool) -> None: @@ -556,6 +692,7 @@ def helm_start(cfg: HelmConfig, dry_run: bool) -> None: "--set", f"salt.master={cfg.master_fqdn}", "--set", f"salt.masterFinger={cfg.master_finger}", "--set", f"salt.minionId={cfg.minion_id}", + "--set", f"salt.minionKeySecretName={cfg.minion_key_secret_name}", "--set", f"image.repository={cfg.image_repository}", "--set", f"image.tag={cfg.image_tag}", ] @@ -592,26 +729,6 @@ def kubectl_exec(namespace: str, pod: str, args: list, dry_run: bool = False, ch dry_run=dry_run, capture=True, check=check, timeout=timeout) -def kubectl_read_minion_pubkey(namespace: str, pod: str, timeout: int, dry_run: bool) -> str: - if dry_run: - return "-----BEGIN PUBLIC KEY-----\nAAAAAAAAAAAAAAAAAAAAAAAA\n-----END PUBLIC KEY-----" - - result = {} - - def _check() -> bool: - pubkey = kubectl_exec(namespace, pod, ["cat", "/etc/salt/pki/minion/minion.pub"], check=False) - if pubkey.startswith("-----BEGIN PUBLIC KEY-----"): - result["pubkey"] = pubkey - return True - return False - - if not wait_until(_check, timeout=timeout, check_interval=2, - message="Waiting for minion to generate its keypair", dry_run=dry_run): - die(f"Timed out waiting for {pod} to generate its minion keypair. " - f"Check `kubectl logs -n {namespace} {pod}`.") - return result["pubkey"] - - def kubectl_is_connected(namespace: str, pod: str, dry_run: bool) -> bool: if dry_run: return True @@ -635,7 +752,7 @@ def kubectl_is_connected(namespace: str, pod: str, dry_run: bool) -> bool: # Main orchestration # -------------------------------------------------------------------------- -TOTAL_STEPS = 7 +TOTAL_STEPS = 6 def build_arg_parser() -> argparse.ArgumentParser: @@ -648,14 +765,17 @@ def build_arg_parser() -> argparse.ArgumentParser: help="Suite API base path (default: /suite-api)") p.add_argument("--insecure", action="store_true", help="Skip TLS certificate verification against VCF Operations") - p.add_argument("--vcf-instance-id", help="VCF instance resource UUID whose master to use") + + p.add_argument("--master-id", help="Salt master ID to use (skips the interactive picker; " + "still validated against GET /api/salt/masters)") + p.add_argument("--master-page-size", type=int, default=1000, + help="Max masters to fetch when listing (default: 1000)") p.add_argument("--deployment", choices=["docker", "kubernetes"], help="Where to run the minion") - p.add_argument("--minion-id", help="Explicit minion ID (default: auto-generated)") - p.add_argument("--minion-id-prefix", default="ext-minion", - help="Prefix for the auto-generated minion ID (default: ext-minion)") + p.add_argument("--key-size", type=int, default=2048, + help="RSA key size (bits) for the minion's keypair (default: 2048, Salt's own default)") # Docker options. Defaults are intentionally None (not the literal # default value) so the script can tell "explicitly passed on the CLI" @@ -673,7 +793,9 @@ def build_arg_parser() -> argparse.ArgumentParser: p.add_argument("--image-tag", help="[k8s] image tag (default: 0.1.0)") p.add_argument("--master-finger-algo", default="sha256", choices=["sha256", "md5"], - help="Hash algorithm for master_finger (default: sha256, matches modern Salt)") + help="[k8s] Hash algorithm for master_finger (default: sha256, matches modern Salt). " + "The Docker path pre-seeds the master's actual public key instead and does not " + "use this.") p.add_argument("--connect-timeout", type=int, default=300, help="Seconds to wait for the minion to connect (default: 300)") p.add_argument("--poll-interval", type=int, default=5, @@ -697,26 +819,71 @@ def build_arg_parser() -> argparse.ArgumentParser: DEFAULT_IMAGE_TAG = "0.1.0" -def onboard_one_minion(client: OpsClient, args: argparse.Namespace, - master_id: str, master_fqdn: str, master_pubkey_b64: str, master_finger: str, +def pem_finger(pem_text: str, sum_type: str = "sha256") -> str: + """ + Reproduces Salt's own salt.utils.crypt.pem_finger(): strip the PEM + header/footer lines, base64-decode the body to raw DER bytes, hash them, + and format as colon-separated hex pairs - the exact string Salt expects + for `master_finger` / SALT_MASTER_FINGER. Only used by the Kubernetes/ + Helm path today - the Docker path pre-seeds the master's actual public + key instead (see helm_start()/docker_start()). + """ + import hashlib + lines = [l for l in pem_text.strip().splitlines() if l.strip()] + if len(lines) < 3: + raise ValueError("Master public key does not look like a PEM block") + body = "".join(lines[1:-1]) + der = base64.b64decode(body) + digest = hashlib.new(sum_type, der).hexdigest() + return ":".join(digest[i:i + 2] for i in range(0, len(digest), 2)) + + +def onboard_one_minion(client: OpsClient, args: argparse.Namespace, master: dict, deployment: str, defaults: dict, index: int) -> dict: """ - Runs steps 4-7 for a single minion and returns a summary dict. + Runs steps 3-6 for a single minion and returns a summary dict. `index` counts minions onboarded in this session (starting at 1). CLI - flags for identity-bearing settings (minion ID, container/volume/release - name) are only honored on the first minion - a container name, PKI - volume, or Helm release can't be reused for a second minion without - colliding, so from the second minion onward this always prompts, with an + flags for identity-bearing settings (container/volume/release name) are + only honored on the first minion - a container name, PKI volume, or + Helm release can't be reused for a second minion without colliding, so + from the second minion onward this always prompts, with an auto-suffixed suggestion ("-2", "-3", ...) to avoid that collision. """ + master_id = master["masterId"] + master_fqdn = master["masterFqdn"] + master_pubkey_b64 = master["masterPublicKey"] + + # ---------------------------------------------------------------- Step 3 + step(3, TOTAL_STEPS, "Generate the minion's RSA keypair") + minion_private_key_pem, minion_public_key_pem = generate_minion_keypair( + key_size=args.key_size, dry_run=args.dry_run) + ok(f"Generated a {args.key_size}-bit RSA keypair (private key never leaves this process)") # ---------------------------------------------------------------- Step 4 - step(4, TOTAL_STEPS, "Start the minion") - suffix = "" if index == 1 else f"-{index}" + step(4, TOTAL_STEPS, "Register the minion's public key as trusted") + if not confirm(f"Register a new minion against master '{master_id}' @ {master_fqdn}?", + assume_yes=args.yes): + die("Aborted by user.", code=0) + if args.dry_run: + minion_id = f"ext-minion-" + info(f"(dry-run) POST /api/salt/minions {{masterId: {master_id}, minionPublicKey: }}") + else: + try: + create_result = client.create_minion(master_id, minion_public_key_pem) + except OpsApiError as e: + die(f"Failed to register the minion's key: {e}") + minion_id = create_result.get("minionId") + if not minion_id: + die(f"Registration reported success but returned no minionId: {create_result}") + key_state = (create_result.get("keyState") or "").upper() + if key_state and key_state != "TRUSTED": + die(f"Registration did not result in a trusted key (keyState={key_state}): {create_result}") + ok(f"Minion registered and trusted: {minion_id}") - minion_id = (args.minion_id if index == 1 else None) or prompt( - "Minion ID", default=f"{args.minion_id_prefix}-{uuid.uuid4()}") + # ---------------------------------------------------------------- Step 5 + step(5, TOTAL_STEPS, "Start the minion") + suffix = "" if index == 1 else f"-{index}" if deployment == "docker": container_name = (args.container_name if index == 1 else None) or prompt( @@ -737,7 +904,7 @@ def onboard_one_minion(client: OpsClient, args: argparse.Namespace, ("Image", image), ("Container name", container_name), ("PKI volume", volume), - ("Salt master", f"{master_fqdn} (master pubkey pre-seeded)"), + ("Salt master", f"{master_fqdn} (master pubkey + minion keypair pre-seeded)"), ]) if not confirm("Proceed with these settings?", assume_yes=args.yes): die("Aborted by user.", code=0) @@ -745,11 +912,14 @@ def onboard_one_minion(client: OpsClient, args: argparse.Namespace, docker_cfg = DockerConfig( image=image, container_name=container_name, volume=volume, master_fqdn=master_fqdn, master_pubkey_b64=master_pubkey_b64, minion_id=minion_id, + minion_private_key_b64=base64.b64encode(minion_private_key_pem.encode()).decode(), + minion_public_key_b64=base64.b64encode(minion_public_key_pem.encode()).decode(), ) docker_start(docker_cfg, dry_run=args.dry_run, assume_yes=args.yes) ok(f"Container '{container_name}' started") defaults["image"] = image pod_name = None + namespace = None else: release_name = (args.release_name if index == 1 else None) or prompt( "Helm release name", default=f"{DEFAULT_RELEASE_NAME}{suffix}") @@ -760,6 +930,10 @@ def onboard_one_minion(client: OpsClient, args: argparse.Namespace, image_tag = args.image_tag or defaults.get("image_tag") or prompt( "Image tag", default=DEFAULT_IMAGE_TAG) + master_finger = pem_finger( + base64.b64decode(master_pubkey_b64).decode("utf-8"), sum_type=args.master_finger_algo) + minion_key_secret_name = f"{release_name}-minion-key" + print_summary("Review before starting the minion", [ ("Deployment", "kubernetes"), ("Minion ID", minion_id), @@ -767,14 +941,24 @@ def onboard_one_minion(client: OpsClient, args: argparse.Namespace, ("Namespace", namespace), ("Image", f"{image_repository}:{image_tag}"), ("Salt master", f"{master_fqdn} (master_finger computed)"), + ("Minion key secret", minion_key_secret_name), ]) if not confirm("Proceed with these settings?", assume_yes=args.yes): die("Aborted by user.", code=0) + kubectl_upsert_minion_key_secret( + namespace, minion_key_secret_name, + minion_private_key_b64=base64.b64encode(minion_private_key_pem.encode()).decode(), + minion_public_key_b64=base64.b64encode(minion_public_key_pem.encode()).decode(), + dry_run=args.dry_run, + ) + ok(f"Minion keypair Secret '{minion_key_secret_name}' created/updated in namespace {namespace}") + helm_cfg = HelmConfig( chart_path=args.chart_path, release_name=release_name, namespace=namespace, image_repository=image_repository, image_tag=image_tag, master_fqdn=master_fqdn, master_finger=master_finger, minion_id=minion_id, + minion_key_secret_name=minion_key_secret_name, ) helm_start(helm_cfg, dry_run=args.dry_run) ok(f"Helm release '{release_name}' installed/upgraded in namespace {namespace}") @@ -784,33 +968,8 @@ def onboard_one_minion(client: OpsClient, args: argparse.Namespace, pod_name = kubectl_get_pod_name(namespace, release_name, dry_run=args.dry_run) ok(f"Pod: {pod_name}") - # ---------------------------------------------------------------- Step 5 - step(5, TOTAL_STEPS, "Read the minion's public key") - if deployment == "docker": - minion_pubkey_pem = docker_read_minion_pubkey(container_name, timeout=60, dry_run=args.dry_run) - else: - minion_pubkey_pem = kubectl_read_minion_pubkey(namespace, pod_name, timeout=60, dry_run=args.dry_run) - ok("Minion public key retrieved (private key never left the minion)") - # ---------------------------------------------------------------- Step 6 - step(6, TOTAL_STEPS, "Trust the minion's key against the master") - if not confirm(f"Register minion '{minion_id}' as trusted against master '{master_id}'?", - assume_yes=args.yes): - die("Aborted by user.", code=0) - if args.dry_run: - info(f"(dry-run) POST /api/salt/minions/{minion_id}/trusted-keys " - f"{{masterId: {master_id}, minionPublicKey: }}") - else: - try: - trust_result = client.add_trusted_key(minion_id, master_id, minion_pubkey_pem) - except OpsApiError as e: - die(f"Failed to register the trusted key: {e}") - if trust_result.get("status", "").upper() not in ("SUCCESS", ""): - die(f"Trust registration reported failure: {trust_result}") - ok("Minion key trusted with the Salt master") - - # ---------------------------------------------------------------- Step 7 - step(7, TOTAL_STEPS, "Wait for the minion to connect") + step(6, TOTAL_STEPS, "Wait for the minion to connect") def _connected() -> bool: if deployment == "docker": @@ -819,10 +978,11 @@ def _connected() -> bool: connected = wait_until(_connected, timeout=args.connect_timeout, check_interval=args.poll_interval, - message="Waiting for the master to accept the minion", dry_run=args.dry_run) + message="Waiting for the minion to connect", dry_run=args.dry_run) if not connected: die(f"Minion did not connect within {args.connect_timeout}s. " - f"Check the master's `salt-key -L` and the minion's logs.") + f"Trust was already registered (minion ID {minion_id}) - this points at a network/" + f"connectivity problem, not a trust problem. Check the minion's logs.") ok("Minion connected to the Salt master") return {"minion_id": minion_id, "deployment": deployment} @@ -858,36 +1018,10 @@ def main() -> None: ok(f"Authenticated to {ops_host}") # ---------------------------------------------------------------- Step 2 - step(2, TOTAL_STEPS, "Resolve the Salt master for your VCF instance") - vcf_instance_id = args.vcf_instance_id or prompt_uuid("VCF instance resource ID (UUID)") - - if args.dry_run: - master = {"masterId": "salt-master-", "masterFqdn": "salt-master.example.com", - "masterPublicKey": base64.b64encode( - b"-----BEGIN PUBLIC KEY-----\nAAAAAAAAAAAAAAAAAAAAAAAA\n-----END PUBLIC KEY-----").decode()} - else: - try: - master = client.get_master_details(vcf_instance_id) - except OpsApiError as e: - die(f"Could not resolve master details: {e}") - - master_id = master["masterId"] - master_fqdn = master["masterFqdn"] - master_pubkey_b64 = master["masterPublicKey"] - master_pubkey_pem = base64.b64decode(master_pubkey_b64).decode("utf-8") - ok(f"Master resolved: {master_id} @ {master_fqdn}") + step(2, TOTAL_STEPS, "List Salt masters and select one") + master = select_master(client, args) - # ---------------------------------------------------------------- Step 3 - # Docker minions are pre-seeded with the master's actual public key - # (SALT_MASTER_PUBKEY_B64) rather than a fingerprint - see - # docker-entrypoint.sh for why. The fingerprint below is still computed - # for the Kubernetes/Helm path (which only supports master_finger today) - # and for your own reference/audit trail. - step(3, TOTAL_STEPS, "Compute master identity fingerprint") - master_finger = pem_finger(master_pubkey_pem, sum_type=args.master_finger_algo) - ok(f"master_finger ({args.master_finger_algo}): {master_finger}") - - # ------------------------------------------------- Steps 4-7 (repeatable) + # ------------------------------------------------- Steps 3-6 (repeatable) deployment = args.deployment or choose( "\nWhere should this minion run?", ["docker", "kubernetes"], default="docker") @@ -895,13 +1029,12 @@ def main() -> None: defaults: dict = {} index = 1 while True: - result = onboard_one_minion(client, args, master_id, master_fqdn, master_pubkey_b64, master_finger, - deployment, defaults, index) + result = onboard_one_minion(client, args, master, deployment, defaults, index) onboarded.append(result) print(f"\n{_C.BOLD}{_C.GREEN}Minion onboarded{_C.RESET}") print(f" Minion ID : {result['minion_id']}") - print(f" Master : {master_id} @ {master_fqdn}") + print(f" Master : {master.get('masterId')} @ {master.get('masterFqdn')}") print(f" Deployment: {result['deployment']}") print(f"\nVerify from the Salt master:\n salt '{result['minion_id']}' test.ping") From 02fed49d80488feca0df6a44fa1f36fe297c3344 Mon Sep 17 00:00:00 2001 From: Praveen T Date: Wed, 2 Sep 2026 14:59:44 +0530 Subject: [PATCH 5/6] Add rotate and list actions to the onboarding script; self-report resourceKind vcf-ops-onboard.py now supports --action {configure,rotate,list}: rotate identifies the target minion by its current public key (read off the running container/pod's PKI dir) and calls the new POST /api/salt/minions/rotate API, then pushes the new keypair into the running instance and restarts it (container recreate for Docker, Secret update + Pod recreate for Kubernetes); list prints trusted minions via GET /api/salt/minions. Also have docker-entrypoint.sh set the vcfops_resource_kind grain to "external" on every start, so VCF Operations' minion listing reports a real resourceKind for these externally managed minions instead of null. --- salt-minion-vcf/scripts/docker-entrypoint.sh | 9 + salt-minion-vcf/scripts/onboarding/README.md | 71 ++- .../scripts/onboarding/vcf-ops-onboard.py | 461 ++++++++++++++++-- 3 files changed, 488 insertions(+), 53 deletions(-) diff --git a/salt-minion-vcf/scripts/docker-entrypoint.sh b/salt-minion-vcf/scripts/docker-entrypoint.sh index 31db7e7..588a678 100755 --- a/salt-minion-vcf/scripts/docker-entrypoint.sh +++ b/salt-minion-vcf/scripts/docker-entrypoint.sh @@ -121,6 +121,15 @@ grains: vcf_executor: true deployment_type: ${DEPLOYMENT_TYPE:-docker} managed_by: salt-minion-vcf + # VCF Operations' minion listing (GET /api/salt/minions) surfaces this as + # resourceKind, read via a bulk get_minion_details grains lookup - it is + # null until this grain is set and synced to the master, which is exactly + # what this line does on every start of this image. "external" (rather + # than a real component kind like vcenter/sddcm) reflects that this is a + # generic, user-managed executor minion, not a VCF appliance component - + # see vcf_grain_keys.py in config-modules for the full set of recognized + # component kinds. + vcfops_resource_kind: ${VCFOPS_RESOURCE_KIND:-external} EOF # Opt-in: make ALL pillar compiles (including those for jobs dispatched from diff --git a/salt-minion-vcf/scripts/onboarding/README.md b/salt-minion-vcf/scripts/onboarding/README.md index e7e98f3..1d79c57 100644 --- a/salt-minion-vcf/scripts/onboarding/README.md +++ b/salt-minion-vcf/scripts/onboarding/README.md @@ -1,12 +1,18 @@ # VCF Operations Onboarding Script -`vcf-ops-onboard.py` is an interactive tool that brings up a `salt-minion-vcf` -instance (Docker **or** Kubernetes/Helm) and registers it as a trusted minion -against a Salt master managed by VMware VCF Operations - without the -minion's private key ever leaving the minion, and without VCF Operations -credentials ever reaching the minion itself. +`vcf-ops-onboard.py` is an interactive tool for managing `salt-minion-vcf` +instances (Docker **or** Kubernetes/Helm) against a Salt master managed by +VMware VCF Operations - without the minion's private key ever leaving the +minion, and without VCF Operations credentials ever reaching the minion +itself. It supports three actions, picked with `--action` or interactively: -## What it does +- **configure** (default) - bring up a *new* minion, already trusted on first + connect. +- **rotate** - rotate the key of an *already-onboarded* minion. +- **list** - read-only listing of trusted minions (master, key/presence + state, resourceKind), handy before rotating one. + +## configure: what it does ```text 1. Log in to VCF Operations @@ -33,6 +39,40 @@ there's no waiting-for-acceptance window and no human needing to run Steps 3-6 can be repeated for multiple minions in one session without re-entering VCF Operations credentials or re-listing masters. +## rotate: what it does + +```text +1. Log in to VCF Operations +2. Identify the running minion (container name, or Helm release/namespace) + and read its CURRENT public key straight off its PKI dir - this is how + the minion is identified server-side too, since the rotate API never + accepts a minion ID as input. Then pick the Salt master to rotate + against (normally the SAME one the minion is already configured for - + picking a different one re-associates the minion's trust record with it, + and the script warns before letting that happen) +3. Generate a fresh RSA keypair locally, and call the rotate API + (POST /api/salt/minions/rotate) with the current and new public keys. + VCF Operations resolves the existing trust record from the current key + and re-registers it in place with the new one +4. Push the new keypair into the running instance and restart it: + - Docker: the container is recreated (env vars holding the keypair are + fixed at container creation, so this is the only way to feed it a new + one); the PKI volume's old key files are cleared first so the fresh + env vars actually get picked up + - Kubernetes: the minion's key Secret is updated, any persisted PKI files + on the Pod are cleared (relevant when `persistence.enabled=true`), and + the Pod is deleted so it's recreated with the new key + Then poll until the master accepts the new connection, same as configure. +``` + +## list: what it does + +A single read-only call to `GET /api/salt/minions`, printed as a table of +minion ID, master ID, key state, presence, and resourceKind (the minion's +`vcfops_resource_kind` grain, if it has reported one - null for a freshly +configured minion until its own bootstrap sets that grain). Use `--state` to +filter by trust state. + ## Interactive features - **Master picker**: every Salt master VCF Operations knows about is listed @@ -102,6 +142,22 @@ python3 scripts/onboarding/vcf-ops-onboard.py \ --ops-user admin \ --master-id salt-master-7a1b2c3d-4e5f-6789-abcd-ef0123456789 \ --deployment docker + +# Rotate an already-onboarded Docker minion's key +python3 scripts/onboarding/vcf-ops-onboard.py \ + --action rotate --deployment docker \ + --ops-host vcfops.example.com --ops-user admin \ + --container-name salt-minion-vcf + +# Rotate an already-onboarded Kubernetes minion's key +python3 scripts/onboarding/vcf-ops-onboard.py \ + --action rotate --deployment kubernetes \ + --ops-host vcfops.example.com --ops-user admin \ + --namespace vcf-salt --release-name vcf-executor + +# List trusted minions +python3 scripts/onboarding/vcf-ops-onboard.py \ + --action list --ops-host vcfops.example.com --ops-user admin ``` See `--help` for the full flag list (container/release naming, image @@ -139,5 +195,6 @@ anything, `-y` to skip confirmation prompts). ## Known limitation There is currently no API to *revoke* a trusted key (deregistration), so this -script only covers onboarding. To remove a minion, use your master's own +script covers onboarding (`configure`), key rotation (`rotate`), and listing +(`list`), but not removal. To remove a minion, use your master's own key-management tooling directly for now. diff --git a/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py b/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py index 09ab316..2d6f029 100755 --- a/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py +++ b/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py @@ -2,42 +2,59 @@ """ vcf-ops-onboard.py -Interactive onboarding tool that registers a new externally managed Salt -minion against a VCF Operations-managed Salt master, then brings up a -salt-minion-vcf instance (Docker or Kubernetes/Helm) that is already -trusted on first connect. - -Flow: - 1. Prompt for VCF Operations (Suite API) connection details and log in. - 2. List every Salt master known to VCF Operations - (GET /api/salt/masters) and interactively select one - by FQDN, with - its key/presence state shown so you don't pick a master that isn't - actually usable. - 3. Generate a fresh RSA keypair for the minion, locally, via `openssl`. - The private key never leaves this process except to be handed - directly to the minion's own runtime (an env var for Docker, or a - Kubernetes Secret this script creates for you) - it is never sent to - VCF Operations, and never written to the console or the audit log. - 4. Register the minion's public key as trusted against the selected - master (POST /api/salt/minions). The minion ID is always assigned by - VCF Operations, not chosen here - the response is the first time this - script (or you) learns what it is. - 5. Start the minion (docker run, or helm install/upgrade), pre-seeded - with that exact keypair and minion ID, and with the master's actual - public key (not just its fingerprint) so it trusts the master - directly on first connect - the same approach VCF's own internal - component minions use. Because the trust relationship was already - established in step 4 *before* the minion ever starts, there is no - manual `salt-key -a` step and no waiting-for-acceptance window. - 6. Poll the minion (already retrying in the background) until it - connects, primarily by watching its logs for the event-driven "Minion - is ready to receive requests" line, falling back to a time-boxed - `salt-call status.master` (the same check the image's own healthcheck/ - readiness probe uses, but that check alone can hang or under-report - - see the comments on docker_is_connected()/kubectl_is_connected()). - -Steps 3-6 can be repeated for multiple minions in one session without -re-entering VCF Operations credentials or re-listing masters. +Interactive tool for managing externally managed Salt minions against a VCF +Operations-managed Salt master. Supports three actions (--action, or picked +interactively): + + configure - registers a NEW minion's key as trusted, then brings up a + salt-minion-vcf instance (Docker or Kubernetes/Helm) that is + already trusted on first connect. + 1. Prompt for VCF Operations (Suite API) connection details and log in. + 2. List every Salt master known to VCF Operations + (GET /api/salt/masters) and interactively select one - by FQDN, with + its key/presence state shown so you don't pick a master that isn't + actually usable. + 3. Generate a fresh RSA keypair for the minion, locally, via `openssl`. + The private key never leaves this process except to be handed + directly to the minion's own runtime (an env var for Docker, or a + Kubernetes Secret this script creates for you) - it is never sent to + VCF Operations, and never written to the console or the audit log. + 4. Register the minion's public key as trusted against the selected + master (POST /api/salt/minions). The minion ID is always assigned by + VCF Operations, not chosen here - the response is the first time this + script (or you) learns what it is. + 5. Start the minion (docker run, or helm install/upgrade), pre-seeded + with that exact keypair and minion ID, and with the master's actual + public key (not just its fingerprint) so it trusts the master + directly on first connect - the same approach VCF's own internal + component minions use. Because the trust relationship was already + established in step 4 *before* the minion ever starts, there is no + manual `salt-key -a` step and no waiting-for-acceptance window. + 6. Poll the minion (already retrying in the background) until it + connects, primarily by watching its logs for the event-driven "Minion + is ready to receive requests" line, falling back to a time-boxed + `salt-call status.master` (the same check the image's own healthcheck/ + readiness probe uses, but that check alone can hang or under-report - + see the comments on docker_is_connected()/kubectl_is_connected()). + Steps 3-6 can be repeated for multiple minions in one session without + re-entering VCF Operations credentials or re-listing masters. + + rotate - rotates the key of an ALREADY-onboarded minion. The minion is + identified by its CURRENT public key (read straight off the + running container/pod's PKI dir), not by minion ID - the rotate + API (POST /api/salt/minions/rotate) never accepts minion ID as + input either. A fresh keypair is generated, registered via the + rotate API (which re-registers the SAME minion record, RaaS-side, + with the new key), and the running instance is then given the + new keypair and restarted: the container is recreated for + Docker (env vars are fixed at container creation, so this is + the only way to feed it a new keypair), or the Pod's key Secret + is updated and the Pod is deleted/recreated for Kubernetes. + + list - a read-only listing of trusted minions (GET /api/salt/minions), + including each minion's live presence status and resourceKind + (its vcfops_resource_kind grain, if it has reported one) - handy + for finding a minion's master/state before rotating its key. Every step is written to a timestamped log file (default: vcf-ops-onboard-.log) in addition to the interactive console @@ -424,6 +441,46 @@ def create_minion(self, master_id: str, minion_public_key_pem: str) -> dict: LOG.debug(f"POST /api/salt/minions response body: {result}") return result + def rotate_minion_key(self, master_id: str, current_minion_public_key_pem: str, + new_minion_public_key_pem: str) -> dict: + """POST /api/salt/minions/rotate + Body: {masterId, currentMinionPublicKey, newMinionPublicKey}. The + minion to rotate is never identified by minionId - it is resolved + server-side from currentMinionPublicKey instead, then re-registered + with newMinionPublicKey. Response: {minionId, masterId, + minionPublicKey, masterPublicKey (base64), masterFqdn, keyState} - + the same shape as create_minion()'s response.""" + result = self._request( + "POST", "/api/salt/minions/rotate", + json_body={ + "masterId": master_id, + "currentMinionPublicKey": current_minion_public_key_pem, + "newMinionPublicKey": new_minion_public_key_pem, + }, + ) + LOG.debug(f"POST /api/salt/minions/rotate response body: {result}") + return result + + def list_minions(self, state: Optional[str] = None, page_size: int = 1000) -> list: + """GET /api/salt/minions -> a page of + {minionId, masterId, minionPublicKey, keyState, presenceStatus, + resourceKind, description, createdAt, updatedAt, acceptedAt, + rejectedAt}. resourceKind is the minion's vcfops_resource_kind grain + (e.g. "vcenter", "sddcm") - null until the minion's own bootstrap + sets it and it syncs to the master.""" + params = {"page": 0, "pageSize": page_size} + if state: + params["state"] = state + result = self._request("GET", "/api/salt/minions", params=params) + minions = result.get("minions", []) + page_info = result.get("pageInfo") or {} + total = page_info.get("totalCount") + if isinstance(total, int) and total > len(minions): + warn(f"VCF Operations reports {total} minion(s) total, but only {len(minions)} were " + f"fetched (page_size={page_size}). Increase --minion-page-size to see the rest.") + LOG.debug(f"GET /api/salt/minions response body: {result}") + return minions + # -------------------------------------------------------------------------- # Master selection @@ -643,6 +700,69 @@ def docker_is_connected(container: str, dry_run: bool) -> bool: return out.strip().lower() == "true" +PKI_MINION_PEM = "/etc/salt/pki/minion/minion.pem" +PKI_MINION_PUB = "/etc/salt/pki/minion/minion.pub" + + +def docker_read_minion_pubkey(container: str, dry_run: bool) -> str: + """Reads the minion's CURRENT public key straight off the running + container's PKI dir - this is what identifies which trust record to + rotate server-side (see OpsClient.rotate_minion_key), since minionId is + never exposed as an input.""" + if dry_run: + return "-----BEGIN PUBLIC KEY-----\n\n-----END PUBLIC KEY-----\n" + out = docker_exec(container, ["cat", PKI_MINION_PUB], check=False) + if not out.strip(): + die(f"Could not read the current public key from '{container}:{PKI_MINION_PUB}'. " + f"Is the container running and already onboarded?") + return out + + +def docker_read_configured_master_fqdn(container: str, dry_run: bool) -> Optional[str]: + """Best-effort read of the minion's currently configured master FQDN, so + the rotate flow can warn if the user is about to select a *different* + master than the one this minion is actually trusted against.""" + if dry_run: + return None + out = docker_exec( + container, + ["sh", "-c", "grep -m1 '^master:' /etc/salt/minion.d/10-master.conf 2>/dev/null || true"], + check=False, + ) + return out.split(":", 1)[1].strip() if out.startswith("master:") else None + + +def docker_clear_minion_keys(container: str, dry_run: bool) -> None: + """Removes the minion's PKI files from the (persistent, named) volume + while the OLD container still exists to exec into. Required before + recreating the container with a new keypair - docker-entrypoint.sh only + seeds SALT_MINION_PRIVATE_KEY_B64/PUBLIC_KEY_B64 into the PKI dir when + minion.pem doesn't already exist, and `docker rm` alone does not remove + the named volume's contents.""" + run(["docker", "exec", container, "rm", "-f", PKI_MINION_PEM, PKI_MINION_PUB], + dry_run=dry_run, check=False) + + +def docker_inspect_minion(container: str, dry_run: bool) -> dict: + """Reads back the image and PKI volume of an already-running container, + so rotate can recreate it identically (aside from the keypair) without + asking the user to re-supply --image/--volume from memory.""" + if dry_run: + return {"image": DEFAULT_IMAGE, "volume": DEFAULT_VOLUME} + out = run(["docker", "inspect", container], dry_run=dry_run, capture=True, check=True) + data = json.loads(out)[0] + image = data["Config"]["Image"] + volume = None + for mount in data.get("Mounts", []): + if mount.get("Destination") == "/etc/salt/pki/minion": + volume = mount.get("Name") or mount.get("Source") + break + if not volume: + die(f"Could not determine the PKI volume mounted on container '{container}' " + f"(expected a mount at /etc/salt/pki/minion).") + return {"image": image, "volume": volume} + + # -------------------------------------------------------------------------- # Kubernetes / Helm deployment # -------------------------------------------------------------------------- @@ -748,17 +868,56 @@ def kubectl_is_connected(namespace: str, pod: str, dry_run: bool) -> bool: return out.strip().lower() == "true" +def kubectl_read_minion_pubkey(namespace: str, pod: str, dry_run: bool) -> str: + """See docker_read_minion_pubkey() - same purpose, Kubernetes path.""" + if dry_run: + return "-----BEGIN PUBLIC KEY-----\n\n-----END PUBLIC KEY-----\n" + out = kubectl_exec(namespace, pod, ["cat", PKI_MINION_PUB], check=False) + if not out.strip(): + die(f"Could not read the current public key from pod '{pod}':{PKI_MINION_PUB}. " + f"Is it running and already onboarded?") + return out + + +def kubectl_read_configured_master_fqdn(namespace: str, pod: str, dry_run: bool) -> Optional[str]: + """See docker_read_configured_master_fqdn() - same purpose, Kubernetes path.""" + if dry_run: + return None + out = kubectl_exec( + namespace, pod, + ["sh", "-c", "grep -m1 '^master:' /etc/salt/minion.d/10-master.conf 2>/dev/null || true"], + check=False, + ) + return out.split(":", 1)[1].strip() if out.startswith("master:") else None + + +def kubectl_clear_minion_keys(namespace: str, pod: str, dry_run: bool) -> None: + """See docker_clear_minion_keys() - same purpose. Necessary when the pki + volume is a PersistentVolumeClaim (persistence.enabled=true in the Helm + chart), since a Pod restart alone would keep the OLD keypair on disk; a + no-op if pki is an emptyDir (deleting the Pod already wipes it).""" + kubectl_exec(namespace, pod, ["rm", "-f", PKI_MINION_PEM, PKI_MINION_PUB], dry_run=dry_run, check=False) + + # -------------------------------------------------------------------------- # Main orchestration # -------------------------------------------------------------------------- -TOTAL_STEPS = 6 +CONFIGURE_TOTAL_STEPS = 6 +ROTATE_TOTAL_STEPS = 4 +LIST_TOTAL_STEPS = 2 def build_arg_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( - description="Onboard a salt-minion-vcf instance against a VCF Operations-managed Salt master.", + description="Configure, rotate, or list salt-minion-vcf instances against a " + "VCF Operations-managed Salt master.", ) + p.add_argument("--action", choices=["configure", "rotate", "list"], + help="What to do: 'configure' onboards a new minion (default), 'rotate' rotates an " + "already-onboarded minion's key, 'list' prints trusted minions known to VCF " + "Operations. Prompted interactively if omitted.") + p.add_argument("--ops-host", help="VCF Operations FQDN or IP") p.add_argument("--ops-user", help="VCF Operations username") p.add_argument("--ops-base-path", default="/suite-api", @@ -770,6 +929,10 @@ def build_arg_parser() -> argparse.ArgumentParser: "still validated against GET /api/salt/masters)") p.add_argument("--master-page-size", type=int, default=1000, help="Max masters to fetch when listing (default: 1000)") + p.add_argument("--minion-page-size", type=int, default=1000, + help="[list] Max minions to fetch when listing (default: 1000)") + p.add_argument("--state", choices=["TRUSTED", "REJECTED"], + help="[list] Filter minions by trust state") p.add_argument("--deployment", choices=["docker", "kubernetes"], help="Where to run the minion") @@ -855,13 +1018,13 @@ def onboard_one_minion(client: OpsClient, args: argparse.Namespace, master: dict master_pubkey_b64 = master["masterPublicKey"] # ---------------------------------------------------------------- Step 3 - step(3, TOTAL_STEPS, "Generate the minion's RSA keypair") + step(3, CONFIGURE_TOTAL_STEPS, "Generate the minion's RSA keypair") minion_private_key_pem, minion_public_key_pem = generate_minion_keypair( key_size=args.key_size, dry_run=args.dry_run) ok(f"Generated a {args.key_size}-bit RSA keypair (private key never leaves this process)") # ---------------------------------------------------------------- Step 4 - step(4, TOTAL_STEPS, "Register the minion's public key as trusted") + step(4, CONFIGURE_TOTAL_STEPS, "Register the minion's public key as trusted") if not confirm(f"Register a new minion against master '{master_id}' @ {master_fqdn}?", assume_yes=args.yes): die("Aborted by user.", code=0) @@ -882,7 +1045,7 @@ def onboard_one_minion(client: OpsClient, args: argparse.Namespace, master: dict ok(f"Minion registered and trusted: {minion_id}") # ---------------------------------------------------------------- Step 5 - step(5, TOTAL_STEPS, "Start the minion") + step(5, CONFIGURE_TOTAL_STEPS, "Start the minion") suffix = "" if index == 1 else f"-{index}" if deployment == "docker": @@ -969,7 +1132,7 @@ def onboard_one_minion(client: OpsClient, args: argparse.Namespace, master: dict ok(f"Pod: {pod_name}") # ---------------------------------------------------------------- Step 6 - step(6, TOTAL_STEPS, "Wait for the minion to connect") + step(6, CONFIGURE_TOTAL_STEPS, "Wait for the minion to connect") def _connected() -> bool: if deployment == "docker": @@ -988,19 +1151,209 @@ def _connected() -> bool: return {"minion_id": minion_id, "deployment": deployment} +# -------------------------------------------------------------------------- +# List trusted minions +# -------------------------------------------------------------------------- + +def list_minions_action(client: OpsClient, args: argparse.Namespace) -> None: + step(2, LIST_TOTAL_STEPS, "List trusted minions") + if args.dry_run: + minions = [{ + "minionId": "", "masterId": "salt-master-", + "keyState": "TRUSTED", "presenceStatus": "PRESENT", "resourceKind": None, + }] + else: + try: + minions = client.list_minions(state=args.state, page_size=args.minion_page_size) + except OpsApiError as e: + die(f"Could not list minions: {e}") + + if not minions: + info("No trusted minions are known to VCF Operations.") + return + + print(f"\n{_C.BOLD}Trusted minions{_C.RESET}") + print(f"{_C.DIM}{'-' * 110}{_C.RESET}") + print(f" {'Minion ID':<38} {'Master ID':<28} {'Key State':<10} {'Presence':<10} Resource Kind") + for m in minions: + print(f" {str(m.get('minionId', '')):<38} {str(m.get('masterId', '')):<28} " + f"{str(m.get('keyState', '')):<10} {str(m.get('presenceStatus', '')):<10} " + f"{m.get('resourceKind') or '(unknown)'}") + print(f"{_C.DIM}{'-' * 110}{_C.RESET}") + ok(f"Listed {len(minions)} minion(s)") + + +# -------------------------------------------------------------------------- +# Rotate an already-onboarded minion's key +# -------------------------------------------------------------------------- + +def _confirm_master_matches(configured_fqdn: Optional[str], master: dict, assume_yes: bool) -> None: + """Warns (and asks for confirmation) if the master picked for rotation + doesn't match the master this minion is actually configured against - + the rotate API re-associates the minion with whichever masterId is + passed, so picking the wrong one would silently move the minion, not + just rotate its key.""" + if not configured_fqdn: + return + if configured_fqdn == master.get("masterFqdn"): + return + if not confirm( + f"This minion is currently configured against master '{configured_fqdn}', but you selected " + f"'{master.get('masterFqdn')}'. Rotating against a different master also RE-ASSOCIATES the " + f"minion's trust record with it. Continue anyway?", default=False, assume_yes=assume_yes): + die("Aborted by user.", code=0) + + +def rotate_minion_docker(client: OpsClient, args: argparse.Namespace) -> None: + step(2, ROTATE_TOTAL_STEPS, "Identify the minion and read its current key") + container = args.container_name or prompt("Container name to rotate", default=DEFAULT_CONTAINER_NAME) + if not args.dry_run and not docker_container_exists(container, args.dry_run): + die(f"No container named '{container}' found.") + + current_pub = docker_read_minion_pubkey(container, args.dry_run) + ok(f"Read current public key from '{container}'") + configured_fqdn = docker_read_configured_master_fqdn(container, args.dry_run) + if configured_fqdn: + info(f"This minion is currently configured against master FQDN: {configured_fqdn}") + + master = select_master(client, args) + _confirm_master_matches(configured_fqdn, master, args.yes) + + step(3, ROTATE_TOTAL_STEPS, "Generate a new keypair and rotate the trusted key") + new_priv, new_pub = generate_minion_keypair(key_size=args.key_size, dry_run=args.dry_run) + ok(f"Generated a new {args.key_size}-bit RSA keypair") + + if not confirm(f"Rotate the key for '{container}' against master '{master.get('masterId')}'? " + f"The container will be recreated to pick up the new key.", assume_yes=args.yes): + die("Aborted by user.", code=0) + + if args.dry_run: + minion_id = "" + info("(dry-run) POST /api/salt/minions/rotate") + else: + try: + result = client.rotate_minion_key(master["masterId"], current_pub, new_pub) + except OpsApiError as e: + die(f"Failed to rotate the minion's key: {e}") + minion_id = result.get("minionId") + key_state = (result.get("keyState") or "").upper() + if key_state and key_state != "TRUSTED": + die(f"Rotation did not result in a trusted key (keyState={key_state}): {result}") + ok(f"Key rotated and trusted for minion: {minion_id}") + + step(4, ROTATE_TOTAL_STEPS, "Recreate the container with the new key and wait for reconnect") + inspected = docker_inspect_minion(container, args.dry_run) + info(f"Recreating '{container}' (image={inspected['image']}, volume={inspected['volume']})") + # Clear the OLD PKI files while the container still exists to exec into - + # docker rm does not touch the named volume's contents, and the + # entrypoint only re-seeds when minion.pem is absent (see + # docker_clear_minion_keys()). + docker_clear_minion_keys(container, args.dry_run) + run(["docker", "rm", "-f", container], dry_run=args.dry_run) + + docker_cfg = DockerConfig( + image=inspected["image"], container_name=container, volume=inspected["volume"], + master_fqdn=master["masterFqdn"], master_pubkey_b64=master["masterPublicKey"], minion_id=minion_id, + minion_private_key_b64=base64.b64encode(new_priv.encode()).decode(), + minion_public_key_b64=base64.b64encode(new_pub.encode()).decode(), + ) + docker_start(docker_cfg, dry_run=args.dry_run, assume_yes=True) + ok(f"Container '{container}' restarted with the new keypair") + + connected = wait_until(lambda: docker_is_connected(container, args.dry_run), + timeout=args.connect_timeout, check_interval=args.poll_interval, + message="Waiting for the minion to reconnect", dry_run=args.dry_run) + if not connected: + die(f"Minion did not reconnect within {args.connect_timeout}s after rotation. " + f"The key was already rotated (minion ID {minion_id}) - this points at a network/" + f"connectivity problem, not a trust problem. Check the container's logs.") + ok("Minion reconnected with the new key") + + +def rotate_minion_kubernetes(client: OpsClient, args: argparse.Namespace) -> None: + step(2, ROTATE_TOTAL_STEPS, "Identify the minion and read its current key") + release_name = args.release_name or prompt("Helm release name to rotate", default=DEFAULT_RELEASE_NAME) + namespace = args.namespace or prompt("Namespace", default=DEFAULT_NAMESPACE) + pod_name = kubectl_get_pod_name(namespace, release_name, dry_run=args.dry_run) + + current_pub = kubectl_read_minion_pubkey(namespace, pod_name, args.dry_run) + ok(f"Read current public key from pod '{pod_name}'") + configured_fqdn = kubectl_read_configured_master_fqdn(namespace, pod_name, args.dry_run) + if configured_fqdn: + info(f"This minion is currently configured against master FQDN: {configured_fqdn}") + + master = select_master(client, args) + _confirm_master_matches(configured_fqdn, master, args.yes) + + step(3, ROTATE_TOTAL_STEPS, "Generate a new keypair and rotate the trusted key") + new_priv, new_pub = generate_minion_keypair(key_size=args.key_size, dry_run=args.dry_run) + ok(f"Generated a new {args.key_size}-bit RSA keypair") + + if not confirm(f"Rotate the key for release '{release_name}' (pod {pod_name}) against master " + f"'{master.get('masterId')}'? The Pod will be restarted.", assume_yes=args.yes): + die("Aborted by user.", code=0) + + if args.dry_run: + minion_id = "" + info("(dry-run) POST /api/salt/minions/rotate") + else: + try: + result = client.rotate_minion_key(master["masterId"], current_pub, new_pub) + except OpsApiError as e: + die(f"Failed to rotate the minion's key: {e}") + minion_id = result.get("minionId") + key_state = (result.get("keyState") or "").upper() + if key_state and key_state != "TRUSTED": + die(f"Rotation did not result in a trusted key (keyState={key_state}): {result}") + ok(f"Key rotated and trusted for minion: {minion_id}") + + step(4, ROTATE_TOTAL_STEPS, "Update the Secret, restart the Pod, and wait for reconnect") + minion_key_secret_name = f"{release_name}-minion-key" + kubectl_upsert_minion_key_secret( + namespace, minion_key_secret_name, + minion_private_key_b64=base64.b64encode(new_priv.encode()).decode(), + minion_public_key_b64=base64.b64encode(new_pub.encode()).decode(), + dry_run=args.dry_run, + ) + ok(f"Minion keypair Secret '{minion_key_secret_name}' updated with the new keypair") + + # Clear any persisted PKI files before restarting - see + # kubectl_clear_minion_keys() for why this matters when persistence is + # enabled (PVC-backed pki volume). + kubectl_clear_minion_keys(namespace, pod_name, args.dry_run) + run(["kubectl", "delete", "pod", "-n", namespace, pod_name], dry_run=args.dry_run) + ok(f"Pod '{pod_name}' restart triggered") + + new_pod_name = kubectl_get_pod_name(namespace, release_name, dry_run=args.dry_run) + connected = wait_until(lambda: kubectl_is_connected(namespace, new_pod_name, args.dry_run), + timeout=args.connect_timeout, check_interval=args.poll_interval, + message="Waiting for the minion to reconnect", dry_run=args.dry_run) + if not connected: + die(f"Minion did not reconnect within {args.connect_timeout}s after rotation. " + f"The key was already rotated (minion ID {minion_id}) - check pod {new_pod_name}'s logs.") + ok("Minion reconnected with the new key") + + def main() -> None: args = build_arg_parser().parse_args() log_file = args.log_file or f"vcf-ops-onboard-{datetime.now():%Y%m%d-%H%M%S}.log" setup_logging(log_file, verbose=args.verbose) LOG.info(f"vcf-ops-onboard started, args={vars(args)}") - print(f"{_C.BOLD}VCF Operations - External Minion Onboarding{_C.RESET}") + print(f"{_C.BOLD}VCF Operations - External Minion Management{_C.RESET}") info(f"Logging full step-by-step detail to: {log_file}") if args.dry_run: warn("Running in --dry-run mode: nothing will actually be executed.") + # Chosen up front (before Step 1 is printed) so the "STEP n/total" header + # can show the right total for whichever action is picked. + action = args.action or choose( + "\nWhat would you like to do?", ["configure", "rotate", "list"], default="configure") + total_steps = {"configure": CONFIGURE_TOTAL_STEPS, "rotate": ROTATE_TOTAL_STEPS, + "list": LIST_TOTAL_STEPS}[action] + # ---------------------------------------------------------------- Step 1 - step(1, TOTAL_STEPS, "Connect to VCF Operations") + step(1, total_steps, "Connect to VCF Operations") ops_host = args.ops_host or prompt("VCF Operations FQDN or IP") ops_user = args.ops_user or prompt("Username") ops_password = prompt("Password", secret=True) @@ -1017,8 +1370,24 @@ def main() -> None: die(f"Login failed: {e}") ok(f"Authenticated to {ops_host}") - # ---------------------------------------------------------------- Step 2 - step(2, TOTAL_STEPS, "List Salt masters and select one") + if action == "list": + list_minions_action(client, args) + print(f"\nFull audit log: {log_file}") + return + + if action == "rotate": + deployment = args.deployment or choose( + "\nWhere is the minion currently running?", ["docker", "kubernetes"], default="docker") + if deployment == "docker": + rotate_minion_docker(client, args) + else: + rotate_minion_kubernetes(client, args) + print(f"\n{_C.BOLD}{_C.GREEN}Minion key rotated{_C.RESET}") + print(f"\nFull audit log: {log_file}") + return + + # ------------------------------------------------------- action == "configure" + step(2, total_steps, "List Salt masters and select one") master = select_master(client, args) # ------------------------------------------------- Steps 3-6 (repeatable) From 4a9242dee6928b6a6bafd0a501d6feda6ef50b69 Mon Sep 17 00:00:00 2001 From: Praveen T Date: Thu, 3 Sep 2026 17:33:21 +0530 Subject: [PATCH 6/6] Fix stale/mismatched keys on rotate and configure: auto-resolve master FQDN and clear PKI files before every container start Two related bugs found while validating rotate end-to-end against a real master: - Master FQDN resolution: internal-only master hostnames often only resolve via a static /etc/hosts entry on the host, which Docker containers don't inherit. docker_start() now resolves the master's FQDN on the host itself and passes --add-host automatically, so no manual IP lookup is needed. - Stale PKI files: -v {volume}:/etc/salt/pki/minion can be a named volume or a bind-mounted host path, either of which outlives `docker rm`. docker-entrypoint.sh only seeds the keypair when minion.pem is absent (so a genuine restart keeps its identity), so reusing a volume/path across separate configure/rotate runs silently kept an OLD keypair instead of the one just registered via the API - and a brand-new minion ID still gets auto-accepted by the real master regardless of which key it presents, so the mismatch never surfaced as a rejection. Both docker_start() and rotate's own cleanup now clear minion.pem/minion.pub before every container start, running as root (-u root) since the target can be a root-owned bind-mounted path that the image's default non-root user can't write to. --- salt-minion-vcf/scripts/onboarding/README.md | 20 +++++ .../scripts/onboarding/vcf-ops-onboard.py | 88 +++++++++++++++++-- 2 files changed, 101 insertions(+), 7 deletions(-) diff --git a/salt-minion-vcf/scripts/onboarding/README.md b/salt-minion-vcf/scripts/onboarding/README.md index 1d79c57..7d29a78 100644 --- a/salt-minion-vcf/scripts/onboarding/README.md +++ b/salt-minion-vcf/scripts/onboarding/README.md @@ -167,6 +167,26 @@ anything, `-y` to skip confirmation prompts). ## Things to validate in your own environment +- **PKI volume reuse (Docker)**: a named Docker volume outlives `docker rm` - if you + reuse the same `--volume` (or the default `salt-minion-vcf-pki`) across separate + `configure`/`rotate` runs, `docker-entrypoint.sh`'s own guard (only seed the + keypair if `minion.pem` doesn't already exist, so a genuine restart keeps its + identity) would otherwise silently keep an OLD, unrelated keypair instead of the + one just registered via the API. The script now clears any leftover + `minion.pem`/`minion.pub` from the volume immediately before every container + start, so this can't happen - no action needed on your part, just worth knowing + why a fresh run always gets a fresh identity even if you reuse the same volume + name. +- **Master FQDN resolution (Docker)**: many on-prem/lab masters have internal-only + hostnames (e.g. `*.vrack.vsphere.internal`) that only resolve via a static entry + in the *host's* `/etc/hosts`, not real DNS - and Docker containers don't inherit + that file automatically, which surfaces as the minion logging + `Master hostname: '' not found or not responsive` even though the host + itself can ping it fine. The script resolves the master's FQDN on the host + (the same way `getent hosts`/`ping` would) and automatically passes it to the + container via `--add-host`, so this is handled for you for any master FQDN - + no manual IP lookup needed. If the host itself can't resolve it either, this + is silently skipped and the container falls back to its own DNS as before. - **VCF Operations auth flow**: the script logs in via `POST /suite-api/api/auth/token/acquire` and sends `Authorization: OpsToken ` on subsequent calls - the same pattern diff --git a/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py b/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py index 2d6f029..1a91f68 100755 --- a/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py +++ b/salt-minion-vcf/scripts/onboarding/vcf-ops-onboard.py @@ -76,6 +76,7 @@ import json import logging import shlex +import socket import ssl import subprocess import sys @@ -625,6 +626,58 @@ def docker_container_exists(name: str, dry_run: bool) -> bool: return out.strip() == name +def resolve_master_fqdn_on_host(master_fqdn: str) -> Optional[str]: + """ + Resolves master_fqdn the same way this host's own resolver would (which + covers a static /etc/hosts entry, not just real DNS) - internal-only + hostnames (e.g. "*.vrack.vsphere.internal" in on-prem/lab environments) + are very often only resolvable via such a static entry on the host, and + Docker containers do NOT inherit the host's /etc/hosts automatically. + Returns None (rather than raising) if the host itself can't resolve it + either - in that case the container is no worse off than before, and + presumably relies on the same DNS Docker's embedded resolver forwards to. + """ + try: + return socket.gethostbyname(master_fqdn) + except OSError: + return None + + +def docker_clear_pki_volume(volume: str, image: str, dry_run: bool) -> None: + """ + Ensures the named PKI volume has no leftover minion.pem/minion.pub from a + PREVIOUS container before this one writes a freshly generated keypair + into it. +

+ A named Docker volume outlives `docker rm` - it is not tied to any one + container's lifecycle. docker-entrypoint.sh only pre-seeds the keypair + from SALT_MINION_PRIVATE_KEY_B64/PUBLIC_KEY_B64 when minion.pem doesn't + already exist (so a genuine restart of the SAME identity correctly keeps + it) - but that means reusing a volume name across separate + configure/rotate runs would otherwise silently keep the OLD identity's + key files, and the container would run on a key that was never the one + just registered via createMinion/rotateMinionKey. This can go + unnoticed: a never-before-seen minion ID still gets auto-accepted by the + real Salt master regardless of which key it presents (auto_accept + doesn't cross-check against RaaS's trust store for a fresh ID), so the + mismatch doesn't surface as a rejection - only as a minion whose actual + key silently doesn't match what VCF Operations thinks is trusted for it. +

+ Both configure and rotate always have a genuinely fresh keypair to write + at this point, so clearing first is always correct here - never confirm, + never destructive to anything the user intended to keep. +

+ Runs as root (-u root): the image's default container user is a non-root + uid, and --volume can also be a bind-mounted host path (e.g. /root/keys) + rather than a Docker-managed named volume - such a path is very commonly + root-owned on the host, and the non-root default user would silently + fail (permission denied) to delete anything there, leaving the stale + files in place with no visible error. + """ + run(["docker", "run", "--rm", "-u", "root", "-v", f"{volume}:/etc/salt/pki/minion", image, + "rm", "-f", PKI_MINION_PEM, PKI_MINION_PUB], dry_run=dry_run, check=False) + + def docker_start(cfg: DockerConfig, dry_run: bool, assume_yes: bool = False) -> None: if docker_container_exists(cfg.container_name, dry_run): warn(f"A container named '{cfg.container_name}' already exists " @@ -636,9 +689,28 @@ def docker_start(cfg: DockerConfig, dry_run: bool, assume_yes: bool = False) -> f"Choose a different --container-name or remove it manually with " f"`docker rm -f {cfg.container_name}`.") + docker_clear_pki_volume(cfg.volume, cfg.image, dry_run) + cmd = [ "docker", "run", "-d", "--name", cfg.container_name, + ] + + # Give the container the SAME resolution for the master's FQDN that this + # host already has - whether that's real DNS or (very commonly, for + # internal-only hostnames) just a static /etc/hosts entry that Docker's + # container would otherwise have no way to see. Generic: works for any + # master FQDN, on any host, without the user needing to look up or supply + # an IP themselves. --add-host only takes effect at container creation, + # which is fine here since both configure and rotate always (re)create + # the container. + resolved_master_ip = resolve_master_fqdn_on_host(cfg.master_fqdn) if not dry_run else None + if resolved_master_ip: + info(f"Resolved master FQDN '{cfg.master_fqdn}' to {resolved_master_ip} on this host - " + f"passing --add-host so the container can resolve it too.") + cmd += ["--add-host", f"{cfg.master_fqdn}:{resolved_master_ip}"] + + cmd += [ "-e", f"SALT_MASTER={cfg.master_fqdn}", "-e", f"SALT_MASTER_PUBKEY_B64={cfg.master_pubkey_b64}", "-e", f"SALT_MINION_ID={cfg.minion_id}", @@ -733,13 +805,15 @@ def docker_read_configured_master_fqdn(container: str, dry_run: bool) -> Optiona def docker_clear_minion_keys(container: str, dry_run: bool) -> None: - """Removes the minion's PKI files from the (persistent, named) volume - while the OLD container still exists to exec into. Required before - recreating the container with a new keypair - docker-entrypoint.sh only - seeds SALT_MINION_PRIVATE_KEY_B64/PUBLIC_KEY_B64 into the PKI dir when - minion.pem doesn't already exist, and `docker rm` alone does not remove - the named volume's contents.""" - run(["docker", "exec", container, "rm", "-f", PKI_MINION_PEM, PKI_MINION_PUB], + """Removes the minion's PKI files from the (persistent, named or + bind-mounted) volume while the OLD container still exists to exec into. + Required before recreating the container with a new keypair - + docker-entrypoint.sh only seeds SALT_MINION_PRIVATE_KEY_B64/PUBLIC_KEY_B64 + into the PKI dir when minion.pem doesn't already exist, and `docker rm` + alone does not remove the volume's contents. Runs as root (-u root) - + see docker_clear_pki_volume() for why (default non-root user, possible + root-owned bind-mounted host path).""" + run(["docker", "exec", "-u", "root", container, "rm", "-f", PKI_MINION_PEM, PKI_MINION_PUB], dry_run=dry_run, check=False)