diff --git a/plugins/cozystack/.claude-plugin/plugin.json b/plugins/cozystack/.claude-plugin/plugin.json index 2215e6b..ea8c0f1 100644 --- a/plugins/cozystack/.claude-plugin/plugin.json +++ b/plugins/cozystack/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "cozystack", - "version": "1.15.0", + "version": "1.15.1", "description": "Cozystack platform skills bundle. Start with cozystack:wizard — it begins with a free-form 'tell me about your setup and goal' question, parses hints, then asks Talos / Ubuntu / Existing, builds a chain, and dispatches downstream skills via a cluster config directory the operator picks (every artifact lives there: inventory.yml, kubeconfig, .state.yaml, cozystack-platform-package.yaml — operator manages git on their own; optional sops opt-in encrypts secret files in-tree). Skills, invoked as cozystack:: wizard (orchestrator + 3-route dispatcher + Phase 4.5 active research + auto-dispatches debug on any failed_at), talos-bootstrap (Talos node prep via talm — Talos-1.12-aware maintenance probe, NAT-provider cert-SAN guardrail before first talm apply, multidoc machine-config with per-node VIP-link IPv4 stubs, etcd bootstrap, kubeconfig fetch, cozystack-tuned shape verification with Phase 11.5 auto-upgrade), talos-reset (cloud-provider terminate+relaunch helper for OCI/AWS/GCP/Hetzner when nodes are unrecoverable from inside; preserves block volumes + secondary VNICs + NSG memberships), ubuntu-bootstrap (wraps cozystack/ansible-cozystack — OS prep + k3s install in one go), cluster-install (Cozystack on a ready cluster — node-readiness, ZFS pool provisioning via privileged DaemonSet on Talos with hostNetwork, extractedprism for kube-apiserver HA, OCI-tag-normalized cozy-installer chart, Platform Package, inline tenants/root ingress patch + LINSTOR pool registration during watch loop with combined HRs-Ready + pools-registered gate, Phase 8.6 default StorageClasses for v1.3.x, Phase 9.1 end-to-end reachability probe), debug (investigate a stuck or broken install — gathers symptoms, classifies operator error / config drift / upstream bug / not-yet-supported, applies fixes or workarounds, drafts upstream issues on approval; never opens PRs or files silently), cluster-upgrade (v1.x patch/minor upgrade with release-notes analysis), package-deploy (dev-loop deploy of a single package with ExternalArtifact support), package-bump (single-package version bump with changelog adaptation), external-app-create (scaffold a new external-apps package), dev-ui-bootstrap (bootstrap a UI dev sandbox — Playwright + Vite dev server for cozystack-ui pointed at a chosen kubeconfig; worktree, @playwright/test, Chromium, playwright.config.ts, dev:e2e / test:e2e scripts, kubectl proxy + Vite on a free port). All skills match the operator's natural language detected from conversation context — code identifiers, commands, file paths, and GitHub-public text stay canonical. All skills follow the same gate-and-confirm discipline: read-only lookups run freely; any mutation needs explicit per-step approval.", "author": { "name": "Cozystack", diff --git a/plugins/cozystack/skills/cluster-install/SKILL.md b/plugins/cozystack/skills/cluster-install/SKILL.md index 8605e17..65f5184 100644 --- a/plugins/cozystack/skills/cluster-install/SKILL.md +++ b/plugins/cozystack/skills/cluster-install/SKILL.md @@ -1,6 +1,6 @@ --- name: cluster-install -description: Use when installing Cozystack on an existing Kubernetes cluster (kubeadm / k3s / RKE2 / managed). Discovers cluster facts, validates node readiness via kubectl debug, recommends an installer + platform variant, gathers values interactively, creates the ZFS pool on each storage node through kubectl debug (cozystack standardises on ZFS for LINSTOR — LVM / LVM-thin paths are not supported), installs extractedprism (per-node kube-apiserver HA proxy, generic variant default), installs the cozy-installer chart, applies the Platform Package, patches the root Tenant for ingress (breaks the OIDC chicken-and-egg), waits until every HelmRelease is Ready, prints a NOTES-style access summary, and offers an issue-template handoff to cozystack/* on fatal failure. Not for Kubernetes bootstrap and not for upgrades — see `cozystack:cluster-upgrade` for that. Talos node-prep is out of scope — `cozystack:cluster-install` refuses if Talos nodes are missing cozystack-tuned extensions and points at `cozystack:talos-bootstrap`. +description: Use when installing Cozystack on an existing Kubernetes cluster (kubeadm / k3s / RKE2 / managed). Discovers cluster facts, validates node readiness via kubectl debug, recommends an installer + platform variant, gathers values interactively, creates the ZFS pool on each storage node through kubectl debug (cozystack standardises on ZFS for LINSTOR — LVM / LVM-thin paths are not supported), installs extractedprism (per-node kube-apiserver HA proxy, generic variant default), installs the cozy-installer chart, applies the Platform Package (on current releases OIDC is enabled after convergence, not at install), patches the root Tenant host + ingress, waits until every HelmRelease is Ready, prints a NOTES-style access summary, and offers an issue-template handoff to cozystack/* on fatal failure. Not for Kubernetes bootstrap and not for upgrades — see `cozystack:cluster-upgrade` for that. Talos node-prep is out of scope — `cozystack:cluster-install` refuses if Talos nodes are missing cozystack-tuned extensions and points at `cozystack:talos-bootstrap`. argument-hint: "[--config-dir=] [--context=] [--installer-version=] [--installer-variant=] [--platform-variant=] [--no-extractedprism] [--api-host=] [--dry-run]" --- @@ -411,10 +411,11 @@ actions on Continue: 3. (if cozy-system namespace exists but unowned) adopt namespace into cozy-system/cozy-installer 4. helm upgrade --install cozy-installer ... --namespace cozy-system --create-namespace (~2 min) 5. wait deploy/cozystack-operator Available; wait CRD packages.cozystack.io Established - 6. kubectl apply --filename /tmp/.../platform-package.yaml - 7. wait root Tenant CR, patch spec.host + spec.ingress=true (~3 min — required for Phase 8 to ever finish; breaks the OIDC chicken-and-egg) + 6. kubectl apply --filename /tmp/.../platform-package.yaml (OIDC OFF on installer ≥1.5.0; ON on v1.4.x — see Phase 8 version gate) + 7. wait root Tenant CR, patch spec.host + spec.ingress=true (~3 min — brings up the root ingress with the right domain) 8. poll HRs every 30s until all Ready=True (~30–60 min) - 9. print access summary + 9. (installer ≥1.5.0 + dashboard auth = OIDC; on v1.4.x OIDC is already on from step 6) once HRs are green and root-ingress-controller is up, enable authentication.oidc.enabled + expose keycloak, then wait for keycloak + gatekeeper rollout (~5–10 min) — see Phase 8 + 10. print access summary options: Continue / Edit values / Cancel ``` @@ -678,14 +679,22 @@ See `references/helmrelease-monitoring.md` for the full polling pattern and stuc This phase merges what used to be Phase 7.5 (root Tenant ingress patch) into the watch loop. The motivation: on a real install the `tenants.apps.cozystack.io` CRD can take longer to install than the wait deadline of a separate phase — sometimes the CRD doesn't exist until after most HRs are Ready, so a fixed 5-minute wait-for-Tenant expires before the CRD lands. Folding the patch into the watch loop makes it event-driven: as soon as the CR appears, the skill patches it and continues monitoring, regardless of where Phase 8 is in its own progression. -**Why the patch is needed**: cozystack's dashboard ships gatekeeper (oauth2-proxy) which, on startup, does OIDC discovery against the **public FQDN** `https://keycloak.${HOST}/realms/cozy/.well-known/openid-configuration` — not an in-cluster service. Without the root ingress controller running, nothing listens on 443, gatekeeper CrashLoopBackOffs, the `cozy-dashboard/dashboard` HR sits in `Unknown: Running 'install' action with timeout of 10m0s` and then `InstallFailed: context deadline exceeded`, Flux remediates and retries forever. `cozy-fluxcd/flux-plunger` has a hard dependency on `cozy-dashboard/dashboard` and stays `False: dependency is not ready`. The phase would never go green. +**Why the patch is needed**: the root ingress controller only starts once `tenants.apps.cozystack.io/root` is patched with `spec.host` + `spec.ingress: true`. Until then nothing serves the public wildcard `*.${HOST}` on 443, so every per-tenant ingress object (`dashboard.${HOST}`, `keycloak.${HOST}`, `api.${HOST}`) renders against an empty/parked domain and no public URL works. The root tenant ships with `spec.host: ""` and does NOT inherit `publishing.host` from the Platform Package (verified on 1.6.x), so an ingress-only patch with an empty host is not enough — set both keys. The CR can appear at any point in the watch loop; patch it event-driven as soon as it lands (loop below). This patch is required for correct public URLs and is a precondition for the OIDC flip described next — but it is NOT what unblocks a stalled OIDC-at-install (that is avoided by installing OIDC-off, see below). -**The dashboard requires OIDC/Keycloak — it is not optional on the supported path.** The "Why the patch is needed" note above describes gatekeeper doing OIDC discovery against Keycloak. But Keycloak only deploys when `authentication.oidc.enabled: true` in the Platform Package — and that key defaults to `false`, with the `isp-full*` overlays NOT turning it on. If the skill enables `ingress` + sets `host` but never enables OIDC, the result on v1.4.2 is: no `cozy-keycloak` namespace, no Keycloak HR, and the dashboard falls back to its non-OIDC `token-proxy` container, which is **broken on v1.4.2** — the container starts, never binds `:8000` (connection refused, zero logs), and is killed by its own `/ping` liveness probe every ~45 s → CrashLoopBackOff → the `cozy-dashboard/dashboard` HR fails install → `flux-plunger` and the rest of the chain hang. The cluster reports 88/90 HRs Ready and looks "almost done" forever. +**Version gate.** The OIDC-off-then-flip sequence below is for `installer_version` **≥ 1.5.0**, where the OIDC-off dashboard converges on a working `token-proxy` container (its `/ping` liveness CrashLoop was fixed by a `startupProbe` backstop in v1.5.0). On **v1.4.x** the token-proxy is broken (`/ping` liveness kills the container → CrashLoopBackOff → the dashboard HR fails install), so a working web dashboard there needs OIDC enabled at install — keep `authentication.oidc.enabled: true` in the Package for v1.4.x and, if the install stalls, fall back to the recovery in `references/known-failures.md`. v1.3.x ships no Keycloak in `isp-full` and the dashboard uses a bundled self-issued OIDC (see `references/provider-pitfalls.md`). The rest of this section is the v1.5.0+ path. -So for a usable dashboard the skill must enable OIDC. This is exactly what cozystack's own e2e (`hack/e2e-install-cozystack.bats`) does — patch the root tenant host, then enable OIDC and expose Keycloak: +**Install with OIDC OFF, converge, THEN flip it on (installer_version ≥ 1.5.0).** On 1.6.x, setting `authentication.oidc.enabled: true` in the Platform Package before the platform has converged DEADLOCKS the install. The flag switches the system bundle to the `oidc` engine variant (`packages/core/platform/templates/bundles/system.yaml` selects `cozystack.cozystack-engine` variant `oidc` when `authentication.oidc.enabled`), which pulls Keycloak into the dashboard's dependency chain: on the `oidc` variant the `dashboard` component `dependsOn: keycloak-configure` and the engine itself `dependsOn cozystack.keycloak, cozystack.keycloak-operator` (`packages/core/platform/sources/cozystack-engine.yaml`); on the `default` variant the dashboard depends only on `cozystack-api`/`cozystack-controller`/`application-definition-crd`. The dashboard's gatekeeper (oauth2-proxy) then also needs to reach Keycloak's public URL, which is only served once the root ingress is up — but the root ingress and the root Tenant CR sit behind the dashboard in the reconcile graph. Net cycle: dashboard needs Keycloak + ingress, ingress needs the dashboard-gated chain, so the install plateaus partway (e.g. ~65% of HRs) and never converges. Patching a tenant CR does NOT rescue this — on the `oidc` variant the tenant CR does not even appear until the dashboard-gated chain unblocks, which with OIDC on it never does. + +The underlying deadlock is fixed on `main` (post-v1.6.0-rc.2) by defaulting the gatekeeper's OIDC backend to the in-cluster Keycloak URL, so the gatekeeper skips external discovery and no longer needs the root ingress at startup; this is not yet in a released tag. OIDC-off-then-flip remains the recommended, e2e-verified sequence on all versions. + +**Supported sequence — exactly what cozystack's own e2e (`hack/e2e-install-cozystack.bats`) does:** + +1. Apply the Platform Package with OIDC **off** — the default (`authentication.oidc.enabled` unset / `false`; the `isp-full*` overlays do not turn it on). The whole platform converges and the dashboard comes up on its non-OIDC `token-proxy` container. +2. Patch `tenants/root` `spec.host` + `spec.ingress: true` (the inline watch-loop patch below) and wait until `root-ingress-controller` is Available (`2/2`) in `tenant-root`. +3. Only THEN enable OIDC, in a second step: ```bash -# Enable OIDC and expose keycloak (do this once, after the Package exists). +# Step 3 — after all HRs are Ready AND root-ingress-controller is Available. kubectl --context $CTX patch package cozystack.cozystack-platform --type merge \ --patch '{"spec":{"components":{"platform":{"values":{"authentication":{"oidc":{"enabled":true}}}}}}}' @@ -695,7 +704,21 @@ kubectl --context $CTX patch package cozystack.cozystack-platform --type merge \ --patch '{"spec":{"components":{"platform":{"values":{"publishing":{"exposedServices":["api","dashboard","keycloak"]}}}}}}' ``` -Better: bake both into the Platform Package CR written in Phase 4/7 from the start (`authentication.oidc.enabled: true` and `keycloak` in `exposedServices`) so there is no second reconcile. The Phase 4 intake should collect a **dashboard auth** decision — OIDC/Keycloak (recommended; the only path with a working dashboard on 1.4.2) vs none (API-only, no web dashboard) — and only enable OIDC when the operator wants the dashboard. When OIDC is enabled, Keycloak needs a working LE cert for `keycloak.${HOST}`, so the same DNS/port-80 preconditions as the dashboard host apply (Phase 4 publishing gate already covers this — just make sure `keycloak.${HOST}` is inside the wildcard). +With the root ingress already up, enabling OIDC now brings up Keycloak, swaps the dashboard's `token-proxy` container for the oauth2-proxy gatekeeper, and gatekeeper reaches Keycloak — by default via the in-cluster keycloak Service (`keycloakInternalUrl`), and even when it is pointed at the public `keycloak.${HOST}` URL those in-cluster requests hairpin through the now-running ingress (verified HTTP 200) — so the gatekeeper rolls out cleanly instead of CrashLooping. Gate the flip on the `auth-proxy` container appearing in `deploy/incloud-web-gatekeeper` and its `rollout status`, not on `hr/dashboard` (which stays stale-Ready from the token-proxy install). + +The Phase 4 intake collects a **dashboard auth** decision — OIDC/Keycloak (recommended; a browser SSO dashboard) vs none (API-only; dashboard stays on the token-proxy container). On installer_version ≥ 1.5.0, even when the operator wants OIDC the skill installs OIDC-off first and performs the flip as a post-convergence step (Phase 5 plan item 9), NOT by baking `authentication.oidc.enabled: true` into the Platform Package from the start; on v1.4.x it bakes OIDC on at install per the version gate above. When OIDC is enabled, Keycloak needs a working LE cert for `keycloak.${HOST}`, so the same DNS/port-80 preconditions as the dashboard host apply — make sure `keycloak.${HOST}` is inside the wildcard (Phase 4 publishing gate already covers this). + +**off→on toggle hazard — flip OIDC on exactly once; do not toggle it back off.** Disabling OIDC after it was enabled marks the `keycloak-configure` HR for deletion; its Helm pre-delete teardown Job fails on RBAC and the HR sticks in `Terminating`, with the Helm release secret stuck in status `uninstalling`. Recovery: + +```bash +kubectl --context $CTX --namespace cozy-keycloak patch hr keycloak-configure \ + --type merge --patch '{"metadata":{"finalizers":null}}' +kubectl --context $CTX --namespace cozy-keycloak delete secret \ + --selector owner=helm,name=keycloak-configure,status=uninstalling +flux --context $CTX --namespace cozy-keycloak reconcile hr keycloak-configure --force +``` + +Installing OIDC-off and flipping it on once avoids this churn entirely. Skip the root-Tenant patch entirely on `isp-hosted` or when the `system` bundle was disabled in Phase 4 — there is no root Tenant CR in those modes. @@ -1086,7 +1109,8 @@ If any phase hits a fatal failure that looks like an upstream bug or doc gap, fo - NEVER bootstrap Talos nodes or invoke `boot-to-talos` / `talm` from inside this skill — that flow lives in `/cozystack:talos-bootstrap`. Refuse and hand off. - NEVER auto-rollback a partially provisioned storage state — print backout commands and let the operator decide. - NEVER accept a custom `publishing.host` without an explicit operator confirmation that they own the domain and will configure wildcard DNS — the HTTP-01 cert solver fails silently otherwise. nip.io patterns skip this gate because nip.io is publicly hosted DNS. -- ALWAYS patch `tenants/root` with BOTH `spec.host=$HOST` and `spec.ingress=true` from inside the Phase 8 watch loop as soon as the CR appears, on `system`-bundle installs. The root tenant ships with `spec.host: ""` and does not inherit `publishing.host`, so an ingress-only patch leaves every per-tenant ingress object rendering against an empty domain. The OIDC chicken-and-egg makes Phase 8 unreachable otherwise — dashboard / keycloak / flux-plunger loop forever, every other downstream HR stalls on the missing root ingress. The CR can appear at any point during the watch loop; do not gate the patch behind a fixed pre-Phase-8 wait. +- On `installer_version` ≥ 1.5.0, do NOT set `authentication.oidc.enabled: true` in the Platform Package at install time. On 1.6.x it switches the system bundle to the `oidc` engine variant, which makes the dashboard `dependsOn` Keycloak; the dashboard then can't converge until the root ingress is up, but the root ingress sits behind the dashboard in the reconcile graph — the install deadlocks partway. Install OIDC-off, let the platform converge and `root-ingress-controller` come up, THEN `kubectl patch package … authentication.oidc.enabled=true` exactly once. Flipping OIDC back off strands `keycloak-configure` in `Terminating` (see Phase 8 recovery). Exception: on v1.4.x the OIDC-off token-proxy dashboard is broken, so OIDC must be enabled at install there (Phase 8 version gate). +- ALWAYS patch `tenants/root` with BOTH `spec.host=$HOST` and `spec.ingress=true` from inside the Phase 8 watch loop as soon as the CR appears, on `system`-bundle installs. The root tenant ships with `spec.host: ""` and does not inherit `publishing.host`, so an ingress-only patch leaves every per-tenant ingress object rendering against an empty domain, and the root ingress controller never serves any public URL (`dashboard.${HOST}`, `keycloak.${HOST}`, …). The CR can appear at any point during the watch loop; do not gate the patch behind a fixed pre-Phase-8 wait. This patch is required for correct public URLs and is a precondition for the post-convergence OIDC flip — it is NOT itself the fix for the OIDC-at-install deadlock (installing OIDC-off is). - ALWAYS read variant overlays and `requirements.md` before declaring "this looks fine" — variant-specific checks (CP-label value, ZFS availability, KubeOVN MASTER_NODES) are easy to miss. - ALWAYS pull live data over cached assumption: `kubectl get` over "I think this is …". - ALWAYS write Phase 4 collected values to disk in `/cozystack-platform-package.yaml` before applying — the file is part of the diagnostic bundle if Phase 8 fails. ZFS pool registration is stored separately under `/.state.yaml` `cozystack.storage.nodes[]` and replayed by the Phase 8 post-Ready hook (there is no `LinstorSatelliteConfiguration` CR for the ZFS path). diff --git a/plugins/cozystack/skills/cluster-install/references/known-failures.md b/plugins/cozystack/skills/cluster-install/references/known-failures.md index 965d7a2..be2c796 100644 --- a/plugins/cozystack/skills/cluster-install/references/known-failures.md +++ b/plugins/cozystack/skills/cluster-install/references/known-failures.md @@ -21,9 +21,9 @@ Pod-level look: ```bash kubectl --context $CTX -n cozy-dashboard get pods -# gatekeeper-... CrashLoopBackOff +# incloud-web-gatekeeper-... CrashLoopBackOff -kubectl --context $CTX -n cozy-dashboard logs deploy/gatekeeper | head -20 +kubectl --context $CTX -n cozy-dashboard logs -l app.kubernetes.io/name=gatekeeper --tail=20 # Unable to fetch OIDC well-known: dial tcp :443: connection refused # (or: x509: certificate signed by unknown authority) ``` @@ -40,35 +40,94 @@ This is a chicken-and-egg of the `isp-full*` variant + OIDC combination, not a b - The cozystack dependency graph is built so gatekeeper can't come up before ingress, and dashboard can't come up before gatekeeper. - But flux-plunger waits on dashboard, which waits on ingress, which waits on the missing manual patch. -`cozystack:cluster-install` Phase 8 patches `tenants/root` with both `spec.host` and `spec.ingress=true` inline as soon as the CR appears in the watch loop, which avoids the trap entirely on a fresh install regardless of when the CRD lands relative to other HRs. +**Primary avoidance (installer ≥ 1.5.0) — install OIDC-off, converge, THEN enable OIDC.** On 1.6.x, `authentication.oidc.enabled: true` switches the system bundle to the `oidc` engine variant, which makes the dashboard `dependsOn` Keycloak (`packages/core/platform/sources/cozystack-engine.yaml`). If OIDC is enabled at install the whole chain deadlocks before the root Tenant CR even appears, so patching the tenant cannot rescue it. `cozystack:cluster-install` therefore applies the Platform Package with OIDC **off**, lets every HR converge, patches `tenants/root` (`spec.host` + `spec.ingress=true`) inline as soon as the CR appears, waits for `root-ingress-controller` to be Available, and only THEN flips `authentication.oidc.enabled=true` (matches cozystack's own `hack/e2e-install-cozystack.bats`). The inline tenant patch is for correct public URLs and to unblock the later OIDC flip — it is not itself the deadlock fix. This is the installer_version ≥ 1.5.0 path; on v1.4.x the OIDC-off token-proxy dashboard is broken, so OIDC is enabled at install there instead (SKILL.md Phase 8 version gate). See SKILL.md Phase 8. **Recovery on an install that has already stalled in Phase 8** +On 1.6.x the stall is almost always OIDC enabled at install, and the root Tenant CR never appeared — so patching the tenant is not the first step (the patch has nothing to target). Take OIDC back off in the Package first, so the dashboard-gated chain converges and the Tenant CR shows up: + +```bash +# Only if authentication.oidc.enabled was set at install time. +kubectl --context $CTX patch package cozystack.cozystack-platform --type merge \ + --patch '{"spec":{"components":{"platform":{"values":{"authentication":{"oidc":{"enabled":false}}}}}}}' +# Disabling OIDC can strand keycloak-configure in Terminating; if that HR sticks, +# recover with the off->on toggle steps in SKILL.md Phase 8. +``` + +Then wait for the root Tenant CR and patch host + ingress: + ```bash kubectl --context $CTX --namespace tenant-root wait tenants.apps.cozystack.io/root \ - --for=jsonpath='{.metadata.name}'=root --timeout=300s + --for=jsonpath='{.metadata.name}'=root --timeout=600s kubectl --context $CTX --namespace tenant-root patch tenants.apps.cozystack.io root \ --type=merge --patch "{\"spec\":{\"ingress\":true,\"host\":\"${HOST}\"}}" ``` -Within ~2 min: +Within ~2 min of the patch: - `root-ingress-controller` pods come up in `tenant-root-ingress` namespace. - External IPs from the LB pool get the wildcard ingress. - `dashboard.${HOST}` / `keycloak.${HOST}` become reachable from the public internet. -- gatekeeper does OIDC discovery successfully, exits CrashLoop. -- dashboard / keycloak HRs reach Ready. -- flux-plunger picks up the dashboard dependency, reaches Ready. +- the dashboard (token-proxy container) and the rest of the chain reach Ready; the cluster converges to `N/N Ready`. -The whole cluster typically reaches `N/N Ready` within 5 min of the patch. +Once every HR is Ready and `root-ingress-controller` is Available, re-enable OIDC exactly once (SKILL.md Phase 8 step 3) and wait for the `incloud-web-gatekeeper` rollout. If the cluster was already OIDC-off (the operator followed the skill and only the tenant patch was missing), skip the disable/re-enable steps and apply the tenant patch alone. -If gatekeeper still crashes after the patch, double-check: +If gatekeeper still crashes after re-enabling OIDC, double-check: 1. DNS: `dig +short keycloak.${HOST}` returns the external IPs. 2. cert-manager certificate for `*.${HOST}` is Ready (gatekeeper rejects self-signed unless `insecureSkipVerify: true`). 3. Port 80 is reachable from the public internet (Let's Encrypt HTTP-01 needs this — without a valid cert, gatekeeper TLS verify still fails). +## CNPG webhook unreachable from the apiserver (Cilium host-BPF stale endpoint) + +**Symptom** + +During a fresh install on Cilium + KubeOVN, Keycloak and every postgres-backed HR stall. The apiserver rejects admission on the cnpg (cloudnative-pg) mutating webhook with `connection refused` to the webhook ClusterIP; the webhook's `failurePolicy: Fail` then blocks every `Cluster` create. The webhook is reachable from ordinary pods but NOT from the apiserver. + +```bash +kubectl --context $CTX get events --all-namespaces | grep -i webhook +# ... failed calling webhook "...cnpg...": ... dial tcp :443: connect: connection refused +``` + +**Cause** + +The cnpg operator flaps on leader-election during early KubeOVN convergence and restarts with a new pod IP. Cilium's HOST-side service BPF keeps the stale (dead) backend for the webhook ClusterIP, so host-network clients (the apiserver) dial a dead endpoint while pod-network clients get the fresh one. Not a cnpg bug and not a config error — a Cilium host-BPF staleness window during install. + +**Diagnostic** — the tell is host-network vs pod-network asymmetry. `nc` to the webhook ClusterIP SUCCEEDS from a normal pod but FAILS from a `hostNetwork` pod (the apiserver's path), while cert-manager's webhook ClusterIP and `10.96.0.1` both SUCCEED from that same host-network pod: + +```bash +# WEBHOOK_CIP = ClusterIP of the cnpg webhook Service in cozy-postgres-operator. +kubectl --context $CTX run nc-pod --rm --stdin --tty --restart=Never \ + --image=busybox -- nc -zvw3 $WEBHOOK_CIP 443 # from pod network → OK +kubectl --context $CTX run nc-host --rm --stdin --tty --restart=Never \ + --overrides='{"spec":{"hostNetwork":true}}' \ + --image=busybox -- nc -zvw3 $WEBHOOK_CIP 443 # from host network → refused +``` + +**Recovery** — force endpoint churn so Cilium resyncs the host BPF: + +```bash +kubectl --context $CTX rollout restart deploy/postgres-operator-cloudnative-pg \ + --namespace cozy-postgres-operator + +# Verify the webhook answers from the apiserver's path again: +kubectl --context $CTX create --dry-run=server --filename - <<'EOF' +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: webhook-probe + namespace: default +spec: + instances: 1 + storage: + size: 1Gi +EOF +# Expect: admitted (dry-run), not "connection refused". +``` + +If it recurs, the same restart clears it. + ## LINSTOR: HR stuck with no storage pool registered **Symptom** diff --git a/plugins/cozystack/skills/cluster-install/references/provider-pitfalls.md b/plugins/cozystack/skills/cluster-install/references/provider-pitfalls.md index eca3267..4b56c37 100644 --- a/plugins/cozystack/skills/cluster-install/references/provider-pitfalls.md +++ b/plugins/cozystack/skills/cluster-install/references/provider-pitfalls.md @@ -15,6 +15,27 @@ How `cozystack:cluster-install` Phase 4 catches this: - Reads `cozystack_intake.external_ips.strategy` from the wizard (default `internal` when `intent_hints.platform: oci`). - Validates against `Node.status.addresses`: when `InternalIP` ≠ `ExternalIP` on a NAT-fronted platform, refuses the `external` strategy and explains the failure mode. +## OCI: current Talos disk images don't boot from qcow2 — import as streamOptimized VMDK + +**Scope note**: Talos node imaging is `cozystack:talos-bootstrap` territory, not `cluster-install`. This lives here as a cross-reference because it bites OCI installs before a cluster ever exists — a non-booting image is discovered as "the whole cluster is dead", not as a node-prep detail. + +**Symptom**: a fresh OCI Custom Image built from a Talos `nocloud`/metal `qcow2` (Talos 1.11.5+) never comes up. With UEFI firmware the instance drops to the UEFI shell; with BIOS firmware the serial console stays empty; the Talos API (`:50000`) never answers, so `talm`/`talosctl` time out. Tracked upstream as siderolabs/talos#12557. + +**Mechanism**: OCI's qcow2 import path produces a disk the instance firmware won't boot for current Talos images. The same Talos disk boots fine when imported in a different container format. + +**Fix**: convert the identical Talos disk to a **streamOptimized VMDK** and import it with `--source-image-type VMDK`, launch profile firmware **BIOS**: + +```bash +# Produce a streamOptimized VMDK from the Talos raw/qcow2 disk. +qemu-img convert -O vmdk -o subformat=streamOptimized \ + talos-oci.raw talos-oci.vmdk +# Upload to object storage, then: +# oci compute image import from-object ... --source-image-type VMDK +# and launch instances with the BIOS launch mode (not UEFI). +``` + +**Practice — verify BOOT before recreating the cluster**: import + launch the new image on a single throwaway instance and confirm the serial console shows a Linux boot line and `enabling system extension schematic ` BEFORE wiring the image into IaC and recreating every node. A non-booting image discovered after a full recreate is a far longer outage than a one-instance smoke test. + ## GCP NAT'd external IPs **Symptom**: same as OCI — `Ready=True`, dashboard unreachable, RST/timeout. @@ -92,7 +113,7 @@ The skill's Phase 5.5 step 7 (pre-existing-data check) catches this before `zpoo **Mechanism**: in v1.3.x the `isp-full` overlay does not enable Keycloak; cozystack's dashboard ships its own self-issued OIDC provider (`cozystack-issuer`). Keycloak is opt-in via a separate bundle. v1.4+ may change this. -**Fix**: not really a fix — clarification. The dashboard works without Keycloak via the bundled self-issued OIDC. Operators expecting external SSO need to layer Keycloak themselves (out of scope for v1 of `cozystack:cluster-install`). +**Fix**: version-dependent. On v1.3.x the dashboard works without Keycloak via the bundled self-issued OIDC (`cozystack-issuer`); external SSO needs Keycloak layered by hand. On v1.4.x the OIDC-off token-proxy dashboard is broken (`/ping` liveness CrashLoop), so Keycloak/OIDC must be enabled at install for a working web dashboard. On v1.5.0+ the token-proxy was fixed (a `startupProbe` backstop landed in v1.5.0), so `cozystack:cluster-install` installs with OIDC OFF and flips it on after the platform converges (SKILL.md Phase 8) — not at install, because on 1.6.x OIDC-at-install deadlocks the platform behind the root-ingress/dashboard cycle. ## `api.` ingress speaks TCP passthrough, not HTTP diff --git a/plugins/cozystack/skills/cluster-install/references/values-template.md b/plugins/cozystack/skills/cluster-install/references/values-template.md index 0fcb3a0..15d5a2f 100644 --- a/plugins/cozystack/skills/cluster-install/references/values-template.md +++ b/plugins/cozystack/skills/cluster-install/references/values-template.md @@ -97,13 +97,13 @@ spec: enabled: true authentication: oidc: - enabled: true # REQUIRED for a working web dashboard. - # Defaults to false; the isp-full* overlays do NOT turn it on. - # When false, no Keycloak is deployed and the dashboard falls back - # to its token-proxy container, which is broken on v1.4.2 (never - # binds :8000, CrashLoops on its own liveness probe). Set true to - # deploy Keycloak and switch the dashboard to oauth2-proxy. Omit - # (leave false) only for an API-only install with no web dashboard. + enabled: false # OIDC OFF at install (installer >=1.5.0). + # On >=1.5.0, true at install switches the system bundle to the + # oidc engine variant (dashboard dependsOn Keycloak) and deadlocks + # the install (~65%-plateau); leave false so the platform converges + # on the token-proxy dashboard, then flip true AFTER convergence + # (SKILL.md Phase 8 step 3). On v1.4.x the token-proxy is broken, so + # set this true at install there instead (SKILL.md Phase 8 gate). networking: podCIDR: "10.244.0.0/16" # cozystack default, from packages/core/platform/values.yaml podGateway: "10.244.0.1" # first IP of podCIDR @@ -117,9 +117,9 @@ spec: exposedServices: - api - dashboard - - keycloak # REQUIRED when authentication.oidc.enabled - # is true — gives Keycloak its public - # ingress + LE cert + issuer URL. + - keycloak # Needed once OIDC is flipped on post-convergence + # (SKILL.md Phase 8 step 3): gives Keycloak its + # public ingress + LE cert. Harmless while off. externalIPs: - 192.0.2.10 exposure: externalIPs # or "loadBalancer"