From 53b1a3fd5c9919db80115b70f2d3cb1c10b91efc Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 20 Jul 2026 12:04:05 +0300 Subject: [PATCH] fix(cluster-install): version-gate OIDC install order; add OCI VMDK and cnpg gotchas Correct the OIDC install-order guidance and add two provider gotchas that each cost a long real-world debug session. OIDC install order, version-gated. On installer_version >= 1.5.0, enabling authentication.oidc.enabled in the Platform Package at install time deadlocks the install: the flag switches the system bundle to the oidc engine variant, which makes the dashboard dependsOn Keycloak. The dashboard's gatekeeper then needs the root ingress, but the root ingress and the root Tenant CR sit behind the dashboard in the reconcile graph, so the platform plateaus partway and never converges; the root Tenant CR does not even appear, so patching it cannot rescue the stall. Reframe Phase 4 / Phase 8 to the supported sequence (matching cozystack's own e2e): apply the Platform Package with OIDC off, let every HelmRelease converge, patch the root Tenant host + ingress, wait for the root ingress controller, and only then flip OIDC on. Flip the canonical Platform Package template default to OIDC-off to match (it previously baked OIDC on at install, which was the actual landmine). Present the root Tenant patch as what brings up the root ingress with the right domain, not as the deadlock fix. Document the off->on toggle hazard (disabling OIDC strands keycloak-configure in Terminating) with revision-agnostic recovery, rewrite the stalled-install recovery to disable OIDC first, and correct the gatekeeper Deployment name to incloud-web-gatekeeper. Keep the v1.4.x path intact: on v1.4.x the OIDC-off token-proxy dashboard is broken (/ping liveness CrashLoop, fixed by a startupProbe backstop in v1.5.0), so OIDC must be enabled at install there. The guidance, the values-template default, and the provider-pitfalls Keycloak note are all version-gated (v1.3.x self-issued OIDC / v1.4.x OIDC-at-install / v1.5.0+ OIDC-off-then-flip) rather than stated absolutely. OCI Talos image boot. Current Talos disk images (1.11.5+) do not boot on OCI when imported from qcow2 (UEFI shell / empty BIOS console, API never answers). The same disk imported as a streamOptimized VMDK with firmware BIOS boots fine. Add the workaround and the practice of verifying boot on a throwaway instance before recreating the whole cluster. cnpg webhook unreachable from the apiserver. On Cilium + KubeOVN the cloudnative-pg operator flaps on leader-election during early KubeOVN convergence and restarts with a new pod IP; Cilium's host-side service BPF keeps the stale endpoint, so the apiserver gets connection refused on the webhook ClusterIP (failurePolicy=Fail blocks keycloak and every postgres-backed release) while pods reach it fine. Add the host-network-vs-pod-network diagnostic and the rollout-restart recovery. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- plugins/cozystack/.claude-plugin/plugin.json | 2 +- .../cozystack/skills/cluster-install/SKILL.md | 44 ++++++++--- .../references/known-failures.md | 79 ++++++++++++++++--- .../references/provider-pitfalls.md | 23 +++++- .../references/values-template.md | 20 ++--- 5 files changed, 136 insertions(+), 32 deletions(-) diff --git a/plugins/cozystack/.claude-plugin/plugin.json b/plugins/cozystack/.claude-plugin/plugin.json index 2215e6b..ea8c0f1 100644 --- a/plugins/cozystack/.claude-plugin/plugin.json +++ b/plugins/cozystack/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "cozystack", - "version": "1.15.0", + "version": "1.15.1", "description": "Cozystack platform skills bundle. Start with cozystack:wizard — it begins with a free-form 'tell me about your setup and goal' question, parses hints, then asks Talos / Ubuntu / Existing, builds a chain, and dispatches downstream skills via a cluster config directory the operator picks (every artifact lives there: inventory.yml, kubeconfig, .state.yaml, cozystack-platform-package.yaml — operator manages git on their own; optional sops opt-in encrypts secret files in-tree). Skills, invoked as cozystack:: wizard (orchestrator + 3-route dispatcher + Phase 4.5 active research + auto-dispatches debug on any failed_at), talos-bootstrap (Talos node prep via talm — Talos-1.12-aware maintenance probe, NAT-provider cert-SAN guardrail before first talm apply, multidoc machine-config with per-node VIP-link IPv4 stubs, etcd bootstrap, kubeconfig fetch, cozystack-tuned shape verification with Phase 11.5 auto-upgrade), talos-reset (cloud-provider terminate+relaunch helper for OCI/AWS/GCP/Hetzner when nodes are unrecoverable from inside; preserves block volumes + secondary VNICs + NSG memberships), ubuntu-bootstrap (wraps cozystack/ansible-cozystack — OS prep + k3s install in one go), cluster-install (Cozystack on a ready cluster — node-readiness, ZFS pool provisioning via privileged DaemonSet on Talos with hostNetwork, extractedprism for kube-apiserver HA, OCI-tag-normalized cozy-installer chart, Platform Package, inline tenants/root ingress patch + LINSTOR pool registration during watch loop with combined HRs-Ready + pools-registered gate, Phase 8.6 default StorageClasses for v1.3.x, Phase 9.1 end-to-end reachability probe), debug (investigate a stuck or broken install — gathers symptoms, classifies operator error / config drift / upstream bug / not-yet-supported, applies fixes or workarounds, drafts upstream issues on approval; never opens PRs or files silently), cluster-upgrade (v1.x patch/minor upgrade with release-notes analysis), package-deploy (dev-loop deploy of a single package with ExternalArtifact support), package-bump (single-package version bump with changelog adaptation), external-app-create (scaffold a new external-apps package), dev-ui-bootstrap (bootstrap a UI dev sandbox — Playwright + Vite dev server for cozystack-ui pointed at a chosen kubeconfig; worktree, @playwright/test, Chromium, playwright.config.ts, dev:e2e / test:e2e scripts, kubectl proxy + Vite on a free port). All skills match the operator's natural language detected from conversation context — code identifiers, commands, file paths, and GitHub-public text stay canonical. All skills follow the same gate-and-confirm discipline: read-only lookups run freely; any mutation needs explicit per-step approval.", "author": { "name": "Cozystack", diff --git a/plugins/cozystack/skills/cluster-install/SKILL.md b/plugins/cozystack/skills/cluster-install/SKILL.md index 8605e17..65f5184 100644 --- a/plugins/cozystack/skills/cluster-install/SKILL.md +++ b/plugins/cozystack/skills/cluster-install/SKILL.md @@ -1,6 +1,6 @@ --- name: cluster-install -description: Use when installing Cozystack on an existing Kubernetes cluster (kubeadm / k3s / RKE2 / managed). Discovers cluster facts, validates node readiness via kubectl debug, recommends an installer + platform variant, gathers values interactively, creates the ZFS pool on each storage node through kubectl debug (cozystack standardises on ZFS for LINSTOR — LVM / LVM-thin paths are not supported), installs extractedprism (per-node kube-apiserver HA proxy, generic variant default), installs the cozy-installer chart, applies the Platform Package, patches the root Tenant for ingress (breaks the OIDC chicken-and-egg), waits until every HelmRelease is Ready, prints a NOTES-style access summary, and offers an issue-template handoff to cozystack/* on fatal failure. Not for Kubernetes bootstrap and not for upgrades — see `cozystack:cluster-upgrade` for that. Talos node-prep is out of scope — `cozystack:cluster-install` refuses if Talos nodes are missing cozystack-tuned extensions and points at `cozystack:talos-bootstrap`. +description: Use when installing Cozystack on an existing Kubernetes cluster (kubeadm / k3s / RKE2 / managed). Discovers cluster facts, validates node readiness via kubectl debug, recommends an installer + platform variant, gathers values interactively, creates the ZFS pool on each storage node through kubectl debug (cozystack standardises on ZFS for LINSTOR — LVM / LVM-thin paths are not supported), installs extractedprism (per-node kube-apiserver HA proxy, generic variant default), installs the cozy-installer chart, applies the Platform Package (on current releases OIDC is enabled after convergence, not at install), patches the root Tenant host + ingress, waits until every HelmRelease is Ready, prints a NOTES-style access summary, and offers an issue-template handoff to cozystack/* on fatal failure. Not for Kubernetes bootstrap and not for upgrades — see `cozystack:cluster-upgrade` for that. Talos node-prep is out of scope — `cozystack:cluster-install` refuses if Talos nodes are missing cozystack-tuned extensions and points at `cozystack:talos-bootstrap`. argument-hint: "[--config-dir=] [--context=] [--installer-version=] [--installer-variant=] [--platform-variant=] [--no-extractedprism] [--api-host=] [--dry-run]" --- @@ -411,10 +411,11 @@ actions on Continue: 3. (if cozy-system namespace exists but unowned) adopt namespace into cozy-system/cozy-installer 4. helm upgrade --install cozy-installer ... --namespace cozy-system --create-namespace (~2 min) 5. wait deploy/cozystack-operator Available; wait CRD packages.cozystack.io Established - 6. kubectl apply --filename /tmp/.../platform-package.yaml - 7. wait root Tenant CR, patch spec.host + spec.ingress=true (~3 min — required for Phase 8 to ever finish; breaks the OIDC chicken-and-egg) + 6. kubectl apply --filename /tmp/.../platform-package.yaml (OIDC OFF on installer ≥1.5.0; ON on v1.4.x — see Phase 8 version gate) + 7. wait root Tenant CR, patch spec.host + spec.ingress=true (~3 min — brings up the root ingress with the right domain) 8. poll HRs every 30s until all Ready=True (~30–60 min) - 9. print access summary + 9. (installer ≥1.5.0 + dashboard auth = OIDC; on v1.4.x OIDC is already on from step 6) once HRs are green and root-ingress-controller is up, enable authentication.oidc.enabled + expose keycloak, then wait for keycloak + gatekeeper rollout (~5–10 min) — see Phase 8 + 10. print access summary options: Continue / Edit values / Cancel ``` @@ -678,14 +679,22 @@ See `references/helmrelease-monitoring.md` for the full polling pattern and stuc This phase merges what used to be Phase 7.5 (root Tenant ingress patch) into the watch loop. The motivation: on a real install the `tenants.apps.cozystack.io` CRD can take longer to install than the wait deadline of a separate phase — sometimes the CRD doesn't exist until after most HRs are Ready, so a fixed 5-minute wait-for-Tenant expires before the CRD lands. Folding the patch into the watch loop makes it event-driven: as soon as the CR appears, the skill patches it and continues monitoring, regardless of where Phase 8 is in its own progression. -**Why the patch is needed**: cozystack's dashboard ships gatekeeper (oauth2-proxy) which, on startup, does OIDC discovery against the **public FQDN** `https://keycloak.${HOST}/realms/cozy/.well-known/openid-configuration` — not an in-cluster service. Without the root ingress controller running, nothing listens on 443, gatekeeper CrashLoopBackOffs, the `cozy-dashboard/dashboard` HR sits in `Unknown: Running 'install' action with timeout of 10m0s` and then `InstallFailed: context deadline exceeded`, Flux remediates and retries forever. `cozy-fluxcd/flux-plunger` has a hard dependency on `cozy-dashboard/dashboard` and stays `False: dependency is not ready`. The phase would never go green. +**Why the patch is needed**: the root ingress controller only starts once `tenants.apps.cozystack.io/root` is patched with `spec.host` + `spec.ingress: true`. Until then nothing serves the public wildcard `*.${HOST}` on 443, so every per-tenant ingress object (`dashboard.${HOST}`, `keycloak.${HOST}`, `api.${HOST}`) renders against an empty/parked domain and no public URL works. The root tenant ships with `spec.host: ""` and does NOT inherit `publishing.host` from the Platform Package (verified on 1.6.x), so an ingress-only patch with an empty host is not enough — set both keys. The CR can appear at any point in the watch loop; patch it event-driven as soon as it lands (loop below). This patch is required for correct public URLs and is a precondition for the OIDC flip described next — but it is NOT what unblocks a stalled OIDC-at-install (that is avoided by installing OIDC-off, see below). -**The dashboard requires OIDC/Keycloak — it is not optional on the supported path.** The "Why the patch is needed" note above describes gatekeeper doing OIDC discovery against Keycloak. But Keycloak only deploys when `authentication.oidc.enabled: true` in the Platform Package — and that key defaults to `false`, with the `isp-full*` overlays NOT turning it on. If the skill enables `ingress` + sets `host` but never enables OIDC, the result on v1.4.2 is: no `cozy-keycloak` namespace, no Keycloak HR, and the dashboard falls back to its non-OIDC `token-proxy` container, which is **broken on v1.4.2** — the container starts, never binds `:8000` (connection refused, zero logs), and is killed by its own `/ping` liveness probe every ~45 s → CrashLoopBackOff → the `cozy-dashboard/dashboard` HR fails install → `flux-plunger` and the rest of the chain hang. The cluster reports 88/90 HRs Ready and looks "almost done" forever. +**Version gate.** The OIDC-off-then-flip sequence below is for `installer_version` **≥ 1.5.0**, where the OIDC-off dashboard converges on a working `token-proxy` container (its `/ping` liveness CrashLoop was fixed by a `startupProbe` backstop in v1.5.0). On **v1.4.x** the token-proxy is broken (`/ping` liveness kills the container → CrashLoopBackOff → the dashboard HR fails install), so a working web dashboard there needs OIDC enabled at install — keep `authentication.oidc.enabled: true` in the Package for v1.4.x and, if the install stalls, fall back to the recovery in `references/known-failures.md`. v1.3.x ships no Keycloak in `isp-full` and the dashboard uses a bundled self-issued OIDC (see `references/provider-pitfalls.md`). The rest of this section is the v1.5.0+ path. -So for a usable dashboard the skill must enable OIDC. This is exactly what cozystack's own e2e (`hack/e2e-install-cozystack.bats`) does — patch the root tenant host, then enable OIDC and expose Keycloak: +**Install with OIDC OFF, converge, THEN flip it on (installer_version ≥ 1.5.0).** On 1.6.x, setting `authentication.oidc.enabled: true` in the Platform Package before the platform has converged DEADLOCKS the install. The flag switches the system bundle to the `oidc` engine variant (`packages/core/platform/templates/bundles/system.yaml` selects `cozystack.cozystack-engine` variant `oidc` when `authentication.oidc.enabled`), which pulls Keycloak into the dashboard's dependency chain: on the `oidc` variant the `dashboard` component `dependsOn: keycloak-configure` and the engine itself `dependsOn cozystack.keycloak, cozystack.keycloak-operator` (`packages/core/platform/sources/cozystack-engine.yaml`); on the `default` variant the dashboard depends only on `cozystack-api`/`cozystack-controller`/`application-definition-crd`. The dashboard's gatekeeper (oauth2-proxy) then also needs to reach Keycloak's public URL, which is only served once the root ingress is up — but the root ingress and the root Tenant CR sit behind the dashboard in the reconcile graph. Net cycle: dashboard needs Keycloak + ingress, ingress needs the dashboard-gated chain, so the install plateaus partway (e.g. ~65% of HRs) and never converges. Patching a tenant CR does NOT rescue this — on the `oidc` variant the tenant CR does not even appear until the dashboard-gated chain unblocks, which with OIDC on it never does. + +The underlying deadlock is fixed on `main` (post-v1.6.0-rc.2) by defaulting the gatekeeper's OIDC backend to the in-cluster Keycloak URL, so the gatekeeper skips external discovery and no longer needs the root ingress at startup; this is not yet in a released tag. OIDC-off-then-flip remains the recommended, e2e-verified sequence on all versions. + +**Supported sequence — exactly what cozystack's own e2e (`hack/e2e-install-cozystack.bats`) does:** + +1. Apply the Platform Package with OIDC **off** — the default (`authentication.oidc.enabled` unset / `false`; the `isp-full*` overlays do not turn it on). The whole platform converges and the dashboard comes up on its non-OIDC `token-proxy` container. +2. Patch `tenants/root` `spec.host` + `spec.ingress: true` (the inline watch-loop patch below) and wait until `root-ingress-controller` is Available (`2/2`) in `tenant-root`. +3. Only THEN enable OIDC, in a second step: ```bash -# Enable OIDC and expose keycloak (do this once, after the Package exists). +# Step 3 — after all HRs are Ready AND root-ingress-controller is Available. kubectl --context $CTX patch package cozystack.cozystack-platform --type merge \ --patch '{"spec":{"components":{"platform":{"values":{"authentication":{"oidc":{"enabled":true}}}}}}}' @@ -695,7 +704,21 @@ kubectl --context $CTX patch package cozystack.cozystack-platform --type merge \ --patch '{"spec":{"components":{"platform":{"values":{"publishing":{"exposedServices":["api","dashboard","keycloak"]}}}}}}' ``` -Better: bake both into the Platform Package CR written in Phase 4/7 from the start (`authentication.oidc.enabled: true` and `keycloak` in `exposedServices`) so there is no second reconcile. The Phase 4 intake should collect a **dashboard auth** decision — OIDC/Keycloak (recommended; the only path with a working dashboard on 1.4.2) vs none (API-only, no web dashboard) — and only enable OIDC when the operator wants the dashboard. When OIDC is enabled, Keycloak needs a working LE cert for `keycloak.${HOST}`, so the same DNS/port-80 preconditions as the dashboard host apply (Phase 4 publishing gate already covers this — just make sure `keycloak.${HOST}` is inside the wildcard). +With the root ingress already up, enabling OIDC now brings up Keycloak, swaps the dashboard's `token-proxy` container for the oauth2-proxy gatekeeper, and gatekeeper reaches Keycloak — by default via the in-cluster keycloak Service (`keycloakInternalUrl`), and even when it is pointed at the public `keycloak.${HOST}` URL those in-cluster requests hairpin through the now-running ingress (verified HTTP 200) — so the gatekeeper rolls out cleanly instead of CrashLooping. Gate the flip on the `auth-proxy` container appearing in `deploy/incloud-web-gatekeeper` and its `rollout status`, not on `hr/dashboard` (which stays stale-Ready from the token-proxy install). + +The Phase 4 intake collects a **dashboard auth** decision — OIDC/Keycloak (recommended; a browser SSO dashboard) vs none (API-only; dashboard stays on the token-proxy container). On installer_version ≥ 1.5.0, even when the operator wants OIDC the skill installs OIDC-off first and performs the flip as a post-convergence step (Phase 5 plan item 9), NOT by baking `authentication.oidc.enabled: true` into the Platform Package from the start; on v1.4.x it bakes OIDC on at install per the version gate above. When OIDC is enabled, Keycloak needs a working LE cert for `keycloak.${HOST}`, so the same DNS/port-80 preconditions as the dashboard host apply — make sure `keycloak.${HOST}` is inside the wildcard (Phase 4 publishing gate already covers this). + +**off→on toggle hazard — flip OIDC on exactly once; do not toggle it back off.** Disabling OIDC after it was enabled marks the `keycloak-configure` HR for deletion; its Helm pre-delete teardown Job fails on RBAC and the HR sticks in `Terminating`, with the Helm release secret stuck in status `uninstalling`. Recovery: + +```bash +kubectl --context $CTX --namespace cozy-keycloak patch hr keycloak-configure \ + --type merge --patch '{"metadata":{"finalizers":null}}' +kubectl --context $CTX --namespace cozy-keycloak delete secret \ + --selector owner=helm,name=keycloak-configure,status=uninstalling +flux --context $CTX --namespace cozy-keycloak reconcile hr keycloak-configure --force +``` + +Installing OIDC-off and flipping it on once avoids this churn entirely. Skip the root-Tenant patch entirely on `isp-hosted` or when the `system` bundle was disabled in Phase 4 — there is no root Tenant CR in those modes. @@ -1086,7 +1109,8 @@ If any phase hits a fatal failure that looks like an upstream bug or doc gap, fo - NEVER bootstrap Talos nodes or invoke `boot-to-talos` / `talm` from inside this skill — that flow lives in `/cozystack:talos-bootstrap`. Refuse and hand off. - NEVER auto-rollback a partially provisioned storage state — print backout commands and let the operator decide. - NEVER accept a custom `publishing.host` without an explicit operator confirmation that they own the domain and will configure wildcard DNS — the HTTP-01 cert solver fails silently otherwise. nip.io patterns skip this gate because nip.io is publicly hosted DNS. -- ALWAYS patch `tenants/root` with BOTH `spec.host=$HOST` and `spec.ingress=true` from inside the Phase 8 watch loop as soon as the CR appears, on `system`-bundle installs. The root tenant ships with `spec.host: ""` and does not inherit `publishing.host`, so an ingress-only patch leaves every per-tenant ingress object rendering against an empty domain. The OIDC chicken-and-egg makes Phase 8 unreachable otherwise — dashboard / keycloak / flux-plunger loop forever, every other downstream HR stalls on the missing root ingress. The CR can appear at any point during the watch loop; do not gate the patch behind a fixed pre-Phase-8 wait. +- On `installer_version` ≥ 1.5.0, do NOT set `authentication.oidc.enabled: true` in the Platform Package at install time. On 1.6.x it switches the system bundle to the `oidc` engine variant, which makes the dashboard `dependsOn` Keycloak; the dashboard then can't converge until the root ingress is up, but the root ingress sits behind the dashboard in the reconcile graph — the install deadlocks partway. Install OIDC-off, let the platform converge and `root-ingress-controller` come up, THEN `kubectl patch package … authentication.oidc.enabled=true` exactly once. Flipping OIDC back off strands `keycloak-configure` in `Terminating` (see Phase 8 recovery). Exception: on v1.4.x the OIDC-off token-proxy dashboard is broken, so OIDC must be enabled at install there (Phase 8 version gate). +- ALWAYS patch `tenants/root` with BOTH `spec.host=$HOST` and `spec.ingress=true` from inside the Phase 8 watch loop as soon as the CR appears, on `system`-bundle installs. The root tenant ships with `spec.host: ""` and does not inherit `publishing.host`, so an ingress-only patch leaves every per-tenant ingress object rendering against an empty domain, and the root ingress controller never serves any public URL (`dashboard.${HOST}`, `keycloak.${HOST}`, …). The CR can appear at any point during the watch loop; do not gate the patch behind a fixed pre-Phase-8 wait. This patch is required for correct public URLs and is a precondition for the post-convergence OIDC flip — it is NOT itself the fix for the OIDC-at-install deadlock (installing OIDC-off is). - ALWAYS read variant overlays and `requirements.md` before declaring "this looks fine" — variant-specific checks (CP-label value, ZFS availability, KubeOVN MASTER_NODES) are easy to miss. - ALWAYS pull live data over cached assumption: `kubectl get` over "I think this is …". - ALWAYS write Phase 4 collected values to disk in `/cozystack-platform-package.yaml` before applying — the file is part of the diagnostic bundle if Phase 8 fails. ZFS pool registration is stored separately under `/.state.yaml` `cozystack.storage.nodes[]` and replayed by the Phase 8 post-Ready hook (there is no `LinstorSatelliteConfiguration` CR for the ZFS path). diff --git a/plugins/cozystack/skills/cluster-install/references/known-failures.md b/plugins/cozystack/skills/cluster-install/references/known-failures.md index 965d7a2..be2c796 100644 --- a/plugins/cozystack/skills/cluster-install/references/known-failures.md +++ b/plugins/cozystack/skills/cluster-install/references/known-failures.md @@ -21,9 +21,9 @@ Pod-level look: ```bash kubectl --context $CTX -n cozy-dashboard get pods -# gatekeeper-... CrashLoopBackOff +# incloud-web-gatekeeper-... CrashLoopBackOff -kubectl --context $CTX -n cozy-dashboard logs deploy/gatekeeper | head -20 +kubectl --context $CTX -n cozy-dashboard logs -l app.kubernetes.io/name=gatekeeper --tail=20 # Unable to fetch OIDC well-known: dial tcp :443: connection refused # (or: x509: certificate signed by unknown authority) ``` @@ -40,35 +40,94 @@ This is a chicken-and-egg of the `isp-full*` variant + OIDC combination, not a b - The cozystack dependency graph is built so gatekeeper can't come up before ingress, and dashboard can't come up before gatekeeper. - But flux-plunger waits on dashboard, which waits on ingress, which waits on the missing manual patch. -`cozystack:cluster-install` Phase 8 patches `tenants/root` with both `spec.host` and `spec.ingress=true` inline as soon as the CR appears in the watch loop, which avoids the trap entirely on a fresh install regardless of when the CRD lands relative to other HRs. +**Primary avoidance (installer ≥ 1.5.0) — install OIDC-off, converge, THEN enable OIDC.** On 1.6.x, `authentication.oidc.enabled: true` switches the system bundle to the `oidc` engine variant, which makes the dashboard `dependsOn` Keycloak (`packages/core/platform/sources/cozystack-engine.yaml`). If OIDC is enabled at install the whole chain deadlocks before the root Tenant CR even appears, so patching the tenant cannot rescue it. `cozystack:cluster-install` therefore applies the Platform Package with OIDC **off**, lets every HR converge, patches `tenants/root` (`spec.host` + `spec.ingress=true`) inline as soon as the CR appears, waits for `root-ingress-controller` to be Available, and only THEN flips `authentication.oidc.enabled=true` (matches cozystack's own `hack/e2e-install-cozystack.bats`). The inline tenant patch is for correct public URLs and to unblock the later OIDC flip — it is not itself the deadlock fix. This is the installer_version ≥ 1.5.0 path; on v1.4.x the OIDC-off token-proxy dashboard is broken, so OIDC is enabled at install there instead (SKILL.md Phase 8 version gate). See SKILL.md Phase 8. **Recovery on an install that has already stalled in Phase 8** +On 1.6.x the stall is almost always OIDC enabled at install, and the root Tenant CR never appeared — so patching the tenant is not the first step (the patch has nothing to target). Take OIDC back off in the Package first, so the dashboard-gated chain converges and the Tenant CR shows up: + +```bash +# Only if authentication.oidc.enabled was set at install time. +kubectl --context $CTX patch package cozystack.cozystack-platform --type merge \ + --patch '{"spec":{"components":{"platform":{"values":{"authentication":{"oidc":{"enabled":false}}}}}}}' +# Disabling OIDC can strand keycloak-configure in Terminating; if that HR sticks, +# recover with the off->on toggle steps in SKILL.md Phase 8. +``` + +Then wait for the root Tenant CR and patch host + ingress: + ```bash kubectl --context $CTX --namespace tenant-root wait tenants.apps.cozystack.io/root \ - --for=jsonpath='{.metadata.name}'=root --timeout=300s + --for=jsonpath='{.metadata.name}'=root --timeout=600s kubectl --context $CTX --namespace tenant-root patch tenants.apps.cozystack.io root \ --type=merge --patch "{\"spec\":{\"ingress\":true,\"host\":\"${HOST}\"}}" ``` -Within ~2 min: +Within ~2 min of the patch: - `root-ingress-controller` pods come up in `tenant-root-ingress` namespace. - External IPs from the LB pool get the wildcard ingress. - `dashboard.${HOST}` / `keycloak.${HOST}` become reachable from the public internet. -- gatekeeper does OIDC discovery successfully, exits CrashLoop. -- dashboard / keycloak HRs reach Ready. -- flux-plunger picks up the dashboard dependency, reaches Ready. +- the dashboard (token-proxy container) and the rest of the chain reach Ready; the cluster converges to `N/N Ready`. -The whole cluster typically reaches `N/N Ready` within 5 min of the patch. +Once every HR is Ready and `root-ingress-controller` is Available, re-enable OIDC exactly once (SKILL.md Phase 8 step 3) and wait for the `incloud-web-gatekeeper` rollout. If the cluster was already OIDC-off (the operator followed the skill and only the tenant patch was missing), skip the disable/re-enable steps and apply the tenant patch alone. -If gatekeeper still crashes after the patch, double-check: +If gatekeeper still crashes after re-enabling OIDC, double-check: 1. DNS: `dig +short keycloak.${HOST}` returns the external IPs. 2. cert-manager certificate for `*.${HOST}` is Ready (gatekeeper rejects self-signed unless `insecureSkipVerify: true`). 3. Port 80 is reachable from the public internet (Let's Encrypt HTTP-01 needs this — without a valid cert, gatekeeper TLS verify still fails). +## CNPG webhook unreachable from the apiserver (Cilium host-BPF stale endpoint) + +**Symptom** + +During a fresh install on Cilium + KubeOVN, Keycloak and every postgres-backed HR stall. The apiserver rejects admission on the cnpg (cloudnative-pg) mutating webhook with `connection refused` to the webhook ClusterIP; the webhook's `failurePolicy: Fail` then blocks every `Cluster` create. The webhook is reachable from ordinary pods but NOT from the apiserver. + +```bash +kubectl --context $CTX get events --all-namespaces | grep -i webhook +# ... failed calling webhook "...cnpg...": ... dial tcp :443: connect: connection refused +``` + +**Cause** + +The cnpg operator flaps on leader-election during early KubeOVN convergence and restarts with a new pod IP. Cilium's HOST-side service BPF keeps the stale (dead) backend for the webhook ClusterIP, so host-network clients (the apiserver) dial a dead endpoint while pod-network clients get the fresh one. Not a cnpg bug and not a config error — a Cilium host-BPF staleness window during install. + +**Diagnostic** — the tell is host-network vs pod-network asymmetry. `nc` to the webhook ClusterIP SUCCEEDS from a normal pod but FAILS from a `hostNetwork` pod (the apiserver's path), while cert-manager's webhook ClusterIP and `10.96.0.1` both SUCCEED from that same host-network pod: + +```bash +# WEBHOOK_CIP = ClusterIP of the cnpg webhook Service in cozy-postgres-operator. +kubectl --context $CTX run nc-pod --rm --stdin --tty --restart=Never \ + --image=busybox -- nc -zvw3 $WEBHOOK_CIP 443 # from pod network → OK +kubectl --context $CTX run nc-host --rm --stdin --tty --restart=Never \ + --overrides='{"spec":{"hostNetwork":true}}' \ + --image=busybox -- nc -zvw3 $WEBHOOK_CIP 443 # from host network → refused +``` + +**Recovery** — force endpoint churn so Cilium resyncs the host BPF: + +```bash +kubectl --context $CTX rollout restart deploy/postgres-operator-cloudnative-pg \ + --namespace cozy-postgres-operator + +# Verify the webhook answers from the apiserver's path again: +kubectl --context $CTX create --dry-run=server --filename - <<'EOF' +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: webhook-probe + namespace: default +spec: + instances: 1 + storage: + size: 1Gi +EOF +# Expect: admitted (dry-run), not "connection refused". +``` + +If it recurs, the same restart clears it. + ## LINSTOR: HR stuck with no storage pool registered **Symptom** diff --git a/plugins/cozystack/skills/cluster-install/references/provider-pitfalls.md b/plugins/cozystack/skills/cluster-install/references/provider-pitfalls.md index eca3267..4b56c37 100644 --- a/plugins/cozystack/skills/cluster-install/references/provider-pitfalls.md +++ b/plugins/cozystack/skills/cluster-install/references/provider-pitfalls.md @@ -15,6 +15,27 @@ How `cozystack:cluster-install` Phase 4 catches this: - Reads `cozystack_intake.external_ips.strategy` from the wizard (default `internal` when `intent_hints.platform: oci`). - Validates against `Node.status.addresses`: when `InternalIP` ≠ `ExternalIP` on a NAT-fronted platform, refuses the `external` strategy and explains the failure mode. +## OCI: current Talos disk images don't boot from qcow2 — import as streamOptimized VMDK + +**Scope note**: Talos node imaging is `cozystack:talos-bootstrap` territory, not `cluster-install`. This lives here as a cross-reference because it bites OCI installs before a cluster ever exists — a non-booting image is discovered as "the whole cluster is dead", not as a node-prep detail. + +**Symptom**: a fresh OCI Custom Image built from a Talos `nocloud`/metal `qcow2` (Talos 1.11.5+) never comes up. With UEFI firmware the instance drops to the UEFI shell; with BIOS firmware the serial console stays empty; the Talos API (`:50000`) never answers, so `talm`/`talosctl` time out. Tracked upstream as siderolabs/talos#12557. + +**Mechanism**: OCI's qcow2 import path produces a disk the instance firmware won't boot for current Talos images. The same Talos disk boots fine when imported in a different container format. + +**Fix**: convert the identical Talos disk to a **streamOptimized VMDK** and import it with `--source-image-type VMDK`, launch profile firmware **BIOS**: + +```bash +# Produce a streamOptimized VMDK from the Talos raw/qcow2 disk. +qemu-img convert -O vmdk -o subformat=streamOptimized \ + talos-oci.raw talos-oci.vmdk +# Upload to object storage, then: +# oci compute image import from-object ... --source-image-type VMDK +# and launch instances with the BIOS launch mode (not UEFI). +``` + +**Practice — verify BOOT before recreating the cluster**: import + launch the new image on a single throwaway instance and confirm the serial console shows a Linux boot line and `enabling system extension schematic ` BEFORE wiring the image into IaC and recreating every node. A non-booting image discovered after a full recreate is a far longer outage than a one-instance smoke test. + ## GCP NAT'd external IPs **Symptom**: same as OCI — `Ready=True`, dashboard unreachable, RST/timeout. @@ -92,7 +113,7 @@ The skill's Phase 5.5 step 7 (pre-existing-data check) catches this before `zpoo **Mechanism**: in v1.3.x the `isp-full` overlay does not enable Keycloak; cozystack's dashboard ships its own self-issued OIDC provider (`cozystack-issuer`). Keycloak is opt-in via a separate bundle. v1.4+ may change this. -**Fix**: not really a fix — clarification. The dashboard works without Keycloak via the bundled self-issued OIDC. Operators expecting external SSO need to layer Keycloak themselves (out of scope for v1 of `cozystack:cluster-install`). +**Fix**: version-dependent. On v1.3.x the dashboard works without Keycloak via the bundled self-issued OIDC (`cozystack-issuer`); external SSO needs Keycloak layered by hand. On v1.4.x the OIDC-off token-proxy dashboard is broken (`/ping` liveness CrashLoop), so Keycloak/OIDC must be enabled at install for a working web dashboard. On v1.5.0+ the token-proxy was fixed (a `startupProbe` backstop landed in v1.5.0), so `cozystack:cluster-install` installs with OIDC OFF and flips it on after the platform converges (SKILL.md Phase 8) — not at install, because on 1.6.x OIDC-at-install deadlocks the platform behind the root-ingress/dashboard cycle. ## `api.` ingress speaks TCP passthrough, not HTTP diff --git a/plugins/cozystack/skills/cluster-install/references/values-template.md b/plugins/cozystack/skills/cluster-install/references/values-template.md index 0fcb3a0..15d5a2f 100644 --- a/plugins/cozystack/skills/cluster-install/references/values-template.md +++ b/plugins/cozystack/skills/cluster-install/references/values-template.md @@ -97,13 +97,13 @@ spec: enabled: true authentication: oidc: - enabled: true # REQUIRED for a working web dashboard. - # Defaults to false; the isp-full* overlays do NOT turn it on. - # When false, no Keycloak is deployed and the dashboard falls back - # to its token-proxy container, which is broken on v1.4.2 (never - # binds :8000, CrashLoops on its own liveness probe). Set true to - # deploy Keycloak and switch the dashboard to oauth2-proxy. Omit - # (leave false) only for an API-only install with no web dashboard. + enabled: false # OIDC OFF at install (installer >=1.5.0). + # On >=1.5.0, true at install switches the system bundle to the + # oidc engine variant (dashboard dependsOn Keycloak) and deadlocks + # the install (~65%-plateau); leave false so the platform converges + # on the token-proxy dashboard, then flip true AFTER convergence + # (SKILL.md Phase 8 step 3). On v1.4.x the token-proxy is broken, so + # set this true at install there instead (SKILL.md Phase 8 gate). networking: podCIDR: "10.244.0.0/16" # cozystack default, from packages/core/platform/values.yaml podGateway: "10.244.0.1" # first IP of podCIDR @@ -117,9 +117,9 @@ spec: exposedServices: - api - dashboard - - keycloak # REQUIRED when authentication.oidc.enabled - # is true — gives Keycloak its public - # ingress + LE cert + issuer URL. + - keycloak # Needed once OIDC is flipped on post-convergence + # (SKILL.md Phase 8 step 3): gives Keycloak its + # public ingress + LE cert. Harmless while off. externalIPs: - 192.0.2.10 exposure: externalIPs # or "loadBalancer"