From 000e5d9272cae48ea8e144fe4804c06f24fb9044 Mon Sep 17 00:00:00 2001 From: Lucas Koontz Date: Wed, 5 Aug 2026 13:21:23 -0700 Subject: [PATCH 1/4] fix: improve release notification sensitivity during freeze cycles Adds logic to distinguish release-blocking failures based on freeze status, sending alerts only during active staging freezes. It introduces configurable alert behavior outside freeze periods to prevent alert fatigue. Enhances ArgoCD PR environment deploy action with configurable timeout and provides detailed diagnostics on failures. --- .github/workflows/notify-main-failure.yml | 122 +++++++++++++++++++++- argocd-pr-env-deploy/action.yml | 35 ++++++- 2 files changed, 149 insertions(+), 8 deletions(-) diff --git a/.github/workflows/notify-main-failure.yml b/.github/workflows/notify-main-failure.yml index 08de674..739d20d 100644 --- a/.github/workflows/notify-main-failure.yml +++ b/.github/workflows/notify-main-failure.yml @@ -16,6 +16,16 @@ # every merge. This is derived from the GitHub API rather than cross-run # state, so it is self-healing (no cache to go stale). # +# `freeze-scoped: true` adds a third outcome for STAGING callers. A staging +# failure is only release-blocking once the freeze window is open; mid-week it is +# an ordinary integration-branch red, and paging the channel for it is what +# teaches people to ignore the channel. So a freeze-scoped failure posts the red +# alert while the `staging-freeze` ruleset is active, and outside the freeze +# posts a muted grey notice instead (`outside-freeze: silent` drops it entirely, +# at the cost of a broken staging branch being able to sit red unnoticed until +# the next freeze opens). Prod and freeze/unfreeze callers leave this alone — a +# failure there is always worth interrupting for. +# # It posts its own message rather than the deploy-notification composite, whose # copy is deploy-specific ("has failed deploying to ...") and reads wrong for # non-deploy pipelines (freeze, CI, docs). It uses the same Slack bot token. @@ -64,6 +74,14 @@ on: description: "'failed' (red alert) or 'recovered' (green, only if the prior run was failing)" type: string default: failed + freeze-scoped: + description: "Scale a 'failed' alert by release-freeze state (see `outside-freeze`). For staging pipelines, where a failure is only release-blocking once the freeze window is open." + type: boolean + default: false + outside-freeze: + description: "What a freeze-scoped failure does while staging is NOT frozen: 'notice' (muted, no panic styling) or 'silent' (post nothing)." + type: string + default: notice runs-on: description: "Runner label for the notify job" type: string @@ -178,6 +196,100 @@ jobs: ;; esac + # A staging pipeline failure means different things depending on where the + # release train is. Once the freeze window is open staging IS the release + # candidate, so a red pipeline blocks the release and is worth interrupting + # the channel for. Mid-week it is the integration branch and the same red + # is routine — panic styling there just teaches people to scroll past it. + # + # The freeze signal is the `staging-freeze` repository ruleset that the + # freeze/unfreeze workflows toggle between `active` and `disabled`, not an + # inference from workflow-run history: a freeze that skipped itself because + # staging had nothing unreleased still concludes `success`, and history + # cannot tell that apart from a real freeze. + # + # Reading rulesets needs admin, which the default workflow token does not + # carry, so this reuses the App that owns the toggle. Every failure path + # below falls through to `frozen=true`: a notify job that cannot establish + # the freeze state must escalate rather than silently downgrade a real + # release-blocking failure. + - name: Mint release-train App token + id: freeze-token + if: inputs.freeze-scoped && inputs.status != 'recovered' + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.RELEASE_APP_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + - name: Check release-freeze state + id: freeze + if: inputs.freeze-scoped && inputs.status != 'recovered' + env: + GH_TOKEN: ${{ steps.freeze-token.outputs.token }} + REPO: ${{ github.repository }} + RULESET_NAME: staging-freeze + OUTSIDE_FREEZE: ${{ inputs.outside-freeze }} + run: | + set -uo pipefail + + escalate() { + echo "::warning::$1 Treating staging as frozen so this failure still escalates." + echo "frozen=true" >> "$GITHUB_OUTPUT" + exit 0 + } + + [ -n "${GH_TOKEN:-}" ] || escalate "Could not mint the release-train App token." + + ENFORCEMENT=$(gh api "repos/${REPO}/rulesets" \ + --jq ".[] | select(.name == \"${RULESET_NAME}\") | .enforcement" 2>&1) \ + || escalate "Could not read rulesets: ${ENFORCEMENT}." + [ -n "${ENFORCEMENT}" ] || escalate "Ruleset '${RULESET_NAME}' not found." + + if [ "${ENFORCEMENT}" = "active" ]; then + echo "frozen=true" >> "$GITHUB_OUTPUT" + echo "Ruleset '${RULESET_NAME}' is active — staging is frozen, escalating." + else + echo "frozen=false" >> "$GITHUB_OUTPUT" + echo "Ruleset '${RULESET_NAME}' is '${ENFORCEMENT}' — staging is not frozen, downgrading to '${OUTSIDE_FREEZE}'." + fi + + # Collapse the three outcomes (recovered / failed / failed-outside-freeze) + # into flat strings. Deciding this in shell keeps the Slack payload from + # growing a second and third nested ternary per field, which is where this + # kind of template stops being reviewable. + - name: Compose message + id: msg + env: + STATUS: ${{ inputs.status }} + FREEZE_SCOPED: ${{ inputs.freeze-scoped }} + FROZEN: ${{ steps.freeze.outputs.frozen }} + OUTSIDE_FREEZE: ${{ inputs.outside-freeze }} + run: | + set -euo pipefail + POST=true + if [ "${STATUS}" = "recovered" ]; then + LEVEL=recovered; COLOR='#00C851'; ICON=':white_check_mark:'; VERB=recovered + PREFIX=Recovered; NOTE='' + elif [ "${FREEZE_SCOPED}" = "true" ] && [ "${FROZEN}" = "false" ]; then + LEVEL=notice; COLOR='#B0B4BA'; ICON=':warning:'; VERB=failed + PREFIX=Failed; NOTE=' (outside the release freeze, not release-blocking)' + if [ "${OUTSIDE_FREEZE}" = "silent" ]; then POST=false; fi + else + LEVEL=alert; COLOR='#FF4444'; ICON=':rotating_light:'; VERB=failed + PREFIX=FAILED; NOTE='' + fi + { + echo "post=${POST}" + echo "level=${LEVEL}" + echo "color=${COLOR}" + echo "icon=${ICON}" + echo "verb=${VERB}" + echo "prefix=${PREFIX}" + echo "note=${NOTE}" + } >> "$GITHUB_OUTPUT" + echo "level=${LEVEL} post=${POST}" + - name: Notify Slack # Failure mode always posts; on a real pipeline, recovery posts only after # a failing run; a manual dispatch always posts (smoke test). @@ -187,22 +299,24 @@ jobs: # reader, and a notification-list preview show, so without it the alert # arrives on a phone as an empty message — which is the one context where # a pipeline failure most needs to be readable. - if: inputs.status != 'recovered' || github.event_name == 'workflow_dispatch' || steps.prev.outputs.should_post == 'true' + # `steps.msg.outputs.post` carries the one extra veto: a freeze-scoped + # failure outside the freeze window when the caller asked for silence. + if: steps.msg.outputs.post == 'true' && (inputs.status != 'recovered' || github.event_name == 'workflow_dispatch' || steps.prev.outputs.should_post == 'true') uses: slackapi/slack-github-action@v1.26.0 with: channel-id: ${{ secrets.SLACK_ENG_CHANNEL_ID }} payload: | { - "text": "${{ inputs.status == 'recovered' && 'Recovered' || 'FAILED' }}: ${{ github.event.repository.name }} ${{ inputs.env-name }} pipeline on ${{ github.ref_name }}", + "text": "${{ steps.msg.outputs.prefix }}: ${{ github.event.repository.name }} ${{ inputs.env-name }} pipeline on ${{ github.ref_name }}${{ steps.msg.outputs.note }}", "attachments": [ { - "color": "${{ inputs.status == 'recovered' && '#00C851' || '#FF4444' }}", + "color": "${{ steps.msg.outputs.color }}", "blocks": [ { "type": "section", "text": { "type": "mrkdwn", - "text": "${{ inputs.status == 'recovered' && ':white_check_mark:' || ':rotating_light:' }} *<${{ github.server_url }}/${{ github.repository }}|${{ github.event.repository.name }}>* — *${{ inputs.env-name }}* pipeline ${{ inputs.status == 'recovered' && 'recovered' || 'failed' }} on `${{ github.ref_name }}`" + "text": "${{ steps.msg.outputs.icon }} *<${{ github.server_url }}/${{ github.repository }}|${{ github.event.repository.name }}>* — *${{ inputs.env-name }}* pipeline ${{ steps.msg.outputs.verb }} on `${{ github.ref_name }}`${{ steps.msg.outputs.note }}" }, "fields": [ { diff --git a/argocd-pr-env-deploy/action.yml b/argocd-pr-env-deploy/action.yml index 867d113..00dd56f 100644 --- a/argocd-pr-env-deploy/action.yml +++ b/argocd-pr-env-deploy/action.yml @@ -33,8 +33,10 @@ # Issues a single `argocd app set` against the parent Application # (`pr--`) with `--helm-set tags.=development-` per # resolved repo, kicks the auto-sync via `argocd app sync --async`, and then -# blocks on `argocd app wait` (up to 30m) so the GH job stays open until -# the env is actually up. +# blocks on `argocd app wait` (`wait-timeout`, 30m default) so the GH job +# stays open until the env is actually up. On timeout it prints every app's +# sync/health plus any Application conditions, because the CLI's own timeout +# message names neither the stuck app nor the reason. # # Rollout state surfaces in the PR via the caller workflow's `environment:` # block (native GH Deployment). The PR sidebar pill is `in_progress` while @@ -66,6 +68,13 @@ inputs: description: ArgoCD CLI version installed when missing from the runner. required: false default: v2.13.1 + wait-timeout: + description: >- + Seconds to wait for the env to reach Synced+Healthy before failing. + Covers a cold env's image pulls and migrations, so lower it only if you + know this repo's envs come up faster. + required: false + default: "1800" runs: using: composite @@ -76,6 +85,7 @@ runs: ARGOCD_AUTH_TOKEN: ${{ inputs.argocd-token }} ARGOCD_VERSION: ${{ inputs.argocd-version }} GH_TOKEN: ${{ inputs.gh-token }} + WAIT_TIMEOUT: ${{ inputs.wait-timeout }} run: | set -euo pipefail @@ -406,5 +416,22 @@ runs: # written by the AppSet template onto the parent and by `prenv.labels` # onto every child, so this one selector covers both. WAIT_SELECTOR="pr-env.mindsdb.com/anchor-repo=${OWN_SLUG},pr-env.mindsdb.com/pr-number=${PR_NUMBER}" - echo "argocd app wait -l ${WAIT_SELECTOR} --sync --health --operation --timeout 1800" - "$ARGOCD_BIN" app wait -l "$WAIT_SELECTOR" --grpc-web --sync --health --operation --timeout 1800 + echo "argocd app wait -l ${WAIT_SELECTOR} --sync --health --operation --timeout ${WAIT_TIMEOUT}" + if "$ARGOCD_BIN" app wait -l "$WAIT_SELECTOR" --grpc-web --sync --health --operation --timeout "$WAIT_TIMEOUT"; then + exit 0 + fi + + # On timeout the CLI cancels its own watch stream and all it prints is + # `rpc error: code = Canceled desc = context canceled`, which names + # neither the stuck app nor the reason. Every Application already + # carries both in its status, so dump that instead. An app stuck + # Healthy-but-never-Synced is a spec ArgoCD cannot converge rather than + # a slow or crash-looping rollout; two apps rendering one Secret shows + # up here as SharedResourceWarning. Never let the diagnostics + # themselves fail the step ahead of the explicit exit below. + echo "::error::PR env did not converge within ${WAIT_TIMEOUT}s. State at timeout:" + "$ARGOCD_BIN" app list -l "$WAIT_SELECTOR" --grpc-web -o json 2>/dev/null | jq -r ' + .[] | " \(.metadata.name) sync=\(.status.sync.status) health=\(.status.health.status)", + (.status.conditions[]? | " \(.type): \(.message)") + ' >&2 || true + exit 1 From 1349119c870a7afb4fe7fadc7946d404d4465692 Mon Sep 17 00:00:00 2001 From: Lucas Koontz Date: Wed, 5 Aug 2026 13:38:42 -0700 Subject: [PATCH 2/4] feat(workflow): add workflow to sync main branch to staging automatically Introduces a new reusable workflow that automatically synchronizes changes from the main branch to the staging branch after any push to main. This ensures that updates, including hotfixes and direct merges to main, are reflected in staging promptly, maintaining consistency and reducing manual reconciliation efforts. --- .github/workflows/sync-main-to-staging.yml | 118 +++++++++++++++++++++ README.md | 34 +++++- 2 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/sync-main-to-staging.yml diff --git a/.github/workflows/sync-main-to-staging.yml b/.github/workflows/sync-main-to-staging.yml new file mode 100644 index 0000000..3083f73 --- /dev/null +++ b/.github/workflows/sync-main-to-staging.yml @@ -0,0 +1,118 @@ +# Reusable workflow: merge `main` back into `staging` after ANY push to main. +# +# `release-unfreeze.yml` already syncs main into staging when the weekly release +# PR merges. That covers the release path and nothing else. A commit that reaches +# main any other way — a hotfix PR, a revert, a direct merge — fires no unfreeze +# (the wrapper's guard requires the merged PR's head to be `staging`), so it sits +# on main until the NEXT weekly release drags it across, up to a week later. +# +# That gap is not cosmetic on squash-merge repos. While main is ahead, the next +# release PR's diff is computed against a main that staging does not contain, so +# the release either reverts the missing commit or has to be reconciled by hand. +# This workflow closes the window: main flows into staging within a minute of +# landing, whatever route it took. +# +# Idempotent by construction: it exits 0 when staging already contains main, so +# it is safe to run on every push, and safe to run alongside the unfreeze sync. +# +# Ordering against release-unfreeze: merging the release PR fires BOTH this +# (push to main) and the unfreeze (pull_request closed). Both push staging with +# the same App, so the CALLER must put both workflows in one `concurrency` group +# to serialise them; the loser then finds staging already current and no-ops. +# The push also retries once on a non-fast-forward, which covers a staging that +# moved between fetch and push for any other reason. +# +# The push uses the `mindsdb-release-train` App token. The App is the ruleset +# bypass actor on staging, so it lands even while staging is frozen — which is +# what you want: a hotfix on main belongs in the release candidate too. +# +# Called by a per-repo wrapper, e.g. +# auth/.github/workflows/sync-main-to-staging.yml +# +# Requires: vars.RELEASE_APP_CLIENT_ID + secrets.RELEASE_APP_PRIVATE_KEY +# (org-provisioned; secret reaches here via `secrets: inherit`). + +name: Sync main to staging + +on: + workflow_call: + inputs: + staging-branch: + description: "Branch to sync into" + type: string + default: staging + base-branch: + description: "Production branch to sync from" + type: string + default: main + runs-on: + description: "Runner label" + type: string + default: ubuntu-latest + +permissions: + contents: read + +jobs: + sync: + runs-on: ${{ inputs.runs-on }} + steps: + - name: Mint release-train App token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.RELEASE_APP_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - name: Sync base branch into staging + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + STAGING: ${{ inputs.staging-branch }} + BASE: ${{ inputs.base-branch }} + run: | + set -euo pipefail + + # Attribute the merge commit to the App's bot identity. The username is + # URL-encoded ('[' -> %5B, ']' -> %5D) for the users API lookup. + APP_USER_ID=$(gh api "/users/${APP_SLUG}%5Bbot%5D" --jq '.id') + git config user.name "${APP_SLUG}[bot]" + git config user.email "${APP_USER_ID}+${APP_SLUG}[bot]@users.noreply.github.com" + + # One attempt, then one retry: the retry exists for the case where + # staging moved after the fetch (a concurrent sync, or an ordinary push + # landing while staging is unfrozen), which shows up as a rejected + # non-fast-forward rather than an error worth failing the run over. + for attempt in 1 2; do + git fetch origin "${BASE}" "${STAGING}" + git checkout -B "${STAGING}" "origin/${STAGING}" + + if git merge-base --is-ancestor "origin/${BASE}" HEAD; then + echo "${STAGING} already contains ${BASE} — nothing to sync." + exit 0 + fi + + git merge --no-ff "origin/${BASE}" \ + -m "Sync ${BASE} into ${STAGING}" + + if git push origin "HEAD:${STAGING}"; then + echo "Pushed ${BASE} -> ${STAGING} on attempt ${attempt}." + { + echo "## Synced \`${BASE}\` into \`${STAGING}\`" + echo "" + echo "\`${STAGING}\` now contains every commit on \`${BASE}\`." + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + echo "Push rejected on attempt ${attempt} — ${STAGING} moved underneath us." + git merge --abort 2>/dev/null || true + done + + echo "::error::Could not fast-forward ${STAGING} after 2 attempts. ${STAGING} is moving faster than this workflow can merge, or the App lost its bypass on ${STAGING}." + exit 1 diff --git a/README.md b/README.md index c272a9c..3e3c926 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Use an action from this repo in your workflow like this: ## Release-train reusable workflows -Three reusable workflows automate the weekly `staging → main` release cycle. +Four reusable workflows automate the weekly `staging → main` release cycle. They live in `.github/workflows/` and are called from ~25-line per-repo wrappers (same pattern as `stale-deploy-label.yml`): @@ -24,6 +24,19 @@ They live in `.github/workflows/` and are called from ~25-line per-repo wrappers | `release-freeze.yml` | `Staging Freeze` | Activates the `staging-freeze` ruleset to lock staging (skips if staging == main) | | `release-pr.yml` | `Create staging to main release PR` | Opens the `staging → main` PR (idempotent) | | `release-unfreeze.yml` | `Staging Unfreeze` | Disables the ruleset when the release PR merges, then syncs `main` back into `staging` | +| `sync-main-to-staging.yml` | `Sync main to staging` | Merges `main` into `staging` after **any** push to main, not just the release merge | + +`release-unfreeze.yml` syncs main back only on the release path, because its wrapper's guard requires the merged PR's head branch to be `staging`. A commit that reaches main any other way (a hotfix PR, a revert, a direct merge) fires nothing, and on a squash-merge repo that leaves the next release PR diffed against a `main` that `staging` does not contain. `sync-main-to-staging.yml` closes that window on `push: main`. + +Both push `staging` as the release-train App, and merging the release PR fires both, so a caller that installs both **must put them in the same `concurrency` group**: + +```yaml +concurrency: + group: sync-main-to-staging + cancel-in-progress: false +``` + +The sync is idempotent (it exits 0 when `staging` already contains `main`), so whichever run loses the race no-ops. The chain is event-driven: `Staging Freeze` finishing fires the release-PR workflow via `workflow_run`; merging that PR fires `Staging Unfreeze`. The @@ -71,6 +84,25 @@ only and a called workflow can never hold more than its caller grants. Without it the lookup is refused and the job stays silent (it never fails the run), so a missing recovery message is the symptom to look for. +### Scoping staging alerts to the freeze window + +A staging failure is not the same event all week. Once the freeze window is open, `staging` is the release candidate and a red pipeline blocks the release. Before it, `staging` is the integration branch and the same red is routine, so paging the channel for it is what teaches people to scroll past the channel. + +`freeze-scoped: true` on a **staging** caller keeps the red `:rotating_light:` alert while the `staging-freeze` ruleset is `active`, and drops it to a muted grey `:warning:` notice outside the window: + +```yaml + with: + env-name: "staging build+deploy" + status: ${{ contains(needs.*.result, 'failure') && 'failed' || 'recovered' }} + freeze-scoped: true +``` + +Add `outside-freeze: silent` to post nothing at all outside the window instead. That trades the noise for a blind spot: a staging pipeline can then sit red unnoticed until the next freeze opens on a branch that no longer builds, so `notice` is the default. + +Freeze state is read from the `staging-freeze` ruleset itself, not inferred from workflow history — a freeze that skipped itself because staging had nothing unreleased still concludes `success`, and history cannot tell that apart from a real freeze. Reading rulesets needs admin, so this reuses the same App that toggles them (`vars.RELEASE_APP_CLIENT_ID` + `secrets.RELEASE_APP_PRIVATE_KEY`); no extra `permissions:` on the caller. If the App token or the ruleset lookup fails, it escalates to the red alert rather than downgrading, so a lookup problem can never silence a real release-blocking failure. + +Leave `freeze-scoped` off for prod and freeze/unfreeze callers: a failure there is always worth interrupting for. + For a freeze/unfreeze wrapper, keep it failure-only: `needs:` its single job, `if: failure()`, the default `status: failed`, no `permissions:` block (the prior-run lookup only runs in recovered mode), and label it accordingly From edbc395064d1fa8cad0e0e80dc07e4bc4708b4cf Mon Sep 17 00:00:00 2001 From: Lucas Koontz Date: Wed, 5 Aug 2026 14:04:06 -0700 Subject: [PATCH 3/4] fix(workflow): adjust freeze-scope defaults for improved alert behavior Changes default handling of staging failures during non-freeze periods to avoid unnecessary noise. Updates the notification logic to post no alerts for staging failures outside freeze windows by default, aligning with intent to minimize alert fatigue. Provides option to switch to a muted notice for users requiring visibility. --- .github/workflows/notify-main-failure.yml | 31 +++++++++++++++-------- README.md | 16 ++++++++++-- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/.github/workflows/notify-main-failure.yml b/.github/workflows/notify-main-failure.yml index 739d20d..01f9a97 100644 --- a/.github/workflows/notify-main-failure.yml +++ b/.github/workflows/notify-main-failure.yml @@ -16,15 +16,24 @@ # every merge. This is derived from the GitHub API rather than cross-run # state, so it is self-healing (no cache to go stale). # -# `freeze-scoped: true` adds a third outcome for STAGING callers. A staging -# failure is only release-blocking once the freeze window is open; mid-week it is -# an ordinary integration-branch red, and paging the channel for it is what -# teaches people to ignore the channel. So a freeze-scoped failure posts the red -# alert while the `staging-freeze` ruleset is active, and outside the freeze -# posts a muted grey notice instead (`outside-freeze: silent` drops it entirely, -# at the cost of a broken staging branch being able to sit red unnoticed until -# the next freeze opens). Prod and freeze/unfreeze callers leave this alone — a -# failure there is always worth interrupting for. +# `freeze-scoped: true` scopes a STAGING caller to the release window. The policy +# this implements: +# +# failure on main -> alert +# failure on staging, freeze OPEN -> alert (staging is the release candidate) +# failure on staging, freeze NOT open -> nothing (ordinary integration-branch red) +# +# Mid-week staging is the integration branch, and paging the channel for it is +# what teaches people to ignore the channel. `outside-freeze: notice` posts a +# muted grey message instead of nothing, for a caller that wants the trail; the +# default is silence. Either way the run is still red in the Actions tab, and +# pipeline-watchdog.yml still covers a staging branch that stops building at all. +# +# Prod and freeze/unfreeze callers leave this alone: a failure there is always +# worth interrupting for. A workflow triggered on BOTH main and staging from one +# notify job cannot hardcode this — pass an expression instead, or a main failure +# gets silenced too: +# freeze-scoped: ${{ github.ref_name == 'staging' }} # # It posts its own message rather than the deploy-notification composite, whose # copy is deploy-specific ("has failed deploying to ...") and reads wrong for @@ -79,9 +88,9 @@ on: type: boolean default: false outside-freeze: - description: "What a freeze-scoped failure does while staging is NOT frozen: 'notice' (muted, no panic styling) or 'silent' (post nothing)." + description: "What a freeze-scoped failure does while staging is NOT frozen: 'silent' (post nothing) or 'notice' (muted, no panic styling)." type: string - default: notice + default: silent runs-on: description: "Runner label for the notify job" type: string diff --git a/README.md b/README.md index 3e3c926..a09bf6f 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,13 @@ missing recovery message is the symptom to look for. A staging failure is not the same event all week. Once the freeze window is open, `staging` is the release candidate and a red pipeline blocks the release. Before it, `staging` is the integration branch and the same red is routine, so paging the channel for it is what teaches people to scroll past the channel. -`freeze-scoped: true` on a **staging** caller keeps the red `:rotating_light:` alert while the `staging-freeze` ruleset is `active`, and drops it to a muted grey `:warning:` notice outside the window: +`freeze-scoped: true` on a **staging** caller implements that policy: + +| Where it failed | Freeze window | Result | +|---|---|---| +| `main` | n/a | red alert | +| `staging` | open | red alert | +| `staging` | not open | nothing posted | ```yaml with: @@ -97,7 +103,13 @@ A staging failure is not the same event all week. Once the freeze window is open freeze-scoped: true ``` -Add `outside-freeze: silent` to post nothing at all outside the window instead. That trades the noise for a blind spot: a staging pipeline can then sit red unnoticed until the next freeze opens on a branch that no longer builds, so `notice` is the default. +Add `outside-freeze: notice` to post a muted grey `:warning:` message instead of nothing, for a caller that wants the trail. Silence is the default; the run is still red in the Actions tab either way, and `pipeline-watchdog.yml` still covers a staging branch that stops building at all. + +A workflow triggered on **both** main and staging from one notify job must not hardcode `true`, or main failures get silenced too. Pass an expression: + +```yaml + freeze-scoped: ${{ github.ref_name == 'staging' }} +``` Freeze state is read from the `staging-freeze` ruleset itself, not inferred from workflow history — a freeze that skipped itself because staging had nothing unreleased still concludes `success`, and history cannot tell that apart from a real freeze. Reading rulesets needs admin, so this reuses the same App that toggles them (`vars.RELEASE_APP_CLIENT_ID` + `secrets.RELEASE_APP_PRIVATE_KEY`); no extra `permissions:` on the caller. If the App token or the ruleset lookup fails, it escalates to the red alert rather than downgrading, so a lookup problem can never silence a real release-blocking failure. From ae5f69c0763778156c375a65ee773a9efe30d07b Mon Sep 17 00:00:00 2001 From: Lucas Koontz Date: Thu, 6 Aug 2026 00:16:36 -0700 Subject: [PATCH 4/4] fix(workflow): improve script execution by avoiding unintended dep installs Add `--no-project` flag to `uv run` command to prevent auto-discovery and unnecessary dependency installations from the calling repository. This change resolves issues with slow execution and failures caused by private git dependencies requiring credentials. --- .github/workflows/workflow-lint.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index fd82352..6607564 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -99,7 +99,12 @@ jobs: DEFAULTS: ${{ inputs.default-permissions }} run: | set -euo pipefail - uv run --with pyyaml --python 3.12 \ + # --no-project because this runs inside the CALLING repo. Without it + # `uv run` discovers that repo's pyproject.toml, builds a .venv and + # installs its entire dependency tree before running a script that only + # needs pyyaml. That is slow, and it fails outright when a caller depends + # on a private git repo, since this job holds no credentials for one. + uv run --no-project --with pyyaml --python 3.12 \ .ci-shared/scripts/workflow_graph.py \ --workflow-dir .github/workflows \ --default-permissions "$DEFAULTS"