diff --git a/.github/workflows/notify-main-failure.yml b/.github/workflows/notify-main-failure.yml index 08de674..739d20d 100644 --- a/.github/workflows/notify-main-failure.yml +++ b/.github/workflows/notify-main-failure.yml @@ -16,6 +16,16 @@ # every merge. This is derived from the GitHub API rather than cross-run # state, so it is self-healing (no cache to go stale). # +# `freeze-scoped: true` adds a third outcome for STAGING callers. A staging +# failure is only release-blocking once the freeze window is open; mid-week it is +# an ordinary integration-branch red, and paging the channel for it is what +# teaches people to ignore the channel. So a freeze-scoped failure posts the red +# alert while the `staging-freeze` ruleset is active, and outside the freeze +# posts a muted grey notice instead (`outside-freeze: silent` drops it entirely, +# at the cost of a broken staging branch being able to sit red unnoticed until +# the next freeze opens). Prod and freeze/unfreeze callers leave this alone — a +# failure there is always worth interrupting for. +# # It posts its own message rather than the deploy-notification composite, whose # copy is deploy-specific ("has failed deploying to ...") and reads wrong for # non-deploy pipelines (freeze, CI, docs). It uses the same Slack bot token. @@ -64,6 +74,14 @@ on: description: "'failed' (red alert) or 'recovered' (green, only if the prior run was failing)" type: string default: failed + freeze-scoped: + description: "Scale a 'failed' alert by release-freeze state (see `outside-freeze`). For staging pipelines, where a failure is only release-blocking once the freeze window is open." + type: boolean + default: false + outside-freeze: + description: "What a freeze-scoped failure does while staging is NOT frozen: 'notice' (muted, no panic styling) or 'silent' (post nothing)." + type: string + default: notice runs-on: description: "Runner label for the notify job" type: string @@ -178,6 +196,100 @@ jobs: ;; esac + # A staging pipeline failure means different things depending on where the + # release train is. Once the freeze window is open staging IS the release + # candidate, so a red pipeline blocks the release and is worth interrupting + # the channel for. Mid-week it is the integration branch and the same red + # is routine — panic styling there just teaches people to scroll past it. + # + # The freeze signal is the `staging-freeze` repository ruleset that the + # freeze/unfreeze workflows toggle between `active` and `disabled`, not an + # inference from workflow-run history: a freeze that skipped itself because + # staging had nothing unreleased still concludes `success`, and history + # cannot tell that apart from a real freeze. + # + # Reading rulesets needs admin, which the default workflow token does not + # carry, so this reuses the App that owns the toggle. Every failure path + # below falls through to `frozen=true`: a notify job that cannot establish + # the freeze state must escalate rather than silently downgrade a real + # release-blocking failure. + - name: Mint release-train App token + id: freeze-token + if: inputs.freeze-scoped && inputs.status != 'recovered' + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.RELEASE_APP_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + - name: Check release-freeze state + id: freeze + if: inputs.freeze-scoped && inputs.status != 'recovered' + env: + GH_TOKEN: ${{ steps.freeze-token.outputs.token }} + REPO: ${{ github.repository }} + RULESET_NAME: staging-freeze + OUTSIDE_FREEZE: ${{ inputs.outside-freeze }} + run: | + set -uo pipefail + + escalate() { + echo "::warning::$1 Treating staging as frozen so this failure still escalates." + echo "frozen=true" >> "$GITHUB_OUTPUT" + exit 0 + } + + [ -n "${GH_TOKEN:-}" ] || escalate "Could not mint the release-train App token." + + ENFORCEMENT=$(gh api "repos/${REPO}/rulesets" \ + --jq ".[] | select(.name == \"${RULESET_NAME}\") | .enforcement" 2>&1) \ + || escalate "Could not read rulesets: ${ENFORCEMENT}." + [ -n "${ENFORCEMENT}" ] || escalate "Ruleset '${RULESET_NAME}' not found." + + if [ "${ENFORCEMENT}" = "active" ]; then + echo "frozen=true" >> "$GITHUB_OUTPUT" + echo "Ruleset '${RULESET_NAME}' is active — staging is frozen, escalating." + else + echo "frozen=false" >> "$GITHUB_OUTPUT" + echo "Ruleset '${RULESET_NAME}' is '${ENFORCEMENT}' — staging is not frozen, downgrading to '${OUTSIDE_FREEZE}'." + fi + + # Collapse the three outcomes (recovered / failed / failed-outside-freeze) + # into flat strings. Deciding this in shell keeps the Slack payload from + # growing a second and third nested ternary per field, which is where this + # kind of template stops being reviewable. + - name: Compose message + id: msg + env: + STATUS: ${{ inputs.status }} + FREEZE_SCOPED: ${{ inputs.freeze-scoped }} + FROZEN: ${{ steps.freeze.outputs.frozen }} + OUTSIDE_FREEZE: ${{ inputs.outside-freeze }} + run: | + set -euo pipefail + POST=true + if [ "${STATUS}" = "recovered" ]; then + LEVEL=recovered; COLOR='#00C851'; ICON=':white_check_mark:'; VERB=recovered + PREFIX=Recovered; NOTE='' + elif [ "${FREEZE_SCOPED}" = "true" ] && [ "${FROZEN}" = "false" ]; then + LEVEL=notice; COLOR='#B0B4BA'; ICON=':warning:'; VERB=failed + PREFIX=Failed; NOTE=' (outside the release freeze, not release-blocking)' + if [ "${OUTSIDE_FREEZE}" = "silent" ]; then POST=false; fi + else + LEVEL=alert; COLOR='#FF4444'; ICON=':rotating_light:'; VERB=failed + PREFIX=FAILED; NOTE='' + fi + { + echo "post=${POST}" + echo "level=${LEVEL}" + echo "color=${COLOR}" + echo "icon=${ICON}" + echo "verb=${VERB}" + echo "prefix=${PREFIX}" + echo "note=${NOTE}" + } >> "$GITHUB_OUTPUT" + echo "level=${LEVEL} post=${POST}" + - name: Notify Slack # Failure mode always posts; on a real pipeline, recovery posts only after # a failing run; a manual dispatch always posts (smoke test). @@ -187,22 +299,24 @@ jobs: # reader, and a notification-list preview show, so without it the alert # arrives on a phone as an empty message — which is the one context where # a pipeline failure most needs to be readable. - if: inputs.status != 'recovered' || github.event_name == 'workflow_dispatch' || steps.prev.outputs.should_post == 'true' + # `steps.msg.outputs.post` carries the one extra veto: a freeze-scoped + # failure outside the freeze window when the caller asked for silence. + if: steps.msg.outputs.post == 'true' && (inputs.status != 'recovered' || github.event_name == 'workflow_dispatch' || steps.prev.outputs.should_post == 'true') uses: slackapi/slack-github-action@v1.26.0 with: channel-id: ${{ secrets.SLACK_ENG_CHANNEL_ID }} payload: | { - "text": "${{ inputs.status == 'recovered' && 'Recovered' || 'FAILED' }}: ${{ github.event.repository.name }} ${{ inputs.env-name }} pipeline on ${{ github.ref_name }}", + "text": "${{ steps.msg.outputs.prefix }}: ${{ github.event.repository.name }} ${{ inputs.env-name }} pipeline on ${{ github.ref_name }}${{ steps.msg.outputs.note }}", "attachments": [ { - "color": "${{ inputs.status == 'recovered' && '#00C851' || '#FF4444' }}", + "color": "${{ steps.msg.outputs.color }}", "blocks": [ { "type": "section", "text": { "type": "mrkdwn", - "text": "${{ inputs.status == 'recovered' && ':white_check_mark:' || ':rotating_light:' }} *<${{ github.server_url }}/${{ github.repository }}|${{ github.event.repository.name }}>* — *${{ inputs.env-name }}* pipeline ${{ inputs.status == 'recovered' && 'recovered' || 'failed' }} on `${{ github.ref_name }}`" + "text": "${{ steps.msg.outputs.icon }} *<${{ github.server_url }}/${{ github.repository }}|${{ github.event.repository.name }}>* — *${{ inputs.env-name }}* pipeline ${{ steps.msg.outputs.verb }} on `${{ github.ref_name }}`${{ steps.msg.outputs.note }}" }, "fields": [ { diff --git a/argocd-pr-env-deploy/action.yml b/argocd-pr-env-deploy/action.yml index 867d113..00dd56f 100644 --- a/argocd-pr-env-deploy/action.yml +++ b/argocd-pr-env-deploy/action.yml @@ -33,8 +33,10 @@ # Issues a single `argocd app set` against the parent Application # (`pr--`) with `--helm-set tags.=development-` per # resolved repo, kicks the auto-sync via `argocd app sync --async`, and then -# blocks on `argocd app wait` (up to 30m) so the GH job stays open until -# the env is actually up. +# blocks on `argocd app wait` (`wait-timeout`, 30m default) so the GH job +# stays open until the env is actually up. On timeout it prints every app's +# sync/health plus any Application conditions, because the CLI's own timeout +# message names neither the stuck app nor the reason. # # Rollout state surfaces in the PR via the caller workflow's `environment:` # block (native GH Deployment). The PR sidebar pill is `in_progress` while @@ -66,6 +68,13 @@ inputs: description: ArgoCD CLI version installed when missing from the runner. required: false default: v2.13.1 + wait-timeout: + description: >- + Seconds to wait for the env to reach Synced+Healthy before failing. + Covers a cold env's image pulls and migrations, so lower it only if you + know this repo's envs come up faster. + required: false + default: "1800" runs: using: composite @@ -76,6 +85,7 @@ runs: ARGOCD_AUTH_TOKEN: ${{ inputs.argocd-token }} ARGOCD_VERSION: ${{ inputs.argocd-version }} GH_TOKEN: ${{ inputs.gh-token }} + WAIT_TIMEOUT: ${{ inputs.wait-timeout }} run: | set -euo pipefail @@ -406,5 +416,22 @@ runs: # written by the AppSet template onto the parent and by `prenv.labels` # onto every child, so this one selector covers both. WAIT_SELECTOR="pr-env.mindsdb.com/anchor-repo=${OWN_SLUG},pr-env.mindsdb.com/pr-number=${PR_NUMBER}" - echo "argocd app wait -l ${WAIT_SELECTOR} --sync --health --operation --timeout 1800" - "$ARGOCD_BIN" app wait -l "$WAIT_SELECTOR" --grpc-web --sync --health --operation --timeout 1800 + echo "argocd app wait -l ${WAIT_SELECTOR} --sync --health --operation --timeout ${WAIT_TIMEOUT}" + if "$ARGOCD_BIN" app wait -l "$WAIT_SELECTOR" --grpc-web --sync --health --operation --timeout "$WAIT_TIMEOUT"; then + exit 0 + fi + + # On timeout the CLI cancels its own watch stream and all it prints is + # `rpc error: code = Canceled desc = context canceled`, which names + # neither the stuck app nor the reason. Every Application already + # carries both in its status, so dump that instead. An app stuck + # Healthy-but-never-Synced is a spec ArgoCD cannot converge rather than + # a slow or crash-looping rollout; two apps rendering one Secret shows + # up here as SharedResourceWarning. Never let the diagnostics + # themselves fail the step ahead of the explicit exit below. + echo "::error::PR env did not converge within ${WAIT_TIMEOUT}s. State at timeout:" + "$ARGOCD_BIN" app list -l "$WAIT_SELECTOR" --grpc-web -o json 2>/dev/null | jq -r ' + .[] | " \(.metadata.name) sync=\(.status.sync.status) health=\(.status.health.status)", + (.status.conditions[]? | " \(.type): \(.message)") + ' >&2 || true + exit 1