From 1513d900f6eda10683864adbaa3dccba9d2a86d2 Mon Sep 17 00:00:00 2001 From: Lucas Koontz Date: Mon, 24 Aug 2026 09:13:39 -0700 Subject: [PATCH] fix(ci): keep GitHub's synthesized runs out of the deploy-branch sweeps (ENG-1549) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The red-branch sweep reports the newest red run per workflow PATH on a deploy branch, and the wide `workflows:` default meant every path with a run. GitHub gives its own products a synthesized path under `dynamic/` and attributes those runs to the default branch, so `dynamic/dependabot/dependabot-updates` read as a pipeline on `main`: anton's channel got ":rotating_light: anton — 1 pipeline(s) still failing" for a failed Dependabot version update, which says nothing about whether `main` is deployed. The wide default now means every workflow FILE, dropping the `dynamic/` namespace (Dependabot updates, default-setup code scanning, Pages builds) from both sweeps. It is a scope decision, not a ban: an allowlist that names one still reports it, so a repo that wants a red default-setup CodeQL scan can ask. All seven watchdog callers take the defaults, so this covers them with no per-repo change. Lucas Koontz. ENG-1549: Alert when a staging release candidate fails to publish Refs: ENG-1549 --- .github/workflows/notify-startup-failure.yml | 12 ++++++-- README.md | 2 ++ scripts/branch_health.py | 29 ++++++++++++++++---- tests/test_branch_health.py | 21 ++++++++++++++ 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/.github/workflows/notify-startup-failure.yml b/.github/workflows/notify-startup-failure.yml index b15a337..c5bffce 100644 --- a/.github/workflows/notify-startup-failure.yml +++ b/.github/workflows/notify-startup-failure.yml @@ -147,7 +147,7 @@ on: type: number default: 30 workflows: - description: "Space-separated workflow paths the red-branch sweep may report on. Empty means every workflow with a run on these branches, which is wider than the set that opted into an in-run alert: the sweep reads run history, not notify wiring, so a red scheduled job or a red one-off check on a deploy branch counts too. Narrow it here if that is not wanted." + description: "Space-separated workflow paths the red-branch sweep may report on. Empty means every workflow FILE with a run on these branches, which is wider than the set that opted into an in-run alert: the sweep reads run history, not notify wiring, so a red scheduled job or a red one-off check on a deploy branch counts too. Narrow it here if that is not wanted. The runs GitHub synthesizes for its own products (`dynamic/...`: Dependabot updates, default-setup code scanning, Pages builds) are out of the wide default; name one here to get it back." type: string default: "" staging-branch: @@ -181,7 +181,7 @@ on: type: number default: 30 workflows: - description: "Workflow paths to report on (space separated); empty means all" + description: "Workflow paths to report on (space separated); empty means every workflow file, excluding GitHub's synthesized `dynamic/...` runs" type: string default: "" staging-branch: @@ -253,8 +253,16 @@ jobs: # changes the display name, the path is stable). The previous run's # conclusion rides along for the message rather than as a filter, so # a failing push into an already-broken branch is still reported. + # + # `dynamic/` is the prefix on the runs GitHub synthesizes for its own + # products — Dependabot updates, default-setup code scanning, Pages + # builds. They have no workflow file, they sit on the default branch, + # and neither sweep is asking about them: "the branch is not deployed" + # is a statement about a pipeline. See SYNTHESIZED_PREFIX in + # scripts/branch_health.py, which drops them from the other sweep. jq --argjson cutoff "${CUTOFF}" --arg branch "${BRANCH}" ' [ .workflow_runs[] + | select(.path | startswith("dynamic/") | not) | select(.conclusion == "success" or .conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "startup_failure") | { id, path, name, conclusion, created_at, head_sha, html_url, diff --git a/README.md b/README.md index 1b893a3..e3caa23 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,8 @@ It is a **backstop, not an echo**: a finding must be at least `min-age-minutes` **Know how wide it is.** The sweep reads run history, not notify wiring, so by default it reports *any* red workflow on a swept branch — including a scheduled job or a one-off check that never opted into an in-run alert. Pass `workflows:` with the paths that matter to narrow it. The sweep never reports the watchdog itself, which it derives from `github.workflow_ref` rather than asking the caller. +**What the wide default leaves out** is the runs GitHub synthesizes for its own products: `dynamic/dependabot/dependabot-updates`, `dynamic/github-code-scanning/codeql`, `dynamic/pages/pages-build-deployment`. They have no workflow file, GitHub attributes them to the default branch, and in a `main` run listing they look exactly like a pipeline. A Dependabot version update that fails is not a statement about whether `main` got deployed, which is the only thing either sweep is asking, so both sweeps drop the `dynamic/` prefix. Naming one in `workflows:` reports it again; an explicit ask wins. + Age is measured from the current attempt's `run_started_at`, not the run's `created_at`. A re-run keeps the original `created_at` forever, so an age window measured from it filters out the very attempt that just failed — and a re-run is the most common way a failure gets fixed, or re-broken. Selection logic lives in `scripts/branch_health.py` with unit tests, because it is date arithmetic plus a two-way trigger and that does not belong in a jq expression. diff --git a/scripts/branch_health.py b/scripts/branch_health.py index 651c6b4..c1d0937 100644 --- a/scripts/branch_health.py +++ b/scripts/branch_health.py @@ -34,7 +34,9 @@ where GitHub rejected the run at load time and no job ran at all, which ``notify-startup-failure`` already sweeps for with a different message and a different thing to check. Splitting them keeps one finding from producing two -alerts. +alerts. Nor does it report the runs GitHub synthesizes for its own products +(Dependabot updates, default-setup code scanning, Pages builds), which sit on the +default branch and look like pipelines but say nothing about a deploy. When it fires, per branch: @@ -80,6 +82,16 @@ CONCLUSIVE = ("success", "failure", "timed_out", "startup_failure") RED = ("failure", "timed_out") +# Runs GitHub synthesizes for its own products live under this prefix: +# `dynamic/dependabot/dependabot-updates`, `dynamic/github-code-scanning/codeql`, +# `dynamic/pages/pages-build-deployment`. They have no workflow file in the repo, +# they are attributed to the default branch, and in a `main` run listing they look +# exactly like a pipeline — so the wide default scope swept them in. A Dependabot +# version update that fails alerted the channel as "main is still red", which says +# nothing about whether main is deployed, and that is the only question this sweep +# is asking. Naming one in `workflows:` still reports it: an explicit ask wins. +SYNTHESIZED_PREFIX = "dynamic/" + # The freeze workflow's per-repo wrapper keeps this name verbatim, because the # release-PR workflow chains off it by name. That makes it a stable handle for # "the window just opened" without this file knowing anything about the schedule, @@ -147,19 +159,26 @@ def freeze_opened_within(freeze_runs: Iterable[dict], *, cutoff: datetime) -> bo def in_scope(path: str, *, only: Sequence[str], exclude: Sequence[str]) -> bool: """Whether this workflow file is one the sweep speaks about. - ``only`` empty means every workflow on the branch, which is the default and is - the widest this gets. It is worth knowing how wide that is: the sweep reads run - history, not the notify wiring, so it reports any red workflow on a deploy + ``only`` empty means every workflow FILE in the repo, which is the default and + is the widest this gets. It is worth knowing how wide that is: the sweep reads + run history, not the notify wiring, so it reports any red workflow on a deploy branch and not only the pipelines that opted into an in-run alert. A repo that wants it narrowed passes `workflows:` with the paths that matter. + What the wide default does NOT include is the runs GitHub synthesizes for its + own products (see ``SYNTHESIZED_PREFIX``), because none of them says anything + about whether the branch got deployed. An allowlist that names one overrides + that, so a repo can still ask for it. + ``exclude`` always carries the sweep's own workflow. A watchdog whose own run went red would otherwise report itself on the next tick, which reads as a pipeline failure and is really just the watchdog. """ if path in exclude: return False - return not only or path in only + if only: + return path in only + return not path.startswith(SYNTHESIZED_PREFIX) def select_red( diff --git a/tests/test_branch_health.py b/tests/test_branch_health.py index 9cd5eb8..8b36daf 100644 --- a/tests/test_branch_health.py +++ b/tests/test_branch_health.py @@ -262,6 +262,27 @@ def test_an_allowlist_narrows_it(self): ) assert [f["path"] for f in findings] == ["a.yml"] + def test_a_synthesized_dependabot_run_is_not_a_pipeline(self): + """A red Dependabot update on `main` is not "main is still red".""" + runs = [run(path="dynamic/dependabot/dependabot-updates", run_id=1)] + assert select(runs, branch="main", frozen=False) == [] + + def test_the_other_synthesized_namespaces_are_out_too(self): + runs = [ + run(path="dynamic/github-code-scanning/codeql", run_id=1), + run(path="dynamic/pages/pages-build-deployment", run_id=2), + ] + assert select(runs, branch="main", frozen=False) == [] + + def test_an_allowlist_that_names_a_synthesized_run_still_reports_it(self): + """The default is a scope decision, not a ban: an explicit ask wins.""" + path = "dynamic/dependabot/dependabot-updates" + findings = health.select_red( + [run(path=path)], branch="main", cutoff=CUTOFF, settled=SETTLED, frozen=False, + staging_branch="staging", only=[path], + ) + assert [f["path"] for f in findings] == [path] + def test_the_sweep_never_reports_itself(self): """A watchdog whose own run went red would otherwise alert on itself.""" runs = [run(path=".github/workflows/pipeline-watchdog.yml", run_id=1)]