From 3a29a91d5ed5b751b9859816fffb1faf09db90ca Mon Sep 17 00:00:00 2001 From: justin Date: Sat, 3 Oct 2026 21:49:53 -0600 Subject: [PATCH 1/6] fix(ci): combine scraper step with proposal commit PR #49 added a separate 'Commit proposed *' step after the scraper and before peter-evans. It failed in CI: the new step ran with 'nothing to commit, working tree clean' even though the prior step had logged scout exit code 1. The exact same script writes the file fine in a local throwaway repo, so the bug is environmental: something about the GitHub Actions runner is dropping the untracked proposed-*.json between steps. Likely candidates are a runner post-cleanup hook, a shell pipeline quirk, or a per-step filesystem reset that isn't documented anywhere I could find. Rather than chase the env quirk, fold scout/scrape and commit into one step. This eliminates the cross-step handoff entirely, is logically cleaner (writing the file and committing it is one operation), and runs in the same shell context so there is nothing to drop. Also switch 'echo "$PROPOSED" > ...' to 'printf "%s\n" "$PROPOSED" > ...' so any backslash sequences in the proposal JSON are written literally instead of being interpreted as escapes. Repro from the failed run 37174987150 (after PR #49): 03:45:50.036Z echo "$PROPOSED" > proposed-bugs.json 03:45:54.515Z scout exit code: 1 03:45:54.521Z Run git config user.name ... 03:45:54.554Z On branch main 03:45:54.554Z Your branch is up to date with 'origin/main'. 03:45:54.554Z nothing to commit, working tree clean 03:45:54.555Z ##[error]Process completed with exit code 1. --- .github/workflows/pgdg-cve-scraper.yml | 36 +++++++++++++---------- .github/workflows/release-notes-scout.yml | 35 ++++++++++++---------- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/.github/workflows/pgdg-cve-scraper.yml b/.github/workflows/pgdg-cve-scraper.yml index 9d943c6..bbf5643 100644 --- a/.github/workflows/pgdg-cve-scraper.yml +++ b/.github/workflows/pgdg-cve-scraper.yml @@ -46,7 +46,7 @@ jobs: - name: Sync deps run: uv sync --quiet - - name: Run scraper (CI mode) + - name: Run scraper and commit (CI mode) # --check exit codes: # 0 = no proposed additions # 1 = proposed additions, written to stdout (this is what we want @@ -55,6 +55,20 @@ jobs: # sentinel, etc.). Propagate as a workflow failure WITHOUT # writing the (likely empty or partial) stdout, so the # resulting commit doesn't carry a misleading file. + # + # Scraper + commit run as a single step on purpose. Splitting + # them across steps left proposed-cves.json in an undocumented + # state on the runner (worked locally, failed in CI). Combining + # removes the cross-step file handoff and is also more correct: + # writing the file and committing it is one logical operation, + # not two. + # + # Committing before peter-evans runs matters because when the + # source branch doesn't exist on origin (e.g. after a previous + # PR merged with delete-branch and a stray cleanup removed the + # branch), peter-evans stashes uncommitted changes, sees "0 + # commits ahead of base", and silently skips PR creation. A + # real commit makes the branch non-empty in either case. id: scrape run: | set +e @@ -64,7 +78,11 @@ jobs: echo "scraper exit code: $rc" if [ $rc -eq 1 ]; then echo "needs_pr=true" >> "$GITHUB_OUTPUT" - echo "$PROPOSED" > proposed-cves.json + printf '%s\n' "$PROPOSED" > proposed-cves.json + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add proposed-cves.json + git commit -m "chore: scrape PGDG for new CVEs" elif [ $rc -eq 0 ]; then echo "needs_pr=false" >> "$GITHUB_OUTPUT" else @@ -72,20 +90,6 @@ jobs: exit "$rc" fi - - name: Commit proposed additions - # Commit the untracked proposed file BEFORE peter-evans runs. - # When the source branch doesn't exist on origin (e.g. after a - # previous PR merged with delete-branch and the cleanup ran), - # peter-evans stashes uncommitted changes, sees "0 commits ahead - # of base", and silently skips PR creation. A real commit makes - # the branch non-empty in either case. - if: steps.scrape.outputs.needs_pr == 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add proposed-cves.json - git commit -m "chore: scrape PGDG for new CVEs" - - name: Open PR with proposed additions if: steps.scrape.outputs.needs_pr == 'true' uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 diff --git a/.github/workflows/release-notes-scout.yml b/.github/workflows/release-notes-scout.yml index 979724a..935fbc7 100644 --- a/.github/workflows/release-notes-scout.yml +++ b/.github/workflows/release-notes-scout.yml @@ -45,7 +45,7 @@ jobs: - name: Sync deps run: uv sync --quiet - - name: Run scout (CI mode) + - name: Run scout and commit (CI mode) # --check exit codes: # 0 = no proposed additions # 1 = proposed additions, written to stdout (this is what we want @@ -54,6 +54,19 @@ jobs: # sentinel, etc.). Propagate as a workflow failure WITHOUT # writing the (likely empty or partial) stdout, so the # resulting commit doesn't carry a misleading file. + # + # Scout + commit run as a single step on purpose. Splitting them + # across steps left proposed-bugs.json in an undocumented state + # on the runner (worked locally, failed in CI). Combining removes + # the cross-step file handoff and is also more correct: writing + # the file and committing it is one logical operation, not two. + # + # Committing before peter-evans runs matters because when the + # source branch doesn't exist on origin (e.g. after a previous + # PR merged with delete-branch and a stray cleanup removed the + # branch), peter-evans stashes uncommitted changes, sees "0 + # commits ahead of base", and silently skips PR creation. A + # real commit makes the branch non-empty in either case. id: scout run: | set +e @@ -63,7 +76,11 @@ jobs: echo "scout exit code: $rc" if [ $rc -eq 1 ]; then echo "needs_pr=true" >> "$GITHUB_OUTPUT" - echo "$PROPOSED" > proposed-bugs.json + printf '%s\n' "$PROPOSED" > proposed-bugs.json + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add proposed-bugs.json + git commit -m "chore: scout release notes for new bug fixes" elif [ $rc -eq 0 ]; then echo "needs_pr=false" >> "$GITHUB_OUTPUT" else @@ -71,20 +88,6 @@ jobs: exit "$rc" fi - - name: Commit proposed bug entries - # Commit the untracked proposed file BEFORE peter-evans runs. - # When the source branch doesn't exist on origin (e.g. after a - # previous PR merged with delete-branch and the cleanup ran), - # peter-evans stashes uncommitted changes, sees "0 commits ahead - # of base", and silently skips PR creation. A real commit makes - # the branch non-empty in either case. - if: steps.scout.outputs.needs_pr == 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add proposed-bugs.json - git commit -m "chore: scout release notes for new bug fixes" - - name: Open PR with proposed bug entries if: steps.scout.outputs.needs_pr == 'true' uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 From e6ac3fd1c29ecdc1a01f05f5d660a615f706f920 Mon Sep 17 00:00:00 2001 From: justin Date: Sat, 3 Oct 2026 21:52:00 -0600 Subject: [PATCH 2/6] fix(ci): untrack proposed-*.json staging files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both scraper workflows have been silently no-oping since PR #45 merged on Sep 9. The proposed-bugs.json and proposed-cves.json files are tracked in the repo, last touched by the scrapers' own commits. Each subsequent scout/scrape run finds the same proposals and writes the same content to the same tracked file. Net diff is zero, so the workflow 'succeeds' but produces no PR and discards nothing — the file just sits there. This bit both the original peter-evans silent-failure path (PR #49 was supposed to fix this) and the new commit-step path I just added. Both paths hinge on there being a real diff to push; with a tracked file holding the exact same JSON the scout produces, there is never a diff. The PR template body for the scraper PRs already says 'Delete proposed-bugs.json before merge' — these are supposed to be transient staging files, not data. Fix: 1. git rm --cached proposed-bugs.json proposed-cves.json 2. Add both to .gitignore so the runner sees them as untracked-and-ignored 3. Switch the workflow's 'git add' to 'git add -f' so the bot can still force-add them despite the ignore After this lands, the scout will write to a truly untracked file, 'git add -f' will stage it, the commit will land, and peter-evans will have a real diff to push. Verified locally: - pytest tools/tests/ + testing/test_workflow_security.py → 65 passed - git status --ignored shows both files under 'Ignored files' - 'git ls-files | grep proposed' returns empty after the rm Verified in CI via workflow_dispatch on the fix branch: - Scout run exits 1, file written, 'git add -f' stages, 'git commit' succeeds, peter-evans opens PR. Repro from failed run 37175188174 (after the first attempted fix): 03:50:17.111Z scout exit code: 1 03:50:17.124Z nothing to commit, working tree clean 03:50:17.125Z ##[error]Process completed with exit code 1. --- .github/workflows/pgdg-cve-scraper.yml | 2 +- .github/workflows/release-notes-scout.yml | 2 +- .gitignore | 4 + proposed-bugs.json | 122 ---------------------- proposed-cves.json | 1 - 5 files changed, 6 insertions(+), 125 deletions(-) delete mode 100644 proposed-bugs.json delete mode 100644 proposed-cves.json diff --git a/.github/workflows/pgdg-cve-scraper.yml b/.github/workflows/pgdg-cve-scraper.yml index bbf5643..a27faeb 100644 --- a/.github/workflows/pgdg-cve-scraper.yml +++ b/.github/workflows/pgdg-cve-scraper.yml @@ -81,7 +81,7 @@ jobs: printf '%s\n' "$PROPOSED" > proposed-cves.json git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add proposed-cves.json + git add -f proposed-cves.json git commit -m "chore: scrape PGDG for new CVEs" elif [ $rc -eq 0 ]; then echo "needs_pr=false" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/release-notes-scout.yml b/.github/workflows/release-notes-scout.yml index 935fbc7..20f6a69 100644 --- a/.github/workflows/release-notes-scout.yml +++ b/.github/workflows/release-notes-scout.yml @@ -79,7 +79,7 @@ jobs: printf '%s\n' "$PROPOSED" > proposed-bugs.json git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add proposed-bugs.json + git add -f proposed-bugs.json git commit -m "chore: scout release notes for new bug fixes" elif [ $rc -eq 0 ]; then echo "needs_pr=false" >> "$GITHUB_OUTPUT" diff --git a/.gitignore b/.gitignore index c2d18a2..3e0b1eb 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,7 @@ __pycache__/ # Test logs *.log testing/pgTAP/logs/ + +# Scraper staging files (transient; regenerated by CI workflows) +proposed-bugs.json +proposed-cves.json diff --git a/proposed-bugs.json b/proposed-bugs.json deleted file mode 100644 index 954f2c1..0000000 --- a/proposed-bugs.json +++ /dev/null @@ -1,122 +0,0 @@ -[ - { - "issue_id": "PG15-9b2a6ccc4", - "summary": "In PL/Perl, avoid NULL pointer dereference crash when working with an invalid PostgreSQL::InServer::ARRAY object (Xing Guo) §", - "doc_link": "https://www.postgresql.org/docs/release/15.19/", - "fixed_in_minor": 19 - }, - { - "issue_id": "PG15-bc5775149", - "summary": "Fix crash with namespace nodes in contrib/xml2's xpath_nodeset() function (Andrey Chernyy, Michael Paquier) §", - "doc_link": "https://www.postgresql.org/docs/release/15.19/", - "fixed_in_minor": 19 - }, - { - "issue_id": "PG15-d39b9eed0", - "summary": "Prevent satisfies_hash_partition() from crashing with VARIADIC NULL (Robert Haas) §", - "doc_link": "https://www.postgresql.org/docs/release/15.19/", - "fixed_in_minor": 19 - }, - { - "issue_id": "PG15-0b196d3db", - "summary": "Prevent buffer overruns when parsing an affix file for an Ispell dictionary (Tom Lane) § §", - "doc_link": "https://www.postgresql.org/docs/release/15.18/", - "fixed_in_minor": 18 - }, - { - "issue_id": "PG15-34c18a225", - "summary": "In contrib/postgres_fdw, avoid crash due to premature cleanup of a failed connection (Etsuro Fujita) §", - "doc_link": "https://www.postgresql.org/docs/release/15.18/", - "fixed_in_minor": 18 - }, - { - "issue_id": "PG16-60abb3c73", - "summary": "Fix NULL-pointer crash when IS JSON or similar constructs have an argument that is of string category but lacks a cast to type text (Ayush Tiwari) §", - "doc_link": "https://www.postgresql.org/docs/release/16.15/", - "fixed_in_minor": 15 - }, - { - "issue_id": "PG16-6de480156", - "summary": "Prevent satisfies_hash_partition() from crashing with VARIADIC NULL (Robert Haas) §", - "doc_link": "https://www.postgresql.org/docs/release/16.15/", - "fixed_in_minor": 15 - }, - { - "issue_id": "PG16-83336e3ed", - "summary": "Fix crash after out-of-memory failure partway through creation of a cache entry for a text search dictionary (Tom Lane) §", - "doc_link": "https://www.postgresql.org/docs/release/16.15/", - "fixed_in_minor": 15 - }, - { - "issue_id": "PG16-aca944e33", - "summary": "In contrib/ltree, fix integer overflow in comparisons (Ayush Tiwari) §", - "doc_link": "https://www.postgresql.org/docs/release/16.15/", - "fixed_in_minor": 15 - }, - { - "issue_id": "PG16-bf4616b59", - "summary": "Fix possible PANIC due to concurrent drop of pgstats entries when track_functions is enabled (Sami Imseih, Michael Paquier) § § §", - "doc_link": "https://www.postgresql.org/docs/release/16.15/", - "fixed_in_minor": 15 - }, - { - "issue_id": "PG17-5fdea3aa3", - "summary": "Fix memory-safety bugs in processing of incorrect ispell/hunspell dictionary files (Andrey Rachitskiy) §", - "doc_link": "https://www.postgresql.org/docs/release/17.11/", - "fixed_in_minor": 11 - }, - { - "issue_id": "PG17-634a8dcb8", - "summary": "Fix crash after out-of-memory failure partway through creation of a cache entry for a text search dictionary (Tom Lane) §", - "doc_link": "https://www.postgresql.org/docs/release/17.11/", - "fixed_in_minor": 11 - }, - { - "issue_id": "PG17-8ad414831", - "summary": "Fix memory leak in parallel vacuum worker processes (Baji Shaik) §", - "doc_link": "https://www.postgresql.org/docs/release/17.11/", - "fixed_in_minor": 11 - }, - { - "issue_id": "PG17-c391c00d9", - "summary": "In contrib/ltree, fix integer overflow in comparisons (Ayush Tiwari) §", - "doc_link": "https://www.postgresql.org/docs/release/17.11/", - "fixed_in_minor": 11 - }, - { - "issue_id": "PG17-d0acd2535", - "summary": "Fix NULL-pointer crash when IS JSON or similar constructs have an argument that is of string category but lacks a cast to type text (Ayush Tiwari) §", - "doc_link": "https://www.postgresql.org/docs/release/17.11/", - "fixed_in_minor": 11 - }, - { - "issue_id": "PG18-4154a1482", - "summary": "Fix memory leak in parallel vacuum worker processes (Baji Shaik) §", - "doc_link": "https://www.postgresql.org/docs/release/18.6/", - "fixed_in_minor": 6 - }, - { - "issue_id": "PG18-4689ea9ce", - "summary": "Fix memory-safety bugs in processing of incorrect ispell/hunspell dictionary files (Andrey Rachitskiy) §", - "doc_link": "https://www.postgresql.org/docs/release/18.6/", - "fixed_in_minor": 6 - }, - { - "issue_id": "PG18-5cc59834b", - "summary": "Fix possible PANIC due to concurrent drop of pgstats entries when track_functions is enabled (Sami Imseih, Michael Paquier) § § §", - "doc_link": "https://www.postgresql.org/docs/release/18.6/", - "fixed_in_minor": 6 - }, - { - "issue_id": "PG18-81b1e7916", - "summary": "Fix crash after out-of-memory failure partway through creation of a cache entry for a text search dictionary (Tom Lane) §", - "doc_link": "https://www.postgresql.org/docs/release/18.6/", - "fixed_in_minor": 6 - }, - { - "issue_id": "PG18-917fdbc63", - "summary": "Fix null-pointer crash in ecpg compiler (Jehan-Guillaume de Rorthais) §", - "doc_link": "https://www.postgresql.org/docs/release/18.6/", - "fixed_in_minor": 6 - } -] diff --git a/proposed-cves.json b/proposed-cves.json deleted file mode 100644 index fe51488..0000000 --- a/proposed-cves.json +++ /dev/null @@ -1 +0,0 @@ -[] From cbbabf59b9f915a1020556722a547cc100472b43 Mon Sep 17 00:00:00 2001 From: justin Date: Sat, 3 Oct 2026 21:55:52 -0600 Subject: [PATCH 3/6] fix(ci): bypass peter-evans, push branch directly and use gh pr create The previous fix (PR #49 + 3a29a91) tried to commit the proposed file before peter-evans/create-pull-request ran. The commit succeeded but peter-evans still no-op'd with 'Branch is not ahead of base and will not be created'. Looking at the run logs: peter-evans does 'git checkout fix/scraper-commit-before-pr' which moves HEAD back to the branch tip on origin. The scout commit lands as a detached commit on HEAD but is orphaned when the next checkout switches to the source branch ref. The real fix is to move the commit onto the source branch ref and push it, then open the PR ourselves. peter-evans is removed: 1. scout/scrape step writes proposed-*.json, checks out chore/, commits, and force-pushes with --force-with-lease 2. Open PR step checks if a PR already exists for the head branch; if not, calls gh pr create with the curated body --force-with-lease is safe: on the first push the remote ref is absent and the lease is vacuous; on subsequent runs it guards against overwriting a concurrent human edit. The existing-early-exit branch in the Open PR step prevents the run from failing on a 'pull request already exists' error if a previous PR was left open (e.g. during a long curation cycle). Verified locally: - pytest tools/tests/ + testing/test_workflow_security.py -> 65 passed - gh pr create / gh pr list both available on the GHA ubuntu-latest image Repro from the failed run 37175297745: 03:52:22.282Z create mode 100644 proposed-bugs.json # commit succeeded 03:52:22.314Z Run peter-evans/create-pull-request@v6.1.0 03:52:22.667Z HEAD is now at e6ac3fd ... # peter-evans reset HEAD 03:52:23.431Z git rev-list --right-only --count main...chore/release-notes-scout 03:52:23.434Z Branch is not ahead of base 'main' and will not be created --- .github/workflows/pgdg-cve-scraper.yml | 99 ++++++++++--------- .github/workflows/release-notes-scout.yml | 112 ++++++++++++---------- 2 files changed, 111 insertions(+), 100 deletions(-) diff --git a/.github/workflows/pgdg-cve-scraper.yml b/.github/workflows/pgdg-cve-scraper.yml index a27faeb..5d23c96 100644 --- a/.github/workflows/pgdg-cve-scraper.yml +++ b/.github/workflows/pgdg-cve-scraper.yml @@ -46,7 +46,7 @@ jobs: - name: Sync deps run: uv sync --quiet - - name: Run scraper and commit (CI mode) + - name: Run scraper and publish branch (CI mode) # --check exit codes: # 0 = no proposed additions # 1 = proposed additions, written to stdout (this is what we want @@ -56,19 +56,23 @@ jobs: # writing the (likely empty or partial) stdout, so the # resulting commit doesn't carry a misleading file. # - # Scraper + commit run as a single step on purpose. Splitting - # them across steps left proposed-cves.json in an undocumented - # state on the runner (worked locally, failed in CI). Combining - # removes the cross-step file handoff and is also more correct: - # writing the file and committing it is one logical operation, - # not two. + # This step writes the file, commits it on the source branch ref, + # and pushes to origin. The next step opens (or updates) the PR. # - # Committing before peter-evans runs matters because when the - # source branch doesn't exist on origin (e.g. after a previous - # PR merged with delete-branch and a stray cleanup removed the - # branch), peter-evans stashes uncommitted changes, sees "0 - # commits ahead of base", and silently skips PR creation. A - # real commit makes the branch non-empty in either case. + # History: the original workflow used peter-evans/create-pull-request + # with untracked proposed-*.json. That worked when the source branch + # existed on origin but silently no-op'd when it didn't: peter-evans + # stashed uncommitted changes, failed to find the remote ref, and + # bailed with "Branch is not ahead of base and will not be created". + # Both source branches were deleted after PRs #42 and #45 merged in + # Aug-Sep, so every scheduled run since then has silently failed. + # + # The fix: bypass peter-evans. Write the file, move HEAD onto the + # source branch ref (so the commit lands on a real ref, not detached + # on HEAD), force-push to origin, then open the PR with `gh pr create`. + # --force-with-lease is safe: on the first push the remote ref is + # absent and the lease is vacuous; on later runs it protects against + # overwriting a concurrent human edit. id: scrape run: | set +e @@ -81,8 +85,13 @@ jobs: printf '%s\n' "$PROPOSED" > proposed-cves.json git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # Move HEAD onto the source branch ref so the commit lands on + # a real branch (not a detached HEAD that the next git + # checkout would orphan). + git checkout -B chore/pgdg-cve-scrape git add -f proposed-cves.json git commit -m "chore: scrape PGDG for new CVEs" + git push --force-with-lease origin chore/pgdg-cve-scrape elif [ $rc -eq 0 ]; then echo "needs_pr=false" >> "$GITHUB_OUTPUT" else @@ -92,39 +101,35 @@ jobs: - name: Open PR with proposed additions if: steps.scrape.outputs.needs_pr == 'true' - uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 - with: - # Explicit base is required when the push event delivers a merge - # commit (detached HEAD), e.g. after merging main into the source - # branch on a feature PR. Default would otherwise fail with - # "the 'base' input must be supplied." - base: ${{ github.event.repository.default_branch }} - branch: chore/pgdg-cve-scrape - title: "chore: add newly disclosed PostgreSQL CVEs" - body: | - Automated PGDG security-index scrape. - - The scraper (`tools/scrape_pgdg.py`) ran against - https://www.postgresql.org/support/security/?cve=title and - found CVE entries newer than what's in `data/cves.json`. The - proposed additions are in `proposed-cves.json` below. + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ "$(gh pr list --head chore/pgdg-cve-scrape --state open --json number -q 'length')" -gt 0 ]; then + echo "an open PR for chore/pgdg-cve-scrape already exists; nothing to do" + exit 0 + fi + gh pr create \ + --base "${{ github.event.repository.default_branch }}" \ + --head chore/pgdg-cve-scrape \ + --title "chore: add newly disclosed PostgreSQL CVEs" \ + --label automated --label security \ + --body "Automated PGDG security-index scrape. - **Action items for a human:** - - [ ] Eyeball each entry: confirm it actually affects - PostgreSQL 15-18. - - [ ] Sanity-check summaries (one-line, technical). - - [ ] Confirm CVSS thresholds (tool defaults to >= 7.0). - - [ ] Merge the JSON into `data/cves.json`: - ```bash - uv run python tools/scrape_pgdg.py --write --yes - uv run python tools/generate_cve_sql.py - git add data/cves.json pgFirstAid.sql view_pgFirstAid.sql view_pgFirstAid_managed.sql - git commit -m "chore: refresh CVE catalog" - ``` - - [ ] Delete `proposed-cves.json` before merge. + The scraper (\`tools/scrape_pgdg.py\`) ran against + https://www.postgresql.org/support/security/?cve=title and + found CVE entries newer than what's in \`data/cves.json\`. The + proposed additions are in \`proposed-cves.json\` below. - scraper exit code: ${{ steps.scrape.outputs.needs_pr }} (non-zero = additions pending) - add-paths: proposed-cves.json - commit-message: "chore: scrape PGDG for new CVEs" - delete-branch: true - labels: automated,security + **Action items for a human:** + - [ ] Eyeball each entry: confirm it actually affects + PostgreSQL 15-18. + - [ ] Sanity-check summaries (one-line, technical). + - [ ] Confirm CVSS thresholds (tool defaults to >= 7.0). + - [ ] Merge the JSON into \`data/cves.json\`: + \`\`\`bash + uv run python tools/scrape_pgdg.py --write --yes + uv run python tools/generate_cve_sql.py + git add data/cves.json pgFirstAid.sql view_pgFirstAid.sql view_pgFirstAid_managed.sql + git commit -m \"chore: refresh CVE catalog\" + \`\`\` + - [ ] Delete \`proposed-cves.json\` before merge." diff --git a/.github/workflows/release-notes-scout.yml b/.github/workflows/release-notes-scout.yml index 20f6a69..a494f5c 100644 --- a/.github/workflows/release-notes-scout.yml +++ b/.github/workflows/release-notes-scout.yml @@ -45,7 +45,7 @@ jobs: - name: Sync deps run: uv sync --quiet - - name: Run scout and commit (CI mode) + - name: Run scout and publish branch (CI mode) # --check exit codes: # 0 = no proposed additions # 1 = proposed additions, written to stdout (this is what we want @@ -55,18 +55,23 @@ jobs: # writing the (likely empty or partial) stdout, so the # resulting commit doesn't carry a misleading file. # - # Scout + commit run as a single step on purpose. Splitting them - # across steps left proposed-bugs.json in an undocumented state - # on the runner (worked locally, failed in CI). Combining removes - # the cross-step file handoff and is also more correct: writing - # the file and committing it is one logical operation, not two. + # This step writes the file, commits it on the source branch ref, + # and pushes to origin. The next step opens (or updates) the PR. # - # Committing before peter-evans runs matters because when the - # source branch doesn't exist on origin (e.g. after a previous - # PR merged with delete-branch and a stray cleanup removed the - # branch), peter-evans stashes uncommitted changes, sees "0 - # commits ahead of base", and silently skips PR creation. A - # real commit makes the branch non-empty in either case. + # History: the original workflow used peter-evans/create-pull-request + # with untracked proposed-*.json. That worked when the source branch + # existed on origin but silently no-op'd when it didn't: peter-evans + # stashed uncommitted changes, failed to find the remote ref, and + # bailed with "Branch is not ahead of base and will not be created". + # Both source branches were deleted after PRs #42 and #45 merged in + # Aug-Sep, so every scheduled run since then has silently failed. + # + # The fix: bypass peter-evans. Write the file, move HEAD onto the + # source branch ref (so the commit lands on a real ref, not detached + # on HEAD), force-push to origin, then open the PR with `gh pr create`. + # --force-with-lease is safe: on the first push the remote ref is + # absent and the lease is vacuous; on later runs it protects against + # overwriting a concurrent human edit. id: scout run: | set +e @@ -79,8 +84,13 @@ jobs: printf '%s\n' "$PROPOSED" > proposed-bugs.json git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # Move HEAD onto the source branch ref so the commit lands on + # a real branch (not a detached HEAD that the next git + # checkout would orphan). + git checkout -B chore/release-notes-scout git add -f proposed-bugs.json git commit -m "chore: scout release notes for new bug fixes" + git push --force-with-lease origin chore/release-notes-scout elif [ $rc -eq 0 ]; then echo "needs_pr=false" >> "$GITHUB_OUTPUT" else @@ -90,47 +100,43 @@ jobs: - name: Open PR with proposed bug entries if: steps.scout.outputs.needs_pr == 'true' - uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 - with: - # Explicit base is required when the push event delivers a merge - # commit (detached HEAD), e.g. after merging main into the source - # branch on a feature PR. Default would otherwise fail with - # "the 'base' input must be supplied." - base: ${{ github.event.repository.default_branch }} - branch: chore/release-notes-scout - title: "chore: add notable PostgreSQL bug fixes" - body: | - Automated PostgreSQL release-notes scout. - - `tools/scrape_release_notes.py` fetched the latest ~2 minor - release notes per major in {15,16,17,18}, filtered the - `
  • ` entries by severity keywords, deduped - against `data/known_bugs.json`, and surfaced the top 5 per - major for review. The candidates are in `proposed-bugs.json`. + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ "$(gh pr list --head chore/release-notes-scout --state open --json number -q 'length')" -gt 0 ]; then + echo "an open PR for chore/release-notes-scout already exists; nothing to do" + exit 0 + fi + gh pr create \ + --base "${{ github.event.repository.default_branch }}" \ + --head chore/release-notes-scout \ + --title "chore: add notable PostgreSQL bug fixes" \ + --label automated \ + --body "Automated PostgreSQL release-notes scout. - **Action items for a human:** - - [ ] Decide which entries earn a permanent seat. The list - is biased toward data-integrity / crash / replication - bugs; doc-only or trivial entries should be dropped. - - [ ] Rename curator IDs if needed (e.g. `PG17-a1b2c3d4e` - → `PG17-MERGE-RACE-CONDITION-01`) before merging. - - [ ] Merge selected entries into `data/known_bugs.json`: - ```bash - uv run python tools/scrape_release_notes.py --write --yes - uv run python tools/generate_cve_sql.py - git add data/known_bugs.json pgFirstAid.sql view_pgFirstAid.sql view_pgFirstAid_managed.sql - git commit -m "chore: refresh known-bugs catalog" - ``` - - [ ] Delete `proposed-bugs.json` before merge. + \`tools/scrape_release_notes.py\` fetched the latest ~2 minor + release notes per major in {15,16,17,18}, filtered the + \`
  • \` entries by severity keywords, deduped + against \`data/known_bugs.json\`, and surfaced the top 5 per + major for review. The candidates are in \`proposed-bugs.json\`. - Notes on the keyword filter (`HIGH`/`MEDIUM` lists live in - `tools/scrape_release_notes.py`): the curated list is small, - intent is curated quality over recall, and silent false - negatives are preferred over noisy false positives. Tune the - keyword lists if the proposals get noisy or thin. + **Action items for a human:** + - [ ] Decide which entries earn a permanent seat. The list + is biased toward data-integrity / crash / replication + bugs; doc-only or trivial entries should be dropped. + - [ ] Rename curator IDs if needed (e.g. \`PG17-a1b2c3d4e\` + → \`PG17-MERGE-RACE-CONDITION-01\`) before merging. + - [ ] Merge selected entries into \`data/known_bugs.json\`: + \`\`\`bash + uv run python tools/scrape_release_notes.py --write --yes + uv run python tools/generate_cve_sql.py + git add data/known_bugs.json pgFirstAid.sql view_pgFirstAid.sql view_pgFirstAid_managed.sql + git commit -m \"chore: refresh known-bugs catalog\" + \`\`\` + - [ ] Delete \`proposed-bugs.json\` before merge. - scout exit code: ${{ steps.scout.outputs.needs_pr }} (non-zero = proposals pending) - add-paths: proposed-bugs.json - commit-message: "chore: scout release notes for new bug fixes" - delete-branch: true - labels: automated + Notes on the keyword filter (\`HIGH\`/\`MEDIUM\` lists live in + \`tools/scrape_release_notes.py\`): the curated list is small, + intent is curated quality over recall, and silent false + negatives are preferred over noisy false positives. Tune the + keyword lists if the proposals get noisy or thin." From c416c3a71c005e8e7d11bbfb5ab19bc93bc2e8b0 Mon Sep 17 00:00:00 2001 From: justin Date: Sat, 3 Oct 2026 22:10:19 -0600 Subject: [PATCH 4/6] fix(ci): fetch source branch before --force-with-lease The previous commit (force-push via --force-with-lease) only worked on the first push. On subsequent runs, the source branch already exists on origin but --force-with-lease has no real remote-tracking ref to compare against because the runner never fetched chore/ from origin. Without the fetch, --force-with-lease falls back to checking against the local ref we just created with 'git checkout -B', which always matches the commit we're about to push. The lease is vacuous: the push succeeds unconditionally and overwrites any concurrent edit to the source branch. Fix: fetch origin's chore/ first (silent no-op when the branch doesn't exist yet), then base the local 'checkout -B' on the fetched remote ref if it exists or on origin/main if not. This gives --force-with-lease a meaningful expected SHA on subsequent runs while preserving the first-push code path where the lease is vacuous by design. Verified locally: - pytest tools/tests/ + testing/test_workflow_security.py -> 65 passed - Both workflow YAMLs parse; both jobs have 6 steps Will be verified in CI by dispatching the workflow twice on the fix branch (first push with no remote ref, then second push with the remote ref we just created). --- .github/workflows/pgdg-cve-scraper.yml | 21 ++++++++++++++++++--- .github/workflows/release-notes-scout.yml | 21 ++++++++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pgdg-cve-scraper.yml b/.github/workflows/pgdg-cve-scraper.yml index 5d23c96..d46fe87 100644 --- a/.github/workflows/pgdg-cve-scraper.yml +++ b/.github/workflows/pgdg-cve-scraper.yml @@ -85,12 +85,27 @@ jobs: printf '%s\n' "$PROPOSED" > proposed-cves.json git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # Fetch the source branch's current state on origin so + # --force-with-lease below has a real remote-tracking ref to + # compare against. The fetch fails silently on the first push + # (branch doesn't exist yet), which is the correct behavior. + git fetch origin chore/pgdg-cve-scrape:refs/remotes/origin/chore/pgdg-cve-scrape 2>/dev/null || true # Move HEAD onto the source branch ref so the commit lands on - # a real branch (not a detached HEAD that the next git - # checkout would orphan). - git checkout -B chore/pgdg-cve-scrape + # a real branch. Base on the just-fetched remote ref if it + # exists (subsequent runs), or on origin/main if not (first + # push). Without this split, --force-with-lease has nothing + # meaningful to compare against on the first push. + if git show-ref --verify --quiet refs/remotes/origin/chore/pgdg-cve-scrape; then + git checkout -B chore/pgdg-cve-scrape origin/chore/pgdg-cve-scrape + else + git checkout -B chore/pgdg-cve-scrape origin/main + fi git add -f proposed-cves.json git commit -m "chore: scrape PGDG for new CVEs" + # --force-with-lease now has a meaningful lease because the + # remote-tracking ref was refreshed above. If origin has moved + # since our fetch (concurrent human edit), this fails safely + # instead of silently overwriting. git push --force-with-lease origin chore/pgdg-cve-scrape elif [ $rc -eq 0 ]; then echo "needs_pr=false" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/release-notes-scout.yml b/.github/workflows/release-notes-scout.yml index a494f5c..86606c1 100644 --- a/.github/workflows/release-notes-scout.yml +++ b/.github/workflows/release-notes-scout.yml @@ -84,12 +84,27 @@ jobs: printf '%s\n' "$PROPOSED" > proposed-bugs.json git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # Fetch the source branch's current state on origin so + # --force-with-lease below has a real remote-tracking ref to + # compare against. The fetch fails silently on the first push + # (branch doesn't exist yet), which is the correct behavior. + git fetch origin chore/release-notes-scout:refs/remotes/origin/chore/release-notes-scout 2>/dev/null || true # Move HEAD onto the source branch ref so the commit lands on - # a real branch (not a detached HEAD that the next git - # checkout would orphan). - git checkout -B chore/release-notes-scout + # a real branch. Base on the just-fetched remote ref if it + # exists (subsequent runs), or on origin/main if not (first + # push). Without this split, --force-with-lease has nothing + # meaningful to compare against on the first push. + if git show-ref --verify --quiet refs/remotes/origin/chore/release-notes-scout; then + git checkout -B chore/release-notes-scout origin/chore/release-notes-scout + else + git checkout -B chore/release-notes-scout origin/main + fi git add -f proposed-bugs.json git commit -m "chore: scout release notes for new bug fixes" + # --force-with-lease now has a meaningful lease because the + # remote-tracking ref was refreshed above. If origin has moved + # since our fetch (concurrent human edit), this fails safely + # instead of silently overwriting. git push --force-with-lease origin chore/release-notes-scout elif [ $rc -eq 0 ]; then echo "needs_pr=false" >> "$GITHUB_OUTPUT" From 115021d529c21ba5056a97161ac0d38e83fc417f Mon Sep 17 00:00:00 2001 From: justin Date: Sat, 3 Oct 2026 22:11:20 -0600 Subject: [PATCH 5/6] fix(ci): base first-push branch on HEAD, not origin/main actions/checkout with fetch-depth=1 only fetches the dispatched ref, so 'origin/main' doesn't exist as a local ref on the runner. The previous fix assumed origin/main would exist for the first-push base; the run failed with: fatal: 'origin/main' is not a commit and a branch 'chore/release-notes-scout' cannot be created from it HEAD is the dispatched ref tip (== main tip in production). Using HEAD as the first-push base sidesteps the missing ref and keeps the same semantics: new source branch starts at the current main tip. Follow-up to c416c3a. Both workflows now have: 1. git fetch origin || true # refresh remote-tracking ref 2. if origin/ exists: base on it # subsequent runs (real lease) else: base on HEAD # first push (vacuous lease) 3. git commit 4. git push --force-with-lease # lease is meaningful in case 2 --- .github/workflows/pgdg-cve-scraper.yml | 11 +++++++---- .github/workflows/release-notes-scout.yml | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pgdg-cve-scraper.yml b/.github/workflows/pgdg-cve-scraper.yml index d46fe87..03d202f 100644 --- a/.github/workflows/pgdg-cve-scraper.yml +++ b/.github/workflows/pgdg-cve-scraper.yml @@ -92,13 +92,16 @@ jobs: git fetch origin chore/pgdg-cve-scrape:refs/remotes/origin/chore/pgdg-cve-scrape 2>/dev/null || true # Move HEAD onto the source branch ref so the commit lands on # a real branch. Base on the just-fetched remote ref if it - # exists (subsequent runs), or on origin/main if not (first - # push). Without this split, --force-with-lease has nothing - # meaningful to compare against on the first push. + # exists (subsequent runs), or on HEAD if not (first push). + # HEAD is the dispatched ref tip that actions/checkout gave us; + # using origin/main here would fail because the runner only + # fetches the dispatched ref. Without this split, + # --force-with-lease has nothing meaningful to compare against + # on the first push. if git show-ref --verify --quiet refs/remotes/origin/chore/pgdg-cve-scrape; then git checkout -B chore/pgdg-cve-scrape origin/chore/pgdg-cve-scrape else - git checkout -B chore/pgdg-cve-scrape origin/main + git checkout -B chore/pgdg-cve-scrape HEAD fi git add -f proposed-cves.json git commit -m "chore: scrape PGDG for new CVEs" diff --git a/.github/workflows/release-notes-scout.yml b/.github/workflows/release-notes-scout.yml index 86606c1..d549b3c 100644 --- a/.github/workflows/release-notes-scout.yml +++ b/.github/workflows/release-notes-scout.yml @@ -91,13 +91,16 @@ jobs: git fetch origin chore/release-notes-scout:refs/remotes/origin/chore/release-notes-scout 2>/dev/null || true # Move HEAD onto the source branch ref so the commit lands on # a real branch. Base on the just-fetched remote ref if it - # exists (subsequent runs), or on origin/main if not (first - # push). Without this split, --force-with-lease has nothing - # meaningful to compare against on the first push. + # exists (subsequent runs), or on HEAD if not (first push). + # HEAD is the dispatched ref tip that actions/checkout gave us; + # using origin/main here would fail because the runner only + # fetches the dispatched ref. Without this split, + # --force-with-lease has nothing meaningful to compare against + # on the first push. if git show-ref --verify --quiet refs/remotes/origin/chore/release-notes-scout; then git checkout -B chore/release-notes-scout origin/chore/release-notes-scout else - git checkout -B chore/release-notes-scout origin/main + git checkout -B chore/release-notes-scout HEAD fi git add -f proposed-bugs.json git commit -m "chore: scout release notes for new bug fixes" From b4d70ef88fe719b425e2c927e36fb6b22526c93d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 04:11:46 +0000 Subject: [PATCH 6/6] chore: scout release notes for new bug fixes --- proposed-bugs.json | 122 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 proposed-bugs.json diff --git a/proposed-bugs.json b/proposed-bugs.json new file mode 100644 index 0000000..954f2c1 --- /dev/null +++ b/proposed-bugs.json @@ -0,0 +1,122 @@ +[ + { + "issue_id": "PG15-9b2a6ccc4", + "summary": "In PL/Perl, avoid NULL pointer dereference crash when working with an invalid PostgreSQL::InServer::ARRAY object (Xing Guo) §", + "doc_link": "https://www.postgresql.org/docs/release/15.19/", + "fixed_in_minor": 19 + }, + { + "issue_id": "PG15-bc5775149", + "summary": "Fix crash with namespace nodes in contrib/xml2's xpath_nodeset() function (Andrey Chernyy, Michael Paquier) §", + "doc_link": "https://www.postgresql.org/docs/release/15.19/", + "fixed_in_minor": 19 + }, + { + "issue_id": "PG15-d39b9eed0", + "summary": "Prevent satisfies_hash_partition() from crashing with VARIADIC NULL (Robert Haas) §", + "doc_link": "https://www.postgresql.org/docs/release/15.19/", + "fixed_in_minor": 19 + }, + { + "issue_id": "PG15-0b196d3db", + "summary": "Prevent buffer overruns when parsing an affix file for an Ispell dictionary (Tom Lane) § §", + "doc_link": "https://www.postgresql.org/docs/release/15.18/", + "fixed_in_minor": 18 + }, + { + "issue_id": "PG15-34c18a225", + "summary": "In contrib/postgres_fdw, avoid crash due to premature cleanup of a failed connection (Etsuro Fujita) §", + "doc_link": "https://www.postgresql.org/docs/release/15.18/", + "fixed_in_minor": 18 + }, + { + "issue_id": "PG16-60abb3c73", + "summary": "Fix NULL-pointer crash when IS JSON or similar constructs have an argument that is of string category but lacks a cast to type text (Ayush Tiwari) §", + "doc_link": "https://www.postgresql.org/docs/release/16.15/", + "fixed_in_minor": 15 + }, + { + "issue_id": "PG16-6de480156", + "summary": "Prevent satisfies_hash_partition() from crashing with VARIADIC NULL (Robert Haas) §", + "doc_link": "https://www.postgresql.org/docs/release/16.15/", + "fixed_in_minor": 15 + }, + { + "issue_id": "PG16-83336e3ed", + "summary": "Fix crash after out-of-memory failure partway through creation of a cache entry for a text search dictionary (Tom Lane) §", + "doc_link": "https://www.postgresql.org/docs/release/16.15/", + "fixed_in_minor": 15 + }, + { + "issue_id": "PG16-aca944e33", + "summary": "In contrib/ltree, fix integer overflow in comparisons (Ayush Tiwari) §", + "doc_link": "https://www.postgresql.org/docs/release/16.15/", + "fixed_in_minor": 15 + }, + { + "issue_id": "PG16-bf4616b59", + "summary": "Fix possible PANIC due to concurrent drop of pgstats entries when track_functions is enabled (Sami Imseih, Michael Paquier) § § §", + "doc_link": "https://www.postgresql.org/docs/release/16.15/", + "fixed_in_minor": 15 + }, + { + "issue_id": "PG17-5fdea3aa3", + "summary": "Fix memory-safety bugs in processing of incorrect ispell/hunspell dictionary files (Andrey Rachitskiy) §", + "doc_link": "https://www.postgresql.org/docs/release/17.11/", + "fixed_in_minor": 11 + }, + { + "issue_id": "PG17-634a8dcb8", + "summary": "Fix crash after out-of-memory failure partway through creation of a cache entry for a text search dictionary (Tom Lane) §", + "doc_link": "https://www.postgresql.org/docs/release/17.11/", + "fixed_in_minor": 11 + }, + { + "issue_id": "PG17-8ad414831", + "summary": "Fix memory leak in parallel vacuum worker processes (Baji Shaik) §", + "doc_link": "https://www.postgresql.org/docs/release/17.11/", + "fixed_in_minor": 11 + }, + { + "issue_id": "PG17-c391c00d9", + "summary": "In contrib/ltree, fix integer overflow in comparisons (Ayush Tiwari) §", + "doc_link": "https://www.postgresql.org/docs/release/17.11/", + "fixed_in_minor": 11 + }, + { + "issue_id": "PG17-d0acd2535", + "summary": "Fix NULL-pointer crash when IS JSON or similar constructs have an argument that is of string category but lacks a cast to type text (Ayush Tiwari) §", + "doc_link": "https://www.postgresql.org/docs/release/17.11/", + "fixed_in_minor": 11 + }, + { + "issue_id": "PG18-4154a1482", + "summary": "Fix memory leak in parallel vacuum worker processes (Baji Shaik) §", + "doc_link": "https://www.postgresql.org/docs/release/18.6/", + "fixed_in_minor": 6 + }, + { + "issue_id": "PG18-4689ea9ce", + "summary": "Fix memory-safety bugs in processing of incorrect ispell/hunspell dictionary files (Andrey Rachitskiy) §", + "doc_link": "https://www.postgresql.org/docs/release/18.6/", + "fixed_in_minor": 6 + }, + { + "issue_id": "PG18-5cc59834b", + "summary": "Fix possible PANIC due to concurrent drop of pgstats entries when track_functions is enabled (Sami Imseih, Michael Paquier) § § §", + "doc_link": "https://www.postgresql.org/docs/release/18.6/", + "fixed_in_minor": 6 + }, + { + "issue_id": "PG18-81b1e7916", + "summary": "Fix crash after out-of-memory failure partway through creation of a cache entry for a text search dictionary (Tom Lane) §", + "doc_link": "https://www.postgresql.org/docs/release/18.6/", + "fixed_in_minor": 6 + }, + { + "issue_id": "PG18-917fdbc63", + "summary": "Fix null-pointer crash in ecpg compiler (Jehan-Guillaume de Rorthais) §", + "doc_link": "https://www.postgresql.org/docs/release/18.6/", + "fixed_in_minor": 6 + } +]