diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ffefc8f..92f390c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -15,9 +15,9 @@ body: - type: input id: version attributes: - label: aurtomator version or commit SHA - description: Tagged release or short SHA of the commit you are running. - placeholder: "v0.2.2 or abc1234" + label: aurtomator version + description: "The version your calling workflows pin, from the `# vX.Y.Z` comment after the SHA." + placeholder: "v0.8.0" validations: required: true @@ -37,7 +37,7 @@ body: attributes: label: Workflow run URL description: If this is a CI-related bug, link to the failing workflow run. - placeholder: "https://github.com///actions/runs/1234567890" + placeholder: "https://github.com///actions/runs/1234567890" - type: input id: strategy diff --git a/.github/workflows/check-updates.yml b/.github/workflows/check-updates.yml index 869d45f..1a90f18 100644 --- a/.github/workflows/check-updates.yml +++ b/.github/workflows/check-updates.yml @@ -617,7 +617,7 @@ jobs: [[ -d "$STATUS_DIR" ]] || exit 0 # gh issue list fails in a repository with Issues turned off. A - # fork like that gets a notice, and any other failure of the + # repository like that gets a notice, and any other failure of the # list still fails the step. if grep -qsE '^(check_failed|update_failed|held|newer):' "$STATUS_DIR"/* || [[ -s "$STATUS_DIR/_alerts" ]]; then has_issues=$(gh api "repos/$GITHUB_REPOSITORY" --jq .has_issues) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 41027de..5eacbee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to aurtomator -aurtomator is a template repository: you fork it and run your own instance. Upstream contributions are welcome and optional. +aurtomator holds the scripts, the strategies and the reusable workflows. A user repository, created from the template [staticwire/aurtomator-seed](https://github.com/staticwire/aurtomator-seed), holds the package YAMLs and calls the workflows of aurtomator at a pinned SHA, see [docs/SETUP.md](docs/SETUP.md#the-calling-workflows). A run takes the scripts and the strategies from aurtomator at that SHA, so a fix or a new strategy comes here by pull request. ## Before opening a PR @@ -44,6 +44,8 @@ bats tests/ --recursive CI runs the suite under bashcov, as `bashcov -- tests/helpers/bashcov-bats.sh tests/ --recursive`. bashcov passes `errexit`, `nounset` and `pipefail` to every script a test starts, so a mock has to work under `set -euo pipefail`. +`ci.yml` also calls the reusable workflows of the same commit: the job `validate` calls `validate.yml`, and the job `pipeline` calls `check-updates.yml` with aurtomator itself as the repository of the packages, see [docs/WORKFLOW.md section 6.4](docs/WORKFLOW.md#64-validate-and-pipeline-jobs). A pull request from a fork of aurtomator gets a read-only token, so `pipeline` runs only for a push and for a pull request from a branch of aurtomator. + ## PR workflow PRs are squash-merged. The squash commit subject on `main` must follow Conventional Commits with a scope, so write the PR title and the commit subjects that way. A PR titled `fix strategy bug` is renamed to something like `fix(strategy): handle empty release list from github-release` before it is merged. @@ -52,7 +54,7 @@ Keep PRs focused: one logical change per PR. Unrelated refactors go in a separat ## Releases -aurtomator follows semantic versioning, and `VERSION` holds the version of the code. After a merge to `main`, release-please (`.github/workflows/release-please.yml`) opens or updates a release PR from the commit types since the last release: `fix` raises the patch number, `feat` the minor one, and before 1.0.0 a breaking change raises the minor one too. A `docs`, `ci`, `chore` or `test` commit opens no release PR and goes out with the next release. The release PR writes the new version to `VERSION` and to the version line of the package table in `README.md`. Its merge tags `vX.Y.Z` and publishes the release with the notes GitHub generates from the merged PRs. A line `Release-As: X.Y.Z` in the body of a squash commit sets another number. +aurtomator follows semantic versioning, and `VERSION` holds the version of the code. After a merge to `main`, release-please (`.github/workflows/release-please.yml`) opens or updates a release PR from the commit types since the last release: `fix` raises the patch number, `feat` the minor one, and before 1.0.0 a breaking change raises the minor one too. A `!` after the type and scope marks a breaking change, as in `feat(workflow)!: run as a reusable workflow`. A `docs`, `ci`, `chore` or `test` commit opens no release PR and goes out with the next release. The release PR writes the new version to `VERSION` and to the version line of the package table in `README.md`. Its merge tags `vX.Y.Z` and publishes the release with the notes GitHub generates from the merged PRs. A line `Release-As: X.Y.Z` in the body of a squash commit sets another number. The workflow runs release-please with the token of a GitHub App installed on the repository, with read and write access to contents, issues and pull requests. It creates the token from the variable `RELEASE_APP_CLIENT_ID` and the secret `RELEASE_APP_PRIVATE_KEY`. With the workflow token instead, GitHub would run the CI of the release PR only after someone with write access approves it. diff --git a/README.md b/README.md index 3ea9569..96dc867 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Lint & Test](https://github.com/staticwire/aurtomator/actions/workflows/ci.yml/badge.svg)](https://github.com/staticwire/aurtomator/actions/workflows/ci.yml) [![codecov](https://codecov.io/github/staticwire/aurtomator/graph/badge.svg)](https://codecov.io/github/staticwire/aurtomator) [![Bash](https://img.shields.io/badge/bash-5%2B-green)](https://www.gnu.org/software/bash/) [![AUR](https://img.shields.io/badge/AUR-automation-orange)](https://aur.archlinux.org/) [![License](https://img.shields.io/badge/license-BSD--3--Clause-blue)](LICENSE) Automated AUR package updates from a single GitHub repo. -[Fork it](https://github.com/staticwire/aurtomator/fork), drop in your packages, let CI handle the rest. +Create a repository from the template [staticwire/aurtomator-seed](https://github.com/staticwire/aurtomator-seed) and drop in your packages. Its workflows call the pipeline of aurtomator at a pinned commit. ## Packages @@ -22,7 +22,8 @@ aurtomator 0.7.1 ```mermaid graph LR - A[GitHub Actions
on schedule] -->|for each package| K[check job, read-only token
run strategy
decide the run] + U[your repository
package YAMLs
calling workflow on schedule] --> R[aurtomator at the pinned SHA
reusable check-updates.yml] + R -->|for each package| K[check job, read-only token
run strategy
decide the run] K -->|new version| B[prepare job, no secrets
clone AUR repo
sed pkgver
updpkgsums
makepkg --printsrcinfo] B --> V[verify job, no secrets
download the sources
check the checksums] V --> C[publish job
check the prepared files
git push to AUR] @@ -32,17 +33,18 @@ graph LR C --> D ``` -Each package has a YAML config pointing to an upstream source. A first job runs the strategies and decides what the run does. When a new version is detected, aurtomator clones the existing AUR repo, bumps `pkgver`, regenerates checksums and `.SRCINFO`, and pushes. A PKGBUILD is code, so these steps run in a job without secrets, and the jobs after it take the statuses from the first job. A second job without secrets downloads the sources and checks the checksums, and the job that pushes checks the result once more. It does not generate PKGBUILDs or modify anything else. A package that starts failing gets a GitHub Issue, and so does a failed run of the workflow. +Each package has a YAML config pointing to an upstream source. Your repository holds these configs and calls the pipeline of aurtomator. A first job runs the strategies and decides what the run does. When a new version is detected, aurtomator clones the existing AUR repo, bumps `pkgver`, regenerates checksums and `.SRCINFO`, and pushes. A PKGBUILD is code, so these steps run in a job without secrets, and the jobs after it take the statuses from the first job. A second job without secrets downloads the sources and checks the checksums, and the job that pushes checks the result once more. It does not generate PKGBUILDs or modify anything else. A package that starts failing gets a GitHub Issue, and so does a failed run of the workflow. ## Quick start -1. Fork this repo (or use as template) -2. Enable Actions and Issues in your fork's Settings ([why?](docs/SETUP.md#after-forking)) -3. Run `./scripts/setup.sh` ([detailed guide](docs/SETUP.md)) -4. Add a package: `cp packages/example.yml.sample packages/.yml`, where `` is the pkgbase on the AUR -5. Edit the copy: set `name: `, a `strategy` and its `upstream` fields, see [Strategies](#strategies) -6. Validate: `./scripts/validate-pkg.sh ` -7. Push and let CI handle the rest +1. Create a repository from [staticwire/aurtomator-seed](https://github.com/staticwire/aurtomator-seed) with "Use this template" +2. Make sure Issues are on in the Settings of the new repository ([why?](docs/SETUP.md#after-creating-the-repository)) +3. Clone aurtomator for the scripts, at the tag the calling workflows pin: `git clone https://github.com/staticwire/aurtomator ~/src/aurtomator` and `git -C ~/src/aurtomator checkout vX.Y.Z`, where `vX.Y.Z` is the version in the `# vX.Y.Z` comment of those workflows +4. Clone the new repository and run `~/src/aurtomator/scripts/setup.sh` from inside it ([detailed guide](docs/SETUP.md)) +5. Add a package: `cp ~/src/aurtomator/packages/example.yml.sample packages/.yml`, where `` is the pkgbase on the AUR +6. Edit the copy: set `name: `, a `strategy` and its `upstream` fields, see [Strategies](#strategies) +7. Validate from inside the new repository: `~/src/aurtomator/scripts/validate-pkg.sh ` +8. Push and let CI handle the rest The setup script configures your identity (git author for AUR commits), SSH access, and GitHub Secrets. With [gh CLI](https://cli.github.com/) it sets the secrets itself. Without it, the script prints manual instructions. GPG commit signing is optional. @@ -99,7 +101,7 @@ Some upstreams name a release asset with more than the version and rebuild a rel ## Updates of aurtomator -`VERSION` holds the version of aurtomator a repository runs, and the package table shows it. When a newer aurtomator release is out, a run links it under the table and opens an issue, which closes once the fork has merged the release. Details are in [docs/WORKFLOW.md section 5.8](docs/WORKFLOW.md#58-version-of-aurtomator). +The line under the package table shows the version a run uses, which is `VERSION` of aurtomator at the SHA the calling workflows pin. Dependabot opens the pull request that moves the pin. A newer release also gets an issue and a link on that line. A run closes the issue once the pin of the calling workflows names that release, or once a newer release is out. Details are in [docs/WORKFLOW.md section 5.8](docs/WORKFLOW.md#58-version-of-aurtomator). ## Resource usage @@ -119,7 +121,7 @@ All packages run in one workflow run. `check` and `issues` run every time, `prep | [lilac] | Large repos (archlinuxcn) | Full build bot, needs own infra | | [aur-auto-update] | Centralized service | Requires bot co-maintainer on AUR | | [Renovate] | Multi-ecosystem | Enterprise-grade, heavy config | -| **aurtomator** | Individual maintainers | Fork and go | +| **aurtomator** | Individual maintainers | Template and a pinned workflow | [nvchecker]: https://github.com/lilydjwg/nvchecker [lilac]: https://github.com/archlinuxcn/lilac diff --git a/SECURITY.md b/SECURITY.md index fd714a5..69c7dad 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,7 +7,7 @@ | latest release | yes | | older releases | no | -There are no backports. A fork is expected to merge fixes from upstream as they land. Unreleased `main` is not supported. +There are no backports. A user repository calls the workflows of aurtomator at a pinned SHA, the SHA of a release tag. It is expected to move the pin to the latest release, and Dependabot opens the pull request that moves it, see [docs/SETUP.md](docs/SETUP.md#the-calling-workflows). Unreleased `main` is not supported. ## Reporting a vulnerability @@ -26,14 +26,14 @@ In scope: - `scripts/`: any defect that allows arbitrary code execution, secret exfiltration, or writes outside the working tree. - `strategies/`: injection through crafted upstream responses, YAML parsing issues, or command injection through package configuration. -- `.github/workflows/`: misconfiguration that leaks the `AUR_SSH_KEY`, `GPG_SIGNING_KEY`, `AUR_GIT_NAME` or `AUR_GIT_EMAIL` secrets into logs, artifacts, forks, pull requests or third-party actions. -- Anything else in this repository that can make a fork push a compromised package to the AUR without the fork maintainer's intent. +- `.github/workflows/`: misconfiguration that leaks the `AUR_SSH_KEY`, `GPG_SIGNING_KEY`, `AUR_GIT_NAME` or `AUR_GIT_EMAIL` secrets into logs, artifacts, pull requests or third-party actions. A user repository passes these secrets to the reusable workflows when it calls them. +- Anything else in this repository that can make a user repository push a compromised package to the AUR without the intent of its maintainer. Out of scope: - Vulnerabilities in external tools that aurtomator calls (`makepkg`, `yq`, `curl`, `git`, `namcap`, `shellcheck`, `shfmt`, `bats`). Report those upstream. -- The contents of AUR packages published by forks. Each fork maintainer is responsible for the packages they publish. -- The fork model itself, for example that a malicious fork can push a bad package. +- The contents of AUR packages published from user repositories. The maintainer of each user repository is responsible for the packages they publish. +- The model itself: a user repository runs the workflows with its own AUR key, and a pin that is not the SHA of a release tag runs the code of the commit it names. - Issues in third-party GitHub Actions. Report them to the action's maintainer. aurtomator pins every action by commit SHA. ## Known limitations @@ -42,4 +42,4 @@ Publish compares a line of the prepared `.SRCINFO` other than `pkgver`, `pkgrel` ## Disclosure -Once a reported issue is fixed, the advisory is published on the repository's Security tab. A CVE assigned through GitHub's CNA is referenced in the published advisory. A fork gets the fix with its next merge from upstream. Watching this repository's releases gives a notification when a fix ships. +Once a reported issue is fixed, the advisory is published on the repository's Security tab. A CVE assigned through GitHub's CNA is referenced in the published advisory. A user repository gets the fix once the pin of its calling workflows moves to a release that holds the fix. Watching this repository's releases gives a notification when a fix ships. diff --git a/docs/SETUP.md b/docs/SETUP.md index 50e9426..7251a75 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -23,37 +23,99 @@ Install everything at once: sudo pacman -S --needed git openssh go-yq curl pacman-contrib namcap github-cli gnupg ``` -## After forking +## After creating the repository + +A repository created from the template [staticwire/aurtomator-seed](https://github.com/staticwire/aurtomator-seed) with "Use this template" holds the two [calling workflows](#the-calling-workflows), `.github/dependabot.yml`, a `.gitignore`, an empty `packages/` directory and a README with the markers that a run fills. ### Required -1. **Enable GitHub Actions**: open the Actions tab of your fork and enable workflows. GitHub disables Actions on forks by default. -2. **Enable the "Check AUR Package Updates" workflow**: open it in the Actions tab and click "Enable workflow". A fork needs this for every scheduled workflow. GitHub also disables a scheduled workflow of a public repository after 60 days without activity in it. The workflow commits only when a status or a version changes, so a fork whose packages stay the same that long loses its schedule and has to enable the workflow again. -3. **Enable Issues**: Settings > General > Features > Issues. Failures, update alerts and newer aurtomator releases are reported as issues. With Issues off, a run prints a notice instead and opens none. -4. **Run setup**: `./scripts/setup.sh` configures identity, SSH, GPG and GitHub Secrets, as described below. +1. **Enable Issues**: Settings > General > Features > Issues. Failures, update alerts and newer aurtomator releases are reported as issues. With Issues off, a run prints a notice instead and opens none. +2. **Check Actions**: the Actions tab lists `Check AUR Package Updates` and `Validate packages`. A workflow shown as disabled there runs again once you open it and click "Enable workflow". In a fork, GitHub disables the workflows until they are enabled in its Actions tab. GitHub also disables a scheduled workflow of a public repository after 60 days without activity in it. The workflow commits only when a status or a version changes, so a repository whose packages stay the same that long loses its schedule and needs "Enable workflow" again. +3. **Set the secrets**: `~/src/aurtomator/scripts/setup.sh`, run from inside a clone of your repository, sets `AUR_SSH_KEY`, `AUR_GIT_NAME`, `AUR_GIT_EMAIL` and, with GPG, `GPG_SIGNING_KEY`, as [Quick setup](#quick-setup-with-gh-cli) describes. The calling workflow passes them to aurtomator by name. ### Recommended -- **Enable Dependabot version updates**: Settings > Advanced Security, then Enable next to "Dependabot version updates". GitHub leaves them off on forks. Dependabot then opens pull requests for new action versions and for the digest of the Arch image the workflows run in. -- **Merge Arch image updates automatically** (optional): Settings > Secrets and variables > Actions > Variables > New repository variable, with the name `CI_IMAGE_AUTO_MERGE` and the value `true`. A new image then reaches the jobs with the secrets after the tests and without a review, see [WORKFLOW.md](WORKFLOW.md#63-ci-image-updates). The merge is a squash merge, so Settings > General > Pull Requests has to allow squash merging. -- Set repository topics if your fork is public. +- **Dependabot version updates**: the `.github/dependabot.yml` of the template turns them on in a repository created from it. Dependabot then opens the pull request that moves the pin of the calling workflows to a new release of aurtomator, see [The calling workflows](#the-calling-workflows). A fork of the template needs the switch: Settings > Advanced Security, then Enable next to "Dependabot version updates". +- Set repository topics if your repository is public. ### Badges -`update-readme.sh` points the README badges (CI, Codecov) at your fork. +`update-readme.sh` writes three badges between the `` and `` markers of the README: `Check AUR Package Updates`, `Validate packages` and `automated by aurtomator`. A run of `Check AUR Package Updates` runs the script and commits the README when it changed. + +The badge URLs name `check-updates.yml` and `validate.yml` in your repository, so the calling workflows keep those file names. + +## The calling workflows + +`.github/workflows/check-updates.yml` runs the update pipeline at minute 17 of every hour, and its `notify` job opens an issue when the run fails: + +```yaml +name: Check AUR Package Updates + +on: + schedule: + - cron: "17 * * * *" + workflow_dispatch: + +concurrency: + group: check-updates + cancel-in-progress: false + +jobs: + check-updates: + uses: staticwire/aurtomator/.github/workflows/check-updates.yml@ # vX.Y.Z + permissions: + contents: write + issues: write + secrets: + AUR_SSH_KEY: ${{ secrets.AUR_SSH_KEY }} + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + AUR_GIT_NAME: ${{ secrets.AUR_GIT_NAME }} + AUR_GIT_EMAIL: ${{ secrets.AUR_GIT_EMAIL }} + + notify: + needs: check-updates + if: ${{ always() && needs.check-updates.result != 'success' }} + uses: staticwire/aurtomator/.github/workflows/notify-failure.yml@ # vX.Y.Z + permissions: + actions: read + issues: write +``` + +`.github/workflows/validate.yml` runs `validate-pkg.sh --all` on every push to `main` and on pull requests: + +```yaml +name: Validate packages + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read -- **Codecov needs a public repository.** A private fork shows the upstream aurtomator badge. -- **A fresh fork shows "no status".** The badges activate after your first commit to `main`, for example adding a package. +jobs: + validate: + uses: staticwire/aurtomator/.github/workflows/validate.yml@ # vX.Y.Z +``` + +The pin after `@` is the SHA of a release tag of aurtomator, and the comment after it names that release. All three `uses:` lines carry the same pin. A tag can be moved and a SHA cannot, so a run always gets the code of that release. `git -C ~/src/aurtomator rev-parse 'vX.Y.Z^{commit}'` prints the SHA of a tag. Dependabot opens the pull request that moves the pin to a new release, and the `Validate packages` run of that pull request checks your packages with the validator of the new release before the merge. After the merge, `git -C ~/src/aurtomator fetch` and `git -C ~/src/aurtomator checkout` with the new tag move the clone to the same release. + +A calling job sets the permissions of the token for the whole called workflow, and a job of the called workflow can lower them but cannot raise them. `check-updates` grants `contents: write` for the commit of the package versions and the README, and `issues: write` for the issues it opens and closes. `notify` grants `actions: read` to list the failed jobs of the run, and `issues: write` to open its issue. `check-updates.yml` has no workflow-level `permissions`, so each job gets its own block and nothing more. `validate.yml` has one job, and its workflow-level `contents: read` covers that job. A calling job that grants less than this makes GitHub reject the file before any job starts, see [Invalid workflow file](#invalid-workflow-file-a-nested-job-requests-more-permissions). ## Quick setup (with gh CLI) -With `gh` installed and authenticated (`gh auth login`), setup is one command: +With `gh` installed and authenticated (`gh auth login`), clone aurtomator for the scripts at the tag the calling workflows pin, then clone your repository and run `setup.sh` from inside it: ```bash -./scripts/setup.sh +git clone https://github.com/staticwire/aurtomator ~/src/aurtomator +git -C ~/src/aurtomator checkout vX.Y.Z +git clone https://github.com// +cd +~/src/aurtomator/scripts/setup.sh ``` -It runs five steps in order: +`vX.Y.Z` is the version in the `# vX.Y.Z` comment of the calling workflows. The script runs five steps in order: 1. **identity**: your name and email for AUR git commits 2. **ssh**: an existing SSH key for the AUR. CI uses the key without a prompt, so the step refuses a key with a passphrase and prints the commands that make a copy without one. @@ -61,11 +123,11 @@ It runs five steps in order: 4. **secrets**: pushes `AUR_SSH_KEY`, `AUR_GIT_NAME`, `AUR_GIT_EMAIL` and, with GPG, `GPG_SIGNING_KEY` to your GitHub repo. It checks the SSH key as the ssh step does, so a key that is missing or has a passphrase stops it before any secret is set. 5. **verify**: checks identity, SSH access to the AUR, GPG signing and the secrets -The script reads the GitHub repo from the `origin` remote and asks for `owner/repo` when that fails. It saves your answers to `.aurtomator.conf` after each step, so a step that fails, and a `verify` that finds a problem, leave the answers given so far for the next run. +The script works on the directory it runs in. The `identity` step prints that directory as `Data directory:`. The `secrets` step sets the secrets of the GitHub repo of its `origin` remote, prints that repo as `Target repo:`, and asks for `owner/repo` when there is no `origin`. A run from inside the clone of aurtomator prints `Target repo: staticwire/aurtomator` and writes `.aurtomator.conf` into the clone. The script saves your answers to `.aurtomator.conf` after each step, so a step that fails, and a `verify` that finds a problem, leave the answers given so far for the next run. ## Identity -The `identity` step sets the git author name and email of your AUR commits, as `git log` shows them in your AUR package repos. They are pushed to GitHub Secrets as `AUR_GIT_NAME` and `AUR_GIT_EMAIL`, so CI commits under the same identity. +The `identity` step sets the git author name and email of your AUR commits, as `git log` shows them in your AUR package repos. They are pushed to the GitHub Secrets of your repository as `AUR_GIT_NAME` and `AUR_GIT_EMAIL`, so CI commits under the same identity. The `# Maintainer:` line in your PKGBUILD is yours to manage. aurtomator does not change it and warns during an update when it is missing. @@ -99,7 +161,7 @@ A "Welcome" message means it works. The connection then closes, which is expecte ### 4. Set GitHub Secrets in the web UI -In your fork, open Settings > Secrets and variables > Actions > New repository secret, and create: +In your repository, open Settings > Secrets and variables > Actions > New repository secret, and create: | Secret name | Value | |------------------|-------------------------------------------------------| @@ -113,12 +175,14 @@ With GPG signing, also create: |-------------------|------------------------------------------------------------| | `GPG_SIGNING_KEY` | Output of `gpg --export-secret-subkeys --armor '!'` | -That output works in CI only for a key without a passphrase. For a key with one, `./scripts/setup.sh gpg secrets` prints a passphrase-free copy, see [Passphrase stripping](#passphrase-stripping-3-pinentry-dialogs). +That output works in CI only for a key without a passphrase. For a key with one, `~/src/aurtomator/scripts/setup.sh gpg secrets` prints a passphrase-free copy, see [Passphrase stripping](#passphrase-stripping-3-pinentry-dialogs). ### 5. Run verify +From inside your repository: + ```bash -./scripts/setup.sh verify +~/src/aurtomator/scripts/setup.sh verify ``` It checks identity, SSH access, GPG signing when configured, and the secrets when `gh` is available. When a check fails, the others still run, and `verify` then prints how many failed and exits with status 1. @@ -155,16 +219,16 @@ Paste the full output, including the `-----BEGIN PGP PRIVATE KEY BLOCK-----` and ### Configure in setup -Pass the key ID: +Pass the key ID, from inside your repository: ```bash -./scripts/setup.sh --gpg-key +~/src/aurtomator/scripts/setup.sh --gpg-key ``` Or run the GPG step and answer `y` to "Enable GPG commit signing? (y/N)": ```bash -./scripts/setup.sh gpg +~/src/aurtomator/scripts/setup.sh gpg ``` ### Passphrase stripping (3 pinentry dialogs) @@ -181,14 +245,16 @@ The second dialog asks again because the temporary keyring runs its own GPG agen Your original key is not modified. With `gh`, the copy is stored as the `GPG_SIGNING_KEY` secret. Without `gh`, the script prints the copy for you to paste as that secret. The temporary keyring is deleted right after, and also when the script stops early: on an error, on `Ctrl-C` or when its terminal closes. -The copy exists only while the script runs, so the `gpg` and `secrets` steps have to run together: `./scripts/setup.sh gpg secrets`. Run alone, `secrets` does not set `GPG_SIGNING_KEY` for a key with a passphrase and prints that command. +The copy exists only while the script runs, so the `gpg` and `secrets` steps have to run together: `~/src/aurtomator/scripts/setup.sh gpg secrets`. Run alone, `secrets` does not set `GPG_SIGNING_KEY` for a key with a passphrase and prints that command. ## Setup script reference ```bash -./scripts/setup.sh [step...] [options] +~/src/aurtomator/scripts/setup.sh [step...] [options] ``` +Run it from inside your repository: the directory it runs in is the one it configures. + ### Steps Without a step, all of them run in order. @@ -204,8 +270,8 @@ Without a step, all of them run in order. Steps can be combined: ```bash -./scripts/setup.sh identity ssh # identity + SSH only -./scripts/setup.sh verify # just verify +~/src/aurtomator/scripts/setup.sh identity ssh # identity + SSH only +~/src/aurtomator/scripts/setup.sh verify # just verify ``` ### Options @@ -223,24 +289,24 @@ Steps can be combined: ```bash # Full interactive setup -./scripts/setup.sh +~/src/aurtomator/scripts/setup.sh # Non-interactive with explicit values -./scripts/setup.sh \ +~/src/aurtomator/scripts/setup.sh \ --git-name "Jane Doe" --git-email "jane@example.com" \ --ssh-key ~/.ssh/aur_ed25519 --gpg-key ABCDEF1234567890 \ --non-interactive # Re-run just secrets after rotating a key -./scripts/setup.sh secrets +~/src/aurtomator/scripts/setup.sh secrets # Verify after manual changes -./scripts/setup.sh verify +~/src/aurtomator/scripts/setup.sh verify ``` ## Configuration file -Setup saves its state to `.aurtomator.conf` in the project root. The scripts source it as a shell file, so `setup.sh` writes every value shell-quoted, and a `"`, `$` or backtick in a name stays part of the name. A file from an older version, with double quotes, still loads. +Setup saves its state to `.aurtomator.conf` in the root of your repository, next to `packages/`. The scripts source it as a shell file, so `setup.sh` writes every value shell-quoted, and a `"`, `$` or backtick in a name stays part of the name. A file from an older version, with double quotes, still loads. ```bash GIT_AUTHOR_NAME='Jane Doe' @@ -256,14 +322,16 @@ AUR_SSH_KEY='/home//.ssh/aur_ed25519' | `GPG_KEY_ID` | GPG key ID for commit signing. Empty string if not used. | | `AUR_SSH_KEY` | Absolute path to the SSH private key for AUR. | -A re-run of `setup.sh` loads this file first and does not ask again for the values it holds. A command line option overrides the saved value, and the new value is saved. The file is in `.gitignore` because it holds local paths. +A re-run of `setup.sh` loads this file first and does not ask again for the values it holds. A command line option overrides the saved value, and the new value is saved. The `.gitignore` of the template lists the file, because it holds local paths. ## Adding your first package +Every command below runs from inside your repository, with the scripts of the clone of aurtomator. + ### 1. Copy the example ```bash -cp packages/example.yml.sample packages/my-package.yml +cp ~/src/aurtomator/packages/example.yml.sample packages/my-package.yml ``` ### 2. Edit the YAML @@ -282,7 +350,7 @@ upstream: ### 3. Validate the config ```bash -./scripts/validate-pkg.sh my-package +~/src/aurtomator/scripts/validate-pkg.sh my-package ``` It checks the YAML syntax, the required fields and the fields each strategy needs. @@ -290,7 +358,7 @@ It checks the YAML syntax, the required fields and the fields each strategy need ### 4. Test the version check ```bash -./scripts/check-update.sh my-package +~/src/aurtomator/scripts/check-update.sh my-package ``` It runs the strategy and prints the latest upstream version. Without a `current` field in the YAML, it also reports that no current version is set. @@ -298,14 +366,14 @@ It runs the strategy and prints the latest upstream version. Without a `current` ### 5. Test the update (dry run) ```bash -./scripts/update-pkg.sh my-package 1.2.3 --dry-run +~/src/aurtomator/scripts/update-pkg.sh my-package 1.2.3 --dry-run ``` It clones the AUR repo, sets `pkgver`, regenerates checksums and `.SRCINFO`, runs `namcap` when it is installed, and shows the PKGBUILD diff **without pushing**. ### 6. Push and let CI take over -Commit the package YAML and push to your fork. The `check-updates` workflow runs hourly, as set in `.github/workflows/check-updates.yml`, and can be started by hand through `workflow_dispatch`. +Commit the package YAML and push to your repository. The push to `main` runs `Validate packages`. `Check AUR Package Updates` runs every hour at minute 17, as set in `.github/workflows/check-updates.yml`, and can be started by hand through `workflow_dispatch`. ## Troubleshooting @@ -343,23 +411,36 @@ Without `gh`, set the secrets in the web UI: see [Manual setup](#manual-setup-wi ### Secrets not set / workflow fails with missing secret -**Symptom:** CI fails with "secret not found" or with SSH key errors. +**Symptom:** The `publish` job fails in `Setup AUR SSH` with `The secret AUR_SSH_KEY is empty`, or with SSH key errors. **Fixes:** -1. Open Settings > Secrets and variables > Actions in your repo. It lists `AUR_SSH_KEY`, `AUR_GIT_NAME`, `AUR_GIT_EMAIL` and, with GPG, `GPG_SIGNING_KEY`. -2. Run `./scripts/setup.sh secrets` to push them again. -3. Secret names are case-sensitive. +1. Open Settings > Secrets and variables > Actions in your repository. It lists `AUR_SSH_KEY`, `AUR_GIT_NAME`, `AUR_GIT_EMAIL` and, with GPG, `GPG_SIGNING_KEY`. +2. Run `~/src/aurtomator/scripts/setup.sh secrets` from inside your repository to push them again. +3. The `check-updates` job of `.github/workflows/check-updates.yml` passes each secret by name, as [The calling workflows](#the-calling-workflows) shows. + +### Invalid workflow file: a nested job requests more permissions + +**Symptom:** A run of a calling workflow fails before any job starts, with `Invalid workflow file` and a line of this form: + +```text +The nested job '' is requesting '', but is only allowed '' +``` + +**Cause:** The calling job grants less than a job of the called workflow requests. A called workflow can lower the permissions of the calling job but cannot raise them. + +**Fix:** Give each job of the calling workflow the `permissions` block that [The calling workflows](#the-calling-workflows) shows: `contents: write` and `issues: write` for `check-updates`, `actions: read` and `issues: write` for `notify`, and `contents: read` in `validate.yml`. ### check-update.sh fails: "Strategy not found" -**Symptom:** `Strategy not found or not executable:` followed by the absolute path of `strategies/xxx.sh` in your checkout +**Symptom:** `Strategy not found or not executable:` followed by the absolute path of `strategies/xxx.sh` in your clone of aurtomator **Fixes:** -1. The `strategy` field in your package YAML must match a file in `strategies/`. `ls strategies/` lists them. -2. Strategy scripts must be executable: `chmod +x strategies/*.sh` -3. `./scripts/validate-pkg.sh my-package` catches config errors before a run. +1. The `strategy` field in your package YAML must match a file in `strategies/` of the clone. `ls ~/src/aurtomator/strategies/` lists them. +2. The clone has the strategies of the tag it is at. `git -C ~/src/aurtomator checkout vX.Y.Z`, with the version the calling workflows pin, gives it the strategies a run uses. +3. Strategy scripts must be executable: `chmod +x ~/src/aurtomator/strategies/*.sh` +4. `~/src/aurtomator/scripts/validate-pkg.sh my-package`, run from inside your repository, catches config errors before a run. ### update-pkg.sh fails: "No PKGBUILD found" diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 5f35f4b..3826986 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -8,6 +8,10 @@ 4. [Update Process](#4-update-process) 5. [CI Workflow: check-updates.yml](#5-ci-workflow-check-updatesyml) 6. [CI Quality Gate: ci.yml](#6-ci-quality-gate-ciyml) + - [6.1 Lint job](#61-lint-job) + - [6.2 Test job](#62-test-job) + - [6.3 CI image updates](#63-ci-image-updates) + - [6.4 Validate and pipeline jobs](#64-validate-and-pipeline-jobs) 7. [Local Testing](#7-local-testing) 8. [Writing Custom Strategies](#8-writing-custom-strategies) @@ -17,9 +21,12 @@ aurtomator bumps the version of AUR packages you already maintain. It does **not ### End-to-end flow +A run involves two repositories. Your repository, created from the template `staticwire/aurtomator-seed`, holds the package YAMLs, the README with the package table and two calling workflows, `.github/workflows/check-updates.yml` and `.github/workflows/validate.yml`. aurtomator holds the scripts, the strategies and the reusable workflows `check-updates.yml`, `notify-failure.yml` and `validate.yml`, and the calling workflows run them at the SHA they pin, see [5.1](#51-trigger). A run pushes the new versions to the AUR and commits the changed package YAMLs and README to your repository. + ```mermaid flowchart TD - A[Schedule / manual trigger] --> K[check job: read-only token, no PKGBUILD] + A[your repository: calling workflow, on schedule or by hand] --> A2[aurtomator at the pinned SHA: reusable check-updates.yml] + A2 --> K[check job: read-only token, no PKGBUILD] K --> B[check-all.sh] B --> C{for each package} C --> D[check-update.sh] @@ -51,7 +58,7 @@ flowchart TD R --> R2[publish-pkg.sh: check the prepared files against the AUR copy] R2 --> R3[git commit + push to AUR] R3 --> U[update-readme.sh] - U --> L[commit via GitHub API] + U --> L[commit to your repository via GitHub API] L --> I[issues job: issues token, no secrets, every run] I --> N[close resolved issues, open new ones] N --> O[done] @@ -1007,14 +1014,21 @@ A download saved under a name without the value, as in `${pkgname}-${pkgver}-amd ### 5.1 Trigger -The workflow runs every hour at minute 17 (`cron: "17 * * * *"`) and by hand through `workflow_dispatch`. GitHub delays scheduled runs at the start of every hour and drops some of them when the load is high, so the schedule avoids minute 0. The `check-updates` concurrency group serializes runs: a new run waits for the running one. +`check-updates.yml` of aurtomator runs only when another workflow calls it: its trigger is `on: workflow_call`. It takes no inputs and declares the secrets `AUR_SSH_KEY`, `GPG_SIGNING_KEY`, `AUR_GIT_NAME` and `AUR_GIT_EMAIL`, each with `required: false`. Only `publish` reads them, and it stops on an empty `AUR_SSH_KEY`, see [5.3](#53-steps). + +The calling workflow `.github/workflows/check-updates.yml` of your repository, which [SETUP.md](SETUP.md#the-calling-workflows) shows in full, carries the trigger: every hour at minute 17 (`cron: "17 * * * *"`) and by hand through `workflow_dispatch`. GitHub delays scheduled runs at the start of every hour and drops some of them when the load is high, so the schedule avoids minute 0. Its job `check-updates` calls the reusable file at the pinned SHA and passes the four secrets by name. The concurrency group `check-updates`, with `cancel-in-progress: false`, serializes runs: a new run waits for the running one. The calling workflow sets the group at its workflow level, so it covers the `notify` job of [5.7](#57-failed-runs) too. + +The `pipeline` job of `ci.yml` calls the same file in aurtomator itself, see [6.4](#64-validate-and-pipeline-jobs). ### 5.2 Permissions and container -The workflow has five jobs after a short `image` job. `image` reads the image from `.github/ci-image/Dockerfile`: `archlinux:base-devel`, pinned by digest, so a new image on Docker Hub reaches the jobs with the secrets only through a reviewed commit. Dependabot updates the digest once a month with a pull request, and `ci.yml` runs the tests in the new image before the merge, see [6.3](#63-ci-image-updates). The five jobs run in that image on `ubuntu-latest`, `issues` with a 10-minute timeout and the others with 30 minutes, and `vercmp` from pacman, which `check-all.sh` requires, is the reason `check`, `verify` and `publish` need Arch too. Every install step upgrades `archlinux-keyring` before the system, because a pinned image ages and an old keyring rejects packages signed with a newer packager key. `makepkg` refuses to run as root, so the steps that touch packages in `prepare` and `publish` run as a `builder` user, and `check` runs the strategies as `builder` as well. `builder` has no `sudo` rights: the steps run as root and switch to `builder` with `sudo -u builder`. `verify` executes no PKGBUILD and runs `check-all.sh` as `builder` too. +A run checks out two repositories. Every job checks out aurtomator at `job.workflow_sha`, the commit of the workflow file that defines the job, which for a job of `check-updates.yml` is the SHA the calling workflow pins. `image` takes a sparse checkout of `.github/ci-image` into the root of the workspace. The other jobs take the whole repository into `aurtomator/`, and `check`, `prepare`, `verify` and `publish` first check out your repository into the root of the workspace. Every step runs in the root of the workspace, so the scripts in `aurtomator/` take the packages, the README and `.status/` from your repository, see [7.1](#71-prerequisites). Inside the called workflow, the `github` context is that of the calling workflow: `github.repository` names your repository, `github.ref_name` the branch of the run, and `github.token` is the token of your repository, with the permissions below. + +The workflow has five jobs after a short `image` job. `image` reads the image from `.github/ci-image/Dockerfile` of aurtomator at `job.workflow_sha`: `archlinux:base-devel`, pinned by digest, so a new image on Docker Hub reaches the jobs with the secrets only through a reviewed commit of aurtomator and a move of the pin. Dependabot updates the digest in aurtomator once a month with a pull request, and `ci.yml` runs the tests in the new image before the merge, see [6.3](#63-ci-image-updates). The five jobs run in that image on `ubuntu-latest`, `issues` with a 10-minute timeout and the others with 30 minutes, and `vercmp` from pacman, which `check-all.sh` requires, is the reason `check`, `verify` and `publish` need Arch too. Every install step upgrades `archlinux-keyring` before the system, because a pinned image ages and an old keyring rejects packages signed with a newer packager key. `makepkg` refuses to run as root, so the steps that touch packages in `prepare` and `publish` run as a `builder` user, and `check` runs the strategies as `builder` as well. `builder` has no `sudo` rights: the steps run as root and switch to `builder` with `sudo -u builder`. `verify` executes no PKGBUILD and runs `check-all.sh` as `builder` too. | Job | Permissions | Secrets | |-----|-------------|---------| +| `check-updates` of your calling workflow | `contents: write` and `issues: write`, the most any job below gets | passes the four secrets by name | | `image` | `contents: read` | none | | `check` | `contents: read` | none | | `prepare` | `contents: read` | none | @@ -1022,75 +1036,87 @@ The workflow has five jobs after a short `image` job. `image` reads the image fr | `publish` | `contents: write` for the status commit | `AUR_SSH_KEY`, `GPG_SIGNING_KEY`, `AUR_GIT_NAME`, `AUR_GIT_EMAIL` | | `issues` | `contents: read`, `issues: write` for the issues of [5.6](#56-issues) | none | +The calling job sets the permissions of the token for the whole call, and a job of the called workflow can only lower them. `check-updates.yml` sets `contents: read` at the workflow level, and `publish` and `issues` set the blocks above. A calling job that grants less than a job requests makes GitHub reject the calling workflow before any job starts, see [SETUP.md](SETUP.md#invalid-workflow-file-a-nested-job-requests-more-permissions). + `check` decides the run. `prepare` and `verify` run only when a package has a new version. `publish` runs for a new version and also for a change in the status table of `README.md`. It runs after a failure of `prepare` or `verify` as well: then it publishes nothing, and every package with a new version gets `update_failed`. In most hours nothing changes, and all three are skipped. `issues` runs in every run in which `check` succeeded, after `publish` when `publish` runs. ### 5.3 Steps -Every job checks out the repository with **Checkout:** `actions/checkout`, pinned by commit SHA like every action in this repository, with `persist-credentials: false`, so the workflow token is not written to `.git/config`. In `image` it is the first step. The five jobs after it install their packages first and check out after that. All of them but `issues` install `git`. The workflow never pushes to GitHub with git: the status commit goes through the API. +Every checkout uses `actions/checkout`, pinned by commit SHA like every action in aurtomator, with `persist-credentials: false`, so the workflow token is not written to `.git/config`. **Checkout** takes your repository at the commit of the run into the root of the workspace, and **Checkout aurtomator** takes `job.workflow_repository` at `job.workflow_sha` into `aurtomator/`. Your repository comes first: its checkout owns the root of the workspace, and `aurtomator/` is a directory inside it. In `image`, the step **Checkout** takes a sparse checkout of `.github/ci-image` of aurtomator instead, after **Check the pin**. The five jobs after it install their packages first and check out after that. All of them but `issues` install `git`. The workflow never pushes to GitHub with git: the status commit goes through the API. The four steps that run `check-all.sh` as `builder` stop workflow commands while it runs. Its output carries text that a PKGBUILD, the AUR or an upstream chose: the error text a git server sends, a line of a prepared `.SRCINFO` that `verify` refuses. The runner takes a line as a workflow command when it starts with `::` after leading whitespace, or when it holds `##[` anywhere. A line `::add-mask::true` makes the runner skip the outputs `prepare` and `work` of `check`, so no job after `check` runs. Root prints `::stop-commands::` with a token of 32 hex digits from `/dev/urandom` before the run and `::::` after it, and the runner takes no command in between. The token stays in a variable of root's shell, so `builder` cannot print it. The output goes through a pipe to `cat`, which ends only when no process of `builder` holds the pipe, so nothing `builder` prints comes after the token. The steps run in bash with `pipefail` and exit with the status of `check-all.sh`. -The four steps use the composite action `.github/actions/run-as-builder`. It adds the workspace to `safe.directory` of root's git, gives the workspace to `builder` and runs `check-all.sh` with the words of its input `args`. Its input `keep-env` names the variables of the step that `sudo --preserve-env` passes to `builder`. The runner reads a local action from the workspace when its step starts, and `builder` owns the workspace once `check-all.sh` ran, so no local action follows `run-as-builder` in a job. Hand over runs after `check-all.sh` in `prepare` and in `verify`, so it stays a step of the workflow: `prepare` defines it with the YAML anchor `&hand-over`, and `verify` repeats it with the alias `*hand-over`. +The four steps use the composite action `.github/actions/run-as-builder` of aurtomator, as `uses: ./aurtomator/.github/actions/run-as-builder`: the runner resolves a `./` path from the root of the workspace. The action adds the workspace to `safe.directory` of root's git, gives the workspace to `builder`, both checkouts included, and runs `scripts/check-all.sh` of its own checkout with the words of its input `args`. It finds that checkout from `GITHUB_ACTION_PATH`, the directory of its `action.yml`. Its input `keep-env` names the variables of the step that `sudo --preserve-env` passes to `builder`. The runner reads a local action from the workspace when its step starts, and `builder` owns the workspace once `check-all.sh` ran, so no local action follows `run-as-builder` in a job. Hand over runs after `check-all.sh` in `prepare` and in `verify`, so it stays a step of the workflow: `prepare` defines it with the YAML anchor `&hand-over`, and `verify` repeats it with the alias `*hand-over`. `image`: -1. **Checkout:** `.github/ci-image` only. -2. **Read image:** takes the `FROM` line of `.github/ci-image/Dockerfile` and fails unless it is `archlinux:base-devel@sha256:` with a 64-character digest. +1. **Check the pin:** fails with `job.workflow_sha is not a commit SHA` unless `job.workflow_sha` is 40 lowercase hex digits. With an empty `ref`, `actions/checkout` takes the default branch of a repository other than the one that triggered the run, so an empty value would run the code of `main` of aurtomator, and no other step would fail. Every other job needs `image`, so this step stops the whole run. +2. **Checkout:** a sparse checkout of `.github/ci-image` of aurtomator, at `job.workflow_sha`. +3. **Read image:** takes the `FROM` line of `.github/ci-image/Dockerfile` and fails unless it is `archlinux:base-devel@sha256:` with a 64-character digest. `check`: 1. **Install dependencies:** `sudo git go-yq`, and the `builder` user. -2. **Check updates:** gives the workspace to `builder` and runs `check-all.sh` as `builder`, with `GITHUB_TOKEN` set to the read-only workflow token. `sudo --preserve-env` passes the token, so it does not appear on a command line. Workflow commands are stopped while it runs. -3. **Decide:** sets the output `prepare` when a status is `new_version`, `available_downgrade` or `new_build`. It runs `update-readme.sh` as `builder` and sets the output `work` for `prepare` and for a change in the status table. -4. **Upload statuses:** uploads `.status/` as the `checked` artifact with a retention of 1 day. +2. **Checkout:** your repository. +3. **Checkout aurtomator:** into `aurtomator/`. +4. **Check updates:** gives the workspace to `builder` and runs `check-all.sh` as `builder`, with `GITHUB_TOKEN` set to the read-only workflow token. `sudo --preserve-env` passes the token, so it does not appear on a command line. Workflow commands are stopped while it runs. +5. **Decide:** sets the output `prepare` when a status is `new_version`, `available_downgrade` or `new_build`. It runs `./aurtomator/scripts/update-readme.sh` as `builder` and sets the output `work` for `prepare` and for a change in the status table. +6. **Upload statuses:** uploads `.status/` as the `checked` artifact with a retention of 1 day. `prepare`: 1. **Install dependencies:** `sudo git openssh namcap jq go-yq pacman-contrib`, and the `builder` user. -2. **Download statuses:** the `checked` artifact to `.status/`. -3. **Prepare updates:** gives the workspace to `builder` and runs `check-all.sh --prepare handoff` as `builder`, without a token. Code from a PKGBUILD can read the environment of every running process of `builder` through `/proc//environ`, so the token stays in `check`. Workflow commands are stopped while it runs. -4. **Hand over:** code from a PKGBUILD has run in the workspace by now, so the step runs nothing from it. A process started by a PKGBUILD can outlive its step, so the step kills every process whose effective or real user is `builder` and waits up to 30 seconds for them to end. A zombie does not count: it runs nothing, and pid 1 of a job container never reaps one. A process that is still running after that fails the step, and so does a process list that `ps` cannot produce. Only after that does root copy the regular files at `prepared//PKGBUILD`, `prepared//.SRCINFO`, `prepared//base` and `status/` from `handoff/` to `upload/`. `find` does not follow symlinks and separates paths with NUL, because a name may hold a newline, so a link to a file outside the workspace does not reach the artifact. A package name never starts with a dot, so `.` and `..` match no path. -5. **Upload handoff:** uploads `upload/` as the `handoff` artifact with `include-hidden-files: true`, because `.SRCINFO` starts with a dot, and a retention of 1 day. +2. **Checkout:** your repository. +3. **Checkout aurtomator:** into `aurtomator/`. +4. **Download statuses:** the `checked` artifact to `.status/`. +5. **Prepare updates:** gives the workspace to `builder` and runs `check-all.sh --prepare handoff` as `builder`, without a token. Code from a PKGBUILD can read the environment of every running process of `builder` through `/proc//environ`, so the token stays in `check`. Workflow commands are stopped while it runs. +6. **Hand over:** code from a PKGBUILD has run in the workspace by now, so the step runs nothing from it. A process started by a PKGBUILD can outlive its step, so the step kills every process whose effective or real user is `builder` and waits up to 30 seconds for them to end. A zombie does not count: it runs nothing, and pid 1 of a job container never reaps one. A process that is still running after that fails the step, and so does a process list that `ps` cannot produce. Only after that does root copy the regular files at `prepared//PKGBUILD`, `prepared//.SRCINFO`, `prepared//base` and `status/` from `handoff/` to `upload/`. `find` does not follow symlinks and separates paths with NUL, because a name may hold a newline, so a link to a file outside the workspace does not reach the artifact. A package name never starts with a dot, so `.` and `..` match no path. +7. **Upload handoff:** uploads `upload/` as the `handoff` artifact with `include-hidden-files: true`, because `.SRCINFO` starts with a dot, and a retention of 1 day. `verify`: 1. **Install dependencies:** `sudo git jq go-yq`, and the `builder` user. -2. **Download handoff:** the `handoff` artifact to `handoff/`. -3. **Verify checksums:** gives the workspace to `builder` and runs `check-all.sh --verify handoff` as `builder`, see [4.4](#44-prepare-and-publish). git and curl read the AUR repo and the sources here. A job container has the Docker socket of the runner, which root can use and `builder` cannot. Workflow commands are stopped while it runs. -4. **Hand over:** the same step as in `prepare`, through the alias `*hand-over`. -5. **Upload verified handoff:** uploads `upload/` as the `verified` artifact with `include-hidden-files: true` and a retention of 1 day. +2. **Checkout:** your repository. +3. **Checkout aurtomator:** into `aurtomator/`. +4. **Download handoff:** the `handoff` artifact to `handoff/`. +5. **Verify checksums:** gives the workspace to `builder` and runs `check-all.sh --verify handoff` as `builder`, see [4.4](#44-prepare-and-publish). git and curl read the AUR repo and the sources here. A job container has the Docker socket of the runner, which root can use and `builder` cannot. Workflow commands are stopped while it runs. +6. **Hand over:** the same step as in `prepare`, through the alias `*hand-over`. +7. **Upload verified handoff:** uploads `upload/` as the `verified` artifact with `include-hidden-files: true` and a retention of 1 day. `publish`: 1. **Install dependencies:** `sudo git github-cli openssh gnupg jq go-yq`, and the `builder` user. -2. **Download statuses:** the `checked` artifact to `.status/`. -3. **Download verified handoff:** the `verified` artifact to `handoff/`, when `verify` ran. -4. **Setup workspace:** git identity for `builder` from the `AUR_GIT_NAME` and `AUR_GIT_EMAIL` secrets, default `aurtomator`. -5. **Setup AUR SSH:** writes `AUR_SSH_KEY` to `/home/builder/.ssh/aur`, writes the Ed25519 host key that publishes to `known_hosts`, and writes an SSH config with `StrictHostKeyChecking yes` and connection multiplexing. A host that answers with another key fails with `Host key verification failed`. When the AUR changes its host key, update the key in `check-updates.yml`. -6. **Setup GPG:** with `GPG_SIGNING_KEY` set, imports the key for `builder`, passes `GPG_KEY_ID` to the later steps and enables commit signing. Without it, the step is skipped. -7. **Publish updates:** runs `check-all.sh --publish handoff` as `builder`, with `GPG_KEY_ID`, `GIT_AUTHOR_NAME` and `GIT_AUTHOR_EMAIL`, and without `GITHUB_TOKEN`, as it runs no strategy. It keeps the statuses of `check` and checks the prepared files before it pushes, see [4.4](#44-prepare-and-publish). Workflow commands are stopped while it runs. -8. **Update README status table:** runs `update-readme.sh` as `builder`. -9. **Commit changes via API:** see [5.5](#55-verified-commit). -10. **Hand over statuses:** also after a failed step, copies the regular files of `.status/` whose names fit a package name, `_alerts` and `_upstream` included, to `statuses/`. A link does not reach the artifact. `builder` reads each file and root writes the copy into its own directory, so a link swapped in after the test reaches nothing that `builder` cannot read. -11. **Upload statuses:** also after a failed step, uploads `statuses/` as the `published` artifact with a retention of 1 day. - -Step 9 runs as root with `GH_TOKEN` set to the workflow token, `github.token`. Steps 10 and 11 run as root without it. +2. **Checkout:** your repository. +3. **Checkout aurtomator:** into `aurtomator/`. +4. **Download statuses:** the `checked` artifact to `.status/`. +5. **Download verified handoff:** the `verified` artifact to `handoff/`, when `verify` ran. +6. **Setup workspace:** git identity for `builder` from the `AUR_GIT_NAME` and `AUR_GIT_EMAIL` secrets, default `aurtomator`. +7. **Setup AUR SSH:** fails with `The secret AUR_SSH_KEY is empty` when the secret is empty or the calling workflow does not pass it. Otherwise it writes `AUR_SSH_KEY` to `/home/builder/.ssh/aur`, writes the Ed25519 host key that publishes to `known_hosts`, and writes an SSH config with `StrictHostKeyChecking yes` and connection multiplexing. A host that answers with another key fails with `Host key verification failed`. The host key is part of `check-updates.yml` of aurtomator, so a new host key of the AUR reaches a run through a release and a move of the pin. +8. **Setup GPG:** with `GPG_SIGNING_KEY` set, imports the key for `builder`, passes `GPG_KEY_ID` to the later steps and enables commit signing. Without it, the step is skipped. +9. **Publish updates:** runs `check-all.sh --publish handoff` as `builder`, with `GPG_KEY_ID`, `GIT_AUTHOR_NAME` and `GIT_AUTHOR_EMAIL`, and without `GITHUB_TOKEN`, as it runs no strategy. It keeps the statuses of `check` and checks the prepared files before it pushes, see [4.4](#44-prepare-and-publish). Workflow commands are stopped while it runs. +10. **Update README status table:** runs `./aurtomator/scripts/update-readme.sh` as `builder`. +11. **Commit changes via API:** see [5.5](#55-verified-commit). +12. **Hand over statuses:** also after a failed step, copies the regular files of `.status/` whose names fit a package name, `_alerts` and `_upstream` included, to `statuses/`. A link does not reach the artifact. `builder` reads each file and root writes the copy into its own directory, so a link swapped in after the test reaches nothing that `builder` cannot read. +13. **Upload statuses:** also after a failed step, uploads `statuses/` as the `published` artifact with a retention of 1 day. + +Step 11 runs as root with `GH_TOKEN` set to the workflow token, `github.token`. Steps 12 and 13 run as root without it. `issues`: -1. **Install dependencies:** `github-cli`. The image has no git, so the job sets `GH_REPO` to the repository for `gh`. -2. **Download statuses of check:** the `checked` artifact to `.status/`, when `publish` was skipped. -3. **Download statuses of publish:** the `published` artifact to `.status/`, when `publish` ran. -4. **Close resolved issues:** see [5.6](#56-issues). -5. **Create issues:** see [5.6](#56-issues). +1. **Install dependencies:** `github-cli`. The image has no git, so the job sets `GH_REPO` to your repository for `gh`. +2. **Checkout aurtomator:** into `aurtomator/`. The job reads no file of your repository, so it does not check it out. Without git in the image, `actions/checkout` downloads the files through the REST API. +3. **Download statuses of check:** the `checked` artifact to `.status/`, when `publish` was skipped. +4. **Download statuses of publish:** the `published` artifact to `.status/`, when `publish` ran. +5. **Close resolved issues:** see [5.6](#56-issues). +6. **Create issues:** see [5.6](#56-issues). -Steps 4 and 5 run as root with `GH_TOKEN` set to the workflow token. No other user runs code in the workspace of this job, so root writes the bodies with `scripts/lib/issue.sh`. +Steps 5 and 6 run as root with `GH_TOKEN` set to the workflow token. No other user runs code in the workspace of this job, so root writes the bodies with `scripts/lib/issue.sh` of `aurtomator/`. ### 5.4 README update `update-readme.sh` rewrites two blocks of `README.md`: -1. **Badges** between `` and ``: CI, Codecov and static badges for the repository in `origin`. A public repository gets its own tokenless Codecov badge, and a private one links the upstream aurtomator badge. When the API gives no clear answer, a rate limit for example, the Codecov badge already in the README stays. A README without one then gets the upstream badge. +1. **Badges** between `` and ``: the badges `Check AUR Package Updates` and `Validate packages`, the status of `check-updates.yml` and `validate.yml` in the repository in `origin`, and the badge `automated by aurtomator`, which links to aurtomator. Without a GitHub `origin`, the block holds the last badge alone. The badge URLs name the two files, so the calling workflows keep those names. 2. **Package table** between `` and ``, built from `.status/`. Each name links to `https://aur.archlinux.org/pkgbase/`. The AUR shows the base of a split package there and sends any other package to `/packages/`. The version of aurtomator follows the table, see [5.8](#58-version-of-aurtomator). The badges are optional. Each marker must be a line of its own, once, with the start marker above the end marker. The script checks this for the table, and for the badges when their start marker is there, and stops before it writes anything otherwise. A line that only quotes a marker in its text is left as it is. @@ -1117,7 +1143,7 @@ The badges are optional. Each marker must be a line of its own, once, with the s | `.status/_error_count` | `0` | Total error count of the run | | `.status/_alerts` | ``, `hold` or `notice`, the note, split by tabs | One line for each alert that fired, see [4.5](#45-update-alerts) | | `.status/_assets` | `` and the asset value, split by a tab | One line for each value found, see [4.6](#46-values-from-release-asset-names) | -| `.status/_upstream` | `newer: 0.7.0` or `current: 0.6.0` | The latest aurtomator release against `VERSION`, see [5.8](#58-version-of-aurtomator) | +| `.status/_upstream` | `newer: 0.7.0` or `current: 0.6.0` | The latest aurtomator release against `VERSION` at the pin, see [5.8](#58-version-of-aurtomator) | #### Packages badge @@ -1143,12 +1169,12 @@ The table is HTML, since a Markdown table cell takes no attributes. GitHub keeps A `git push` from the workflow would show as unverified, so the step commits through the GitHub Git Data API: -1. List the changed tracked files with `git diff --name-only -z`, and stop when there is none. The job writes only `README.md` and `packages/.yml`, so a change to any other tracked file fails the step and nothing is committed. +1. List the changed tracked files with `git diff --name-only -z`, and stop when there is none. The checkout in `aurtomator/` is untracked in your repository and is not listed. The job writes only `README.md` and `packages/.yml`, so a change to any other tracked file fails the step and nothing is committed. 2. Read the branch head and its tree. A head other than the commit the job checked out means a commit was pushed during the run. The step then lists the tree of the new head, and when a changed file differs there from the checked-out commit, or is gone, it prints a notice and commits nothing, because the tree of step 3 replaces each file as a whole and would drop that change. The next run starts from the new head and writes the statuses again. A package that this run pushed to the AUR keeps its old `last_updated`: the next run finds the version in the AUR and writes only `current`. When the pushed commits changed none of these files, the step commits on the new head. 3. Create a blob for every changed file, a tree on the base tree, and a commit. 4. Move the branch ref with `force=false`. -The message is `chore: update package versions [skip ci]`. `[skip ci]` keeps the commit from starting `ci.yml`. GitHub marks the commit as verified because the Actions bot creates it through the API. +The step commits to `github.repository` on `github.ref_name`, your repository and the branch of the run, with the `contents: write` that the calling job grants. The message is `chore: update package versions [skip ci]`. `[skip ci]` keeps the commit from starting the workflows that run on a push: the calling `validate.yml` in your repository, `ci.yml` in aurtomator. GitHub marks the commit as verified because the Actions bot creates it through the API. ### 5.6 Issues @@ -1165,23 +1191,23 @@ An alert opens an issue as well, see [4.5](#45-update-alerts). Its notes come fr - A `held:` status opens an issue titled `✋ : update held` with the label `update-held`, unless an open `update-held` issue has that title. The body holds the held version and the notes of every alert that fired, and asks the maintainer to publish the version by hand and set `current`. - An `updated:` or `downgrade:` status of a package with a line in `.status/_alerts` opens an issue titled `🔔 : notice` with the label `update-notice`, unless an open `update-notice` issue has that title. The body holds the published version and the notes. The maintainer closes it once the edit is done. -A newer aurtomator release, `newer: ` in `.status/_upstream`, opens an issue titled `🚀 aurtomator available` with the label `aurtomator-update`, unless an open `aurtomator-update` issue has that title. The body comes from `upstream_issue_body`: the release, the version in `VERSION`, a link to the release notes and the run. See [5.8](#58-version-of-aurtomator). +A newer aurtomator release, `newer: ` in `.status/_upstream`, opens an issue titled `🚀 aurtomator available` with the label `aurtomator-update`, unless an open `aurtomator-update` issue has that title. The body comes from `upstream_issue_body`: the release, the version the run uses, a link to the release notes and the run. See [5.8](#58-version-of-aurtomator). The job runs also when the status table stays the same, so an issue follows the statuses of every run. A package that starts failing gets its issue in that run, and an issue closed while the package still fails comes back in the next run. -The step before it, `Close resolved issues`, closes the issue of a package that recovered, with a comment that names the run and shows the new status in a code block, as the issue body shows the error. An open `❌ : check failed` closes when the status is anything but `check_failed`. An open `❌ : update failed` closes when the status is `up_to_date`, `updated`, `downgrade` or `rebuilt`, so the version is in the AUR. An open `✋ : update held` closes on the same statuses, so a failed check keeps it open. An open `🚀 aurtomator available` closes when `.status/_upstream` holds anything but `newer: `: `VERSION` holds the release, or a newer one is out and gets its own issue. A run without `.status/_upstream` closes none of them. The close step comes first because an open issue of a package holds back a new one: a package whose check passes and whose update fails gets its check issue closed and an update issue opened in the same run. It runs neither after a failed step nor after a failed `publish`, so a status that `publish` did not write closes nothing, and with Issues turned off it does nothing. +The step before it, `Close resolved issues`, closes the issue of a package that recovered, with a comment that names the run and shows the new status in a code block, as the issue body shows the error. An open `❌ : check failed` closes when the status is anything but `check_failed`. An open `❌ : update failed` closes when the status is `up_to_date`, `updated`, `downgrade` or `rebuilt`, so the version is in the AUR. An open `✋ : update held` closes on the same statuses, so a failed check keeps it open. An open `🚀 aurtomator available` closes when `.status/_upstream` holds anything but `newer: `: `VERSION` at the pin has reached the latest release, or a newer one is out and gets its own issue. A run without `.status/_upstream` closes none of them. The close step comes first because an open issue of a package holds back a new one: a package whose check passes and whose update fails gets its check issue closed and an update issue opened in the same run. It runs neither after a failed step nor after a failed `publish`, so a status that `publish` did not write closes nothing, and with Issues turned off it does nothing. ### 5.7 Failed runs -The job `issues` reports failed packages, and a run that fails in `image` or `check` never reaches it. `notify-failure.yml` runs on `workflow_run` when a run of `Check AUR Package Updates` completes, and its job runs only when the conclusion of that run is `failure` or `timed_out`. It opens one issue titled `❌ Check AUR Package Updates failed`, with a link to the run, its branch and the names of the failed jobs, and opens no second one while that issue is open. The branch stands in a code span whose fence is longer than any run of backticks in its name, so a backtick in the name cannot end the span and turn the text after it into a mention. A repository with Issues turned off gets a notice instead, as in [5.6](#56-issues). The workflow has `actions: read` to list the jobs and `issues: write`, and checks out nothing. +The job `issues` reports failed packages, and a run that fails in `image` or `check` never reaches it. Your calling workflow has a second job for that case, `notify`, with `needs: check-updates` and `if: ${{ always() && needs.check-updates.result != 'success' }}`. It calls the reusable `notify-failure.yml` at the same pin and grants `actions: read` to list the jobs of the run and `issues: write`. The condition lets `notify` run after every result but `success`, and its one step, `Open an issue for the failed run`, decides from the jobs of the run. -GitHub starts a `workflow_run` workflow only from its file on the default branch. +The step lists the jobs of the run and takes those with the conclusion `failure` or `timed_out`. It opens one issue titled `❌ Check AUR Package Updates failed`, with a link to the run, its branch and the names of those jobs, and opens no second one while that issue is open. When the list names no such job, the run was canceled: the step prints the notice `No job of the run failed, so the canceled run gets no issue` and opens nothing. When the list cannot be read, the issue says `jobs that could not be listed` in place of the names. The branch stands in a code span whose fence is longer than any run of backticks in its name, so a backtick in the name cannot end the span and turn the text after it into a mention. A repository with Issues turned off gets a notice instead, as in [5.6](#56-issues). The job runs on `ubuntu-latest` with a 5-minute timeout and checks out nothing. ### 5.8 Version of aurtomator -`VERSION` in the root holds the version of aurtomator that a repository runs. It is the only record of the version that reaches a fork, which gets no tags of the upstream repository, and a template copy, which gets one commit. The release PR of aurtomator writes it, see `CONTRIBUTING.md`. +`VERSION` in the root of aurtomator holds its version, and the release PR of aurtomator writes it, see `CONTRIBUTING.md`. A run reads it from the checkout in `aurtomator/`, so the version a run uses is that of the SHA the calling workflows pin, the SHA of a release tag. Dependabot opens the pull request in your repository that moves the pin to a new release, see [SETUP.md](SETUP.md#the-calling-workflows). -`check-all.sh` logs the version first. In a check run and with `--update`, it reads the latest release of `staticwire/aurtomator` from the GitHub API, with `GITHUB_TOKEN` when it is set, and compares it with `VERSION` by `vercmp`. It writes `newer: ` or `current: ` to `.status/_upstream` and logs a line when the release is newer. A failed request, or a tag other than `vX.Y.Z`, logs a warning and writes nothing, and the run goes on. `--prepare` and `--publish` run no request and carry `.status/_upstream` of the check run. A repository without `VERSION` asks nothing. +`check-all.sh` logs the version first. In a check run and with `--update`, it reads the latest release of `staticwire/aurtomator` from the GitHub API, with `GITHUB_TOKEN` when it is set, and compares it with `VERSION` by `vercmp`. It writes `newer: ` or `current: ` to `.status/_upstream` and logs a line when the release is newer. A failed request, or a tag other than `vX.Y.Z`, logs a warning and writes nothing, and the run goes on. `--prepare` and `--publish` run no request and carry `.status/_upstream` of the check run. Without `VERSION`, it asks nothing. `update-readme.sh` writes the version under the package table, and with `newer:` a link to the release: @@ -1189,9 +1215,9 @@ GitHub starts a `workflow_run` workflow only from its file on the default branch aurtomator 0.6.0 ([0.7.0 available](https://github.com/staticwire/aurtomator/releases/tag/v0.7.0)) ``` -The comment marks the line for the release PR of aurtomator, which writes the new version there in the README of the upstream repository. Its check run then finds the README as `update-readme.sh` writes it, and has nothing to commit to a `main` that only takes pull requests. A run that could not read the release writes the line without the link. +The comment marks the line for the release PR of aurtomator, which writes the new version there in the README of aurtomator. The `pipeline` job of `ci.yml` then finds the README as `update-readme.sh` writes it, and has nothing to commit to a `main` that only takes pull requests, see [6.4](#64-validate-and-pipeline-jobs). A run that could not read the release writes the line without the link. -A newer release also opens an issue, and the run closes it once `VERSION` holds the release, see [5.6](#56-issues). A fork updates by merging the release into its `main`. +A newer release also opens an issue, and a run closes it once the pin names that release or a newer one is out, see [5.6](#56-issues). ## 6. CI Quality Gate: ci.yml @@ -1207,16 +1233,15 @@ shfmt -d -i 2 -ci scripts/*.sh scripts/lib/*.sh strategies/*.sh tests/*.bats tes actionlint ``` -`-x` follows the `source` directives, and `.shellcheckrc` makes shellcheck look for their files next to the script. `-S warning` sets the minimum severity. `-d` fails on any formatting diff, `-i 2` sets two-space indentation, and `-ci` indents `case` branches. `actionlint` checks every file in `.github/workflows/`: the syntax, the expressions, the `needs` of each job, and the `run:` blocks with `shellcheck`. +`-x` follows the `source` directives, and `.shellcheckrc` makes shellcheck look for their files next to the script. `-S warning` sets the minimum severity. `-d` fails on any formatting diff, `-i 2` sets two-space indentation, and `-ci` indents `case` branches. `actionlint` checks every file in `.github/workflows/`: the syntax, the expressions, the `needs` of each job, and the `run:` blocks with `shellcheck`. It reads `.github/actionlint.yaml`, which ignores one message in `check-updates.yml` and `validate.yml`: actionlint 1.7.12 does not know the properties `job.workflow_repository` and `job.workflow_sha` and reports them as not defined. The entries go once a release of actionlint knows them. `actionlint` does not check the `run:` blocks of a composite action. The step `shellcheck the actions` takes them from every `.github/actions/*/action.yml` with the `yq` of the runner image and runs `shellcheck -s bash` on them at its default severity, so an unquoted variable fails the step as it fails `actionlint` in a workflow. ### 6.2 Test job -15-minute timeout. It runs in the image that the `image` job of `ci.yml` reads, the same job as in [5.2](#52-permissions-and-container). It upgrades `archlinux-keyring`, installs `bash bats git go-yq jq curl openssh namcap ruby ruby-erb` and the gems `bashcov` 4.0.0 and `simplecov-cobertura` 4.0.0, then runs: +15-minute timeout. It runs in the image that the `image` job of `ci.yml` reads. That job is the `image` job of [5.2](#52-permissions-and-container) without `Check the pin`, and its checkout takes the commit of the run. It upgrades `archlinux-keyring`, installs `bash bats git go-yq jq curl openssh namcap ruby ruby-erb` and the gems `bashcov` 4.0.0 and `simplecov-cobertura` 4.0.0, then runs: ```bash -./scripts/validate-pkg.sh --all bashcov -- tests/helpers/bashcov-bats.sh tests/ --recursive ``` @@ -1228,11 +1253,19 @@ The two tests of the `namcap` check in `update-pkg.sh` run the real `namcap`, an ### 6.3 CI image updates -Once a month, when Docker Hub has a new `archlinux:base-devel`, Dependabot opens a pull request that changes the digest in `.github/ci-image/Dockerfile`. The `image` job reads the digest from the pull request, so `test` runs in the new image before the merge. +Once a month, when Docker Hub has a new `archlinux:base-devel`, Dependabot opens a pull request in aurtomator that changes the digest in `.github/ci-image/Dockerfile`. The `image` jobs read the digest from the pull request, so `test` and the two calls of [6.4](#64-validate-and-pipeline-jobs) run in the new image before the merge. + +The `merge-image` job merges such a pull request without a review. It is off until `CI_IMAGE_AUTO_MERGE`, a variable of the aurtomator repository, is `true`. It runs after `lint`, `test`, `validate` and `pipeline` passed, only for a pull request that `dependabot[bot]` opened, on an event Dependabot caused, and gets `contents: write` and `pull-requests: write`. A pull request that changes any other file, a Dependabot update of an action for example, stays open with a notice. Otherwise the job squash-merges it with `--match-head-commit`, so a commit pushed after the tests stops the merge. + +With the variable set, a new image reaches `main` of aurtomator after the tests and without a review. Your repository gets it when its pin moves to a release that holds it. A branch rule that requires an approval, or an up-to-date branch after `main` has moved, makes the merge fail, and the pull request waits for a manual merge. + +### 6.4 Validate and pipeline jobs -The `merge-image` job merges such a pull request without a review. It is off until the repository variable `CI_IMAGE_AUTO_MERGE` is `true`, see [SETUP.md](SETUP.md#recommended). It runs after `lint` and `test` passed, only for a pull request that `dependabot[bot]` opened, on an event Dependabot caused, and gets `contents: write` and `pull-requests: write`. A pull request that changes any other file, a Dependabot update of an action for example, stays open with a notice. Otherwise the job squash-merges it with `--match-head-commit`, so a commit pushed after the tests stops the merge. +The two jobs call the reusable workflows of the same commit, with the `./.github/workflows/` form of `uses`, so a change to them runs end to end before the merge. Neither passes a secret. -With the variable set, a new image reaches the jobs with the secrets after the tests and without a review. A branch rule that requires an approval, or an up-to-date branch after `main` has moved, makes the merge fail, and the pull request waits for a manual merge. +`validate` calls `validate.yml` with `contents: read`. Your repository calls the same workflow on every push to `main` and on pull requests, see [SETUP.md](SETUP.md#the-calling-workflows). After the `image` job of [5.2](#52-permissions-and-container), its job `validate` runs in the image with a 10-minute timeout. It upgrades `archlinux-keyring` before the system, installs `git` and `go-yq`, runs **Checkout** and **Checkout aurtomator** as in [5.3](#53-steps), and runs `./aurtomator/scripts/validate-pkg.sh --all` in the step `Validate packages`. It creates no `builder`: `validate-pkg.sh` reads YAML and runs no PKGBUILD. In aurtomator, `packages/` holds `example.yml.sample` alone, so the job there checks the call itself: the pin check, the two checkouts and the container. In your repository, the pull request that moves the pin runs the validator of the new release against your packages before the merge. + +`pipeline` calls `check-updates.yml` with `contents: write` and `issues: write`, with aurtomator itself as the repository of the packages. No package exists there, so `image`, `check` and `issues` run, `prepare` and `verify` are skipped, and `publish` is skipped as well, because the README that `update-readme.sh` writes equals the committed one. The test `update-readme: the README of aurtomator is what the script writes for it` in `tests/scripts.bats` fails when the two differ. The job shows on GitHub what the BATS tests cannot: that the runner finds the action in `aurtomator/`, that `GITHUB_ACTION_PATH` leads the action to the scripts, that the checkouts at `job.workflow_sha` work in the containers, and that the artifacts pass between the jobs. It runs for a push and for a pull request from a branch of aurtomator, `github.event.pull_request.head.repo.full_name == github.repository`, because a pull request from a fork of aurtomator gets a read-only token. ## 7. Local Testing @@ -1244,10 +1277,21 @@ sudo pacman -S --needed git curl go-yq pacman-contrib namcap sudo pacman -S --needed bats shellcheck shfmt ``` +The scripts run from a clone of aurtomator at the tag the calling workflows pin, `vX.Y.Z` in their `# vX.Y.Z` comment, so a local run uses the strategies and the checks of the version a run in CI uses: + +```bash +git clone https://github.com/staticwire/aurtomator ~/src/aurtomator +git -C ~/src/aurtomator checkout vX.Y.Z +``` + +After a merge that moves the pin, `git -C ~/src/aurtomator fetch` and `git -C ~/src/aurtomator checkout` with the new tag move the clone to the same release. + +The commands of 7.2 to 7.5 run from inside your repository. A script takes the packages, `.status/`, `.aurtomator.conf` and `README.md` from the working directory, or from `AURTOMATOR_DATA_DIR` when it is set, and stops with `AURTOMATOR_DATA_DIR is not a directory: ` when that names no directory. + ### 7.2 Check a single package ```bash -./scripts/check-update.sh my-package +~/src/aurtomator/scripts/check-update.sh my-package ``` The log goes to stderr. When an update is available, the new version is printed on stdout: @@ -1268,7 +1312,7 @@ When the package is up to date, stdout stays empty: ### 7.3 Dry-run an update ```bash -./scripts/update-pkg.sh my-package 2.0.0 --dry-run +~/src/aurtomator/scripts/update-pkg.sh my-package 2.0.0 --dry-run ``` It runs the steps of [4.2](#42-step-by-step) up to the check of `PKGBUILD` and `.SRCINFO`, prints the PKGBUILD diff and does not push. @@ -1276,7 +1320,7 @@ It runs the steps of [4.2](#42-step-by-step) up to the check of `PKGBUILD` and ` ### 7.4 Push an update ```bash -./scripts/update-pkg.sh my-package 2.0.0 +~/src/aurtomator/scripts/update-pkg.sh my-package 2.0.0 ``` It runs the full cycle, including `git push` to the AUR with your SSH key. @@ -1285,13 +1329,13 @@ It runs the full cycle, including `git push` to the AUR with your SSH key. ```bash # Check only -./scripts/check-all.sh +~/src/aurtomator/scripts/check-all.sh # Check and push updates -./scripts/check-all.sh --update +~/src/aurtomator/scripts/check-all.sh --update # Check and dry-run updates -./scripts/check-all.sh --update --dry-run +~/src/aurtomator/scripts/check-all.sh --update --dry-run ``` The last line is a summary: @@ -1302,6 +1346,8 @@ The last line is a summary: ### 7.6 Running BATS tests +The tests here and the linters of 7.7 check aurtomator itself, and run in the root of a clone of aurtomator: + ```bash # Run all tests bats tests/ --recursive --timing @@ -1335,7 +1381,7 @@ actionlint ## 8. Writing Custom Strategies -A strategy is a bash script in `strategies/` that finds the latest upstream version of a package. +Your repository holds no strategies. A run takes them from `strategies/` of aurtomator at the pinned SHA, so a new strategy comes into aurtomator by a pull request, as `CONTRIBUTING.md` describes. A strategy is a bash script in `strategies/` that finds the latest upstream version of a package. ### 8.1 Interface contract @@ -1344,7 +1390,7 @@ A strategy is a bash script in `strategies/` that finds the latest upstream vers - **Failure:** an error on stderr, a non-zero exit code. - **Version format:** `check-update.sh` rejects a version with any character outside `A-Za-z0-9._+`, and a version without a digit, such as `latest`. `pkgver_dash` replaces the dashes before that check, see [section 2](#2-package-yaml-format). -The script can rely on `yq` and `curl`. `scripts/lib/common.sh` provides YAML helpers, `http_get` for requests and logging, and sets `AURTOMATOR_DIR` to the project root. +The script can rely on `yq` and `curl`. `scripts/lib/common.sh` provides YAML helpers, `http_get` for requests and logging. It sets `AURTOMATOR_DIR` to the root of the checkout of aurtomator and `AURTOMATOR_DATA_DIR` to the directory with the packages, see [7.1](#71-prerequisites). ### 8.2 Template diff --git a/packages/example.yml.sample b/packages/example.yml.sample index d1cf766..3a89f63 100644 --- a/packages/example.yml.sample +++ b/packages/example.yml.sample @@ -1,4 +1,5 @@ -# Example package config. Copy to packages/.yml and edit. +# Example package config. Copy it from the clone of aurtomator to +# packages/.yml of your repository and edit the copy. # # Top-level fields: # name - pkgbase on the AUR, the name of its git repository, equal to diff --git a/tests/workflows.bats b/tests/workflows.bats index 8b00143..2ee6e6f 100644 --- a/tests/workflows.bats +++ b/tests/workflows.bats @@ -1558,3 +1558,76 @@ VALIDATE="$GITHUB_DIR/workflows/validate.yml" [ "$status" -eq 0 ] [ "$output" = "10" ] } + +SETUP_DOC="$BATS_TEST_DIRNAME/../docs/SETUP.md" + +# Prints the inside of fenced yaml block $1, 1 or 2, of the section "The +# calling workflows" in docs/SETUP.md +calling_workflow() { + local text + text=$(sed -n '/^## The calling workflows$/,/^## /p' "$SETUP_DOC") + if [ "$1" -eq 2 ]; then + text=$(sed '0,/^```yaml$/d' <<<"$text" | sed '0,/^```$/d') + fi + sed -n '/^```yaml$/,/^```$/{/^```$/q;/^```yaml$/d;p}' <<<"$text" +} + +# The template repository is created from these two blocks, with the pin +# of a release in place of and vX.Y.Z +@test "template: the calling workflows in SETUP.md pin the three files and grant what the jobs need" { + local check="$BATS_TEST_TMPDIR/check-updates.yml" validate="$BATS_TEST_TMPDIR/validate.yml" + local workflows=staticwire/aurtomator/.github/workflows + calling_workflow 1 >"$check" + calling_workflow 2 >"$validate" + [ -s "$check" ] + [ -s "$validate" ] + + run yq '.name' "$check" + [ "$status" -eq 0 ] + [ "$output" = "Check AUR Package Updates" ] + run yq '.on.schedule[0].cron' "$check" + [ "$output" = "17 * * * *" ] + run yq '.on | has("workflow_dispatch")' "$check" + [ "$output" = "true" ] + run yq '.concurrency.group' "$check" + [ "$output" = "check-updates" ] + run yq '.concurrency.cancel-in-progress' "$check" + [ "$output" = "false" ] + run yq 'has("permissions")' "$check" + [ "$output" = "false" ] + run yq '.jobs.check-updates.uses' "$check" + [ "$output" = "$workflows/check-updates.yml@" ] + run yq '.jobs.check-updates.uses | line_comment' "$check" + [ "$output" = "vX.Y.Z" ] + run yq -o=json -I=0 '.jobs.check-updates.permissions' "$check" + [ "$output" = '{"contents":"write","issues":"write"}' ] + run yq -o=json -I=0 '.jobs.check-updates.secrets | keys | sort' "$check" + [ "$output" = '["AUR_GIT_EMAIL","AUR_GIT_NAME","AUR_SSH_KEY","GPG_SIGNING_KEY"]' ] + run yq '.jobs.check-updates.secrets | to_entries | .[] | select(.value != "${{ secrets." + .key + " }}") | .key' "$check" + [ "$status" -eq 0 ] + [ -z "$output" ] + run yq '.jobs.notify.needs' "$check" + [ "$output" = "check-updates" ] + run yq '.jobs.notify.if' "$check" + [ "$output" = "\${{ always() && needs.check-updates.result != 'success' }}" ] + run yq '.jobs.notify.uses' "$check" + [ "$output" = "$workflows/notify-failure.yml@" ] + run yq '.jobs.notify.uses | line_comment' "$check" + [ "$output" = "vX.Y.Z" ] + run yq -o=json -I=0 '.jobs.notify.permissions' "$check" + [ "$output" = '{"actions":"read","issues":"write"}' ] + + run yq '.name' "$validate" + [ "$status" -eq 0 ] + [ "$output" = "Validate packages" ] + run yq -o=json -I=0 '.on.push.branches' "$validate" + [ "$output" = '["main"]' ] + run yq '.on | has("pull_request")' "$validate" + [ "$output" = "true" ] + run yq -o=json -I=0 '.permissions' "$validate" + [ "$output" = '{"contents":"read"}' ] + run yq '.jobs.validate.uses' "$validate" + [ "$output" = "$workflows/validate.yml@" ] + run yq '.jobs.validate.uses | line_comment' "$validate" + [ "$output" = "vX.Y.Z" ] +}