From 51eb2572f310eb670c1cd5c30e5d0e0965520f69 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 13 Aug 2026 13:39:57 -0400 Subject: [PATCH] In-repo build artifacts survive the CI cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate rebuilds every crate that lives in the repository on every run: our two host binaries with their LTO links (~2m20), the pinned iroh checkout's crates in the host profile (~50s), and the upstream relay (~2m20) — about 5 of the gate's 7.5 minutes. Two mechanisms held them there, and neither is fixed by the other. Swatinem/rust-cache drops, from each cached target directory, every package whose manifest lives inside that workspace root, which is all of them; cache-workspace-crates keeps the members. And cargo decides a path dependency is stale by mtime, while a checkout (ours) or a clone (setup.sh's) writes every source with the current time — so a restored artifact is always older than the sources it was built from, and keeping it changes nothing. scripts/restore-mtimes.py dates each tracked file by the commit that last touched it, which is stable across runs and machines, so unchanged sources stay older than the artifacts built from them. Files that differ from HEAD keep their mtimes: backdating a modified file would hide the modification from cargo. The full history the dating needs also arrives now (fetch-depth: 0); a depth-1 checkout knows only the tip commit and would date every file alike. The workflow joins the cache key, because it decides what the cache holds and a save skipped on an exact-key hit would otherwise pin the contents chosen by a superseded configuration — this change included. --- .github/workflows/ci.yml | 28 ++++++++++++- scripts/restore-mtimes.py | 82 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 2 deletions(-) create mode 100755 scripts/restore-mtimes.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed0e767..3e6bb1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,13 @@ jobs: timeout-minutes: 45 steps: - name: Checkout code + # Full history: scripts/restore-mtimes.py dates each file by the + # commit that last touched it, and a depth-1 checkout knows only + # the tip commit — it would date every file alike, defeating the + # freshness the step exists to restore. uses: actions/checkout@v5 + with: + fetch-depth: 0 - name: Install Rust toolchain # The toolchain version and wasm target are pinned in @@ -41,10 +47,25 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} + - name: Date sources by their last commit + # Cargo decides a path dependency is stale when its sources are + # newer than its artifact, and a checkout (ours) or a clone + # (setup.sh's) stamps every file with the current time — so + # restored artifacts always lose to freshly written sources, and + # every in-repo crate recompiles however good the cache is. + run: ./scripts/restore-mtimes.py . .deps/* + - name: Cache cargo uses: Swatinem/rust-cache@v2 with: cache-all-crates: true + # Keep the artifacts of crates that live in the repository — + # ours and the pinned checkouts' — which the action drops by + # default. They are the expensive units left: the two host + # binaries and their LTO links, and the upstream relay. This + # only pays off with the mtime step above; without it cargo + # rejects every restored in-repo artifact on sight. + cache-workspace-crates: true # The action keys on workspace-MEMBER manifests and lockfiles; # the virtual workspace ROOT manifests are never read. But the # root is where cargo requires [profile.*] to live (and where @@ -61,8 +82,11 @@ jobs: # thus every unit hash. A shape change with an unrotated key # leaves every run full-matching a cache whose artifacts no # longer match any unit the gate builds (how #73's split - # build-hosts recompiled ~360 host-profile crates per run). - key: gate-inputs-${{ hashFiles('Cargo.toml', '.deps/iroh/Cargo.toml', 'justfile', '.github/justfile') }} + # build-hosts recompiled ~360 host-profile crates per run). The + # workflow joins them because it decides what the cache holds: + # a save skipped on an exact-key hit would otherwise pin the + # contents chosen by a superseded configuration. + key: gate-inputs-${{ hashFiles('Cargo.toml', '.deps/iroh/Cargo.toml', 'justfile', '.github/justfile', '.github/workflows/ci.yml') }} # setup.sh has already installed the pinned tools into # ~/.cargo/bin when this restore runs; a cached bin/ would roll # them back to whatever versions main's cache holds. Excluding diff --git a/scripts/restore-mtimes.py b/scripts/restore-mtimes.py new file mode 100755 index 0000000..09a29a1 --- /dev/null +++ b/scripts/restore-mtimes.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Set every tracked file's mtime to the time of the last commit that touched it. + +Cargo decides whether a path dependency is fresh by comparing source mtimes +against the artifact's. A CI checkout stamps every file with the checkout +time, so restored build artifacts are always older than the sources they were +built from and every in-repo crate recompiles. Commit times are stable across +runs and across machines, so the same source content gets the same mtime on +every checkout and unchanged crates stay fresh. + +Files whose content differs from HEAD keep their mtimes: backdating a modified +file would hide the modification from cargo and produce a build that does not +match the source. + +Usage: restore-mtimes.py [REPO ...] +""" + +import os +import subprocess +import sys + + +def git(repo, *args): + # stderr is inherited: git's own message is the useful one when a path + # under .deps is not a checkout. + return subprocess.run( + ["git", "-C", repo, "-c", "core.quotePath=false", *args], + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout + + +def restore(repo): + pending = {p for p in git(repo, "ls-files").splitlines() if p} + pending -= {p for p in git(repo, "diff", "--name-only", "HEAD").splitlines() if p} + + log = subprocess.Popen( + [ + "git", + "-C", + repo, + "-c", + "core.quotePath=false", + "log", + "--pretty=format:\x01%ct", + "--name-only", + "--no-renames", + ], + stdout=subprocess.PIPE, + text=True, + ) + + stamped = 0 + timestamp = None + for line in log.stdout: + line = line.rstrip("\n") + if line.startswith("\x01"): + timestamp = int(line[1:]) + continue + # A path git had to quote (embedded newline or control character) + # cannot be recovered from this stream; leave the file alone. + if not line or line.startswith('"') or line not in pending: + continue + pending.discard(line) + path = os.path.join(repo, line) + try: + os.utime(path, (timestamp, timestamp)) + stamped += 1 + except OSError: + pass + if not pending: + break + + log.stdout.close() + log.terminate() + log.wait() + print(f"{repo}: {stamped} files stamped", file=sys.stderr) + + +for repo in sys.argv[1:] or ["."]: + restore(repo)