fix(ci): stop restoring read-only rustc metadata caches

.github/workflows/hotpath-coverage.yml

@@ -32,6 +32,10 @@ on:
       - ".cargo/**"
       - ".config/nextest.toml"
       - ".github/workflows/hotpath-coverage.yml"
+      - "scripts/check-rust-cache-lineage.py"
+      - "scripts/drop-prefix-restored-rust-artifacts.sh"
+      - "scripts/rust_cache_lineage.py"
+      - "scripts/test-rust-cache-lineage.py"
   workflow_dispatch:
 
 concurrency:
@@ -46,6 +50,9 @@ env:
   # themselves before building a guard, and this makes that explicit at the
   # job boundary too.
   HOTPATH_METRICS_SERVER_OFF: "1"
+  # rustc writes .rmeta read-only and updates them in place when incremental
+  # compilation is on. A restored rust-cache blob then fails the compile.
+  CARGO_INCREMENTAL: "0"
 
 jobs:
   slice-tests:
@@ -55,7 +62,26 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
+
+      - name: Check rust-cache restore lineage
+        run: |
+          python3 scripts/test-rust-cache-lineage.py
+          python3 scripts/check-rust-cache-lineage.py
+
+      # Unique v1-rust shared-key so this job never prefix-restores another
+      # lane's (or a kache-era v0-rust) compiler tree. rustc writes .rmeta
+      # read-only; a lockfile-prefix restore of those files fails the compile.
+      - name: Cache Rust dependency artifacts
+        id: rust-cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: v1-rust
+          shared-key: hotpath-coverage-slice
+          cache-workspace-crates: "false"
+          cache-on-failure: true
+
+      - name: Discard prefix-restored compiler artifacts
+        run: scripts/drop-prefix-restored-rust-artifacts.sh "${{ steps.rust-cache.outputs.cache-hit }}"
 
       # `test-helpers` only unlocks the registered-store harness the
       # coverage test drives; it gates no production code.
@@ -77,7 +103,18 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
+
+      - name: Cache Rust dependency artifacts
+        id: rust-cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: v1-rust
+          shared-key: hotpath-coverage-sessions
+          cache-workspace-crates: "false"
+          cache-on-failure: true
+
+      - name: Discard prefix-restored compiler artifacts
+        run: scripts/drop-prefix-restored-rust-artifacts.sh "${{ steps.rust-cache.outputs.cache-hit }}"
 
       - name: Sessions slice tests (hotpath off)
         run: |

.github/workflows/hotpath-profile.yml

@@ -28,6 +28,10 @@ on:
       - ".cargo/**"
       - "benchmark_data/index-bench/**"
       - ".github/workflows/hotpath-profile.yml"
+      - "scripts/check-rust-cache-lineage.py"
+      - "scripts/drop-prefix-restored-rust-artifacts.sh"
+      - "scripts/rust_cache_lineage.py"
+      - "scripts/test-rust-cache-lineage.py"
   workflow_dispatch:
 
 concurrency:
@@ -49,6 +53,9 @@ env:
   # Hotpath binds a localhost metrics server by default; the benchmark turns
   # it off itself, and this makes that explicit at the job boundary too.
   HOTPATH_METRICS_SERVER_OFF: "1"
+  # rustc writes .rmeta read-only and updates them in place when incremental
+  # compilation is on. A restored rust-cache blob then fails the compile.
+  CARGO_INCREMENTAL: "0"
   # The file whose presence marks a checkout as carrying the benchmark.
   INDEX_BENCH_SOURCE: crates/tracedecay-query/src/bin/tracedecay_index_bench.rs
 
@@ -62,7 +69,26 @@ jobs:
           fetch-depth: 0
 
       - uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
+
+      - name: Check rust-cache restore lineage
+        run: |
+          python3 scripts/test-rust-cache-lineage.py
+          python3 scripts/check-rust-cache-lineage.py
+
+      # Isolated v1-rust lineage: this job used to restore a job-name key
+      # that prefix-matched other Linux rust-cache blobs, then rustc could
+      # not overwrite the restored read-only .rmeta files.
+      - name: Cache Rust dependency artifacts
+        id: rust-cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: v1-rust
+          shared-key: hotpath-profile
+          cache-workspace-crates: "false"
+          cache-on-failure: true
+
+      - name: Discard prefix-restored compiler artifacts
+        run: scripts/drop-prefix-restored-rust-artifacts.sh "${{ steps.rust-cache.outputs.cache-hit }}"
 
       - name: Create metrics directory
         run: mkdir -p /tmp/metrics
@@ -121,11 +147,14 @@ jobs:
           HOTPATH_OUTPUT_FORMAT: json
           HOTPATH_OUTPUT_PATH: /tmp/metrics/base_timing.json
           HOTPATH_REPORT: functions-timing
+          # A separate tree from HEAD's restored/compiled target/: rustc
+          # cannot overwrite the read-only .rmeta files that compile left.
+          CARGO_TARGET_DIR: target-base
         run: |
           cargo build --locked -p tracedecay-query --bin tracedecay-index-bench \
             --no-default-features --features production,hotpath
           timeout "${INDEX_BENCH_TIMEOUT_SECONDS}" \
-            target/debug/tracedecay-index-bench | tee /tmp/metrics/base_workload.json
+            target-base/debug/tracedecay-index-bench | tee /tmp/metrics/base_workload.json
 
       # A timing delta only means something if both sides indexed the same
       # bytes. The sealed state digest is a function of the admitted corpus

.github/workflows/hotpath-runtime-core.yml

@@ -31,6 +31,10 @@ on:
       - Cargo.toml
       - Cargo.lock
       - .github/workflows/hotpath-runtime-core.yml
+      - scripts/check-rust-cache-lineage.py
+      - scripts/drop-prefix-restored-rust-artifacts.sh
+      - scripts/rust_cache_lineage.py
+      - scripts/test-rust-cache-lineage.py
   workflow_dispatch:
 
 concurrency:
@@ -62,12 +66,25 @@ jobs:
 
       - uses: ./.github/actions/setup-linux-mold
 
-      - name: Cache Rust build outputs
+      - name: Check rust-cache restore lineage
+        run: |
+          python3 scripts/test-rust-cache-lineage.py
+          python3 scripts/check-rust-cache-lineage.py
+
+      # v1-rust invalidates the v0-rust blobs this lane restored, whose
+      # rustc-readonly .rmeta files the feature-on rebuild could not write.
+      - name: Cache Rust dependency artifacts
+        id: rust-cache
         uses: Swatinem/rust-cache@v2
         with:
+          prefix-key: v1-rust
           shared-key: hotpath-runtime-core
+          cache-workspace-crates: "false"
           cache-on-failure: true
 
+      - name: Discard prefix-restored compiler artifacts
+        run: scripts/drop-prefix-restored-rust-artifacts.sh "${{ steps.rust-cache.outputs.cache-hit }}"
+
       # The default dependency graph must resolve the hotpath crate with the
       # profiler compiled out (macros-only: expected activated feature set is
       # exactly "threads"). Any hotpath* feature on that crate here means a

.github/workflows/pr-run-cleanup.yml

@@ -44,10 +44,15 @@ jobs:
       - uses: actions/checkout@v7
         with:
           persist-credentials: false
-          sparse-checkout: scripts
+          sparse-checkout: |
+            scripts
+            .github/workflows
 
       - name: Test the superseded-entry selection
-        run: python3 scripts/test-prune-superseded-actions-caches.py
+        run: |
+          python3 scripts/test-prune-superseded-actions-caches.py
+          python3 scripts/test-rust-cache-lineage.py
+          python3 scripts/check-rust-cache-lineage.py
 
       - name: Delete the pull request's superseded cache entries
         env:

scripts/check-rust-cache-lineage.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Enforce rust-cache restore lineage on the hosted hotpath lanes.
+
+Those lanes used to restore ``v0-rust-`` prefix matches (often another
+lockfile generation, or a kache-era blob) and then fail to overwrite
+rustc's read-only ``.rmeta`` files. The contract is: a unique shared-key
+under ``v1-rust``, dependency artifacts only, and drop ``target/`` unless
+the restore was an exact cache hit. No chmod, no skipped jobs.
+
+Stdlib only: the pull-request cache-hygiene job checkouts scripts and
+workflows onto a bare runner.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from rust_cache_lineage import PREFIX_KEY
+
+WORKFLOWS = Path(__file__).resolve().parents[1] / ".github" / "workflows"
+DROP_SCRIPT = "scripts/drop-prefix-restored-rust-artifacts.sh"
+
+# job id → required shared-key. These lanes compile after a rust-cache restore
+# and previously inherited incompatible read-only compiler outputs.
+HOSTED_COMPILER_LANES: dict[str, dict[str, str]] = {
+    "hotpath-coverage.yml": {
+        "slice-tests": "hotpath-coverage-slice",
+        "sessions-slice-tests": "hotpath-coverage-sessions",
+    },
+    "hotpath-runtime-core.yml": {
+        "git-authority": "hotpath-runtime-core",
+    },
+    "hotpath-profile.yml": {
+        "profile": "hotpath-profile",
+    },
+}
+
+JOB_HEADER = re.compile(r"^  ([A-Za-z0-9_-]+):\s*$", re.MULTILINE)
+RUST_CACHE_USES = re.compile(r"^\s+uses: Swatinem/rust-cache@", re.MULTILINE)
+
+
+def fail(message: str) -> None:
+    print(f"rust-cache lineage policy violation: {message}", file=sys.stderr)
+    raise SystemExit(1)
+
+
+def job_bodies(text: str) -> dict[str, str]:
+    """Split a workflow into job-id → body (text until the next top-level job)."""
+    matches = list(JOB_HEADER.finditer(text))
+    jobs_marker = text.find("\njobs:\n")
+    if jobs_marker < 0:
+        fail("workflow has no jobs:")
+    start = jobs_marker + len("\njobs:\n")
+    bodies: dict[str, str] = {}
+    job_matches = [match for match in matches if match.start() >= start]
+    for index, match in enumerate(job_matches):
+        end = job_matches[index + 1].start() if index + 1 < len(job_matches) else len(text)
+        bodies[match.group(1)] = text[match.start() : end]
+    return bodies
+
+
+def assert_hosted_job(workflow: str, job_id: str, shared_key: str, body: str) -> None:
+    caches = list(RUST_CACHE_USES.finditer(body))
+    if len(caches) != 1:
+        fail(f"{workflow}:{job_id} must have exactly one rust-cache step, found {len(caches)}")
+    if f"prefix-key: {PREFIX_KEY}" not in body:
+        fail(
+            f"{workflow}:{job_id} rust-cache prefix-key must be {PREFIX_KEY!r} "
+            f"so v0-rust blobs are never prefix-restored"
+        )
+    if f"shared-key: {shared_key}" not in body:
+        fail(f"{workflow}:{job_id} rust-cache shared-key must be {shared_key!r}")
+    if 'cache-workspace-crates: "false"' not in body and "cache-workspace-crates: false" not in body:
+        fail(f"{workflow}:{job_id} must set cache-workspace-crates: false")
+    if "id: rust-cache" not in body:
+        fail(f"{workflow}:{job_id} rust-cache step must have id: rust-cache")
+    drop = f'{DROP_SCRIPT} "${{{{ steps.rust-cache.outputs.cache-hit }}}}"'
+    if drop not in body:
+        fail(f"{workflow}:{job_id} must drop target/ after a rust-cache prefix restore via {DROP_SCRIPT}")
+
+
+def main() -> int:
+    seen_shared_keys: dict[str, str] = {}
+    for filename, jobs in HOSTED_COMPILER_LANES.items():
+        path = WORKFLOWS / filename
+        if not path.is_file():
+            fail(f"missing workflow {filename}")
+        text = path.read_text(encoding="utf-8")
+        if "CARGO_INCREMENTAL: \"0\"" not in text and "CARGO_INCREMENTAL: '0'" not in text:
+            fail(f"{filename} must set CARGO_INCREMENTAL=0 so rustc does not update .rmeta in place")
+        bodies = job_bodies(text)
+        for job_id, shared_key in jobs.items():
+            body = bodies.get(job_id)
+            if body is None:
+                fail(f"{filename} is missing job {job_id}")
+            assert_hosted_job(filename, job_id, shared_key, body)
+            owner = f"{filename}:{job_id}"
+            previous = seen_shared_keys.get(shared_key)
+            if previous is not None:
+                fail(f"shared-key {shared_key!r} is reused by {previous} and {owner}")
+            seen_shared_keys[shared_key] = owner
+        if filename == "hotpath-profile.yml":
+            if "CARGO_TARGET_DIR: target-base" not in text:
+                fail(
+                    "hotpath-profile base compile must use CARGO_TARGET_DIR=target-base "
+                    "so it cannot overwrite HEAD's restored read-only .rmeta"
+                )
+            if "name: Base profile (timing)" not in text:
+                fail("hotpath-profile.yml is missing the base profile compile step")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/drop-prefix-restored-rust-artifacts.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Drop compiled artifacts after a rust-cache prefix restore.
+#
+# rust-cache's restore key omits the lockfile hash, so a miss on the exact
+# key still unpacks another generation's target/. rustc writes .rmeta as
+# mode 0444; a later compile of the same crate hash then fails with
+# "failed to write ... .rmeta". The cargo registry from the restore stays
+# — only the incompatible compiler outputs are discarded. Never chmod.
+set -euo pipefail
+
+cache_hit="${1-}"
+target_dir="${2:-target}"
+
+if [ "$cache_hit" = "true" ]; then
+  echo "exact rust-cache hit; keeping ${target_dir}"
+  exit 0
+fi
+
+if [ -e "$target_dir" ]; then
+  echo "dropping prefix-restored ${target_dir} (cache-hit=${cache_hit:-empty})"
+  rm -rf -- "$target_dir"
+else
+  echo "no ${target_dir} to drop (cache-hit=${cache_hit:-empty})"
+fi

scripts/prune-superseded-actions-caches.py

@@ -7,10 +7,10 @@
 because a newer entry with the same restore lineage exists on the same ref.
 
 A lineage is the part of a key the workflow restores by prefix:
-Swatinem/rust-cache saves `v0-rust-<prefix>-<env hash>-<lockfile hash>` and
-restores `v0-rust-<prefix>-<env hash>-` by prefix, never across an env hash,
-so both an older lockfile generation and an older toolchain generation are
-unreachable.
+Swatinem/rust-cache saves `v<N>-rust-<prefix>-<env hash>-<lockfile hash>` and
+restores `v<N>-rust-<prefix>-<env hash>-` by prefix. rustc writes `.rmeta`
+read-only, so an older lockfile or generation of the same lane is not a
+usable compile cache — only the newest entry of that lane can be restored.
 
 Keys outside that shape (setup-node's `node-cache-…`, arbitrary
 `actions/cache` keys) are left alone: the same prefix can legitimately carry
@@ -20,21 +20,14 @@
 from __future__ import annotations
 
 import json
-import re
 import sys
 from collections import defaultdict
 from collections.abc import Iterable, Iterator
+from pathlib import Path
 from typing import Any
 
-LINEAGES = (re.compile(r"^(?P<lineage>v0-rust-.+)-[0-9a-f]{8}-[0-9a-f]{8}$"),)
-
-
-def lineage_of(key: str) -> str | None:
-    for pattern in LINEAGES:
-        match = pattern.match(key)
-        if match is not None:
-            return match.group("lineage")
-    return None
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from rust_cache_lineage import lineage_of
 
 
 def superseded(entries: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: