fix: don't materialize rules_python when reading the artifacts manifest

.github/workflows/create_archive_and_notes.sh

@@ -44,12 +44,13 @@ git archive \
   "--prefix=${PREFIX}/" \
   "--add-file=MODULE.bazel" \
   "--prefix=${PREFIX}/python/private/" \
-  "--add-file=artifacts_manifest.json" \
+  "--add-file=artifacts_manifest.bzl" \
   "--prefix=${PREFIX}/" \
   "$TAG" \
   . \
   ":!MODULE.bazel" \
-  ":!python/private/artifacts_manifest.json" | gzip > "$ARCHIVE"
+  ":!python/private/artifacts_manifest.bzl" |
+  gzip > "$ARCHIVE"
 SHA=$(shasum -a 256 "$ARCHIVE" | awk '{print $1}')
 
 cat > release_notes.txt << EOF

.github/workflows/release_build_artifacts.yaml

@@ -4,7 +4,7 @@
 #   see dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the
 #   files for other Linux CPUs. macOS and Windows files need those OSes' SDKs,
 #   so they're built on those OSes.
-# - `SHA256SUMS` and `artifacts_manifest.json` for the built platform binaries.
+# - `SHA256SUMS` and `artifacts_manifest.bzl` for the built platform binaries.
 # - The release source archive (`rules_python-<ref>.tar.gz`) and
 #   `release_notes.txt`.
 # - The `bazel_runfiles` wheel (`//python/runfiles:wheel.dist`).
@@ -187,14 +187,14 @@ jobs:
           hashFiles('release-artifacts/*') != ''
         run: >-
           dev/release_artifacts/generate_artifacts_manifest.py
-          "$REF" release-artifacts artifacts_manifest.json
+          "$REF" release-artifacts artifacts_manifest.bzl
 
       - name: Upload artifacts manifest
-        if: hashFiles('artifacts_manifest.json') != ''
+        if: hashFiles('artifacts_manifest.bzl') != ''
         uses: actions/upload-artifact@v7
         with:
           name: release-manifest
-          path: artifacts_manifest.json
+          path: artifacts_manifest.bzl
           if-no-files-found: error
 
       - name: Create release archive and notes

dev/release_artifacts/generate_artifacts_manifest.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Generates an artifacts_manifest.json file for built release artifacts."""
+"""Generates an artifacts_manifest.bzl file for built release artifacts."""
 
 from __future__ import annotations
 
@@ -86,6 +86,19 @@ def generate_manifest(
     return {"artifacts": dict(sorted(artifacts.items()))}
 
 
+def render_bzl(manifest: dict[str, dict[str, list[ArtifactEntry]]]) -> str:
+    manifest_json = json.dumps(manifest, indent=1, sort_keys=True)
+    # Embedded in a Starlark string literal, so it must not end the literal
+    # early or rely on escape sequences.
+    if '"""' in manifest_json or "\\" in manifest_json:
+        raise ValueError("Manifest JSON cannot be embedded in a .bzl string")
+    return (
+        '"""Manifest of prebuilt tool binaries for this release."""\n'
+        "\n"
+        f'ARTIFACTS_MANIFEST_JSON = """\n{manifest_json}\n"""\n'
+    )
+
+
 def _main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("tag", help="Release tag name (e.g. 2.4.0).")
@@ -97,7 +110,7 @@ def _main(argv: list[str] | None = None) -> int:
     parser.add_argument(
         "output_file",
         type=Path,
-        help="Path to the artifacts_manifest.json file to write.",
+        help="Path to the artifacts_manifest.bzl file to write.",
     )
     parser.add_argument(
         "repo",
@@ -108,14 +121,12 @@ def _main(argv: list[str] | None = None) -> int:
     args = parser.parse_args(argv)
     try:
         manifest = generate_manifest(args.tag, args.artifacts_dir, args.repo)
+        manifest_bzl = render_bzl(manifest)
     except ValueError as err:
         print(f"::error::{err}")
         return 1
     args.output_file.parent.mkdir(parents=True, exist_ok=True)
-    args.output_file.write_text(
-        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
-        encoding="utf-8",
-    )
+    args.output_file.write_text(manifest_bzl, encoding="utf-8")
     return 0
 
 

python/extensions/config.bzl

@@ -88,7 +88,7 @@ def _config_impl(module_ctx):
             if mod.is_root:
                 explicit_init_py_modules[""] = str(tag.default)
 
-    configure_tool_toolchains(module_ctx)
+    configure_tool_toolchains()
     internal_config_repo(
         name = "rules_python_internal",
         transition_setting_generators = transition_setting_generators,

python/private/BUILD.bazel

@@ -251,6 +251,11 @@ py_library(
     ],
 )
 
+bzl_library(
+    name = "artifacts_manifest",
+    srcs = ["artifacts_manifest.bzl"],
+)
+
 bzl_library(
     name = "attr_builders",
     srcs = ["attr_builders.bzl"],

python/private/artifacts_manifest.bzl

@@ -0,0 +1,16 @@
+"""Manifest of prebuilt tool binaries.
+
+Empty in git, so source checkouts build the tools from source. The release
+process replaces this file in the release archive with one whose
+`ARTIFACTS_MANIFEST_JSON` holds the JSON manifest of the release's binaries.
+
+It's a .bzl file so the `config` extension can load it rather than read it by
+Label, which would make Bazel's remote repository contents cache materialize
+the entire repository. Once https://github.com/bazelbuild/bazel/pull/31493
+lands, reading by Label no longer materializes, and this can go back to being
+a plain JSON file.
+"""
+
+# A JSON string rather than Starlark values, so the builtin manifest is parsed
+# by the same code path as a custom prebuilt tools manifest.
+ARTIFACTS_MANIFEST_JSON = ""

python/private/toolchains/BUILD.bazel

@@ -58,6 +58,7 @@ bzl_library(
     name = "repo",
     srcs = ["repo.bzl"],
     deps = [
+        "//python/private:artifacts_manifest",
         "//python/private:bazel_tools",
         "//python/private:text_util",
         "//python/private:toolchain_types",

python/private/toolchains/repo.bzl

@@ -3,10 +3,13 @@ toolchains.
 """
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
+
+# Loaded rather than read by Label: reading a file of @rules_python by Label
+# makes --experimental_remote_repo_contents_cache materialize the whole repo.
+load("//python/private:artifacts_manifest.bzl", "ARTIFACTS_MANIFEST_JSON")
 load("//python/private:text_util.bzl", "render")
 load("//python/private:toolchain_types.bzl", "EXE_ZIP_MAKER_TOOLCHAIN_TYPE")
 
-_ARTIFACTS_MANIFEST_LABEL = Label("//python/private:artifacts_manifest.json")
 _IS_RUST_EXE_ZIP_MAKER_ENABLED = (
     "@rules_python//python/private/toolchains:is_rust_exe_zip_maker_enabled"
 )
@@ -110,38 +113,34 @@ Mapping from string index to exec_compatible_with constraint labels.
 
 # Called by _config_impl in //python/extensions:config.bzl.
 def configure_tool_toolchains(
-        mctx,
         http_file = http_file,
         prebuilt_exe_zip_maker_toolchain_impl_repo = (
             prebuilt_exe_zip_maker_toolchain_impl_repo
         ),
-        tools_toolchains_repo = tools_toolchains_repo):
+        tools_toolchains_repo = tools_toolchains_repo,
+        artifacts_manifest_json = ARTIFACTS_MANIFEST_JSON):
     """Configures prebuilt or from-source toolchains in `@rules_python_tools`.
 
     Called by the `config` module extension (`//python/extensions:config.bzl`).
-    If `//python/private:artifacts_manifest.json` contains non-comment JSON
-    content (as in release archives), creates an `http_file` repo and a
+    If `ARTIFACTS_MANIFEST_JSON` from `//python/private:artifacts_manifest.bzl`
+    is non-empty (as in release archives), creates an `http_file` repo and a
     `prebuilt_exe_zip_maker_toolchain_impl_repo` repo for each entry in the
     manifest, and registers `toolchain()` declarations in
     `@rules_python_tools//toolchains`.
-    Otherwise (when the manifest file contains only `#`-comments, as in git
-    checkouts and source builds), registers the from-source `toolchain()`
-    declaration in `@rules_python_tools//toolchains`.
+    Otherwise (as in git checkouts and source builds), registers the
+    from-source `toolchain()` declaration in `@rules_python_tools//toolchains`.
 
     Args:
-        mctx: The `module_ctx` object.
         http_file: Repository rule callable for downloading prebuilt tool
             binaries (injectable for testing).
         prebuilt_exe_zip_maker_toolchain_impl_repo: Repository rule callable for
             creating `py_exe_zip_maker_toolchain` repos (injectable for testing).
         tools_toolchains_repo: Repository rule callable for creating
             `@rules_python_tools` (injectable for testing).
+        artifacts_manifest_json: The JSON manifest of prebuilt tool binaries,
+            or empty to build from source (injectable for testing).
     """
-    manifest_content = "\n".join([
-        line
-        for line in mctx.read(_ARTIFACTS_MANIFEST_LABEL).splitlines()
-        if not line.strip().startswith("#")
-    ]).strip()
+    manifest_content = artifacts_manifest_json.strip()
     toolchain_names = []
     toolchain_impls = []
     exec_compatible_with = {}

tests/exe_zip_maker_toolchain/exe_zip_maker_toolchain_tests.bzl

@@ -125,20 +125,14 @@ def _test_configure_tool_toolchains_from_source(name):
     )
 
 def _test_configure_tool_toolchains_from_source_impl(env):
-    manifest_label = str(Label("//python/private:artifacts_manifest.json"))
     http_file = mocks.repo()
     toolchain_repo = mocks.repo()
     tools_toolchains_repo = mocks.repo()
-    mctx = mocks.mctx(
-        mock_files = {
-            manifest_label: "# Stub manifest with only comments.\n# Another comment.\n\n",
-        },
-    )
     configure_tool_toolchains(
-        mctx,
         http_file = http_file.rule,
         prebuilt_exe_zip_maker_toolchain_impl_repo = toolchain_repo.rule,
         tools_toolchains_repo = tools_toolchains_repo.rule,
+        artifacts_manifest_json = "\n",
     )
     env.expect.that_collection(http_file.calls).contains_exactly([])
     env.expect.that_collection(toolchain_repo.calls).contains_exactly([])
@@ -164,7 +158,6 @@ def _test_configure_tool_toolchains_prebuilt(name):
     )
 
 def _test_configure_tool_toolchains_prebuilt_impl(env):
-    manifest_label = str(Label("//python/private:artifacts_manifest.json"))
     manifest_json = json.encode({
         "artifacts": {
             "exe_zip_maker": [
@@ -192,12 +185,11 @@ def _test_configure_tool_toolchains_prebuilt_impl(env):
     http_file = mocks.repo()
     toolchain_repo = mocks.repo()
     tools_toolchains_repo = mocks.repo()
-    mctx = mocks.mctx(mock_files = {manifest_label: manifest_json})
     configure_tool_toolchains(
-        mctx,
         http_file = http_file.rule,
         prebuilt_exe_zip_maker_toolchain_impl_repo = toolchain_repo.rule,
         tools_toolchains_repo = tools_toolchains_repo.rule,
+        artifacts_manifest_json = manifest_json,
     )
     env.expect.that_collection(http_file.calls).contains_exactly([
         {

tests/integration/prebuilt_toolchain_test.py

@@ -34,7 +34,7 @@ def setUp(self):
 
         # Simulate a release archive where tool dependencies (such as
         # rules_rust) are dev-only and prebuilt toolchains are configured via
-        # artifacts_manifest.json.
+        # artifacts_manifest.bzl.
         module_bazel = self.rules_python_dir / "MODULE.bazel"
         module_content = module_bazel.read_text(encoding="utf-8")
         self.assertIn("\nTOOL_DEPS_DEV_ONLY = False\n", module_content)
@@ -66,9 +66,9 @@ def setUp(self):
                 ],
             },
         }
-        manifest_path = self.rules_python_dir / "python/private/artifacts_manifest.json"
+        manifest_path = self.rules_python_dir / "python/private/artifacts_manifest.bzl"
         manifest_path.write_text(
-            json.dumps(manifest, indent=2) + "\n",
+            f'ARTIFACTS_MANIFEST_JSON = """{json.dumps(manifest, indent=1)}"""\n',
             encoding="utf-8",
         )
 

tests/workflows/release_artifacts_test.py

@@ -20,7 +20,7 @@ def test_generate_artifacts_manifest(tmp_path: Path):
 
     artifacts_dir = tmp_path / "release-artifacts"
     artifacts_dir.mkdir(parents=True)
-    manifest_file = tmp_path / "artifacts_manifest.json"
+    manifest_file = tmp_path / "artifacts_manifest.bzl"
 
     linux_bin = artifacts_dir / "exe_zip_maker-x86_64-unknown-linux-gnu"
     linux_bin.write_bytes(b"linux-binary-contents")
@@ -45,7 +45,10 @@ def test_generate_artifacts_manifest(tmp_path: Path):
         check=True,
     )
 
-    manifest = json.loads(manifest_file.read_text(encoding="utf-8"))
+    manifest_bzl = manifest_file.read_text(encoding="utf-8")
+    _, marker, manifest_json = manifest_bzl.partition('ARTIFACTS_MANIFEST_JSON = """')
+    assert marker
+    manifest = json.loads(manifest_json.removesuffix('"""\n'))
     base_url = "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0"
     assert manifest == {
         "artifacts": {
@@ -93,8 +96,8 @@ def test_create_archive_and_notes_with_manifest(tmp_path: Path):
     repo_dir.mkdir()
     shutil.copy2(module_bazel, repo_dir / "MODULE.bazel")
     (repo_dir / "python/private").mkdir(parents=True)
-    (repo_dir / "python/private/artifacts_manifest.json").write_text(
-        "# stub\n", encoding="utf-8"
+    (repo_dir / "python/private/artifacts_manifest.bzl").write_text(
+        'ARTIFACTS_MANIFEST_JSON = ""\n', encoding="utf-8"
     )
     (repo_dir / "README.md").write_text("# rules_python\n", encoding="utf-8")
 
@@ -114,7 +117,7 @@ def test_create_archive_and_notes_with_manifest(tmp_path: Path):
             "git",
             "add",
             "MODULE.bazel",
-            "python/private/artifacts_manifest.json",
+            "python/private/artifacts_manifest.bzl",
             "README.md",
         ],
         cwd=repo_dir,
@@ -128,10 +131,8 @@ def test_create_archive_and_notes_with_manifest(tmp_path: Path):
         check=True,
     )
 
-    manifest_content = '{"artifacts": {"exe_zip_maker": []}}\n'
-    (repo_dir / "artifacts_manifest.json").write_text(
-        manifest_content, encoding="utf-8"
-    )
+    manifest_content = 'ARTIFACTS_MANIFEST_JSON = """{"artifacts": {}}"""\n'
+    (repo_dir / "artifacts_manifest.bzl").write_text(manifest_content, encoding="utf-8")
 
     subprocess.run(
         [str(archive_script), "2.4.0"],
@@ -146,12 +147,11 @@ def test_create_archive_and_notes_with_manifest(tmp_path: Path):
         names = tar.getnames()
         assert names.count("rules_python-2.4.0/MODULE.bazel") == 1
         assert (
-            names.count("rules_python-2.4.0/python/private/artifacts_manifest.json")
-            == 1
+            names.count("rules_python-2.4.0/python/private/artifacts_manifest.bzl") == 1
         )
 
         manifest_member = tar.extractfile(
-            "rules_python-2.4.0/python/private/artifacts_manifest.json"
+            "rules_python-2.4.0/python/private/artifacts_manifest.bzl"
         )
         assert manifest_member is not None
         archived_manifest = manifest_member.read().decode("utf-8")

tests/workflows/release_build_artifacts_test.py

@@ -120,8 +120,8 @@ def git(cwd: Path, *args: str) -> None:
         "TOOL_DEPS_DEV_ONLY = False\n", encoding="utf-8"
     )
     (origin / "python/private").mkdir(parents=True)
-    (origin / "python/private/artifacts_manifest.json").write_text(
-        "# stub\n", encoding="utf-8"
+    (origin / "python/private/artifacts_manifest.bzl").write_text(
+        'ARTIFACTS_MANIFEST_JSON = ""\n', encoding="utf-8"
     )
     (origin / "python/features.bzl").write_text(
         '_VERSION_PRIVATE = "$Format:%(describe:tags=true)$"\n'
@@ -133,7 +133,7 @@ def git(cwd: Path, *args: str) -> None:
         "add",
         ".gitattributes",
         "MODULE.bazel",
-        "python/private/artifacts_manifest.json",
+        "python/private/artifacts_manifest.bzl",
         "python/features.bzl",
     )
     git(origin, "commit", "-m", "Release commit")
@@ -149,7 +149,9 @@ def git(cwd: Path, *args: str) -> None:
     # Clone without tags at the later commit (matches actions/checkout default)
     clone = tmp_path / "clone"
     git(tmp_path, "clone", "--no-tags", str(origin), str(clone))
-    (clone / "artifacts_manifest.json").write_text("{}\n", encoding="utf-8")
+    (clone / "artifacts_manifest.bzl").write_text(
+        'ARTIFACTS_MANIFEST_JSON = "{}"\n', encoding="utf-8"
+    )
 
     # Git < 2.35 (e.g. Debian 11 / Ubuntu 22.04 on Buildkite) does not support
     # %(describe:tags=true) and leaves the literal placeholder unexpanded.