feat(toolchain): package standard library into zip file for hermetic runtimes

docs/api/rules_python/python/config_settings/index.md

@@ -427,6 +427,26 @@ is created.
 :::
 ::::
 
+::::{bzl:flag} zip_stdlib
+Controls whether the Python standard library is packaged into a zip file.
+
+When enabled (`yes`), hermetic toolchain runtimes package standard library
+`.py` files into a zip file (e.g. `python<major><minor>.zip` or
+`python<major><minor>t.zip` for free-threaded builds) and exclude them from
+individual on-disk runtime files.
+
+When disabled (`no`), standard library files are not zipped and remain on
+disk in the runfiles.
+
+Values:
+* `yes`: (default) Package the standard library into a zip file.
+* `no`: Do not package the standard library into a zip file; retain individual
+  files on disk.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+::::
+
 
 ## Removed Flags
 

news/4146.added.md

@@ -0,0 +1,3 @@
+(toolchain) Added {obj}`//python/config_settings:zip_stdlib` flag to control
+whether the standard library is packaged into a zip file
+([#1653](https://github.com/bazel-contrib/rules_python/issues/1653)).

news/4146.changed.md

@@ -0,0 +1,3 @@
+(toolchain) The Python standard library is zipped by default. Use
+{obj}`--zip_stdlib=no` to disable
+([#1653](https://github.com/bazel-contrib/rules_python/issues/1653)).

python/config_settings/BUILD.bazel

@@ -13,6 +13,7 @@ load(
     "ValidateTestMainFlag",
     "VenvsSitePackages",
     "VenvsUseDeclareSymlinkFlag",
+    "ZipStdlibFlag",
     rp_string_flag = "string_flag",
 )
 load("//python/private:visibility.bzl", "NOT_ACTUALLY_PUBLIC")  # buildifier: disable=bzl-visibility
@@ -230,3 +231,11 @@ string_flag(
     scope = "universal",
     visibility = ["//visibility:public"],
 )
+
+string_flag(
+    name = "zip_stdlib",
+    build_setting_default = ZipStdlibFlag.YES,
+    scope = "universal",
+    values = ZipStdlibFlag.flag_values(),
+    visibility = ["//visibility:public"],
+)

python/private/BUILD.bazel

@@ -346,6 +346,7 @@ bzl_library(
     name = "config_settings",
     srcs = ["config_settings.bzl"],
     deps = [
+        ":common_labels",
         ":text_util",
         ":version",
         ":visibility",
@@ -397,8 +398,11 @@ bzl_library(
     name = "hermetic_runtime_repo_setup",
     srcs = ["hermetic_runtime_repo_setup.bzl"],
     deps = [
+        ":common_labels",
         ":py_exec_tools_toolchain",
+        ":util",
         ":version",
+        ":zip_stdlib",
         "//python:py_runtime",
         "//python:py_runtime_pair",
         "//python/cc:py_cc_toolchain",
@@ -822,6 +826,7 @@ bzl_library(
     srcs = ["rule_builders.bzl"],
     deps = [
         ":builders_util",
+        ":util",
         "@bazel_skylib//lib:types",
     ],
 )
@@ -1036,3 +1041,8 @@ bzl_library(
     name = "visibility",
     srcs = ["visibility.bzl"],
 )
+
+bzl_library(
+    name = "zip_stdlib",
+    srcs = ["zip_stdlib.bzl"],
+)

python/private/builders_util.bzl

@@ -134,20 +134,3 @@ def kwargs_getter_mandatory(kwargs):
 def kwargs_setter_mandatory(kwargs):
     """Creates a `kwargs_setter` for the `mandatory` key."""
     return kwargs_setter(kwargs, "mandatory")
-
-def list_add_unique(add_to, others, convert = None):
-    """Bulk add values to a list if not already present.
-
-    Args:
-        add_to: {type}`list[T]` the list to add values to. It is modified
-            in-place.
-        others: {type}`collection[collection[T]]` collection of collections of
-            the values to add.
-        convert: {type}`callable | None` function to convert the values to add.
-    """
-    existing = {v: None for v in add_to}
-    for values in others:
-        for value in values:
-            value = convert(value) if convert else value
-            if value not in existing:
-                add_to.append(value)

python/private/common_labels.bzl

@@ -35,4 +35,5 @@ labels = struct(
     VENVS_SITE_PACKAGES = str(Label("//python/config_settings:venvs_site_packages")),
     VENVS_USE_DECLARE_SYMLINK = str(Label("//python/config_settings:venvs_use_declare_symlink")),
     VISIBLE_FOR_TESTING = str(Label("//python/private:visible_for_testing")),
+    ZIP_STDLIB = str(Label("//python/config_settings:zip_stdlib")),
 )

python/private/config_settings.bzl

@@ -17,6 +17,7 @@
 
 load("@bazel_skylib//lib:selects.bzl", "selects")
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
+load(":common_labels.bzl", "labels")
 load(":text_util.bzl", "render")
 load(":version.bzl", "version")
 load(":visibility.bzl", "NOT_ACTUALLY_PUBLIC")
@@ -178,6 +179,16 @@ def construct_config_settings(
         flag_values = {freethreaded: "no"},
         visibility = NOT_ACTUALLY_PUBLIC,
     )
+    native.config_setting(
+        name = "_is_zip_stdlib_yes",
+        flag_values = {labels.ZIP_STDLIB: "yes"},
+        visibility = NOT_ACTUALLY_PUBLIC,
+    )
+    native.config_setting(
+        name = "_is_zip_stdlib_no",
+        flag_values = {labels.ZIP_STDLIB: "no"},
+        visibility = NOT_ACTUALLY_PUBLIC,
+    )
 
 def _python_version_flag_impl(ctx):
     value = ctx.build_setting_value

python/private/flags.bzl

@@ -261,3 +261,12 @@ LibcFlag = FlagEnum(
     MUSL = "musl",
     get_value = _libc_flag_get_value,
 )
+
+# Decides if the standard library should be packaged into a zip file.
+# buildifier: disable=name-conventions
+ZipStdlibFlag = FlagEnum(
+    # Zip the standard library.
+    YES = "yes",
+    # Do not zip the standard library.
+    NO = "no",
+)