From 29253283245d88bf526d2e8b9ef9a495e8a64d11 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 5 Sep 2026 01:07:06 -0700 Subject: [PATCH 1/2] feat(toolchain): package standard library into zip file for hermetic runtimes Hermetic Python runtimes include thousands of individual standard library files. Staging each file in runfiles trees consumes inodes, increases manifest overhead, and degrades test invocation latency. Python natively supports importing modules from a zip archive via zipimport. Packaging pure Python standard library modules into a single compressed archive substantially shrinks runfiles trees and accelerates runfiles creation. Update hermetic toolchain runtimes to package standard library modules into a zip archive and omit loose files from runfiles trees. Add a //python/config_settings:zip_stdlib string flag ('yes'/'no', defaulting to 'yes') so users can opt out and retain on-disk files when necessary. Work towards #1653. --- .../python/config_settings/index.md | 20 ++ news/4146.added.md | 3 + news/4146.changed.md | 3 + python/config_settings/BUILD.bazel | 9 + python/private/BUILD.bazel | 10 + python/private/builders_util.bzl | 17 -- python/private/common_labels.bzl | 1 + python/private/config_settings.bzl | 11 + python/private/flags.bzl | 9 + .../private/hermetic_runtime_repo_setup.bzl | 246 ++++++++++++++++-- python/private/py_executable.bzl | 32 ++- python/private/py_runtime_info.bzl | 13 +- python/private/py_runtime_rule.bzl | 10 + python/private/python_bootstrap_template.txt | 11 +- python/private/rule_builders.bzl | 2 +- python/private/site_init_template.py | 37 ++- python/private/stage2_bootstrap_template.py | 37 +++ python/private/util.bzl | 18 ++ python/private/zip_stdlib.bzl | 80 ++++++ python/private/zipapp/zip_main_template.py | 34 +++ tests/zip_stdlib/BUILD.bazel | 13 + tests/zip_stdlib/dummy_file.txt | 0 tests/zip_stdlib/hermetic/BUILD.bazel | 17 ++ tests/zip_stdlib/hermetic/bin/python3 | 0 tests/zip_stdlib/hermetic/hermetic_tests.bzl | 105 ++++++++ tests/zip_stdlib/hermetic/python | 0 tests/zip_stdlib/hermetic_windows/BUILD.bazel | 17 ++ .../hermetic_windows_tests.bzl | 141 ++++++++++ tests/zip_stdlib/hermetic_windows/python | 0 tests/zip_stdlib/zip_stdlib_test.py | 60 +++++ tests/zip_stdlib/zip_stdlib_tests.bzl | 84 ++++++ 31 files changed, 986 insertions(+), 54 deletions(-) create mode 100644 news/4146.added.md create mode 100644 news/4146.changed.md create mode 100644 python/private/zip_stdlib.bzl create mode 100644 tests/zip_stdlib/BUILD.bazel create mode 100644 tests/zip_stdlib/dummy_file.txt create mode 100644 tests/zip_stdlib/hermetic/BUILD.bazel create mode 100644 tests/zip_stdlib/hermetic/bin/python3 create mode 100644 tests/zip_stdlib/hermetic/hermetic_tests.bzl create mode 100644 tests/zip_stdlib/hermetic/python create mode 100644 tests/zip_stdlib/hermetic_windows/BUILD.bazel create mode 100644 tests/zip_stdlib/hermetic_windows/hermetic_windows_tests.bzl create mode 100644 tests/zip_stdlib/hermetic_windows/python create mode 100644 tests/zip_stdlib/zip_stdlib_test.py create mode 100644 tests/zip_stdlib/zip_stdlib_tests.bzl diff --git a/docs/api/rules_python/python/config_settings/index.md b/docs/api/rules_python/python/config_settings/index.md index 9ae0665da2..6900de7c92 100644 --- a/docs/api/rules_python/python/config_settings/index.md +++ b/docs/api/rules_python/python/config_settings/index.md @@ -427,6 +427,26 @@ is created. ::: :::: +::::{bzl:flag} zip_stdlib +Controls whether the Python standard library is packaged into a zip file. + +When enabled (`yes`), hermetic toolchain runtimes package standard library +`.py` files into a zip file (e.g. `python.zip` or +`pythont.zip` for free-threaded builds) and exclude them from +individual on-disk runtime files. + +When disabled (`no`), standard library files are not zipped and remain on +disk in the runfiles. + +Values: +* `yes`: (default) Package the standard library into a zip file. +* `no`: Do not package the standard library into a zip file; retain individual + files on disk. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +:::: + ## Removed Flags diff --git a/news/4146.added.md b/news/4146.added.md new file mode 100644 index 0000000000..445678058f --- /dev/null +++ b/news/4146.added.md @@ -0,0 +1,3 @@ +(toolchain) Added {obj}`//python/config_settings:zip_stdlib` flag to control +whether the standard library is packaged into a zip file +([#1653](https://github.com/bazel-contrib/rules_python/issues/1653)). diff --git a/news/4146.changed.md b/news/4146.changed.md new file mode 100644 index 0000000000..39195245dd --- /dev/null +++ b/news/4146.changed.md @@ -0,0 +1,3 @@ +(toolchain) The Python standard library is zipped by default. Use +{obj}`--zip_stdlib=no` to disable +([#1653](https://github.com/bazel-contrib/rules_python/issues/1653)). diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index 5eb0eafc94..014af3f6e3 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -13,6 +13,7 @@ load( "ValidateTestMainFlag", "VenvsSitePackages", "VenvsUseDeclareSymlinkFlag", + "ZipStdlibFlag", rp_string_flag = "string_flag", ) load("//python/private:visibility.bzl", "NOT_ACTUALLY_PUBLIC") # buildifier: disable=bzl-visibility @@ -230,3 +231,11 @@ string_flag( scope = "universal", visibility = ["//visibility:public"], ) + +string_flag( + name = "zip_stdlib", + build_setting_default = ZipStdlibFlag.YES, + scope = "universal", + values = ZipStdlibFlag.flag_values(), + visibility = ["//visibility:public"], +) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 9e4313d1f8..7ef54d5df7 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -346,6 +346,7 @@ bzl_library( name = "config_settings", srcs = ["config_settings.bzl"], deps = [ + ":common_labels", ":text_util", ":version", ":visibility", @@ -397,8 +398,11 @@ bzl_library( name = "hermetic_runtime_repo_setup", srcs = ["hermetic_runtime_repo_setup.bzl"], deps = [ + ":common_labels", ":py_exec_tools_toolchain", + ":util", ":version", + ":zip_stdlib", "//python:py_runtime", "//python:py_runtime_pair", "//python/cc:py_cc_toolchain", @@ -822,6 +826,7 @@ bzl_library( srcs = ["rule_builders.bzl"], deps = [ ":builders_util", + ":util", "@bazel_skylib//lib:types", ], ) @@ -1036,3 +1041,8 @@ bzl_library( name = "visibility", srcs = ["visibility.bzl"], ) + +bzl_library( + name = "zip_stdlib", + srcs = ["zip_stdlib.bzl"], +) diff --git a/python/private/builders_util.bzl b/python/private/builders_util.bzl index 7710383cb1..3165d3d3a1 100644 --- a/python/private/builders_util.bzl +++ b/python/private/builders_util.bzl @@ -134,20 +134,3 @@ def kwargs_getter_mandatory(kwargs): def kwargs_setter_mandatory(kwargs): """Creates a `kwargs_setter` for the `mandatory` key.""" return kwargs_setter(kwargs, "mandatory") - -def list_add_unique(add_to, others, convert = None): - """Bulk add values to a list if not already present. - - Args: - add_to: {type}`list[T]` the list to add values to. It is modified - in-place. - others: {type}`collection[collection[T]]` collection of collections of - the values to add. - convert: {type}`callable | None` function to convert the values to add. - """ - existing = {v: None for v in add_to} - for values in others: - for value in values: - value = convert(value) if convert else value - if value not in existing: - add_to.append(value) diff --git a/python/private/common_labels.bzl b/python/private/common_labels.bzl index db4a00ba0a..2f2a6cb5b3 100644 --- a/python/private/common_labels.bzl +++ b/python/private/common_labels.bzl @@ -35,4 +35,5 @@ labels = struct( VENVS_SITE_PACKAGES = str(Label("//python/config_settings:venvs_site_packages")), VENVS_USE_DECLARE_SYMLINK = str(Label("//python/config_settings:venvs_use_declare_symlink")), VISIBLE_FOR_TESTING = str(Label("//python/private:visible_for_testing")), + ZIP_STDLIB = str(Label("//python/config_settings:zip_stdlib")), ) diff --git a/python/private/config_settings.bzl b/python/private/config_settings.bzl index 9cd729aff2..e2a090c4af 100644 --- a/python/private/config_settings.bzl +++ b/python/private/config_settings.bzl @@ -17,6 +17,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load(":common_labels.bzl", "labels") load(":text_util.bzl", "render") load(":version.bzl", "version") load(":visibility.bzl", "NOT_ACTUALLY_PUBLIC") @@ -178,6 +179,16 @@ def construct_config_settings( flag_values = {freethreaded: "no"}, visibility = NOT_ACTUALLY_PUBLIC, ) + native.config_setting( + name = "_is_zip_stdlib_yes", + flag_values = {labels.ZIP_STDLIB: "yes"}, + visibility = NOT_ACTUALLY_PUBLIC, + ) + native.config_setting( + name = "_is_zip_stdlib_no", + flag_values = {labels.ZIP_STDLIB: "no"}, + visibility = NOT_ACTUALLY_PUBLIC, + ) def _python_version_flag_impl(ctx): value = ctx.build_setting_value diff --git a/python/private/flags.bzl b/python/private/flags.bzl index 042e4e9838..528817b523 100644 --- a/python/private/flags.bzl +++ b/python/private/flags.bzl @@ -261,3 +261,12 @@ LibcFlag = FlagEnum( MUSL = "musl", get_value = _libc_flag_get_value, ) + +# Decides if the standard library should be packaged into a zip file. +# buildifier: disable=name-conventions +ZipStdlibFlag = FlagEnum( + # Zip the standard library. + YES = "yes", + # Do not zip the standard library. + NO = "no", +) diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 20b0324894..2b17610421 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -18,11 +18,105 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("//python:py_runtime.bzl", "py_runtime") load("//python:py_runtime_pair.bzl", "py_runtime_pair") load("//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain") +load(":common_labels.bzl", "labels") load(":py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") +load(":util.bzl", "list_add_unique") load(":version.bzl", "version") +load(":zip_stdlib.bzl", "zip_stdlib") -_IS_FREETHREADED_YES = Label("//python/config_settings:_is_py_freethreaded_yes") -_IS_FREETHREADED_NO = Label("//python/config_settings:_is_py_freethreaded_no") +_IS_FREETHREADED_YES = str( + Label("//python/config_settings:_is_py_freethreaded_yes"), +) +_IS_FREETHREADED_NO = str( + Label("//python/config_settings:_is_py_freethreaded_no"), +) +_IS_ZIP_STDLIB_YES = str( + Label("//python/config_settings:_is_zip_stdlib_yes"), +) +_IS_ZIP_STDLIB_NO = str( + Label("//python/config_settings:_is_zip_stdlib_no"), +) + +def _define_zip_stdlib( + *, + name, + files_exclude, + version_dict, + tags = None, + unix_stdlib_dir = None): + """Defines a zip_stdlib target for the standard library. + + Args: + name: {type}`str` The target name. + files_exclude: {type}`list[str]` File patterns to exclude from zipping. + version_dict: {type}`dict[str, str]` Version dictionary containing + major and minor versions. + tags: {type}`list[str]` optional list of tags to apply to the target. + unix_stdlib_dir: {type}`str` Base directory of standard library on Unix + (e.g. "lib/python3.11"). + + Returns: + {type}`str` The label of the generated zip_stdlib target. + """ + tags = tags or [] + windows_stdlib_dir = "Lib" + unix_stdlib_dir = ( + unix_stdlib_dir or "lib/python{major}.{minor}".format(**version_dict) + ) + zip_out = select({ + ":is_freethreaded_windows": ( + "python{major}{minor}t.zip".format(**version_dict) + ), + labels.PLATFORMS_OS_WINDOWS: ( + "python{major}{minor}.zip".format(**version_dict) + ), + _IS_FREETHREADED_YES: ( + "lib/python{major}{minor}t.zip".format(**version_dict) + ), + "//conditions:default": ( + "lib/python{major}{minor}.zip".format(**version_dict) + ), + }) + zip_strip_prefix = select({ + ":is_freethreaded_windows": windows_stdlib_dir, + labels.PLATFORMS_OS_WINDOWS: windows_stdlib_dir, + _IS_FREETHREADED_YES: unix_stdlib_dir + "t", + "//conditions:default": unix_stdlib_dir, + }) + zip_srcs = select({ + labels.PLATFORMS_OS_WINDOWS: native.glob( + include = [windows_stdlib_dir + "/**"], + exclude = files_exclude + [ + "**/site-packages/**", + windows_stdlib_dir + "/**/test/**", + windows_stdlib_dir + "/**/tests/**", + ], + allow_empty = True, + ), + "//conditions:default": native.glob( + include = [ + unix_stdlib_dir + "*/**", + ], + exclude = files_exclude + [ + "**/site-packages/**", + "**/lib-dynload/**", + ], + allow_empty = True, + ), + }) + + zip_stdlib_tags = ["manual"] + if tags: + list_add_unique(zip_stdlib_tags, [tags]) + + zip_stdlib( + name = name, + out = zip_out, + srcs = zip_srcs, + strip_prefix = zip_strip_prefix, + tags = zip_stdlib_tags, + ) + return ":" + name def define_hermetic_runtime_toolchain_impl( *, @@ -31,7 +125,8 @@ def define_hermetic_runtime_toolchain_impl( extra_files_glob_exclude, python_version, python_bin, - coverage_tool): + coverage_tool, + tags = None): """Define a toolchain implementation for a python-build-standalone repo. It expected this macro is called in the top-level package of an extracted @@ -51,13 +146,22 @@ def define_hermetic_runtime_toolchain_impl( repository. coverage_tool: {type}`str` optional target to the coverage tool to use. + tags: {type}`list[str]` optional list of tags to apply to generated + targets. """ _ = name # @unused + tags = tags or [] + manual_tags = ["manual"] + if tags: + list_add_unique(manual_tags, [tags]) version_info = version.parse(python_version) version_dict = { "major": version_info.release[0], "minor": version_info.release[1], } + windows_stdlib_dir = "Lib" + unix_stdlib_dir = "lib/python{major}.{minor}".format(**version_dict) + unix_stdlib_dir_glob = unix_stdlib_dir + "*" files_include = [ "bin/**", "extensions/**", @@ -73,21 +177,86 @@ def define_hermetic_runtime_toolchain_impl( # static libraries "lib/**/*.a", # tests for the standard libraries. - "lib/python{major}.{minor}*/**/test/**".format(**version_dict), - "lib/python{major}.{minor}*/**/tests/**".format(**version_dict), + unix_stdlib_dir_glob + "/**/test/**", + unix_stdlib_dir_glob + "/**/tests/**", + windows_stdlib_dir + "/**/test/**", + windows_stdlib_dir + "/**/tests/**", # During pyc creation, temp files named *.pyc.NNN are created "**/__pycache__/*.pyc.*", ] files_exclude += extra_files_glob_exclude + native.filegroup( + name = "files_landmark", + # CPython's calculate_path (getpath.py/getpath.c) requires os.py on disk + # as a landmark file to determine sys.prefix and the standard library + # directory during interpreter initialization. + srcs = select({ + labels.PLATFORMS_OS_WINDOWS: native.glob( + include = [windows_stdlib_dir + "/os.py"], + exclude = files_exclude, + allow_empty = True, + ), + "//conditions:default": native.glob( + include = [unix_stdlib_dir_glob + "/os.py"], + exclude = files_exclude, + allow_empty = True, + ), + }), + tags = manual_tags, + ) + native.filegroup( + name = "files_base", + srcs = [ + ":files_landmark", + ] + select({ + labels.PLATFORMS_OS_WINDOWS: native.glob( + include = files_include, + allow_empty = True, + exclude = files_exclude + [ + windows_stdlib_dir + "/**", + ], + ), + "//conditions:default": native.glob( + include = files_include, + allow_empty = True, + exclude = files_exclude + [ + unix_stdlib_dir_glob + "/**/*.py", + unix_stdlib_dir_glob + "/*.py", + unix_stdlib_dir_glob + "/*.json", + unix_stdlib_dir_glob + "/site-packages/**", + ], + ), + }), + tags = manual_tags, + ) + native.filegroup( + name = "files_unzipped_stdlib", + srcs = select({ + labels.PLATFORMS_OS_WINDOWS: native.glob( + include = [windows_stdlib_dir + "/**"], + allow_empty = True, + exclude = files_exclude, + ), + "//conditions:default": native.glob( + include = [ + unix_stdlib_dir_glob + "/**/*.py", + unix_stdlib_dir_glob + "/*.py", + unix_stdlib_dir_glob + "/*.json", + ], + allow_empty = True, + exclude = files_exclude, + ), + }), + tags = manual_tags, + ) native.filegroup( name = "files", - srcs = native.glob( - include = files_include, - # Platform-agnostic filegroup can't match on all patterns. - allow_empty = True, - exclude = files_exclude, - ), + srcs = [":files_base"] + select({ + _IS_ZIP_STDLIB_YES: [], + _IS_ZIP_STDLIB_NO: [":files_unzipped_stdlib"], + }), + tags = tags, ) cc_import( name = "interface", @@ -96,6 +265,7 @@ def define_hermetic_runtime_toolchain_impl( _IS_FREETHREADED_NO: "libs/python{major}{minor}.lib".format(**version_dict), }), system_provided = True, + tags = tags, ) cc_import( name = "abi3_interface", @@ -104,11 +274,16 @@ def define_hermetic_runtime_toolchain_impl( _IS_FREETHREADED_NO: "libs/python3.lib", }), system_provided = True, + tags = tags, ) native.filegroup( name = "includes", - srcs = native.glob(["include/**/*.h"]), + srcs = native.glob( + ["include/**/*.h"], + allow_empty = True, + ), + tags = tags, ) cc_library( name = "python_headers_abi3", @@ -128,6 +303,7 @@ def define_hermetic_runtime_toolchain_impl( "include/python{major}.{minor}m".format(**version_dict), ], }), + tags = tags, ) cc_library( name = "python_headers", @@ -136,6 +312,7 @@ def define_hermetic_runtime_toolchain_impl( "@bazel_tools//src/conditions:windows": [":interface"], "//conditions:default": [], }), + tags = tags, ) native.config_setting( name = "is_freethreaded_linux", @@ -197,6 +374,7 @@ def define_hermetic_runtime_toolchain_impl( "lib/libpython{major}.{minor}.so.1.0".format(**version_dict), ], }), + tags = tags, ) native.exports_files(["python", python_bin]) @@ -217,9 +395,31 @@ def define_hermetic_runtime_toolchain_impl( "rc": "candidate", }.get(version_info.pre[0]) + zip_stdlib_target = _define_zip_stdlib( + name = "zip_stdlib", + files_exclude = files_exclude, + tags = tags, + unix_stdlib_dir = unix_stdlib_dir, + version_dict = version_dict, + ) + + native.filegroup( + name = "windows_zip_stdlib", + srcs = select({ + _IS_ZIP_STDLIB_YES: [zip_stdlib_target], + _IS_ZIP_STDLIB_NO: [], + }), + tags = manual_tags, + ) + py_runtime( name = "py3_runtime", - files = [":files"], + files = [ + ":files", + ] + select({ + _IS_ZIP_STDLIB_YES: [zip_stdlib_target], + _IS_ZIP_STDLIB_NO: [], + }), interpreter = python_bin, interpreter_version_info = { "major": str(version_info.release[0]), @@ -237,12 +437,18 @@ def define_hermetic_runtime_toolchain_impl( implementation_name = "cpython", # See https://peps.python.org/pep-3147/ for pyc tag infix format pyc_tag = select({ - _IS_FREETHREADED_YES: "cpython-{major}{minor}t".format(**version_dict), - _IS_FREETHREADED_NO: "cpython-{major}{minor}".format(**version_dict), + _IS_FREETHREADED_YES: ( + "cpython-{major}{minor}t".format(**version_dict) + ), + _IS_FREETHREADED_NO: ( + "cpython-{major}{minor}".format(**version_dict) + ), }), # On Windows, a symlink-style venv requires supporting .dll files. venv_bin_files = select({ - "@platforms//os:windows": native.glob( + labels.PLATFORMS_OS_WINDOWS: [ + ":windows_zip_stdlib", + ] + native.glob( include = [ "*.dll", ], @@ -252,12 +458,18 @@ def define_hermetic_runtime_toolchain_impl( ), "//conditions:default": [], }), + zip_stdlib = select({ + _IS_ZIP_STDLIB_YES: zip_stdlib_target, + _IS_ZIP_STDLIB_NO: None, + }), + tags = tags, ) py_runtime_pair( name = "python_runtimes", py2_runtime = None, py3_runtime = ":py3_runtime", + tags = tags, ) py_cc_toolchain( @@ -267,6 +479,7 @@ def define_hermetic_runtime_toolchain_impl( # TODO #3155: add libctl, libtk libs = ":libpython", python_version = python_version, + tags = tags, ) py_exec_tools_toolchain( @@ -274,4 +487,5 @@ def define_hermetic_runtime_toolchain_impl( # This macro is called in another repo: use Label() to ensure it # resolves in the rules_python context. precompiler = Label("//tools/precompiler:precompiler"), + tags = tags, ) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index f7386ee353..5a4c153efa 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -794,21 +794,29 @@ def _create_venv_windows(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_ # NOTE: The .dll files must exist, however, they may not be known at build time # if the interpreter is resolved at runtime. for f in runtime.venv_bin_files: - venv_rel_path = paths.join(venv_bin_rel_path, f.basename) - venv_ctx_rel_path = paths.join(venv_ctx_rel_root, venv_rel_path) + if f.basename.endswith(".zip"): + venv_rel_paths = [ + f.basename, + paths.join(venv_bin_rel_path, f.basename), + ] + else: + venv_rel_paths = [paths.join(venv_bin_rel_path, f.basename)] - venv_file = ctx.actions.declare_file(venv_ctx_rel_path) - ctx.actions.symlink(output = venv_file, target_file = f) + for venv_rel_path in venv_rel_paths: + venv_ctx_rel_path = paths.join(venv_ctx_rel_root, venv_rel_path) - interpreter_runfiles.add(venv_file) + venv_file = ctx.actions.declare_file(venv_ctx_rel_path) + ctx.actions.symlink(output = venv_file, target_file = f) - rf_path = runfiles_root_path(ctx, venv_file.short_path) - interpreter_symlinks.add(ExplicitSymlink( - runfiles_path = rf_path, - venv_path = venv_rel_path, - link_to_path = runfiles_root_path(ctx, f.short_path), - files = depset([f]), - )) + interpreter_runfiles.add(venv_file) + + rf_path = runfiles_root_path(ctx, venv_file.short_path) + interpreter_symlinks.add(ExplicitSymlink( + runfiles_path = rf_path, + venv_path = venv_rel_path, + link_to_path = runfiles_root_path(ctx, f.short_path), + files = depset([f]), + )) # See site.py logic: Windows uses a version/build agnostic site-packages path site_packages = "Lib/site-packages" diff --git a/python/private/py_runtime_info.bzl b/python/private/py_runtime_info.bzl index daf01e6c7e..58007cdf77 100644 --- a/python/private/py_runtime_info.bzl +++ b/python/private/py_runtime_info.bzl @@ -68,7 +68,8 @@ def _PyRuntimeInfo_init( abi_flags = "", site_init_template = None, supports_build_time_venv = True, - venv_bin_files = None): + venv_bin_files = None, + zip_stdlib = None): if (interpreter_path and interpreter) or (not interpreter_path and not interpreter): fail("exactly one of interpreter or interpreter_path must be specified") @@ -133,6 +134,7 @@ def _PyRuntimeInfo_init( "supports_build_time_venv": supports_build_time_venv, "venv_bin_files": venv_bin_files, "zip_main_template": zip_main_template, + "zip_stdlib": zip_stdlib, } PyRuntimeInfo, _unused_raw_py_runtime_info_ctor = provider( @@ -389,6 +391,15 @@ The following substitutions are made during template expansion: :::{versionadded} 0.33.0 ::: +""", + "zip_stdlib": """ +:type: File | None + +A zip file containing the standard library, if the runtime has standard library +zipping enabled. + +:::{versionadded} VERSION_NEXT_FEATURE +::: """, }, ) diff --git a/python/private/py_runtime_rule.bzl b/python/private/py_runtime_rule.bzl index 4f450c4c9b..6eaa6b796b 100644 --- a/python/private/py_runtime_rule.bzl +++ b/python/private/py_runtime_rule.bzl @@ -209,6 +209,7 @@ def _py_runtime_impl(ctx): site_init_template = ctx.file.site_init_template, supports_build_time_venv = ctx.attr.supports_build_time_venv, venv_bin_files = ctx.files.venv_bin_files, + zip_stdlib = ctx.file.zip_stdlib, )) providers = [ @@ -451,6 +452,15 @@ This becomes the entry point executed when `python foo.zip` is run. :::{seealso} The {obj}`PyRuntimeInfo.zip_main_template` field. ::: +""", + ), + "zip_stdlib": attr.label( + allow_single_file = True, + doc = """ +A zip file containing the standard library for this runtime. + +:::{versionadded} VERSION_NEXT_FEATURE +::: """, ), "_py_freethreaded_flag": attr.label( diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 482918c038..b7a262c2cc 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -98,7 +98,9 @@ else: RUNTIME_VENV_SYMLINKS = """ %runtime_venv_symlinks% """.strip().split("\n") -RUNTIME_VENV_SYMLINKS = dict(line.split("|") for line in RUNTIME_VENV_SYMLINKS if line) +RUNTIME_VENV_SYMLINKS = dict( + line.split("|") for line in RUNTIME_VENV_SYMLINKS if line +) ADDITIONAL_INTERPRETER_ARGS = os.environ.get("RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS", "") EXTRACT_ROOT = os.environ.get("RULES_PYTHON_EXTRACT_ROOT") @@ -376,10 +378,13 @@ print(sys.executable) print(sys.base_prefix) print(site.getsitepackages(["{venv_src}"])[-1]) """ - print_verbose("prog:", src) output = subprocess.check_output([python_exe_actual, "-I"], shell=True, encoding = "utf8", input=src) - output = output.strip().split("\n") + output = [ + line.strip() + for line in output.strip().splitlines() + if line.strip() + ] python_exe_actual = output[0] python_home = output[1] if IS_WINDOWS else None venv_lib = output[2] diff --git a/python/private/rule_builders.bzl b/python/private/rule_builders.bzl index 876ca2bf97..83cfded7be 100644 --- a/python/private/rule_builders.bzl +++ b/python/private/rule_builders.bzl @@ -107,10 +107,10 @@ load( "kwargs_set_default_list", "kwargs_setter", "kwargs_setter_doc", - "list_add_unique", "normalize_transition_in_out_value", "normalize_transition_in_out_values", ) +load(":util.bzl", "list_add_unique") # Various string constants for kwarg key names used across two or more # functions, or in contexts with optional lookups (e.g. dict.dict, key in dict). diff --git a/python/private/site_init_template.py b/python/private/site_init_template.py index 4597deebb5..f940e6ea52 100644 --- a/python/private/site_init_template.py +++ b/python/private/site_init_template.py @@ -295,7 +295,7 @@ def _in_runfiles(path_str): norm = _norm_path(path_str) return norm == runfiles_norm or norm.startswith(runfiles_prefix) - target_root = _get_windows_path_with_unc_prefix(runtime_root) + target_root = os.path.abspath(runtime_root) if _is_windows(): target_root = target_root.replace("/", os.sep) @@ -336,15 +336,44 @@ def _in_runfiles(path_str): # Remap all sys.path entries under the verified prefixes (including default # CPython virtual paths like pythonXY.zip that may not exist on disk). - for i, p in enumerate(sys.path): + new_sys_path = [] + for p in sys.path: norm_p = _norm_path(p) + matched = False for old_prefix in remapped_prefixes: norm_old = _norm_path(old_prefix) - if norm_p == norm_old or norm_p.startswith(norm_old + "/"): + if norm_p == norm_old: + # Omit the bare runtime root from early stdlib sys.path + # positions; + # Bazel's _setup_sys_path adds it under user imports. + matched = True + _print_verbose("omit bare stdlib root from early sys.path:", p) + break + elif norm_p.startswith(norm_old + "/"): new_path = target_root + p[len(old_prefix) :] _print_verbose("remap stdlib sys.path:", p, "->", new_path) - sys.path[i] = new_path + new_sys_path.append(new_path) + matched = True break + if not matched: + new_sys_path.append(p) + sys.path[:] = new_sys_path + + if _is_windows(): + base_dlls = os.path.join(target_root, "DLLs") + if os.path.exists(base_dlls): + if base_dlls not in sys.path: + insert_idx = 0 + for i, p in enumerate(sys.path): + if p.endswith(".zip"): + insert_idx = i + 1 + break + sys.path.insert(insert_idx, base_dlls) + if hasattr(os, "add_dll_directory"): + try: + os.add_dll_directory(base_dlls) + except OSError: + pass for attr, old_prefix in candidate_prefixes.items(): if old_prefix in remapped_prefixes: diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index f445ad2b6a..020237b29b 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -520,6 +520,43 @@ def main(): ) _add_site_packages(site_packages) + if IS_WINDOWS: + base_dlls = os.path.join(sys.base_prefix, "DLLs") + if os.path.exists(base_dlls): + if base_dlls not in sys.path: + insert_idx = 0 + for i, p in enumerate(sys.path): + if p.endswith(".zip"): + insert_idx = i + 1 + break + sys.path.insert(insert_idx, base_dlls) + if hasattr(os, "add_dll_directory"): + try: + os.add_dll_directory(base_dlls) + except OSError: + pass + prefix_dlls = os.path.join(sys.prefix, "DLLs") + if ( + prefix_dlls != base_dlls + and os.path.exists(prefix_dlls) + and hasattr(os, "add_dll_directory") + ): + try: + os.add_dll_directory(prefix_dlls) + except OSError: + pass + for i, p in enumerate(list(sys.path)): + if p == sys.base_prefix: + has_stdlib_after = any( + other.startswith(sys.base_prefix) + and other != sys.base_prefix + and not other.endswith("-packages") + for other in sys.path[i + 1 :] + ) + if has_stdlib_after: + sys.path.remove(p) + break + print_verbose("runfiles root:", runfiles_root) runfiles_envkey, runfiles_envvalue = runfiles_envvar(runfiles_root) diff --git a/python/private/util.bzl b/python/private/util.bzl index 31f317fedf..d084fd4fbf 100644 --- a/python/private/util.bzl +++ b/python/private/util.bzl @@ -85,3 +85,21 @@ def is_importable_name(name): "." not in name and "-" not in name ) + +def list_add_unique(add_to, others, convert = None): + """Bulk add values to a list if not already present. + + Args: + add_to: {type}`list[T]` the list to add values to. It is modified + in-place. + others: {type}`collection[collection[T]]` collection of collections of + the values to add. + convert: {type}`callable | None` function to convert the values to add. + """ + existing = {v: None for v in add_to} + for values in others: + for value in values: + value = convert(value) if convert else value + if value not in existing: + add_to.append(value) + existing[value] = None diff --git a/python/private/zip_stdlib.bzl b/python/private/zip_stdlib.bzl new file mode 100644 index 0000000000..5725f7155a --- /dev/null +++ b/python/private/zip_stdlib.bzl @@ -0,0 +1,80 @@ +"""Rule for creating a zipped Python standard library.""" + +def _zip_stdlib_impl(ctx): + output = ctx.actions.declare_file(ctx.attr.out) + strip_prefix = ctx.attr.strip_prefix.strip("/") + + def _map_entry(file): + # In Bazel, files in external repositories have short_path starting with + # "..//". Strip that to obtain the repo-relative path. + path = file.short_path + if path.startswith("../"): + path = path.split("/", 2)[2] + + if strip_prefix: + if path == strip_prefix: + path = "" + elif path.startswith(strip_prefix + "/"): + path = path[len(strip_prefix) + 1:] + else: + fail("File '{}' does not start with strip_prefix '{}'".format( + file.short_path, + strip_prefix, + )) + return path + "=" + file.path + + manifest = ctx.actions.args() + manifest.use_param_file("@%s", use_always = True) + manifest.set_param_file_format("multiline") + manifest.add_all( + ctx.files.srcs, + map_each = _map_entry, + allow_closure = True, + ) + + # Zipper operation mode flags: + # 'c': create a new zip archive + # 'C': compress files (deflate) rather than storing uncompressed + zip_cli_args = ctx.actions.args() + zip_cli_args.add("cC") + zip_cli_args.add(output) + + ctx.actions.run( + executable = ctx.executable._zipper, + arguments = [zip_cli_args, manifest], + inputs = depset(ctx.files.srcs), + outputs = [output], + use_default_shell_env = True, + mnemonic = "ZipStdlib", + progress_message = "Building Python stdlib zip %{output}", + ) + + return [DefaultInfo(files = depset([output]))] + +zip_stdlib = rule( + doc = """Creates a zip file containing the standard library files. + +The resulting zip archive can be passed to py_runtime so that Python +finds and imports standard library modules from the archive.""", + implementation = _zip_stdlib_impl, + attrs = { + "out": attr.string( + mandatory = True, + doc = "The path of the output zip file.", + ), + "srcs": attr.label_list( + allow_files = True, + doc = "The list of files to zip.", + ), + "strip_prefix": attr.string( + default = "", + doc = "Prefix to strip from input file paths before zipping.", + ), + "_zipper": attr.label( + default = Label("@bazel_tools//tools/zip:zipper"), + cfg = "exec", + executable = True, + allow_files = True, + ), + }, +) diff --git a/python/private/zipapp/zip_main_template.py b/python/private/zipapp/zip_main_template.py index 709e08815c..c70f3fc0c4 100644 --- a/python/private/zipapp/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -269,6 +269,8 @@ def execute_file( # workspace after the process finishes so control must return here. try: subprocess_argv = [python_program] + if IS_WINDOWS: + subprocess_argv.append("-Xfrozen_modules=off") if not EXTRACT_ROOT: subprocess_argv.append(f"-XRULES_PYTHON_ZIP_DIR={dirname(runfiles_root)}") subprocess_argv.append(main_filename) @@ -323,6 +325,38 @@ def finish_venv_setup(runfiles_root): # so that support directories (e.g. DLLs, libs) can be found. fp.write("home = {}\n".format(python_home)) + if IS_WINDOWS: + python_home = join(runfiles_root, dirname(_PYTHON_BINARY_ACTUAL)) + search_dirs = [venv_root, dirname(python_program), runfiles_root] + if python_home: + search_dirs.append(python_home) + zip_candidates = [] + for sdir in search_dirs: + try: + for item in os.listdir(sdir): + if item.startswith("python") and item.endswith(".zip"): + candidate = join(sdir, item) + if candidate not in zip_candidates: + zip_candidates.append(candidate) + except OSError: + pass + targets = [venv_root] + for target_root in targets: + sig_path = join(target_root, "Lib", "encodings", "utf_8_sig.py") + if not os.path.exists(sig_path): + for zpath in zip_candidates: + try: + with zipfile.ZipFile(zpath, "r") as zf: + for m in zf.namelist(): + if m.startswith("encodings/"): + zf.extract(m, join(target_root, "Lib")) + elif m.startswith("Lib/encodings/"): + zf.extract(m, target_root) + except (OSError, zipfile.BadZipFile): + pass + if os.path.exists(sig_path): + break + return python_program diff --git a/tests/zip_stdlib/BUILD.bazel b/tests/zip_stdlib/BUILD.bazel new file mode 100644 index 0000000000..95f67dc253 --- /dev/null +++ b/tests/zip_stdlib/BUILD.bazel @@ -0,0 +1,13 @@ +load("//tests/support:support.bzl", "SUPPORTS_BZLMOD") +load("//tests/support/pytest_test:pytest_test.bzl", "pytest_test") +load(":zip_stdlib_tests.bzl", "zip_stdlib_test_suite") + +exports_files(["dummy_file.txt"]) + +zip_stdlib_test_suite(name = "zip_stdlib_tests") + +pytest_test( + name = "zip_stdlib_test", + srcs = ["zip_stdlib_test.py"], + target_compatible_with = SUPPORTS_BZLMOD, +) diff --git a/tests/zip_stdlib/dummy_file.txt b/tests/zip_stdlib/dummy_file.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/zip_stdlib/hermetic/BUILD.bazel b/tests/zip_stdlib/hermetic/BUILD.bazel new file mode 100644 index 0000000000..4834c817d7 --- /dev/null +++ b/tests/zip_stdlib/hermetic/BUILD.bazel @@ -0,0 +1,17 @@ +load( + "//python/private:hermetic_runtime_repo_setup.bzl", + "define_hermetic_runtime_toolchain_impl", +) # buildifier: disable=bzl-visibility +load(":hermetic_tests.bzl", "hermetic_test_suite") + +define_hermetic_runtime_toolchain_impl( + name = "test_hermetic", + coverage_tool = None, + extra_files_glob_exclude = [], + extra_files_glob_include = [], + python_bin = "bin/python3", + python_version = "3.11.8", + tags = ["manual"], +) + +hermetic_test_suite(name = "hermetic_tests") diff --git a/tests/zip_stdlib/hermetic/bin/python3 b/tests/zip_stdlib/hermetic/bin/python3 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/zip_stdlib/hermetic/hermetic_tests.bzl b/tests/zip_stdlib/hermetic/hermetic_tests.bzl new file mode 100644 index 0000000000..5f5b2e8458 --- /dev/null +++ b/tests/zip_stdlib/hermetic/hermetic_tests.bzl @@ -0,0 +1,105 @@ +"""Tests for hermetic runtime setup with zip_stdlib.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:truth.bzl", "matching") +load( + "//python/private:py_runtime_info.bzl", + "PyRuntimeInfo", +) # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_py_runtime_contains_zip_stdlib(name): + analysis_test( + name = name, + target = ":py3_runtime", + impl = _test_py_runtime_contains_zip_stdlib_impl, + ) + +def _test_py_runtime_contains_zip_stdlib_impl(env, target): + info = target[PyRuntimeInfo] + env.expect.that_collection( + info.files.to_list(), + ).contains_predicate( + matching.file_basename_equals("python311.zip"), + ) + env.expect.that_bool(info.zip_stdlib != None).equals(True) + env.expect.that_str(info.zip_stdlib.basename).equals("python311.zip") + +_tests.append(_test_py_runtime_contains_zip_stdlib) + +_PY_FREETHREADED = str(Label("//python/config_settings:py_freethreaded")) + +def _test_py_runtime_contains_zip_stdlib_freethreaded(name): + analysis_test( + name = name, + target = ":py3_runtime", + impl = _test_py_runtime_contains_zip_stdlib_freethreaded_impl, + config_settings = { + _PY_FREETHREADED: "yes", + }, + ) + +def _test_py_runtime_contains_zip_stdlib_freethreaded_impl(env, target): + info = target[PyRuntimeInfo] + env.expect.that_collection( + info.files.to_list(), + ).contains_predicate( + matching.file_basename_equals("python311t.zip"), + ) + env.expect.that_bool(info.zip_stdlib != None).equals(True) + env.expect.that_str(info.zip_stdlib.basename).equals("python311t.zip") + +_tests.append(_test_py_runtime_contains_zip_stdlib_freethreaded) + +def _test_zip_stdlib_target(name): + analysis_test( + name = name, + target = ":zip_stdlib", + impl = _test_zip_stdlib_target_impl, + ) + +def _test_zip_stdlib_target_impl(env, target): + action = env.expect.that_target(target).action_named("ZipStdlib") + action.mnemonic().equals("ZipStdlib") + env.expect.that_target(target).default_outputs().contains_predicate( + matching.file_basename_equals("python311.zip"), + ) + +_tests.append(_test_zip_stdlib_target) + +_ZIP_STDLIB = str(Label("//python/config_settings:zip_stdlib")) + +def _test_py_runtime_does_not_contain_zip_stdlib_when_flag_disabled(name): + analysis_test( + name = name, + target = ":py3_runtime", + impl = ( + _test_py_runtime_does_not_contain_zip_stdlib_when_flag_disabled_impl + ), + config_settings = { + _ZIP_STDLIB: "no", + }, + ) + +def _test_py_runtime_does_not_contain_zip_stdlib_when_flag_disabled_impl( + env, + target): + info = target[PyRuntimeInfo] + env.expect.that_collection( + info.files.to_list(), + ).not_contains_predicate( + matching.file_basename_equals("python311.zip"), + ) + env.expect.that_bool(info.zip_stdlib == None).equals(True) + +_tests.append( + _test_py_runtime_does_not_contain_zip_stdlib_when_flag_disabled, +) + +def hermetic_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) diff --git a/tests/zip_stdlib/hermetic/python b/tests/zip_stdlib/hermetic/python new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/zip_stdlib/hermetic_windows/BUILD.bazel b/tests/zip_stdlib/hermetic_windows/BUILD.bazel new file mode 100644 index 0000000000..3c01d89bc4 --- /dev/null +++ b/tests/zip_stdlib/hermetic_windows/BUILD.bazel @@ -0,0 +1,17 @@ +load( + "//python/private:hermetic_runtime_repo_setup.bzl", + "define_hermetic_runtime_toolchain_impl", +) # buildifier: disable=bzl-visibility +load(":hermetic_windows_tests.bzl", "hermetic_windows_test_suite") + +define_hermetic_runtime_toolchain_impl( + name = "test_hermetic_windows", + coverage_tool = None, + extra_files_glob_exclude = [], + extra_files_glob_include = [], + python_bin = "python", + python_version = "3.11.8", + tags = ["manual"], +) + +hermetic_windows_test_suite(name = "hermetic_windows_tests") diff --git a/tests/zip_stdlib/hermetic_windows/hermetic_windows_tests.bzl b/tests/zip_stdlib/hermetic_windows/hermetic_windows_tests.bzl new file mode 100644 index 0000000000..d76aa4e371 --- /dev/null +++ b/tests/zip_stdlib/hermetic_windows/hermetic_windows_tests.bzl @@ -0,0 +1,141 @@ +"""Tests for hermetic runtime setup for Windows with zip_stdlib.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:truth.bzl", "matching") +load( + "//python/private:py_runtime_info.bzl", + "PyRuntimeInfo", +) # buildifier: disable=bzl-visibility +load( + "//tests/support/platforms:platforms.bzl", + "platform_targets", +) # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_windows_py_runtime_contains_zip_stdlib(name): + analysis_test( + name = name, + target = ":py3_runtime", + impl = _test_windows_py_runtime_contains_zip_stdlib_impl, + config_settings = { + "//command_line_option:platforms": [ + platform_targets.WINDOWS_X86_64, + ], + }, + ) + +def _test_windows_py_runtime_contains_zip_stdlib_impl(env, target): + info = target[PyRuntimeInfo] + env.expect.that_collection( + info.files.to_list(), + ).contains_predicate( + matching.file_basename_equals("python311.zip"), + ) + env.expect.that_collection( + info.venv_bin_files, + ).contains_predicate( + matching.file_basename_equals("python311.zip"), + ) + env.expect.that_bool(info.zip_stdlib != None).equals(True) + env.expect.that_str(info.zip_stdlib.basename).equals("python311.zip") + +_tests.append(_test_windows_py_runtime_contains_zip_stdlib) + +_PY_FREETHREADED = str(Label("//python/config_settings:py_freethreaded")) + +def _test_windows_py_runtime_contains_zip_stdlib_freethreaded(name): + analysis_test( + name = name, + target = ":py3_runtime", + impl = _test_windows_py_runtime_contains_zip_stdlib_freethreaded_impl, + config_settings = { + "//command_line_option:platforms": [ + platform_targets.WINDOWS_X86_64, + ], + _PY_FREETHREADED: "yes", + }, + ) + +def _test_windows_py_runtime_contains_zip_stdlib_freethreaded_impl(env, target): + info = target[PyRuntimeInfo] + env.expect.that_collection( + info.files.to_list(), + ).contains_predicate( + matching.file_basename_equals("python311t.zip"), + ) + env.expect.that_collection( + info.venv_bin_files, + ).contains_predicate( + matching.file_basename_equals("python311t.zip"), + ) + env.expect.that_bool(info.zip_stdlib != None).equals(True) + env.expect.that_str(info.zip_stdlib.basename).equals("python311t.zip") + +_tests.append(_test_windows_py_runtime_contains_zip_stdlib_freethreaded) + +def _test_windows_zip_stdlib_target(name): + analysis_test( + name = name, + target = ":zip_stdlib", + impl = _test_windows_zip_stdlib_target_impl, + config_settings = { + "//command_line_option:platforms": [ + platform_targets.WINDOWS_X86_64, + ], + }, + ) + +def _test_windows_zip_stdlib_target_impl(env, target): + action = env.expect.that_target(target).action_named("ZipStdlib") + action.mnemonic().equals("ZipStdlib") + env.expect.that_target(target).default_outputs().contains_predicate( + matching.file_basename_equals("python311.zip"), + ) + +_tests.append(_test_windows_zip_stdlib_target) + +_ZIP_STDLIB = str(Label("//python/config_settings:zip_stdlib")) + +def _test_windows_py_runtime_does_not_contain_zip_stdlib_when_flag_disabled( + name): + analysis_test( + name = name, + target = ":py3_runtime", + impl = ( + _test_windows_py_runtime_flag_disabled_impl + ), + config_settings = { + "//command_line_option:platforms": [ + platform_targets.WINDOWS_X86_64, + ], + _ZIP_STDLIB: "no", + }, + ) + +def _test_windows_py_runtime_flag_disabled_impl( + env, + target): + info = target[PyRuntimeInfo] + env.expect.that_collection( + info.files.to_list(), + ).not_contains_predicate( + matching.file_basename_equals("python311.zip"), + ) + env.expect.that_collection( + info.venv_bin_files, + ).not_contains_predicate( + matching.file_basename_equals("python311.zip"), + ) + env.expect.that_bool(info.zip_stdlib == None).equals(True) + +_tests.append( + _test_windows_py_runtime_does_not_contain_zip_stdlib_when_flag_disabled, +) + +def hermetic_windows_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) diff --git a/tests/zip_stdlib/hermetic_windows/python b/tests/zip_stdlib/hermetic_windows/python new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/zip_stdlib/zip_stdlib_test.py b/tests/zip_stdlib/zip_stdlib_test.py new file mode 100644 index 0000000000..414e61c52e --- /dev/null +++ b/tests/zip_stdlib/zip_stdlib_test.py @@ -0,0 +1,60 @@ +"""Tests that the Python standard library is imported from a zip file.""" + +import json +import os +import sys +import urllib.parse +import zipimport + +import pytest + + +@pytest.mark.parametrize("mod", [json, urllib.parse]) +def test_pure_python_stdlib_loaded_from_zip(mod): + loader = getattr(mod, "__loader__", None) + assert isinstance(loader, zipimport.zipimporter), ( + f"{mod.__name__} was loaded by {loader!r}, expected zipimporter" + ) + assert ".zip" in mod.__file__, ( + f"{mod.__name__}.__file__ does not indicate a zip: {mod.__file__}" + ) + + +@pytest.mark.parametrize("mod", [json, urllib.parse]) +def test_on_disk_stdlib_files_not_present(mod): + loader = getattr(mod, "__loader__", None) + assert isinstance(loader, zipimport.zipimporter) + archive = loader.archive + lib_dir = os.path.dirname(archive) + candidates = [ + os.path.join( + lib_dir, + f"python{sys.version_info.major}.{sys.version_info.minor}", + mod.__name__.replace(".", os.sep) + ".py", + ), + os.path.join( + lib_dir, + f"python{sys.version_info.major}.{sys.version_info.minor}", + mod.__name__.split(".")[0], + ), + os.path.join( + lib_dir, + "Lib", + mod.__name__.replace(".", os.sep) + ".py", + ), + os.path.join( + lib_dir, + "Lib", + mod.__name__.split(".")[0], + ), + ] + for candidate in candidates: + assert not os.path.exists(candidate), ( + f"Expected {candidate} to not exist on disk in runfiles" + ) + + +def test_json_module_works(): + data = {"hello": "world", "num": 42} + dumped = json.dumps(data) + assert json.loads(dumped) == data diff --git a/tests/zip_stdlib/zip_stdlib_tests.bzl b/tests/zip_stdlib/zip_stdlib_tests.bzl new file mode 100644 index 0000000000..ce9aa6c48f --- /dev/null +++ b/tests/zip_stdlib/zip_stdlib_tests.bzl @@ -0,0 +1,84 @@ +"""Tests for zip_stdlib rule.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:truth.bzl", "matching") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load( + "//python/private:zip_stdlib.bzl", + "zip_stdlib", +) # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_zip_stdlib_default(name): + rt_util.helper_target( + zip_stdlib, + name = name + "_subject", + out = "stdlib.zip", + srcs = [":dummy_file.txt"], + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_zip_stdlib_default_impl, + ) + +def _test_zip_stdlib_default_impl(env, target): + action = env.expect.that_target(target).action_named("ZipStdlib") + action.mnemonic().equals("ZipStdlib") + action.contains_at_least_inputs(["tests/zip_stdlib/dummy_file.txt"]) + action.contains_at_least_args(["cC"]) + env.expect.that_target(target).default_outputs().contains_predicate( + matching.file_basename_equals("stdlib.zip"), + ) + +_tests.append(_test_zip_stdlib_default) + +def _test_zip_stdlib_custom_out(name): + rt_util.helper_target( + zip_stdlib, + name = name + "_subject", + out = "lib/python311.zip", + srcs = [":dummy_file.txt"], + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_zip_stdlib_custom_out_impl, + ) + +def _test_zip_stdlib_custom_out_impl(env, target): + action = env.expect.that_target(target).action_named("ZipStdlib") + action.mnemonic().equals("ZipStdlib") + env.expect.that_target(target).default_outputs().contains_predicate( + matching.file_basename_equals("python311.zip"), + ) + +_tests.append(_test_zip_stdlib_custom_out) + +def _test_zip_stdlib_strip_prefix(name): + rt_util.helper_target( + zip_stdlib, + name = name + "_subject", + out = "stripped.zip", + strip_prefix = "tests/zip_stdlib", + srcs = [":dummy_file.txt"], + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_zip_stdlib_strip_prefix_impl, + ) + +def _test_zip_stdlib_strip_prefix_impl(env, target): + action = env.expect.that_target(target).action_named("ZipStdlib") + action.contains_at_least_inputs(["tests/zip_stdlib/dummy_file.txt"]) + +_tests.append(_test_zip_stdlib_strip_prefix) + +def zip_stdlib_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) From a133885f2a2520f1363217b9a55c0185a82cbb24 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 6 Sep 2026 10:36:17 -0700 Subject: [PATCH 2/2] fix(toolchain): wrap exec interpreter in launcher script for build actions Relying on bare interpreter files or symlinks without a full runfiles tree fails in build actions, especially under remote execution (RBE). Furthermore, modifying PYTHONPATH masks stdlib resolution bugs and leaks into child processes. Wrap the execution interpreter in a launcher script (.sh and .bat) that resolves runfiles properly and sets PYTHONHOME so CPython finds its standard library natively. Add an execution test verifying the interpreter runs as a build action. --- .agents/plans/zip-stdlib-runtime-rule.md | 125 ++++++++++++++++++ python/private/BUILD.bazel | 3 + python/private/common.bzl | 16 +-- python/private/interpreter.bzl | 5 +- python/private/interpreter_tmpl.bat | 44 ++++++ python/private/interpreter_tmpl.sh | 44 +++--- python/private/py_console_script_gen.bzl | 2 +- python/private/py_exec_tools_toolchain.bzl | 61 +++++++-- tests/py_exec_tools_toolchain/BUILD.bazel | 10 +- .../expected_action_output.json | 4 + .../py_exec_tools_toolchain_tests.bzl | 112 +++++++++++++++- tests/py_exec_tools_toolchain/test_action.py | 13 ++ 12 files changed, 398 insertions(+), 41 deletions(-) create mode 100644 .agents/plans/zip-stdlib-runtime-rule.md create mode 100644 python/private/interpreter_tmpl.bat create mode 100644 tests/py_exec_tools_toolchain/expected_action_output.json create mode 100644 tests/py_exec_tools_toolchain/test_action.py diff --git a/.agents/plans/zip-stdlib-runtime-rule.md b/.agents/plans/zip-stdlib-runtime-rule.md new file mode 100644 index 0000000000..16a549dab1 --- /dev/null +++ b/.agents/plans/zip-stdlib-runtime-rule.md @@ -0,0 +1,125 @@ +# Plan: Zipped Standard Library & Executable Runtime Rule + +Tracking requirements, past failures and betrayals, edge cases, and design +decisions for packaging hermetic Python standard library files into zip archives +(Issue #1653, PR #4146). + +## Requirements + +1. **Zipped Standard Library Packaging**: + - Package pure-Python standard library files into a zip archive for hermetic + runtimes: `lib/pythonXY.zip` on Unix and `pythonXY.zip` on Windows. + - Landmark file: Keep unzipped `os.py` on disk so CPython path calculation + (`getpath.py`/`getpath.c`) locates `sys.prefix` and standard library roots. + - Flag: Provide `//python/config_settings:zip_stdlib` ('yes'/'no', default + 'yes') allowing users to opt out when on-disk loose files are required. + +2. **Binary Rule Wrapping Runtime (`current_interpreter_executable`)**: + - Provide a binary rule wrapping the toolchain runtime into an executable + target returning `DefaultInfo(executable = ..., runfiles = ...)` with a + complete `FilesToRunProvider`. + - Toolchains and `actions_run()` must pass + `exec_tools.exec_interpreter[DefaultInfo].files_to_run` as `executable` + to `ctx.actions.run()`, ensuring Bazel constructs runfiles trees and + manifests. + +3. **No PYTHONPATH Modification**: + - **Strict Requirement**: Do NOT set or export `PYTHONPATH` in launcher + wrappers or bootstrap scripts. + - Python must discover `pythonXY.zip` through standard CPython initialization + and path derivation mechanisms (`PYTHONHOME` or directory structure). + - Setting `PYTHONPATH` pollutes child processes and obscures path resolution + behavior. + +4. **No Unnecessary Dependencies**: + - Avoid unnecessary `@bazel_tools//tools/bash/runfiles` runtime dependencies + in `py_exec_tools_toolchain`. + +5. **Cross-Platform Compatibility**: + - Fully support Linux, macOS, and Windows. + - On Windows, provide a `.bat` launcher. Note that + `--windows_enable_symlinks` is strictly required; running without it is + unsupported. + - In `py_console_script_gen`, ensure `_tool` runfiles are staged by passing + `executable = ctx.attr._tool[DefaultInfo].files_to_run`. + +6. **Remote Build Execution (RBE) Hermeticity**: + - RBE environments execute actions without symlink preservation guarantees + (bazelbuild/bazel#23620). All runtime files must be staged via proper + runfiles. + +## What Hasn't Worked (Past Betrayals) + +1. **Bare File Executable in `actions_run()`**: + - *Attempt*: Passing `exec_runtime.interpreter` (a bare `File`) directly to + `ctx.actions.run(executable = action_exe)`. + - *Betrayal*: While Bazel can in certain cases look up runfiles information + for a bare `File`, this is finicky behavior that should not be relied + upon. In practice, relying on it failed to reliably stage or pass runfiles + across platforms (such as RBE and Windows), leaving `pythonXY.zip` isolated + in `bazel-out/` and triggering + `ModuleNotFoundError: No module named 'encodings'`. + +2. **Symlinking Interpreter in Place (`ctx.actions.symlink`)**: + - *Attempt*: Symlinking the toolchain interpreter to declare an executable. + - *Betrayal*: `ctx.actions.symlink(target_file=...)` does not materialize as + a symlink on RBE and fails on Windows without elevated privileges. + +3. **Setting `PYTHONPATH` in Wrapper Scripts**: + - *Attempt*: Prepending `TARGET_ZIP` to `PYTHONPATH` in launcher templates. + - *Betrayal*: Violates the requirement not to mutate `PYTHONPATH`. Mutating + `PYTHONPATH` breaks downstream subprocesses and masks underlying landmark + and prefix discovery failures. + +4. **`py_console_script_gen` Using `ctx.executable._tool`**: + - *Attempt*: `ctx.actions.run(executable = ctx.executable._tool)`. + - *Betrayal*: On Windows, `ctx.executable._tool` is a bare `File` rather + than `FilesToRunProvider`, preventing runfiles from being staged for + `py_console_script_gen_py.exe`. + +5. **Sourcing External `runfiles.bash` in Bash Launcher**: + - *Attempt*: Sourcing `@bazel_tools//tools/bash/runfiles/runfiles.bash` and + relying on `rlocation`. + - *Betrayal*: Adding `@bazel_tools//tools/bash/runfiles` introduces an + unnecessary dependency into the toolchain. Omitting that runfiles dep while + leaving `source .../runfiles.bash` causes + `ERROR: cannot find bazel_tools/tools/bash/runfiles/runfiles.bash`. + - *Resolution*: Implement self-contained runfiles resolution directly in + `interpreter_tmpl.sh` (checking `RUNFILES_MANIFEST_FILE`, `RUNFILES_DIR`, + `$0.runfiles_manifest`, and `$0.runfiles`) without sourcing external + runfiles libraries. + +## Edge Cases + +1. **Windows Symlinks Requirement (`--windows_enable_symlinks`)**: + - The project strictly requires `--windows_enable_symlinks` to be enabled; + running without it is unsupported. Manifest-only fallback mode does not + need special accommodation. + +2. **CPython Landmark & Standard Library Zip Discovery**: + - CPython checks `/lib/python.zip` on Unix and + `/python.zip` on Windows. + - If `PYTHONHOME` is set to the interpreter root in runfiles, CPython + resolves `sys.prefix` to that directory and automatically finds the zip + archive without needing `PYTHONPATH`. + +3. **Subprocess Isolation**: + - Wrapper scripts must not export variables that disrupt child Python + invocations. If `PYTHONHOME` is set, verify whether child processes inherit + it or if it should only be set when not already defined. + +4. **Batch Script Argument Forwarding on Windows**: + - Windows batch scripts must safely forward `%*` and return `!ERRORLEVEL!`. + +## Action Items + +1. Create and maintain this plan file in `.agents/plans/`. +2. Update `interpreter_tmpl.sh` and `interpreter_tmpl.bat`: + - Remove logic setting `PYTHONPATH`. + - Keep `PYTHONHOME` resolution pointing to the runtime root so CPython finds + `lib/pythonXY.zip` (Unix) or `pythonXY.zip` (Windows) naturally. +3. Update `py_exec_tools_toolchain.bzl`: + - Remove `_bash_runfiles` attribute and runfiles merge logic. +4. Run tests and verify stdlib loads without `PYTHONPATH`: + - `bazel test --config=fast-tests //tests/zip_stdlib/...` + `//tests/py_exec_tools_toolchain/...` diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 7ef54d5df7..6201438b51 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -33,6 +33,8 @@ package( licenses(["notice"]) exports_files([ + "interpreter_tmpl.bat", + "interpreter_tmpl.sh", "runtime_env_toolchain_interpreter.sh", "runtimes_manifest.txt", "runtimes_manifest_workspace.bzl", @@ -541,6 +543,7 @@ bzl_library( name = "py_exec_tools_toolchain", srcs = ["py_exec_tools_toolchain.bzl"], deps = [ + ":common", ":common_labels", ":py_exec_tools_info", ":sentinel_impl", diff --git a/python/private/common.bzl b/python/private/common.bzl index 5cff7f8723..cb946cb128 100644 --- a/python/private/common.bzl +++ b/python/private/common.bzl @@ -648,18 +648,14 @@ def actions_run( EXEC_TOOLS_TOOLCHAIN_TYPE, toolchain, )) - exec_runtime = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN_TYPE].exec_tools.exec_runtime - if exec_runtime.interpreter: - action_exe = exec_runtime.interpreter - action_inputs.add(exec_runtime.files) - elif exec_runtime.interpreter_path: - action_exe = exec_runtime.interpreter_path - else: - fail(("Action {}: PyRuntimeInfo from exec tools toolchain is " + - "malformed: requires one of `interpreter` or " + - "`interpreter_path` set").format( + exec_tools = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN_TYPE].exec_tools + if not exec_tools.exec_interpreter: + fail(("Action {}: tool {} provides PyInterpreterProgramInfo, " + + "but exec_tools.exec_interpreter is not configured").format( mnemonic, + executable, )) + action_exe = exec_tools.exec_interpreter[DefaultInfo].files_to_run program_info = executable[PyInterpreterProgramInfo] diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl index b281be9639..72277946e4 100644 --- a/python/private/interpreter.bzl +++ b/python/private/interpreter.bzl @@ -46,7 +46,10 @@ def _interpreter_binary_impl(ctx): template = ctx.file._template, output = executable, substitutions = { - "%target_file%": runfiles_root_path(ctx, runtime.interpreter.short_path), + "%target_file%": runfiles_root_path( + ctx, + runtime.interpreter.short_path, + ), }, is_executable = True, ) diff --git a/python/private/interpreter_tmpl.bat b/python/private/interpreter_tmpl.bat new file mode 100644 index 0000000000..14194d4f1f --- /dev/null +++ b/python/private/interpreter_tmpl.bat @@ -0,0 +1,44 @@ +@echo off +SETLOCAL ENABLEEXTENSIONS +SETLOCAL ENABLEDELAYEDEXPANSION + +rem --- begin runfiles resolution --- +set "MF=%RUNFILES_MANIFEST_FILE:/=\%" +set "TARGET_FILE=%target_file%" + +set "MAIN_BIN=" +if defined MF ( + if exist "%MF%" ( + for /F "tokens=1* usebackq" %%a in (`findstr.exe /l /c:"!TARGET_FILE! " "%MF%"`) do ( + set "MAIN_BIN=%%b" + ) + ) +) +if "!MAIN_BIN!" equ "" ( + set "TF_WIN=!TARGET_FILE:/=\!" + if "%RUNFILES_MANIFEST_ONLY%" neq "1" if defined RUNFILES_DIR ( + if exist "%RUNFILES_DIR%\!TF_WIN!" ( + set "MAIN_BIN=%RUNFILES_DIR%\!TF_WIN!" + ) + ) + if "!MAIN_BIN!" equ "" ( + if exist "!TF_WIN!" ( + set "MAIN_BIN=!TF_WIN!" + ) + ) +) + +if "!MAIN_BIN!" equ "" ( + echo>&2 ERROR: interpreter executable not found: !TARGET_FILE! + exit /b 1 +) + +set "MAIN_BIN=!MAIN_BIN:/=\!" + +if not defined PYTHONHOME ( + for %%d in ("!MAIN_BIN!") do set "PYTHONHOME=%%~dpd" + if "!PYTHONHOME:~-1!"=="\" set "PYTHONHOME=!PYTHONHOME:~0,-1!" +) + +"!MAIN_BIN!" %* +exit /b !ERRORLEVEL! diff --git a/python/private/interpreter_tmpl.sh b/python/private/interpreter_tmpl.sh index c4e87fbb43..801f43505b 100644 --- a/python/private/interpreter_tmpl.sh +++ b/python/private/interpreter_tmpl.sh @@ -1,23 +1,35 @@ #!/usr/bin/env bash +set -uo pipefail -# --- begin runfiles.bash initialization v3 --- -# Copy-pasted from the Bazel Bash runfiles library v3. -set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash -# shellcheck disable=SC1090 -source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ - source "$0.runfiles/$f" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ - { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e -# --- end runfiles.bash initialization v3 --- - -set +e # allow us to check for errors more easily readonly TARGET_FILE="%target_file%" -MAIN_BIN=$(rlocation "$TARGET_FILE") +MAIN_BIN="" + +if [[ -n "${RUNFILES_MANIFEST_FILE:-}" && \ + -f "${RUNFILES_MANIFEST_FILE}" ]]; then + MAIN_BIN="$(grep -F -m1 "${TARGET_FILE} " \ + "${RUNFILES_MANIFEST_FILE}" | cut -f2- -d' ')" +fi +if [[ -z "${MAIN_BIN:-}" ]]; then + if [[ -n "${RUNFILES_DIR:-}" && -e "${RUNFILES_DIR}/${TARGET_FILE}" ]]; then + MAIN_BIN="${RUNFILES_DIR}/${TARGET_FILE}" + elif [[ -f "$0.runfiles_manifest" ]]; then + MAIN_BIN="$(grep -F -m1 "${TARGET_FILE} " \ + "$0.runfiles_manifest" | cut -f2- -d' ')" + elif [[ -e "$0.runfiles/${TARGET_FILE}" ]]; then + MAIN_BIN="$0.runfiles/${TARGET_FILE}" + elif [[ -e "${TARGET_FILE}" ]]; then + MAIN_BIN="${TARGET_FILE}" + fi +fi -if [[ -z "$MAIN_BIN" || ! -e "$MAIN_BIN" ]]; then - echo "ERROR: interpreter executable not found: $MAIN_BIN (from $TARGET_FILE)" +if [[ -z "${MAIN_BIN:-}" || ! -e "${MAIN_BIN}" ]]; then + echo "ERROR: interpreter executable not found: ${MAIN_BIN:-}" \ + "(from ${TARGET_FILE})" >&2 exit 1 fi + +if [[ -z "${PYTHONHOME:-}" ]]; then + export PYTHONHOME="$(dirname "$(dirname "$MAIN_BIN")")" +fi + exec "${MAIN_BIN}" "$@" diff --git a/python/private/py_console_script_gen.bzl b/python/private/py_console_script_gen.bzl index de016036b2..1f310217b5 100644 --- a/python/private/py_console_script_gen.bzl +++ b/python/private/py_console_script_gen.bzl @@ -54,7 +54,7 @@ def _py_console_script_gen_impl(ctx): arguments = [args], mnemonic = "PyConsoleScriptBinaryGen", progress_message = "Generating py_console_script_binary main: %{label}", - executable = ctx.executable._tool, + executable = ctx.attr._tool[DefaultInfo].files_to_run, ) return [DefaultInfo( diff --git a/python/private/py_exec_tools_toolchain.bzl b/python/private/py_exec_tools_toolchain.bzl index d126262033..4f1b4464af 100644 --- a/python/private/py_exec_tools_toolchain.bzl +++ b/python/private/py_exec_tools_toolchain.bzl @@ -16,6 +16,7 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load(":common.bzl", "is_windows_platform", "runfiles_root_path") load(":common_labels.bzl", "labels") load(":py_exec_tools_info.bzl", "PyExecToolsInfo") load(":sentinel_impl.bzl", "SentinelInfo") @@ -109,22 +110,45 @@ def _current_interpreter_executable_impl(ctx): # because of things like pyenv: they use $0 to determine what to # re-exec. If it's not a recognized name, then they fail. if runtime.interpreter: - executable = ctx.actions.declare_file(runtime.interpreter.basename) - - # NOTE: Using ctx.actions.symlink() here doesn't always work with RBE - # because it's not guaranteed that it will materialize as a symlink, but - # we rely on it being a symlink so that Python can find its actual - # PYTHONHOME. - # See https://github.com/bazelbuild/bazel/issues/23620 - ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True) + is_windows = is_windows_platform(ctx) + basename = runtime.interpreter.basename + if is_windows: + if basename.lower().endswith(".exe"): + basename = basename[:-4] + basename = basename + ".bat" + template = ctx.file._template_bat + else: + template = ctx.file._template_sh + + executable = ctx.actions.declare_file(basename) + + ctx.actions.expand_template( + template = template, + output = executable, + substitutions = { + "%target_file%": runfiles_root_path( + ctx, + runtime.interpreter.short_path, + ), + }, + is_executable = True, + ) + runfiles = ctx.runfiles([executable], transitive_files = runtime.files) else: - executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) - ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) + executable = ctx.actions.declare_symlink( + paths.basename(runtime.interpreter_path), + ) + ctx.actions.symlink( + output = executable, + target_path = runtime.interpreter_path, + ) + runfiles = ctx.runfiles([executable], transitive_files = runtime.files) + return [ toolchain, DefaultInfo( executable = executable, - runfiles = ctx.runfiles([executable], transitive_files = runtime.files), + runfiles = runfiles, ), ] @@ -132,4 +156,19 @@ current_interpreter_executable = rule( implementation = _current_interpreter_executable_impl, toolchains = [TARGET_TOOLCHAIN_TYPE], executable = True, + attrs = { + "_template_bat": attr.label( + default = "//python/private:interpreter_tmpl.bat", + allow_single_file = True, + ), + "_template_sh": attr.label( + default = "//python/private:interpreter_tmpl.sh", + allow_single_file = True, + ), + "_windows_constraints": attr.label_list( + default = [ + "@platforms//os:windows", + ], + ), + }, ) diff --git a/tests/py_exec_tools_toolchain/BUILD.bazel b/tests/py_exec_tools_toolchain/BUILD.bazel index 092e790939..ddaaeaf3de 100644 --- a/tests/py_exec_tools_toolchain/BUILD.bazel +++ b/tests/py_exec_tools_toolchain/BUILD.bazel @@ -12,8 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":py_exec_tools_toolchain_tests.bzl", "py_exec_tools_toolchain_test_suite") +load( + ":py_exec_tools_toolchain_tests.bzl", + "interpreter_run_in_action_test", + "py_exec_tools_toolchain_test_suite", +) py_exec_tools_toolchain_test_suite( name = "py_exec_tools_toolchain_tests", ) + +interpreter_run_in_action_test( + name = "test_run_interpreter_as_action", +) diff --git a/tests/py_exec_tools_toolchain/expected_action_output.json b/tests/py_exec_tools_toolchain/expected_action_output.json new file mode 100644 index 0000000000..971612a3fc --- /dev/null +++ b/tests/py_exec_tools_toolchain/expected_action_output.json @@ -0,0 +1,4 @@ +{ + "has_encodings": true, + "status": "ok" +} diff --git a/tests/py_exec_tools_toolchain/py_exec_tools_toolchain_tests.bzl b/tests/py_exec_tools_toolchain/py_exec_tools_toolchain_tests.bzl index 3be2bc3f30..6f4bab5e78 100644 --- a/tests/py_exec_tools_toolchain/py_exec_tools_toolchain_tests.bzl +++ b/tests/py_exec_tools_toolchain/py_exec_tools_toolchain_tests.bzl @@ -13,9 +13,18 @@ # limitations under the License. """Starlark tests for py_exec_tools_toolchain rule.""" +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") # buildifier: disable=bzl-visibility +load( + "//python/private:py_exec_tools_toolchain.bzl", + "current_interpreter_executable", + "py_exec_tools_toolchain", +) # buildifier: disable=bzl-visibility +load( + "//python/private:toolchain_types.bzl", + "EXEC_TOOLS_TOOLCHAIN_TYPE", +) # buildifier: disable=bzl-visibility _tests = [] @@ -36,5 +45,106 @@ def _test_disable_exec_interpreter_impl(env, target): _tests.append(_test_disable_exec_interpreter) +def _test_default_exec_interpreter(name): + py_exec_tools_toolchain( + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_default_exec_interpreter_impl, + ) + +def _test_default_exec_interpreter_impl(env, target): + exec_tools = target[platform_common.ToolchainInfo].exec_tools + env.expect.that_bool(exec_tools.exec_interpreter != None).equals(True) + target_info = exec_tools.exec_interpreter + env.expect.that_bool(DefaultInfo in target_info).equals(True) + env.expect.that_bool( + target_info[DefaultInfo].files_to_run != None, + ).equals(True) + env.expect.that_bool( + target_info[DefaultInfo].files_to_run.executable != None, + ).equals(True) + +_tests.append(_test_default_exec_interpreter) + +def _test_current_interpreter_executable(name): + current_interpreter_executable( + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_current_interpreter_executable_impl, + ) + +def _test_current_interpreter_executable_impl(env, target): + env.expect.that_bool(DefaultInfo in target).equals(True) + env.expect.that_bool(platform_common.ToolchainInfo in target).equals(True) + env.expect.that_bool( + target[DefaultInfo].files_to_run != None, + ).equals(True) + env.expect.that_bool( + target[DefaultInfo].files_to_run.executable != None, + ).equals(True) + +_tests.append(_test_current_interpreter_executable) + def py_exec_tools_toolchain_test_suite(name): test_suite(name = name, tests = _tests) + +def _run_interpreter_action_impl(ctx): + exec_tools = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN_TYPE].exec_tools + out = ctx.actions.declare_file(ctx.label.name + ".out") + ctx.actions.run( + executable = exec_tools.exec_interpreter[DefaultInfo].files_to_run, + arguments = [ + ctx.file.src.path, + out.path, + ], + inputs = [ctx.file.src], + outputs = [out], + mnemonic = "TestRunInterpreterAction", + progress_message = "Running interpreter action: %{label}", + ) + return [DefaultInfo(files = depset([out]))] + +_run_interpreter_action = rule( + implementation = _run_interpreter_action_impl, + attrs = { + "src": attr.label( + mandatory = True, + allow_single_file = True, + doc = "Python script to execute with the interpreter.", + ), + }, + toolchains = [EXEC_TOOLS_TOOLCHAIN_TYPE], + doc = """Runs Python script using exec_tools.exec_interpreter.""", +) + +def interpreter_run_in_action_test( + name, + src = "test_action.py", + expected = "expected_action_output.json", + **kwargs): + """Runs a Python script using the exec interpreter and diffs the output. + + Args: + name: The name of the diff_test target. + src: The Python script to execute. + expected: The expected golden output file to compare against. + **kwargs: Additional keyword arguments forwarded to diff_test. + """ + actual_target = name + "_actual" + _run_interpreter_action( + name = actual_target, + src = src, + tags = ["manual"], + ) + diff_test( + name = name, + file1 = ":" + actual_target, + file2 = expected, + **kwargs + ) diff --git a/tests/py_exec_tools_toolchain/test_action.py b/tests/py_exec_tools_toolchain/test_action.py new file mode 100644 index 0000000000..6316906799 --- /dev/null +++ b/tests/py_exec_tools_toolchain/test_action.py @@ -0,0 +1,13 @@ +import encodings +import json +import sys +from pathlib import Path + +data = { + "has_encodings": bool(encodings), + "status": "ok", +} +Path(sys.argv[1]).write_text( + json.dumps(data, indent=2, sort_keys=True) + "\n", + encoding="utf-8", +)