From 3723cbd9992871718dda8afae8da3b9a1317ec40 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 27 Oct 2026 09:39:11 -0700 Subject: [PATCH] fix(bzlmod): support Python versions not in MINOR_MAPPING Configuring a Python patch version newer than the built-in `MINOR_MAPPING` failed in the `python` and `pip` Bzlmod extensions. Manifest files loaded via `add_runtime_manifest_files` omitted the release tag from default download URLs, `single_version_override` did not require `sha256` and `urls` when introducing a new version, and `pip.parse` failed to match host interpreters when toolchain and pip version granularities differed (such as `X.Y` vs. `X.Y.Z` or versions inferred from `pyproject.toml`). Update the `python` extension to include the release build version in default manifest URLs, require `sha256` and `urls` for new single-version overrides, and verify requested toolchains exist in `tool_versions`. Update `pip.parse` to resolve host interpreters across exact, full, and `major.minor` version names and key index URL settings by the resolved Python version. Fixes #4358 --- news/4358.fixed.md | 4 + python/private/pypi/hub_builder.bzl | 60 +++++++++--- python/private/python.bzl | 22 ++++- tests/pypi/extension/extension_tests.bzl | 92 ++++++++++--------- tests/python/python_tests.bzl | 9 ++ .../runtime_manifests_tests.bzl | 65 +++++++++++++ 6 files changed, 191 insertions(+), 61 deletions(-) create mode 100644 news/4358.fixed.md diff --git a/news/4358.fixed.md b/news/4358.fixed.md new file mode 100644 index 0000000000..f8ea0c2d82 --- /dev/null +++ b/news/4358.fixed.md @@ -0,0 +1,4 @@ +(bzlmod) Fixed configuring and using Python patch versions newer than the +built-in `MINOR_MAPPING` in the {obj}`python` and {obj}`pip` extensions, +including `add_runtime_manifest_files` download URLs and host interpreter +resolution ([#4358](https://github.com/bazel-contrib/rules_python/issues/4358)). diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index cbf4877d65..2500a3483b 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -181,7 +181,11 @@ def _pip_parse(self, module_ctx, pip_attr, python_version = None): if (pip_attr.python_interpreter_target == None and not pip_attr.python_interpreter and - _host_interpreter_name(python_version) not in self._available_interpreters): + not _find_host_interpreter_name( + python_version = python_version, + full_python_version = full_python_version, + available_interpreters = self._available_interpreters, + )): # The version has a toolchain, but none of its interpreters runs on # this host, e.g. there is no CPython build for this os and cpu. Treat # it like a version missing from minor_mapping instead of failing the @@ -198,7 +202,7 @@ def _pip_parse(self, module_ctx, pip_attr, python_version = None): )) return - _set_get_index_urls(self, module_ctx, pip_attr) + _set_get_index_urls(self, module_ctx, pip_attr, python_version) self._platforms[python_version] = _platforms( module_ctx, python_version = full_python_version, @@ -212,7 +216,11 @@ def _pip_parse(self, module_ctx, pip_attr, python_version = None): module_ctx, pip_attr = pip_attr, python_version = python_version, - enable_pipstar_extract = bool(self._config.enable_pipstar_extract or self._get_index_urls.get(python_version)), + full_python_version = full_python_version, + enable_pipstar_extract = bool( + self._config.enable_pipstar_extract or + self._get_index_urls.get(python_version), + ), ) ### end of PUBLIC methods @@ -367,7 +375,7 @@ def _add_whl_library(self, *, python_version, whl, repo): ### end of setters, below we have various functions to implement the public methods -def _set_get_index_urls(self, mctx, pip_attr): +def _set_get_index_urls(self, mctx, pip_attr, python_version): # Resolve the index URL through envsubst so the ``$VAR`` / ``${VAR:-default}`` # form is honored when deciding whether the experimental index-url mode is # active. Without this, an unsubstituted template like ``$RULES_PYTHON_PIP_INDEX_URL`` @@ -385,7 +393,6 @@ def _set_get_index_urls(self, mctx, pip_attr): # here return False - python_version = pip_attr.python_version self._use_downloader.setdefault(python_version, {}).update({ normalize_name(s): False for s in pip_attr.simpleapi_skip @@ -425,11 +432,30 @@ def _select_default_index(index, default): def _host_interpreter_name(python_version): return "python_{}_host".format(python_version.replace(".", "_")) -def _detect_interpreter(self, pip_attr, python_version): +def _find_host_interpreter_name( + *, + python_version, + full_python_version, + available_interpreters): + for version_str in [ + python_version, + full_python_version, + _major_minor_version(python_version), + ]: + name = _host_interpreter_name(version_str) + if name in available_interpreters: + return name + return None + +def _detect_interpreter(self, pip_attr, python_version, full_python_version): python_interpreter_target = pip_attr.python_interpreter_target if python_interpreter_target == None and not pip_attr.python_interpreter: - python_name = _host_interpreter_name(python_version) - if python_name not in self._available_interpreters: + python_name = _find_host_interpreter_name( + python_version = python_version, + full_python_version = full_python_version, + available_interpreters = self._available_interpreters, + ) + if not python_name: fail(( "Unable to find interpreter for pip hub '{hub_name}' for " + "python_version={version}: Make sure a corresponding " + @@ -438,7 +464,7 @@ def _detect_interpreter(self, pip_attr, python_version): ).format( hub_name = self.name, version = python_version, - python_name = python_name, + python_name = _host_interpreter_name(python_version), labels = " \n".join(self._available_interpreters), )) python_interpreter_target = self._available_interpreters[python_name] @@ -507,6 +533,7 @@ def _create_whl_repos( *, pip_attr, python_version, + full_python_version, enable_pipstar_extract = False): """create all of the whl repositories @@ -515,6 +542,7 @@ def _create_whl_repos( module_ctx: {type}`module_ctx`. pip_attr: {type}`struct` - the struct that comes from the tag class iteration. python_version: {type}`str` - the resolved python version for this pip.parse call. + full_python_version: {type}`str` - the full X.Y.Z python version. enable_pipstar_extract: {type}`bool` - enable the pipstar extraction or not. """ logger = self._logger @@ -530,16 +558,13 @@ def _create_whl_repos( uv_lock = pip_attr.uv_lock, extra_pip_args = pip_attr.extra_pip_args, platforms = sorted(platforms), # here we only need keys - python_version = full_version( - version = python_version, - minor_mapping = self._minor_mapping, - ), + python_version = full_python_version, logger = logger, ), uv_lock = pip_attr.uv_lock, platforms = platforms, extra_pip_args = pip_attr.extra_pip_args, - get_index_urls = self._get_index_urls.get(pip_attr.python_version), + get_index_urls = self._get_index_urls.get(python_version), toml_decode = getattr(self._config, "toml_decode", None), logger = logger, ) @@ -561,7 +586,12 @@ def _create_whl_repos( pip_attr = pip_attr, ) - interpreter = _detect_interpreter(self, pip_attr, python_version) + interpreter = _detect_interpreter( + self, + pip_attr, + python_version, + full_python_version, + ) for whl in requirements_by_platform: whl_library_args = common_args | _whl_library_args( diff --git a/python/private/python.bzl b/python/private/python.bzl index 8037963f31..6d8237b24e 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -275,7 +275,8 @@ def _python_impl(module_ctx): minor_mapping = py.config.minor_mapping, fail_on_err = False, ) - if not full_python_version: + if (not full_python_version or + full_python_version not in py.config.default["tool_versions"]): logger.info(lambda: ( "The actual toolchain for python_version '{version}' " + "has not been registered, but was requested, please configure a toolchain " + @@ -589,6 +590,14 @@ def _process_single_version_overrides(*, tag, _fail = fail, default): available_versions = default["tool_versions"] kwargs = default.setdefault("kwargs", {}) + if tag.python_version not in available_versions: + if not (tag.sha256 and tag.urls): + _fail(( + "When introducing a new python_version '{}', 'sha256' and " + + "'urls' must be specified" + ).format(tag.python_version)) + return + if tag.sha256 or tag.urls: if not (tag.sha256 and tag.urls): _fail("Both `sha256` and `urls` overrides need to be provided together") @@ -836,7 +845,16 @@ def _populate_from_pbs_manifest( if "://" in location: urls = [location] else: - urls = ["{}/{}".format(b_url, location) for b_url in base_download_urls] + urls = [] + for b_url in base_download_urls: + if ("/" not in location and + entry.build_version and + b_url in DEFAULT_RELEASE_BASE_URLS): + urls.append( + "{}/{}/{}".format(b_url, entry.build_version, location), + ) + else: + urls.append("{}/{}".format(b_url, location)) strip_prefix = "python/install" if entry.archive_flavor == "full" else "python" diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index b3a2ebf975..11d4c31913 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -190,53 +190,57 @@ _tests.append(_test_simple) def _test_pip_parse_pyproject_toml(env): # pip.parse() reads the version from pyproject.toml's requires-python when - # python_version is not set explicitly. - pypi = _parse_modules( - env, - module_ctx = _pypi_mock_mctx( - _mod( - name = "rules_python", - parse = [ - _parse( - hub_name = "pypi", - pyproject_toml = "pyproject.toml", - simpleapi_skip = ["simple"], - requirements_lock = "requirements.txt", + # python_version is not set explicitly. Verify it works both when the + # toolchain was registered by full version (python_3_15_19_host) and when + # registered by minor version (python_3_15_host). + for available_interpreters in [ + {"python_3_15_19_host": "unit_test_interpreter_target"}, + {"python_3_15_host": "unit_test_interpreter_target"}, + ]: + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "rules_python", + parse = [ + _parse( + hub_name = "pypi", + pyproject_toml = "pyproject.toml", + simpleapi_skip = ["simple"], + requirements_lock = "requirements.txt", + ), + ], + ), + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": "[project]\nrequires-python = \"==3.15.19\"\n", + }, + ), + available_interpreters = available_interpreters, + minor_mapping = {"3.15": "3.15.19"}, + ) + + # Resolves identically to passing python_version = "3.15.19" explicitly: + # the full version drives interpreter selection, hub naming uses major.minor. + pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) + pypi.hub_whl_map().contains_exactly({"pypi": { + "simple": { + "pypi_315_simple": [ + whl_config_setting( + version = "3.15", ), ], - ), - os_name = "linux", - arch_name = "x86_64", - mock_files = { - "pyproject.toml": "[project]\nrequires-python = \"==3.15.19\"\n", }, - ), - available_interpreters = { - "python_3_15_19_host": "unit_test_interpreter_target", - }, - minor_mapping = {"3.15": "3.15.19"}, - ) - - # Resolves identically to passing python_version = "3.15.19" explicitly: - # the full version drives interpreter selection, hub naming uses major.minor. - pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) - pypi.hub_whl_map().contains_exactly({"pypi": { - "simple": { - "pypi_315_simple": [ - whl_config_setting( - version = "3.15", - ), - ], - }, - }}) - pypi.whl_libraries().contains_exactly({ - "pypi_315_simple": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", - }, - }) + }}) + pypi.whl_libraries().contains_exactly({ + "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", + }, + }) _tests.append(_test_pip_parse_pyproject_toml) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index fda82af036..e6956fa91e 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -883,6 +883,15 @@ def _test_single_version_override_errors(env): ], want_error = "Only a single 'python.single_version_override' can be present for '3.12.4'", ), + struct( + overrides = [ + python_ext.single_version_override( + distutils_content = "foo", + python_version = "3.13.99", + ), + ], + want_error = "When introducing a new python_version '3.13.99', 'sha256' and 'urls' must be specified", + ), ]: errors = [] parse_modules( diff --git a/tests/python_bzlmod_ext/runtime_manifests_tests.bzl b/tests/python_bzlmod_ext/runtime_manifests_tests.bzl index 99525817e4..682afa1a8e 100644 --- a/tests/python_bzlmod_ext/runtime_manifests_tests.bzl +++ b/tests/python_bzlmod_ext/runtime_manifests_tests.bzl @@ -167,6 +167,71 @@ def _test_dynamic_manifest_files_impl(env, target): _tests.append(_test_dynamic_manifest_files) +def _test_dynamic_manifest_files_default_base_urls(name): + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_dynamic_manifest_files_default_base_urls_impl, + ) + +def _test_dynamic_manifest_files_default_base_urls_impl(env, target): + _ = target # @unused + + root_module = python_ext.module( + name = "runtime_manifests", + override = [ + python_ext.override( + add_runtime_manifest_files = [ + Label("//:SHA256SUMS"), + ], + ), + ], + toolchain = [ + python_ext.toolchain( + python_version = "3.13", + ), + ], + ) + + mock_mctx = python_ext.mctx( + modules = [root_module], + mock_files = { + str(Label("//:SHA256SUMS")): """ +cba3455e11ee12f8cb085a27e96ae8aa5492e153263e6577ad89213a46646427 cpython-3.13.99+20261026-x86_64-unknown-linux-gnu-install_only.tar.gz +""", + }, + ) + + res = parse_modules( + module_ctx = mock_mctx, + logger = _mock_logger, + ) + + tool_versions = res.config.default["tool_versions"] + env.expect.that_bool("3.13.99" in tool_versions).equals(True) + env.expect.that_str(res.config.minor_mapping["3.13"]).equals("3.13.99") + + version_info = tool_versions["3.13.99"] + env.expect.that_dict(version_info).contains_exactly({ + "sha256": { + "x86_64-unknown-linux-gnu": "cba3455e11ee12f8cb085a27e96ae8aa5492e153263e6577ad89213a46646427", + }, + "strip_prefix": { + "x86_64-unknown-linux-gnu": "python", + }, + "url": { + "x86_64-unknown-linux-gnu": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/cpython-3.13.99+20261026-x86_64-unknown-linux-gnu-install_only.tar.gz", + ], + }, + }) + +_tests.append(_test_dynamic_manifest_files_default_base_urls) + def runtime_manifests_test_suite(name): test_suite( name = name,