fix(bzlmod): support Python versions not in MINOR_MAPPING

news/4358.fixed.md

@@ -0,0 +1,4 @@
+(bzlmod) Fixed configuring and using Python patch versions newer than the
+built-in `MINOR_MAPPING` in the {obj}`python` and {obj}`pip` extensions,
+including `add_runtime_manifest_files` download URLs and host interpreter
+resolution ([#4358](https://github.com/bazel-contrib/rules_python/issues/4358)).

python/private/pypi/hub_builder.bzl

@@ -181,7 +181,11 @@ def _pip_parse(self, module_ctx, pip_attr, python_version = None):
 
     if (pip_attr.python_interpreter_target == None and
         not pip_attr.python_interpreter and
-        _host_interpreter_name(python_version) not in self._available_interpreters):
+        not _find_host_interpreter_name(
+            python_version = python_version,
+            full_python_version = full_python_version,
+            available_interpreters = self._available_interpreters,
+        )):
         # The version has a toolchain, but none of its interpreters runs on
         # this host, e.g. there is no CPython build for this os and cpu. Treat
         # it like a version missing from minor_mapping instead of failing the
@@ -198,7 +202,7 @@ def _pip_parse(self, module_ctx, pip_attr, python_version = None):
         ))
         return
 
-    _set_get_index_urls(self, module_ctx, pip_attr)
+    _set_get_index_urls(self, module_ctx, pip_attr, python_version)
     self._platforms[python_version] = _platforms(
         module_ctx,
         python_version = full_python_version,
@@ -212,7 +216,11 @@ def _pip_parse(self, module_ctx, pip_attr, python_version = None):
         module_ctx,
         pip_attr = pip_attr,
         python_version = python_version,
-        enable_pipstar_extract = bool(self._config.enable_pipstar_extract or self._get_index_urls.get(python_version)),
+        full_python_version = full_python_version,
+        enable_pipstar_extract = bool(
+            self._config.enable_pipstar_extract or
+            self._get_index_urls.get(python_version),
+        ),
     )
 
 ### end of PUBLIC methods
@@ -367,7 +375,7 @@ def _add_whl_library(self, *, python_version, whl, repo):
 
 ### end of setters, below we have various functions to implement the public methods
 
-def _set_get_index_urls(self, mctx, pip_attr):
+def _set_get_index_urls(self, mctx, pip_attr, python_version):
     # Resolve the index URL through envsubst so the ``$VAR`` / ``${VAR:-default}``
     # form is honored when deciding whether the experimental index-url mode is
     # active. Without this, an unsubstituted template like ``$RULES_PYTHON_PIP_INDEX_URL``
@@ -385,7 +393,6 @@ def _set_get_index_urls(self, mctx, pip_attr):
         # here
         return False
 
-    python_version = pip_attr.python_version
     self._use_downloader.setdefault(python_version, {}).update({
         normalize_name(s): False
         for s in pip_attr.simpleapi_skip
@@ -425,11 +432,30 @@ def _select_default_index(index, default):
 def _host_interpreter_name(python_version):
     return "python_{}_host".format(python_version.replace(".", "_"))
 
-def _detect_interpreter(self, pip_attr, python_version):
+def _find_host_interpreter_name(
+        *,
+        python_version,
+        full_python_version,
+        available_interpreters):
+    for version_str in [
+        python_version,
+        full_python_version,
+        _major_minor_version(python_version),
+    ]:
+        name = _host_interpreter_name(version_str)
+        if name in available_interpreters:
+            return name
+    return None
+
+def _detect_interpreter(self, pip_attr, python_version, full_python_version):
     python_interpreter_target = pip_attr.python_interpreter_target
     if python_interpreter_target == None and not pip_attr.python_interpreter:
-        python_name = _host_interpreter_name(python_version)
-        if python_name not in self._available_interpreters:
+        python_name = _find_host_interpreter_name(
+            python_version = python_version,
+            full_python_version = full_python_version,
+            available_interpreters = self._available_interpreters,
+        )
+        if not python_name:
             fail((
                 "Unable to find interpreter for pip hub '{hub_name}' for " +
                 "python_version={version}: Make sure a corresponding " +
@@ -438,7 +464,7 @@ def _detect_interpreter(self, pip_attr, python_version):
             ).format(
                 hub_name = self.name,
                 version = python_version,
-                python_name = python_name,
+                python_name = _host_interpreter_name(python_version),
                 labels = "  \n".join(self._available_interpreters),
             ))
         python_interpreter_target = self._available_interpreters[python_name]
@@ -507,6 +533,7 @@ def _create_whl_repos(
         *,
         pip_attr,
         python_version,
+        full_python_version,
         enable_pipstar_extract = False):
     """create all of the whl repositories
 
@@ -515,6 +542,7 @@ def _create_whl_repos(
         module_ctx: {type}`module_ctx`.
         pip_attr: {type}`struct` - the struct that comes from the tag class iteration.
         python_version: {type}`str` - the resolved python version for this pip.parse call.
+        full_python_version: {type}`str` - the full X.Y.Z python version.
         enable_pipstar_extract: {type}`bool` - enable the pipstar extraction or not.
     """
     logger = self._logger
@@ -530,16 +558,13 @@ def _create_whl_repos(
             uv_lock = pip_attr.uv_lock,
             extra_pip_args = pip_attr.extra_pip_args,
             platforms = sorted(platforms),  # here we only need keys
-            python_version = full_version(
-                version = python_version,
-                minor_mapping = self._minor_mapping,
-            ),
+            python_version = full_python_version,
             logger = logger,
         ),
         uv_lock = pip_attr.uv_lock,
         platforms = platforms,
         extra_pip_args = pip_attr.extra_pip_args,
-        get_index_urls = self._get_index_urls.get(pip_attr.python_version),
+        get_index_urls = self._get_index_urls.get(python_version),
         toml_decode = getattr(self._config, "toml_decode", None),
         logger = logger,
     )
@@ -561,7 +586,12 @@ def _create_whl_repos(
         pip_attr = pip_attr,
     )
 
-    interpreter = _detect_interpreter(self, pip_attr, python_version)
+    interpreter = _detect_interpreter(
+        self,
+        pip_attr,
+        python_version,
+        full_python_version,
+    )
 
     for whl in requirements_by_platform:
         whl_library_args = common_args | _whl_library_args(

python/private/python.bzl

@@ -275,7 +275,8 @@ def _python_impl(module_ctx):
             minor_mapping = py.config.minor_mapping,
             fail_on_err = False,
         )
-        if not full_python_version:
+        if (not full_python_version or
+            full_python_version not in py.config.default["tool_versions"]):
             logger.info(lambda: (
                 "The actual toolchain for python_version '{version}' " +
                 "has not been registered, but was requested, please configure a toolchain " +
@@ -589,6 +590,14 @@ def _process_single_version_overrides(*, tag, _fail = fail, default):
     available_versions = default["tool_versions"]
     kwargs = default.setdefault("kwargs", {})
 
+    if tag.python_version not in available_versions:
+        if not (tag.sha256 and tag.urls):
+            _fail((
+                "When introducing a new python_version '{}', 'sha256' and " +
+                "'urls' must be specified"
+            ).format(tag.python_version))
+            return
+
     if tag.sha256 or tag.urls:
         if not (tag.sha256 and tag.urls):
             _fail("Both `sha256` and `urls` overrides need to be provided together")
@@ -836,7 +845,16 @@ def _populate_from_pbs_manifest(
         if "://" in location:
             urls = [location]
         else:
-            urls = ["{}/{}".format(b_url, location) for b_url in base_download_urls]
+            urls = []
+            for b_url in base_download_urls:
+                if ("/" not in location and
+                    entry.build_version and
+                    b_url in DEFAULT_RELEASE_BASE_URLS):
+                    urls.append(
+                        "{}/{}/{}".format(b_url, entry.build_version, location),
+                    )
+                else:
+                    urls.append("{}/{}".format(b_url, location))
 
         strip_prefix = "python/install" if entry.archive_flavor == "full" else "python"
 

tests/pypi/extension/extension_tests.bzl

@@ -190,53 +190,57 @@ _tests.append(_test_simple)
 
 def _test_pip_parse_pyproject_toml(env):
     # pip.parse() reads the version from pyproject.toml's requires-python when
-    # python_version is not set explicitly.
-    pypi = _parse_modules(
-        env,
-        module_ctx = _pypi_mock_mctx(
-            _mod(
-                name = "rules_python",
-                parse = [
-                    _parse(
-                        hub_name = "pypi",
-                        pyproject_toml = "pyproject.toml",
-                        simpleapi_skip = ["simple"],
-                        requirements_lock = "requirements.txt",
+    # python_version is not set explicitly. Verify it works both when the
+    # toolchain was registered by full version (python_3_15_19_host) and when
+    # registered by minor version (python_3_15_host).
+    for available_interpreters in [
+        {"python_3_15_19_host": "unit_test_interpreter_target"},
+        {"python_3_15_host": "unit_test_interpreter_target"},
+    ]:
+        pypi = _parse_modules(
+            env,
+            module_ctx = _pypi_mock_mctx(
+                _mod(
+                    name = "rules_python",
+                    parse = [
+                        _parse(
+                            hub_name = "pypi",
+                            pyproject_toml = "pyproject.toml",
+                            simpleapi_skip = ["simple"],
+                            requirements_lock = "requirements.txt",
+                        ),
+                    ],
+                ),
+                os_name = "linux",
+                arch_name = "x86_64",
+                mock_files = {
+                    "pyproject.toml": "[project]\nrequires-python = \"==3.15.19\"\n",
+                },
+            ),
+            available_interpreters = available_interpreters,
+            minor_mapping = {"3.15": "3.15.19"},
+        )
+
+        # Resolves identically to passing python_version = "3.15.19" explicitly:
+        # the full version drives interpreter selection, hub naming uses major.minor.
+        pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
+        pypi.hub_whl_map().contains_exactly({"pypi": {
+            "simple": {
+                "pypi_315_simple": [
+                    whl_config_setting(
+                        version = "3.15",
                     ),
                 ],
-            ),
-            os_name = "linux",
-            arch_name = "x86_64",
-            mock_files = {
-                "pyproject.toml": "[project]\nrequires-python = \"==3.15.19\"\n",
             },
-        ),
-        available_interpreters = {
-            "python_3_15_19_host": "unit_test_interpreter_target",
-        },
-        minor_mapping = {"3.15": "3.15.19"},
-    )
-
-    # Resolves identically to passing python_version = "3.15.19" explicitly:
-    # the full version drives interpreter selection, hub naming uses major.minor.
-    pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
-    pypi.hub_whl_map().contains_exactly({"pypi": {
-        "simple": {
-            "pypi_315_simple": [
-                whl_config_setting(
-                    version = "3.15",
-                ),
-            ],
-        },
-    }})
-    pypi.whl_libraries().contains_exactly({
-        "pypi_315_simple": {
-            "config_load": "@pypi//:config.bzl",
-            "dep_template": "@pypi//{name}:{target}",
-            "python_interpreter_target": "unit_test_interpreter_target",
-            "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf",
-        },
-    })
+        }})
+        pypi.whl_libraries().contains_exactly({
+            "pypi_315_simple": {
+                "config_load": "@pypi//:config.bzl",
+                "dep_template": "@pypi//{name}:{target}",
+                "python_interpreter_target": "unit_test_interpreter_target",
+                "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf",
+            },
+        })
 
 _tests.append(_test_pip_parse_pyproject_toml)
 

tests/python/python_tests.bzl

@@ -883,6 +883,15 @@ def _test_single_version_override_errors(env):
             ],
             want_error = "Only a single 'python.single_version_override' can be present for '3.12.4'",
         ),
+        struct(
+            overrides = [
+                python_ext.single_version_override(
+                    distutils_content = "foo",
+                    python_version = "3.13.99",
+                ),
+            ],
+            want_error = "When introducing a new python_version '3.13.99', 'sha256' and 'urls' must be specified",
+        ),
     ]:
         errors = []
         parse_modules(

tests/python_bzlmod_ext/runtime_manifests_tests.bzl

@@ -167,6 +167,71 @@ def _test_dynamic_manifest_files_impl(env, target):
 
 _tests.append(_test_dynamic_manifest_files)
 
+def _test_dynamic_manifest_files_default_base_urls(name):
+    rt_util.helper_target(
+        native.filegroup,
+        name = name + "_subject",
+    )
+    analysis_test(
+        name = name,
+        target = name + "_subject",
+        impl = _test_dynamic_manifest_files_default_base_urls_impl,
+    )
+
+def _test_dynamic_manifest_files_default_base_urls_impl(env, target):
+    _ = target  # @unused
+
+    root_module = python_ext.module(
+        name = "runtime_manifests",
+        override = [
+            python_ext.override(
+                add_runtime_manifest_files = [
+                    Label("//:SHA256SUMS"),
+                ],
+            ),
+        ],
+        toolchain = [
+            python_ext.toolchain(
+                python_version = "3.13",
+            ),
+        ],
+    )
+
+    mock_mctx = python_ext.mctx(
+        modules = [root_module],
+        mock_files = {
+            str(Label("//:SHA256SUMS")): """
+cba3455e11ee12f8cb085a27e96ae8aa5492e153263e6577ad89213a46646427  cpython-3.13.99+20261026-x86_64-unknown-linux-gnu-install_only.tar.gz
+""",
+        },
+    )
+
+    res = parse_modules(
+        module_ctx = mock_mctx,
+        logger = _mock_logger,
+    )
+
+    tool_versions = res.config.default["tool_versions"]
+    env.expect.that_bool("3.13.99" in tool_versions).equals(True)
+    env.expect.that_str(res.config.minor_mapping["3.13"]).equals("3.13.99")
+
+    version_info = tool_versions["3.13.99"]
+    env.expect.that_dict(version_info).contains_exactly({
+        "sha256": {
+            "x86_64-unknown-linux-gnu": "cba3455e11ee12f8cb085a27e96ae8aa5492e153263e6577ad89213a46646427",
+        },
+        "strip_prefix": {
+            "x86_64-unknown-linux-gnu": "python",
+        },
+        "url": {
+            "x86_64-unknown-linux-gnu": [
+                "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/cpython-3.13.99+20261026-x86_64-unknown-linux-gnu-install_only.tar.gz",
+            ],
+        },
+    })
+
+_tests.append(_test_dynamic_manifest_files_default_base_urls)
+
 def runtime_manifests_test_suite(name):
     test_suite(
         name = name,