diff --git a/docs/concepts/bootstrapper-architecture.rst b/docs/concepts/bootstrapper-architecture.rst index 37d7d5493..86a2cc91b 100644 --- a/docs/concepts/bootstrapper-architecture.rst +++ b/docs/concepts/bootstrapper-architecture.rst @@ -16,6 +16,12 @@ Bootstrap Phase Flow Every package passes through a sequence of phases. Source packages traverse the full pipeline; prebuilt wheels skip the build phases. +``pypi-prebuilt`` and ``hook-prebuilt`` use the selected source resolver to +download a fresh wheel in ``PrepareSource``. The legacy ``pre_built`` setting +keeps its existing wheel download path. Both routes update the local mirror +and record the package as prebuilt in the graph and build order. +When a local wheel is already available, bootstrap reuses it before asking +the selected provider to resolve a fresh wheel. .. code-block:: text diff --git a/docs/concepts/package-settings.rst b/docs/concepts/package-settings.rst index c516ff93a..e588b7f96 100644 --- a/docs/concepts/package-settings.rst +++ b/docs/concepts/package-settings.rst @@ -42,6 +42,16 @@ cannot be combined with package-level ``download_source`` or ``resolver_dist``. An explicit ``pre_built: false`` also conflicts; an omitted ``pre_built`` does not. +``PackageBuildInfo.pre_built`` follows the selected ``source`` resolver's +``resolves_prebuilt_wheel`` capability. Without ``source``, it reads the +legacy variant ``pre_built`` setting. Configured prebuilt resolvers download +fresh wheels through their ``download()`` contract; legacy prebuilt packages +keep the wheel downloader. + +.. versionchanged:: 0.102.0 + Configured prebuilt source resolvers now select the prebuilt pipeline + during bootstrap and later builds. + .. versionchanged:: 0.102.0 Source and legacy settings mixed across package and variant levels now raise a validation error when package settings load. diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 4702b957b..18d4cf590 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -43,6 +43,8 @@ the cache already contains a matching version. The configured wheel cache server is not a version-resolution fallback. After a version has been selected, the bootstrapper checks the cache server separately for a previously built wheel. +For prebuilt packages, the local wheel server can supply a cached wheel +before a fresh resolution through the selected source provider. Provider Hierarchy ------------------ @@ -132,7 +134,15 @@ The separate ``step prepare-source`` invocation accepts ``--source-kind`` and ``step download-source-archive --output-format json`` exposes that result's actual path and kind without changing its default path-only output. Without ``--source-kind``, it keeps the legacy archive behavior. -Integrating prebuilt detection into the bootstrapper remains a separate step. +Configured prebuilt resolvers select the prebuilt bootstrap path. Fresh +external wheels use their ``download()`` contract and are staged in the +prebuilt wheel directory. Bootstrap, build, and download-sequence use the +direct wheel downloader for local mirror URLs, so they do not rerun the +configured download hook. Test-mode fallback also downloads directly. +When a fresh configured wheel shares a filename with an older cached wheel, +the local mirror and later build replay use the fresh staged wheel. +Legacy build downloads still pass through the build wheel directory before +the mirror moves them to downloads. See :ref:`download_source_hook` for the return contract and compatibility rules. .. versionchanged:: 0.102.0 @@ -143,7 +153,9 @@ See :ref:`download_source_hook` for the return contract and compatibility rules. ``build_sdist: tarball``. PyPI sdist resolvers repack a tarball. Both configured strategies take precedence over an installed ``build_sdist`` override and log a warning when it is ignored. Legacy - settings continue to use the override. + settings continue to use the override. Configured prebuilt profiles use + their source provider after a local cache check, and local mirror URLs + bypass configured download hooks during replay. Version Filtering Window ------------------------- diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index 8ee267511..6e8407627 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -10,9 +10,11 @@ import threading import typing +import requests.exceptions from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name from packaging.version import Version +from resolvelib.resolvers import ResolverException from . import resolver, sources, wheels from .dependency_graph import DependencyGraph @@ -171,13 +173,17 @@ def _resolve_and_extend( parent_req=parent_req, ) + source_resolver = self.ctx.package_build_info(req).source_resolver + configured_prebuilt = ( + source_resolver is not None and source_resolver.resolves_prebuilt_wheel + ) if cached_resolution: logger.debug( f"resolved from previous bootstrap: {len(cached_resolution)} version(s)" ) results = cached_resolution - elif pre_built: - # Resolve prebuilt wheel + elif pre_built and not configured_prebuilt: + # Legacy prebuilt resolution also serves test-mode source-build fallback. wheel_server_urls = wheels.get_wheel_server_urls( self.ctx, req, cache_wheel_server_url=resolver.PYPI_SERVER_URL ) @@ -188,30 +194,39 @@ def _resolve_and_extend( req_type=req_type, ) else: - # Resolve source (sdist) - provider = sources.get_source_provider( - ctx=self.ctx, - req=req, - sdist_server_url=resolver.PYPI_SERVER_URL, - req_type=req_type, - ) - max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx) - age_fallback = ( - resolver.AgeFallback.NEWEST - if self.multiple_versions - else resolver.AgeFallback.ALL - ) - results = resolver.find_all_matching_from_provider( - provider, - req, - max_age_cutoff=max_age_cutoff, - age_fallback=age_fallback, - ) + results = [] + if configured_prebuilt and self.ctx.wheel_server_url: + try: + results = wheels.resolve_all_cached_wheels( + ctx=self.ctx, + req=req, + cache_server_url=self.ctx.wheel_server_url, + ) + except ResolverException: + logger.debug("no matching wheel in local cache") + except requests.exceptions.RequestException as err: + logger.warning("could not check local wheel cache: %s", err) if not results: - logger.warning( - "resolver returned no results (%s version mode)", - "multiple" if self.multiple_versions else "single", + # The configured source provider resolves either source archives + # or prebuilt wheels, according to its selected capability. + provider = sources.get_source_provider( + ctx=self.ctx, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + req_type=req_type, + ) + max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx) + age_fallback = ( + resolver.AgeFallback.NEWEST + if self.multiple_versions + else resolver.AgeFallback.ALL + ) + results = resolver.find_all_matching_from_provider( + provider, + req, + max_age_cutoff=max_age_cutoff, + age_fallback=age_fallback, ) if results: diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index c524324c7..0cdeb5cdf 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -470,8 +470,12 @@ def _download_prebuilt( resolved_version: Version, wheel_url: str, ) -> tuple[pathlib.Path, pathlib.Path]: - result = _cache.bg_prepare_prebuilt( - self.ctx, req, req_type, resolved_version, wheel_url + result = _cache.bg_prepare_test_mode_prebuilt( + self.ctx, + req, + req_type, + resolved_version, + wheel_url, ) assert result.wheel_filename is not None assert result.unpack_dir is not None diff --git a/src/fromager/bootstrapper/_cache.py b/src/fromager/bootstrapper/_cache.py index 089895f5b..e241a7d63 100644 --- a/src/fromager/bootstrapper/_cache.py +++ b/src/fromager/bootstrapper/_cache.py @@ -246,10 +246,34 @@ def bg_prepare_prebuilt( resolved_version: Version, wheel_url: str, ) -> PreparedSourceData: - """Background-safe prebuilt download: no Bootstrapper state accessed.""" + """Download a prebuilt wheel and make it available to the local mirror.""" # Thread-safe: paths include {req.name}-{resolved_version} (unique per package), # mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked. logger.info(f"using pre-built wheel for {req_type} requirement") + wheel_filename = wheels.download_prebuilt_wheel( + ctx=ctx, + req=req, + version=resolved_version, + download_url=wheel_url, + direct_download_directory=ctx.wheels_prebuilt, + ) + unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version) + if ctx.package_build_info(req).source_resolver is not None: + server.update_wheel_mirror(ctx, preferred_prebuilt_wheel=wheel_filename) + else: + server.update_wheel_mirror(ctx) + return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) + + +def bg_prepare_test_mode_prebuilt( + ctx: context.WorkContext, + req: Requirement, + req_type: RequirementType, + resolved_version: Version, + wheel_url: str, +) -> PreparedSourceData: + """Download a test-mode fallback wheel directly and update the mirror.""" + logger.info(f"using pre-built wheel for {req_type} requirement") wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version) server.update_wheel_mirror(ctx) diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 70d549484..24ceab603 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -362,17 +362,18 @@ def _build( if prebuilt: if not wheel_filename: logger.info("downloading prebuilt wheel") - wheel_filename = wheels.download_wheel( + wheel_filename = wheels.download_prebuilt_wheel( + ctx=wkctx, req=req, - wheel_url=source_download_url, - output_directory=wkctx.wheels_build, + version=resolved_version, + download_url=source_download_url, + direct_download_directory=wkctx.wheels_build, ) else: # already downloaded prebuilt wheel use_exiting_wheel = True - # Run hooks for prebuilt wheels. At this point wheel_filename should - # be set either from _is_wheel_built() or download_wheel(). + # Run hooks for both cached and freshly downloaded prebuilt wheels. hooks.run_prebuilt_wheel_hooks( ctx=wkctx, req=req, @@ -446,11 +447,19 @@ def _build( root_logger.removeHandler(file_handler) file_handler.close() - server.update_wheel_mirror(wkctx) + if ( + prebuilt + and pbi.source_resolver is not None + and wheel_filename.parent == wkctx.wheels_prebuilt + ): + server.update_wheel_mirror(wkctx, preferred_prebuilt_wheel=wheel_filename) + else: + server.update_wheel_mirror(wkctx) - # After we update the wheel mirror, the built file has - # moved to a new directory. - wheel_filename = wkctx.wheels_downloads / wheel_filename.name + # The mirror moves wheels from the build directory to downloads; + # configured prebuilt wheels stay in the prebuilt directory. + if wheel_filename.parent != wkctx.wheels_prebuilt: + wheel_filename = wkctx.wheels_downloads / wheel_filename.name return BuildSequenceEntry( name=canonicalize_name(req.name), @@ -529,8 +538,8 @@ def _is_wheel_built( logger.info("found candidate wheel %s", url) try: wheel_basename = downloads.extract_filename_from_url(url) - _, _, build_tag_from_name, wheel_tags = parse_wheel_filename( - wheel_basename + normalized_name, _, build_tag_from_name, wheel_tags = ( + parse_wheel_filename(wheel_basename) ) except Exception: logger.debug("could not parse candidate wheel %s", url, exc_info=True) @@ -567,9 +576,23 @@ def _is_wheel_built( continue wheel_filename: pathlib.Path | None = None - if wkctx.wheel_server_url and url.startswith(wkctx.wheel_server_url): + if wheels.is_local_wheel_url(url, wkctx.wheel_server_url): logging.debug("found wheel on local server") wheel_filename = wkctx.wheels_downloads / wheel_basename + mirror_wheel = wkctx.wheel_server_dir / normalized_name / wheel_basename + if mirror_wheel.is_symlink(): + mirror_target = mirror_wheel.resolve() + if ( + mirror_target.parent + in ( + wkctx.wheels_downloads, + wkctx.wheels_prebuilt, + ) + and mirror_target.is_file() + ): + # The mirror may point to a fresh prebuilt wheel while + # an older copy remains in downloads. + wheel_filename = mirror_target if not wheel_filename.exists(): logger.info("wheel not found in local cache, preparing to download") wheel_filename = None diff --git a/src/fromager/commands/download_sequence.py b/src/fromager/commands/download_sequence.py index 177440fd5..43876263b 100644 --- a/src/fromager/commands/download_sequence.py +++ b/src/fromager/commands/download_sequence.py @@ -8,7 +8,7 @@ from packaging.version import Version from .. import context, progress, read, sources, threading_utils, wheels -from ..log import requirement_ctxvar +from ..log import req_ctxvar_context logger = logging.getLogger(__name__) @@ -65,54 +65,60 @@ def download_sequence( def download_one(entry: dict[str, typing.Any]) -> None: req = Requirement(f"{entry['dist']}=={entry['version']}") - token = requirement_ctxvar.set(req) - - if entry["prebuilt"]: - if include_wheels: - wheels.download_wheel(req, entry["source_url"], wkctx.wheels_prebuilt) - else: - logger.info(f"{entry['dist']}: uses a pre-built wheel, skipping") - return - - if entry["source_url_type"] == "sdist": - try: - sources.download_source( - ctx=wkctx, - req=req, - version=Version(entry["version"]), - download_url=entry["source_url"], - ) - except Exception as err: - logger.error(f"failed to download sdist for {req}: {err}") - if not ignore_missing_sdists: - raise RuntimeError(f"Failed to download sdist for {req}") from err - else: - logger.info( - f"{entry['dist']}: uses a {entry['source_url_type']} downloader, skipping" - ) - - if include_wheels: - try: - if use_cache_resolution: - resolved_url, _ = wheels.resolve_cached_wheel( + with req_ctxvar_context(req): + if entry["prebuilt"]: + if include_wheels: + wheels.download_prebuilt_wheel( ctx=wkctx, req=req, - cache_server_url=wheel_server_url, + version=Version(entry["version"]), + download_url=entry["source_url"], + direct_download_directory=wkctx.wheels_prebuilt, ) else: - resolved_url, _ = wheels.resolve_prebuilt_wheel( + logger.info(f"{entry['dist']}: uses a pre-built wheel, skipping") + return + + if entry["source_url_type"] == "sdist": + try: + sources.download_source( ctx=wkctx, req=req, - wheel_server_urls=[wheel_server_url], + version=Version(entry["version"]), + download_url=entry["source_url"], ) - wheels.download_wheel( - req=req, - wheel_url=resolved_url, - output_directory=wkctx.wheels_downloads, + except Exception as err: + logger.error(f"failed to download sdist for {req}: {err}") + if not ignore_missing_sdists: + raise RuntimeError( + f"Failed to download sdist for {req}" + ) from err + else: + logger.info( + f"{entry['dist']}: uses a {entry['source_url_type']} downloader, skipping" ) - except Exception as err: - logger.error(f"failed to download wheel for {req}: {err}") - requirement_ctxvar.reset(token) + + if include_wheels: + try: + if use_cache_resolution: + resolved_url, _ = wheels.resolve_cached_wheel( + ctx=wkctx, + req=req, + cache_server_url=wheel_server_url, + ) + else: + resolved_url, _ = wheels.resolve_prebuilt_wheel( + ctx=wkctx, + req=req, + wheel_server_urls=[wheel_server_url], + ) + wheels.download_wheel( + req=req, + wheel_url=resolved_url, + output_directory=wkctx.wheels_downloads, + ) + except Exception as err: + logger.error(f"failed to download wheel for {req}: {err}") num_items = len(build_order) logger.debug( diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index ba62f9bf9..74d55a45b 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -163,7 +163,10 @@ def has_customizations(self) -> bool: @property def pre_built(self) -> bool: - """Does the variant use pre-build wheels?""" + """Whether the selected source or legacy variant uses prebuilt wheels.""" + source_resolver = self.source_resolver + if source_resolver is not None: + return source_resolver.resolves_prebuilt_wheel vi = self._ps.variants.get(self.variant) if vi is not None: return vi.pre_built diff --git a/src/fromager/server.py b/src/fromager/server.py index 7d282f871..17e03ae8f 100644 --- a/src/fromager/server.py +++ b/src/fromager/server.py @@ -10,6 +10,7 @@ import textwrap import threading import typing +import uuid from urllib.parse import quote import uvicorn @@ -59,7 +60,10 @@ def run_wheel_server( @with_thread_lock() -def update_wheel_mirror(ctx: context.WorkContext) -> None: +def update_wheel_mirror( + ctx: context.WorkContext, *, preferred_prebuilt_wheel: pathlib.Path | None = None +) -> None: + """Refresh the local index, preferring a newly staged wheel when provided.""" for wheel in ctx.wheels_build.glob("*.whl"): logger.info("adding %s to local wheel server", wheel.name) downloads_dest_filename = ctx.wheels_downloads / wheel.name @@ -88,6 +92,26 @@ def update_wheel_mirror(ctx: context.WorkContext) -> None: simple_dest_filename.parent.mkdir(parents=True, exist_ok=True) simple_dest_filename.symlink_to(relpath) + # A fresh configured wheel can share a filename with an older download. + # Switch the index link atomically so readers always see a complete wheel. + if preferred_prebuilt_wheel is not None: + (normalized_name, _, _, _) = parse_wheel_filename(preferred_prebuilt_wheel.name) + simple_dest_filename = ( + ctx.wheel_server_dir / normalized_name / preferred_prebuilt_wheel.name + ) + if simple_dest_filename.resolve() != preferred_prebuilt_wheel.resolve(): + relpath = os.path.relpath( + preferred_prebuilt_wheel, simple_dest_filename.parent + ) + temporary_link = simple_dest_filename.with_name( + f".{simple_dest_filename.name}.{uuid.uuid4().hex}" + ) + try: + temporary_link.symlink_to(relpath) + os.replace(temporary_link, simple_dest_filename) + finally: + temporary_link.unlink(missing_ok=True) + class SimpleHTMLIndex: """Simple HTML Repository API (1.0) diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index 943edea8d..b1a36cb29 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -32,6 +32,7 @@ requirements_file, resolver, sbom, + sources, ) from .pkgmetadata.pep376 import verbatim_dist_name @@ -520,6 +521,64 @@ def download_wheel( return wheel_filename +def is_local_wheel_url(wheel_url: str, wheel_server_url: str | None) -> bool: + """Return whether a wheel URL belongs to the local wheel mirror. + + .. versionadded:: 0.102.0 + """ + if not wheel_server_url: + return False + return wheel_url.startswith(wheel_server_url.rstrip("/") + "/") + + +def download_prebuilt_wheel( + *, + ctx: context.WorkContext, + req: Requirement, + version: Version, + download_url: str, + direct_download_directory: pathlib.Path, +) -> pathlib.Path: + """Download a prebuilt wheel via its resolver or the direct wheel path. + + Configured external wheels use the resolver download contract. Legacy + wheels and local-mirror URLs use the direct wheel downloader. + + .. versionadded:: 0.102.0 + """ + source_resolver = ctx.package_build_info(req).source_resolver + if source_resolver is not None and not source_resolver.resolves_prebuilt_wheel: + raise ValueError( + f"{req}: source resolver {source_resolver.provider!r} " + "does not resolve prebuilt wheels" + ) + if source_resolver is None or is_local_wheel_url( + download_url, ctx.wheel_server_url + ): + return download_wheel(req, download_url, direct_download_directory) + + downloaded: packagesettings.DownloadedSource = sources.download_source( + ctx=ctx, req=req, version=version, download_url=download_url + ) + wheel_path = downloaded.path + if wheel_path.parent != ctx.wheels_prebuilt: + ctx.wheels_prebuilt.mkdir(parents=True, exist_ok=True) + staged_path = ctx.wheels_prebuilt / wheel_path.name + with tempfile.NamedTemporaryFile( + dir=ctx.wheels_prebuilt, + prefix=f".tmp.{wheel_path.name}.", + delete=False, + ) as temporary_file: + temporary_path = pathlib.Path(temporary_file.name) + try: + shutil.copy2(wheel_path, temporary_path) + os.replace(temporary_path, staged_path) + finally: + temporary_path.unlink(missing_ok=True) + return staged_path + return wheel_path + + def get_wheel_server_urls( ctx: context.WorkContext, req: Requirement, *, cache_wheel_server_url: str | None ) -> list[str]: diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index af359313c..f6fa5b570 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -2,11 +2,14 @@ from unittest.mock import MagicMock, patch +import pytest +import requests.exceptions from packaging.requirements import Requirement from packaging.utils import canonicalize_name from packaging.version import Version +from resolvelib.resolvers import ResolverException -from fromager import resolver +from fromager import packagesettings, resolver from fromager.bootstrap_requirement_resolver import BootstrapRequirementResolver from fromager.context import WorkContext from fromager.dependency_graph import DependencyGraph @@ -433,6 +436,182 @@ def test_resolve_from_graph_name_fallback_returns_none_for_missing_package( assert result is None +def test_configured_pypi_prebuilt_resolves_from_selected_index( + tmp_context: WorkContext, +) -> None: + """Fresh prebuilt resolution uses the configured index provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", + "source:\n provider: pypi-prebuilt\n" + " index_url: https://custom.test/simple/\n", + ) + ) + expected = [("https://custom.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch( + "fromager.bootstrap_requirement_resolver.resolver.find_all_matching_from_provider", + return_value=expected, + ) as find_matches, + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_prebuilt_wheels" + ) as legacy, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + provider = find_matches.call_args.args[0] + assert isinstance(provider, resolver.PyPIProvider) + assert provider.sdist_server_url == "https://custom.test/simple/" + legacy.assert_not_called() + + +def test_configured_hook_prebuilt_uses_hook_provider( + tmp_context: WorkContext, +) -> None: + """Fresh prebuilt resolution calls the selected hook provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", "source:\n provider: hook-prebuilt\n" + ) + ) + provider = resolver.PyPIProvider(sdist_server_url="https://hook.test/simple/") + expected = [("https://hook.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch.object( + packagesettings.HookPrebuiltResolver, + "resolver_provider", + return_value=provider, + ) as configured, + patch( + "fromager.bootstrap_requirement_resolver.resolver.find_all_matching_from_provider", + return_value=expected, + ) as find_matches, + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_prebuilt_wheels" + ) as legacy, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + configured.assert_called_once() + assert find_matches.call_args.args[0] is provider + legacy.assert_not_called() + + +def test_configured_prebuilt_reuses_local_wheel_cache( + tmp_context: WorkContext, +) -> None: + """A local wheel is resolved before the configured upstream provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", "source:\n provider: pypi-prebuilt\n" + ) + ) + tmp_context.wheel_server_url = "https://local.test/simple/" + expected = [("https://local.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_cached_wheels", + return_value=expected, + ) as cache, + patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider" + ) as configured, + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_prebuilt_wheels", + return_value=[], + ) as legacy, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + cache.assert_called_once_with( + ctx=tmp_context, req=req, cache_server_url=tmp_context.wheel_server_url + ) + configured.assert_not_called() + legacy.assert_not_called() + + +@pytest.mark.parametrize( + "cache_error", + [ + ResolverException("cache miss"), + requests.exceptions.ConnectionError("cache down"), + ], +) +def test_configured_prebuilt_cache_miss_uses_selected_provider( + tmp_context: WorkContext, cache_error: Exception +) -> None: + """A local cache miss or network error falls through to the source provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", + "source:\n provider: pypi-prebuilt\n" + " index_url: https://custom.test/simple/\n", + ) + ) + tmp_context.wheel_server_url = "https://local.test/simple/" + expected = [("https://custom.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_cached_wheels", + side_effect=cache_error, + ) as cache, + patch( + "fromager.bootstrap_requirement_resolver.resolver.find_all_matching_from_provider", + return_value=expected, + ) as find_matches, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + cache.assert_called_once() + assert find_matches.call_args.args[0].sdist_server_url == ( + "https://custom.test/simple/" + ) + + +def test_configured_prebuilt_unexpected_cache_error_propagates( + tmp_context: WorkContext, +) -> None: + """An unexpected cache failure is not mistaken for a cache miss.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", "source:\n provider: pypi-prebuilt\n" + ) + ) + tmp_context.wheel_server_url = "https://local.test/simple/" + with ( + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_cached_wheels", + side_effect=ValueError("invalid cache response"), + ), + patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider" + ) as configured, + pytest.raises(ValueError, match="invalid cache response"), + ): + BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + configured.assert_not_called() + + @patch("fromager.resolver.find_all_matching_from_provider") def test_resolve_auto_routes_to_prebuilt( mock_resolve: MagicMock, @@ -444,6 +623,7 @@ def test_resolve_auto_routes_to_prebuilt( # Mock package build info to return pre_built=True mock_pbi = MagicMock() mock_pbi.pre_built = True + mock_pbi.source_resolver = None mock_pbi.wheel_server_url = None mock_pbi.resolver_min_release_age = None diff --git a/tests/test_bootstrap_requirement_resolver_multiple.py b/tests/test_bootstrap_requirement_resolver_multiple.py index ce9482def..5cf208cff 100644 --- a/tests/test_bootstrap_requirement_resolver_multiple.py +++ b/tests/test_bootstrap_requirement_resolver_multiple.py @@ -25,6 +25,7 @@ def tmp_context(tmp_path: Path) -> WorkContext: ctx.package_build_info = MagicMock() pbi = MagicMock() pbi.pre_built = False + pbi.source_resolver = None pbi.resolver_include_sdists = True pbi.resolver_include_wheels = False pbi.resolver_ignore_platform = False diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 0d775339c..fc2cda9d3 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -1,3 +1,4 @@ +import concurrent.futures import json import logging import pathlib @@ -12,7 +13,7 @@ from packaging.version import Version from resolvelib.resolvers import ResolverException -from fromager import bootstrapper, log +from fromager import bootstrapper, log, sources, wheels from fromager.bootstrapper._build import Build from fromager.bootstrapper._cache import ( _download_wheel_from_cache, @@ -38,10 +39,21 @@ from fromager.bootstrapper._work_item import WorkItem from fromager.context import WorkContext from fromager.dependency_graph import ROOT -from fromager.packagesettings import DownloadedSource, DownloadKind, WheelSettings +from fromager.packagesettings import ( + DownloadedSource, + DownloadKind, + PackageSettings, + WheelSettings, +) from fromager.requirements_file import RequirementType, SourceType +def _configure_package(ctx: WorkContext, yaml_str: str) -> None: + """Apply one package setting to the test context.""" + ps = PackageSettings.from_string("pkg", yaml_str) + ctx.settings._package_settings[ps.name] = ps + + def test_phase_subclass_without_phase_attribute_raises() -> None: """Concrete Phase subclass missing 'phase' raises TypeError at class definition.""" with pytest.raises(TypeError, match="must define the 'phase' class attribute"): @@ -63,6 +75,206 @@ def helper(self) -> None: ... assert _AbstractMiddle.__abstractmethods__ +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_bootstrap_prebuilt_uses_configured_download( + tmp_context: WorkContext, provider: str +) -> None: + """Bootstrap consumes the configured wheel path and updates its mirror.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_prebuilt_wheel", return_value=wheel) as download, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror") as mirror, + ): + result = bg_prepare_prebuilt( + tmp_context, req, RequirementType.TOP_LEVEL, version, url + ) + + assert result.wheel_filename == wheel + download.assert_called_once_with( + ctx=tmp_context, + req=req, + version=version, + download_url=url, + direct_download_directory=tmp_context.wheels_prebuilt, + ) + mirror.assert_called_once_with(tmp_context, preferred_prebuilt_wheel=wheel) + + +def test_bootstrap_configured_prebuilt_reuses_local_wheel( + tmp_context: WorkContext, +) -> None: + """A local mirror URL bypasses the configured download hook.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + tmp_context.wheel_server_url = "https://local.test/simple/" + url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_wheel", return_value=wheel) as cached, + patch.object(sources, "download_source") as configured, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror") as mirror, + ): + result = bg_prepare_prebuilt( + tmp_context, req, RequirementType.TOP_LEVEL, version, url + ) + + assert result.wheel_filename == wheel + cached.assert_called_once_with(req, url, tmp_context.wheels_prebuilt) + configured.assert_not_called() + mirror.assert_called_once_with(tmp_context, preferred_prebuilt_wheel=wheel) + + +@pytest.mark.parametrize("local_cache", [False, True]) +def test_bootstrap_prebuilt_rejects_source_only_resolver( + tmp_context: WorkContext, local_cache: bool +) -> None: + """Do not treat a configured sdist resolver as a legacy wheel source.""" + _configure_package(tmp_context, "source:\n provider: pypi-sdist\n") + req = Requirement("pkg==1.0") + if local_cache: + tmp_context.wheel_server_url = "https://local.test/simple/" + url = ( + "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + if local_cache + else "https://pkg.test/pkg-1.0-py3-none-any.whl" + ) + with ( + patch.object(wheels, "download_wheel") as direct, + patch.object(sources, "download_source") as configured, + pytest.raises(ValueError, match="does not resolve prebuilt wheels"), + ): + bg_prepare_prebuilt( + tmp_context, + req, + RequirementType.TOP_LEVEL, + Version("1.0"), + url, + ) + + direct.assert_not_called() + configured.assert_not_called() + + +def test_legacy_bootstrap_keeps_direct_download(tmp_context: WorkContext) -> None: + """Legacy bootstrap still uses the wheel downloader.""" + _configure_package(tmp_context, "variants:\n cpu:\n pre_built: true\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_wheel", return_value=wheel) as direct, + patch.object(sources, "download_source") as configured, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror") as mirror, + ): + result = bg_prepare_prebuilt( + tmp_context, req, RequirementType.TOP_LEVEL, version, url + ) + + assert result.wheel_filename == wheel + direct.assert_called_once_with(req, url, tmp_context.wheels_prebuilt) + configured.assert_not_called() + mirror.assert_called_once_with(tmp_context) + + +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_bootstrap_records_configured_prebuilt_in_graph_and_order( + tmp_context: WorkContext, provider: str +) -> None: + """The selected source makes bootstrap skip build phases and persist its flag.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + bt = bootstrapper.Bootstrapper(tmp_context) + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + with patch.object(bt, "resolve_versions", return_value=[(url, version)]): + bt._resolve_and_add_top_level(req) + + wi = WorkItem( + req=req, + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + source_url=url, + resolved_version=version, + ) + next_phases = Start(wi).run(bt) + assert wi.pbi_pre_built is True + assert len(next_phases) == 1 + assert isinstance(next_phases[0], PrepareSource) + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_prebuilt_wheel", return_value=wheel), + patch.object(sources, "prepare_source") as prepare_source, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror"), + ): + background_work = next_phases[0].background_work(bt) + assert background_work is not None + prepared = background_work() + prepare_source.assert_not_called() + future: concurrent.futures.Future[typing.Any] = concurrent.futures.Future() + future.set_result(prepared) + next_phases[0].bg_future = future + install_phases = next_phases[0].run(bt) + assert len(install_phases) == 1 + assert isinstance(install_phases[0], ProcessInstallDeps) + assert wi.build_result is not None + assert wi.build_result.source_type is SourceType.PREBUILT + with ( + patch( + "fromager.bootstrapper._process_install_deps.hooks.run_post_bootstrap_hooks" + ), + patch( + "fromager.bootstrapper._process_install_deps._get_install_dependencies", + return_value=[], + ), + patch.object(bt, "create_unresolved_work_items", return_value=[]), + ): + install_phases[0].run(bt) + bt.finalize() + + assert tmp_context.dependency_graph.nodes["pkg==1.0"].pre_built is True + build_order = json.loads(bt._build_order_filename.read_text()) + assert build_order[0]["prebuilt"] is True + assert build_order[0]["source_url"] == url + + +def test_test_mode_source_resolver_uses_legacy_prebuilt_fallback( + tmp_context: WorkContext, +) -> None: + """A failed source build can use the forced legacy wheel fallback.""" + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://fallback.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + PackageSettings.from_string("pkg", "source:\n provider: pypi-sdist\n") + ) + bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) + with ( + patch.object(bt._resolver, "resolve", return_value=[(url, version)]), + patch( + "fromager.bootstrapper._cache.wheels.download_wheel", return_value=wheel + ) as legacy, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror"), + ): + result = bt._handle_test_mode_failure( + req=req, + resolved_version=version, + req_type=RequirementType.TOP_LEVEL, + build_error=RuntimeError("source build failed"), + ) + + assert result is not None + assert result.source_type is SourceType.PREBUILT + assert result.wheel_filename == wheel + legacy.assert_called_once_with(req, url, tmp_context.wheels_prebuilt) + + def test_seen(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testdist") @@ -949,8 +1161,6 @@ def test_check_write_error_logs_on_failure( bt = bootstrapper.Bootstrapper(tmp_context) # Arrange: create a future that has already failed with an OSError - import concurrent.futures - fut: concurrent.futures.Future[int] = concurrent.futures.Future() fut.set_exception(OSError("disk full")) diff --git a/tests/test_commands.py b/tests/test_commands.py index 6b4ae406c..61b752e60 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,4 @@ +import concurrent.futures import json import pathlib import subprocess @@ -5,13 +6,335 @@ from unittest.mock import Mock, patch import click +import pytest from click.testing import CliRunner from packaging.requirements import Requirement from packaging.version import Version -from fromager import packagesettings -from fromager.commands import bootstrap, build, step +from fromager import packagesettings, server, wheels +from fromager.commands import bootstrap, build, download_sequence, step from fromager.context import WorkContext +from fromager.log import requirement_ctxvar + + +def _configure_package(ctx: WorkContext, yaml_str: str) -> None: + """Apply one package setting to the test context.""" + ps = packagesettings.PackageSettings.from_string("pkg", yaml_str) + ctx.settings._package_settings[ps.name] = ps + + +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_build_order_replay_uses_configured_download( + tmp_context: WorkContext, provider: str +) -> None: + """A later build run selects the configured wheel download contract.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=None), + patch.object(wheels, "download_prebuilt_wheel", return_value=wheel) as download, + patch.object(build.wheels, "download_wheel") as legacy, + patch.object(build.hooks, "run_prebuilt_wheel_hooks") as hooks, + patch.object(build.server, "update_wheel_mirror") as mirror, + ): + result = build._build(tmp_context, version, req, url, False, None) + + assert result.prebuilt is True + assert result.wheel_filename == wheel + download.assert_called_once_with( + ctx=tmp_context, + req=req, + version=version, + download_url=url, + direct_download_directory=tmp_context.wheels_build, + ) + legacy.assert_not_called() + hooks.assert_called_once() + mirror.assert_called_once_with(tmp_context, preferred_prebuilt_wheel=wheel) + + +def test_build_replay_local_mirror_url_skips_download_hook( + tmp_context: WorkContext, +) -> None: + """Build replay downloads a local mirror URL directly and runs wheel hooks.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + tmp_context.wheel_server_url = "https://local.test/simple/" + req = Requirement("pkg==1.0") + url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_build / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=None), + patch.object(wheels, "download_wheel", return_value=wheel) as direct, + patch("fromager.wheels.sources.download_source") as configured, + patch.object(build.hooks, "run_prebuilt_wheel_hooks") as hooks, + patch.object(build.server, "update_wheel_mirror"), + ): + result = build._build(tmp_context, Version("1.0"), req, url, False, None) + + assert result.wheel_filename == tmp_context.wheels_downloads / wheel.name + direct.assert_called_once_with(req, url, tmp_context.wheels_build) + configured.assert_not_called() + hooks.assert_called_once() + + +def test_fresh_configured_prebuilt_replaces_old_mirror_entry( + tmp_context: WorkContext, +) -> None: + """A forced hook download makes its new wheel visible in the mirror.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + wheel_name = "pkg-1.0-py3-none-any.whl" + old_wheel = tmp_context.wheels_downloads / wheel_name + old_wheel.write_bytes(b"old wheel") + server.update_wheel_mirror(tmp_context) + + fresh_wheel = tmp_context.wheels_prebuilt / wheel_name + fresh_wheel.write_bytes(b"fresh wheel") + with ( + patch.object(wheels, "download_prebuilt_wheel", return_value=fresh_wheel), + patch.object(build.hooks, "run_prebuilt_wheel_hooks"), + ): + result = build._build( + tmp_context, + Version("1.0"), + req, + f"https://pkg.test/{wheel_name}", + True, + None, + ) + + mirror_wheel = tmp_context.wheel_server_dir / "pkg" / wheel_name + assert result.wheel_filename == fresh_wheel + assert mirror_wheel.resolve() == fresh_wheel + assert mirror_wheel.read_bytes() == b"fresh wheel" + assert old_wheel.read_bytes() == b"old wheel" + + tmp_context.wheel_server_url = "https://local.test/simple/" + mirror_url = f"https://local.test/simple/pkg/{wheel_name}" + with ( + patch.object( + wheels, + "resolve_all_cached_wheels", + return_value=[(mirror_url, Version("1.0"))], + ), + patch.object(wheels, "download_wheel") as direct, + ): + cached_wheel = build._is_wheel_built(tmp_context, "pkg", Version("1.0")) + + assert cached_wheel == fresh_wheel + direct.assert_not_called() + + +def test_configured_prebuilt_reuses_cached_wheel(tmp_context: WorkContext) -> None: + """A cache hit avoids a fresh resolver download while retaining hooks.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + cached = tmp_context.wheels_downloads / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=cached), + patch.object(wheels, "download_prebuilt_wheel") as download, + patch.object(build.hooks, "run_prebuilt_wheel_hooks") as hooks, + patch.object(build.server, "update_wheel_mirror"), + ): + result = build._build(tmp_context, version, req, url, False, None) + + assert result.wheel_filename == cached + assert result.skipped is True + download.assert_not_called() + hooks.assert_called_once() + + +def test_download_sequence_replays_configured_prebuilt( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """Download sequence uses the saved URL with the selected source resolver.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [{"dist": "pkg", "version": "1.0", "prebuilt": True, "source_url": url}] + ) + ) + with ( + patch.object(wheels, "download_prebuilt_wheel") as download, + patch.object(download_sequence.wheels, "download_wheel") as legacy, + ): + result = cli_runner.invoke( + download_sequence.download_sequence, + [ + str(build_order), + "https://index.test/simple", + "--include-wheels", + "--num-threads", + "1", + ], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + download.assert_called_once_with( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + download_url=url, + direct_download_directory=tmp_context.wheels_prebuilt, + ) + legacy.assert_not_called() + + +def test_download_sequence_local_mirror_url_skips_download_hook( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """Download replay fetches a local mirror wheel without invoking its hook.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + tmp_context.wheel_server_url = "https://local.test/simple/" + url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [{"dist": "pkg", "version": "1.0", "prebuilt": True, "source_url": url}] + ) + ) + with ( + patch.object(wheels, "download_wheel") as direct, + patch("fromager.wheels.sources.download_source") as configured, + ): + result = cli_runner.invoke( + download_sequence.download_sequence, + [ + str(build_order), + "https://index.test/simple", + "--include-wheels", + "--num-threads", + "1", + ], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + direct.assert_called_once_with( + Requirement("pkg==1.0"), url, tmp_context.wheels_prebuilt + ) + configured.assert_not_called() + + +@pytest.mark.parametrize("download_fails", [False, True]) +def test_download_sequence_clears_requirement_context( + cli_runner: CliRunner, + tmp_context: WorkContext, + tmp_path: pathlib.Path, + download_fails: bool, +) -> None: + """A prebuilt return or error leaves no package context on the worker.""" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": True, + "source_url": "https://pkg.test/pkg-1.0-py3-none-any.whl", + } + ] + ) + ) + failure = ValueError("download failed") if download_fails else None + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + with ( + patch.object( + download_sequence.futures, + "ThreadPoolExecutor", + return_value=executor, + ), + patch.object(wheels, "download_prebuilt_wheel", side_effect=failure), + ): + result = cli_runner.invoke( + download_sequence.download_sequence, + [ + str(build_order), + "https://index.test/simple", + "--include-wheels", + "--num-threads", + "1", + ], + obj=tmp_context, + ) + assert executor.submit(requirement_ctxvar.get, None).result() is None + + if download_fails: + assert isinstance(result.exception, ValueError) + else: + assert result.exit_code == 0, result.output + + +def test_download_sequence_rejects_stale_prebuilt_flag( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """A source-only resolver cannot replay a prebuilt build-order entry.""" + _configure_package(tmp_context, "source:\n provider: pypi-sdist\n") + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": True, + "source_url": "https://pkg.test/pkg-1.0-py3-none-any.whl", + } + ] + ) + ) + with ( + patch.object(wheels, "download_wheel") as direct, + patch("fromager.wheels.sources.download_source") as configured, + ): + result = cli_runner.invoke( + download_sequence.download_sequence, + [ + str(build_order), + "https://index.test/simple", + "--include-wheels", + "--num-threads", + "1", + ], + obj=tmp_context, + ) + + assert isinstance(result.exception, ValueError) + assert "does not resolve prebuilt wheels" in str(result.exception) + configured.assert_not_called() + direct.assert_not_called() + + +def test_legacy_prebuilt_keeps_direct_download(tmp_context: WorkContext) -> None: + """Legacy pre_built still downloads into the build wheel directory.""" + _configure_package(tmp_context, "variants:\n cpu:\n pre_built: true\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_build / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=None), + patch.object(build.wheels, "download_wheel", return_value=wheel) as legacy, + patch("fromager.wheels.sources.download_source") as configured, + patch.object(build.hooks, "run_prebuilt_wheel_hooks"), + patch.object(build.server, "update_wheel_mirror") as mirror, + ): + result = build._build(tmp_context, version, req, url, False, None) + + assert result.prebuilt is True + assert result.wheel_filename == tmp_context.wheels_downloads / wheel.name + legacy.assert_called_once_with(req, url, tmp_context.wheels_build) + configured.assert_not_called() + mirror.assert_called_once_with(tmp_context) def get_option_names(cmd: click.Command) -> typing.Iterable[str]: diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 610f2097f..ce18db756 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -1255,6 +1255,31 @@ def test_pbi_source_resolver_variant_overrides_package( assert isinstance(pbi.source_resolver, PyPIPrebuiltResolver) +@pytest.mark.parametrize( + ("yaml_str", "expected"), + [ + ("source:\n provider: pypi-prebuilt\n", True), + ("source:\n provider: hook-prebuilt\n", True), + ("source:\n provider: pypi-sdist\n", False), + ("variants:\n cpu:\n pre_built: true\n", True), + ], +) +def test_pbi_prebuilt_uses_selected_source_or_legacy_variant( + tmp_path: pathlib.Path, yaml_str: str, expected: bool +) -> None: + """Read configured source capability before the legacy variant flag.""" + ps = PackageSettings.from_string("test-pkg", yaml_str) + settings = Settings( + settings=SettingsFile(), + package_settings=[ps], + variant="cpu", + patches_dir=tmp_path, + max_jobs=1, + ) + + assert settings.package_build_info("test-pkg").pre_built is expected + + # -- source vs legacy mutual exclusivity validation -------------------------- diff --git a/tests/test_server.py b/tests/test_server.py index d507682ec..c5cf7cec0 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -86,6 +86,44 @@ def test_update_wheel_mirror_prebuilt_wheels( assert symlink.is_file() +def test_update_wheel_mirror_prefers_fresh_prebuilt_wheel( + tmp_context: context.WorkContext, +) -> None: + """An explicit fresh wheel replaces an old mirror link with the same name.""" + wheel_name = "foo-1.0-py3-none-any.whl" + old_wheel = _create_fake_wheel(tmp_context.wheels_downloads, wheel_name) + server.update_wheel_mirror(tmp_context) + mirror_wheel = tmp_context.wheel_server_dir / "foo" / wheel_name + + fresh_wheel = _create_fake_wheel(tmp_context.wheels_prebuilt, wheel_name) + server.update_wheel_mirror(tmp_context) + assert mirror_wheel.resolve() == old_wheel + + server.update_wheel_mirror(tmp_context, preferred_prebuilt_wheel=fresh_wheel) + assert mirror_wheel.resolve() == fresh_wheel + assert old_wheel.is_file() + + +def test_update_wheel_mirror_failed_relink_keeps_old_wheel( + tmp_context: context.WorkContext, +) -> None: + """A failed atomic switch leaves the existing mirror link intact.""" + wheel_name = "foo-1.0-py3-none-any.whl" + old_wheel = _create_fake_wheel(tmp_context.wheels_downloads, wheel_name) + server.update_wheel_mirror(tmp_context) + fresh_wheel = _create_fake_wheel(tmp_context.wheels_prebuilt, wheel_name) + mirror_wheel = tmp_context.wheel_server_dir / "foo" / wheel_name + + with ( + patch("fromager.server.os.replace", side_effect=OSError("replace failed")), + pytest.raises(OSError, match="replace failed"), + ): + server.update_wheel_mirror(tmp_context, preferred_prebuilt_wheel=fresh_wheel) + + assert mirror_wheel.resolve() == old_wheel + assert not list(mirror_wheel.parent.glob(f".{wheel_name}.*")) + + def test_update_wheel_mirror_cleans_dangling_symlinks( tmp_context: context.WorkContext, ) -> None: diff --git a/tests/test_wheels.py b/tests/test_wheels.py index 91bb698e2..a4d3f2260 100644 --- a/tests/test_wheels.py +++ b/tests/test_wheels.py @@ -9,10 +9,231 @@ from packaging.tags import Tag from packaging.version import Version -from fromager import build_environment, context, downloads, wheels +from fromager import build_environment, context, downloads, packagesettings, wheels from fromager.packagesettings import SbomSettings, Settings, SettingsFile, WheelSettings +def _configure_package(ctx: context.WorkContext, yaml_str: str) -> None: + """Apply one package setting to the test context.""" + ps = packagesettings.PackageSettings.from_string("pkg", yaml_str) + ctx.settings._package_settings[ps.name] = ps + + +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_configured_prebuilt_download_uses_resolver_default( + tmp_context: context.WorkContext, provider: str +) -> None: + """Both profiles use their own default when no download hook exists.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + req = Requirement("pkg==1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + expected = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch("fromager.overrides.find_override_method", return_value=None), + patch("fromager.downloads.download_wheel", return_value=expected) as default, + ): + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url=url, + direct_download_directory=tmp_context.wheels_build, + ) + + assert result == expected + default.assert_called_once_with( + destination_dir=tmp_context.wheels_prebuilt, url=url + ) + + +def test_hook_prebuilt_download_stages_hook_result( + tmp_context: context.WorkContext, +) -> None: + """Use the hook's path and make its wheel visible to the local mirror.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + hook_path = tmp_context.work_dir / "pkg-1.0-py3-none-any.whl" + hook_path.write_bytes(b"wheel") + calls: list[str] = [] + + def hook(download_url: str) -> packagesettings.DownloadedSource: + calls.append(download_url) + return packagesettings.DownloadedSource( + path=hook_path, kind=packagesettings.DownloadKind.prebuilt_wheel + ) + + with ( + patch("fromager.overrides.find_override_method", return_value=hook), + patch("fromager.downloads.download_wheel") as default, + ): + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url=url, + direct_download_directory=tmp_context.wheels_build, + ) + + assert result == tmp_context.wheels_prebuilt / hook_path.name + assert result.read_bytes() == b"wheel" + assert calls == [url] + default.assert_not_called() + + +def test_hook_prebuilt_local_mirror_url_uses_direct_download( + tmp_context: context.WorkContext, +) -> None: + """A local mirror URL does not invoke the configured download hook.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + tmp_context.wheel_server_url = "https://local.test/simple/" + req = Requirement("pkg==1.0") + url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + expected = tmp_context.wheels_build / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_wheel", return_value=expected) as direct, + patch("fromager.wheels.sources.download_source") as configured, + ): + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url=url, + direct_download_directory=tmp_context.wheels_build, + ) + + assert result == expected + direct.assert_called_once_with(req, url, tmp_context.wheels_build) + configured.assert_not_called() + + +def test_hook_prebuilt_new_result_replaces_staged_wheel( + tmp_context: context.WorkContext, +) -> None: + """A fresh hook result replaces an older staged wheel with the same name.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + wheel_name = "pkg-1.0-py3-none-any.whl" + hook_path = tmp_context.work_dir / wheel_name + hook_path.write_bytes(b"new wheel") + staged_path = tmp_context.wheels_prebuilt / wheel_name + staged_path.write_bytes(b"old wheel") + + with patch( + "fromager.overrides.find_override_method", return_value=lambda: hook_path + ): + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url=f"https://pkg.test/{wheel_name}", + direct_download_directory=tmp_context.wheels_build, + ) + + assert result == staged_path + assert staged_path.read_bytes() == b"new wheel" + assert hook_path.read_bytes() == b"new wheel" + assert not list(tmp_context.wheels_prebuilt.glob(f".tmp.{wheel_name}.*")) + + +def test_hook_prebuilt_staging_failure_preserves_existing_wheel( + tmp_context: context.WorkContext, +) -> None: + """A failed copy leaves the old staged wheel intact and removes the temp file.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + wheel_name = "pkg-1.0-py3-none-any.whl" + hook_path = tmp_context.work_dir / wheel_name + hook_path.write_bytes(b"new wheel") + staged_path = tmp_context.wheels_prebuilt / wheel_name + staged_path.write_bytes(b"old wheel") + + with ( + patch( + "fromager.overrides.find_override_method", return_value=lambda: hook_path + ), + patch("fromager.wheels.shutil.copy2", side_effect=OSError("copy failed")), + pytest.raises(OSError, match="copy failed"), + ): + wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + download_url=f"https://pkg.test/{wheel_name}", + direct_download_directory=tmp_context.wheels_build, + ) + + assert staged_path.read_bytes() == b"old wheel" + assert not list(tmp_context.wheels_prebuilt.glob(f".tmp.{wheel_name}.*")) + + +def test_local_wheel_url_requires_path_boundary() -> None: + """A different index sharing a URL prefix is not the local mirror.""" + assert wheels.is_local_wheel_url( + "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl", + "https://local.test/simple/", + ) + assert not wheels.is_local_wheel_url( + "https://local.test/simple-other/pkg/pkg-1.0-py3-none-any.whl", + "https://local.test/simple/", + ) + + +def test_configured_prebuilt_rejects_other_kind( + tmp_context: context.WorkContext, +) -> None: + """An invalid hook result fails without trying a different downloader.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + + def hook() -> packagesettings.DownloadedSource: + return packagesettings.DownloadedSource( + path=tmp_context.work_dir / "pkg-1.0.tar.gz", + kind=packagesettings.DownloadKind.sdist, + ) + + with ( + patch("fromager.overrides.find_override_method", return_value=hook), + patch("fromager.downloads.download_wheel") as default, + pytest.raises(ValueError, match="unsupported kind 'sdist'"), + ): + wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url="https://pkg.test/pkg-1.0-py3-none-any.whl", + direct_download_directory=tmp_context.wheels_build, + ) + default.assert_not_called() + + +def test_hook_prebuilt_failure_does_not_fall_back( + tmp_context: context.WorkContext, +) -> None: + """Keep the hook failure visible and avoid the default wheel fetch.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + failure = OSError("hook download failed") + + def hook() -> pathlib.Path: + raise failure + + with ( + patch("fromager.overrides.find_override_method", return_value=hook), + patch("fromager.downloads.download_wheel") as default, + pytest.raises(RuntimeError, match="download_source hook failed") as exc_info, + ): + wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url="https://pkg.test/pkg-1.0-py3-none-any.whl", + direct_download_directory=tmp_context.wheels_build, + ) + + assert exc_info.value.__cause__ is failure + default.assert_not_called() + + @patch("pyproject_hooks.BuildBackendHookCaller.build_wheel") def test_default_build_wheel( mock_build_wheel: Mock,