From ed032d1f57742602c3a0bb82e5ea6e5bf1f3eff5 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 7 Jul 2026 13:26:31 +0200 Subject: [PATCH] feat(resolver): add download method to all resolver classes Add `_download()` to `AbstractResolver` that dispatches to the appropriate download helper based on a mandatory `download_kind` argument. Each concrete resolver implements a public `download()` that delegates with its specific `DownloadKind`. - `PyPIDownloadResolver` accepts `download_kind` as a mandatory Pydantic field (restricted to `sdist | tarball`) - `NotAvailableResolver` and `AbstractHookResolver` override `download()` to raise explicitly - Replace the old `download_kind` ClassVar with a `download_kinds` frozenset ClassVar describing which kinds each resolver can return - Replace `resolves_prebuilt_wheel` property with a ClassVar - Validate `download_kind` against `download_kinds` in `_download()` Co-Authored-By: Claude Signed-off-by: Christian Heimes --- docs/reference/config-reference.rst | 22 +-- src/fromager/packagesettings/_resolver.py | 222 +++++++++++++++++++-- tests/test_packagesettings_resolver.py | 225 ++++++++++++++++++++-- 3 files changed, 425 insertions(+), 44 deletions(-) diff --git a/docs/reference/config-reference.rst b/docs/reference/config-reference.rst index 625a6a23..d81a37fb 100644 --- a/docs/reference/config-reference.rst +++ b/docs/reference/config-reference.rst @@ -31,46 +31,46 @@ Source Resolver .. autopydantic_model:: PyPISDistResolver :inherited-members: AbstractPyPIResolver, CooldownMixin - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autopydantic_model:: PyPIPrebuiltResolver :inherited-members: AbstractPyPIResolver, CooldownMixin - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autopydantic_model:: PyPIDownloadResolver :inherited-members: AbstractPyPIResolver, CooldownMixin - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autopydantic_model:: PyPIGitResolver :inherited-members: AbstractPyPIResolver, CooldownMixin - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autopydantic_model:: GitHubTagDownloadResolver :inherited-members: AbstractGitSourceResolver, CooldownMixin - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autopydantic_model:: GitHubTagCloneResolver :inherited-members: AbstractGitSourceResolver, CooldownMixin - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autopydantic_model:: GitLabTagDownloadResolver :inherited-members: AbstractGitSourceResolver, CooldownMixin - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autopydantic_model:: GitLabTagCloneResolver :inherited-members: AbstractGitSourceResolver, CooldownMixin - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autopydantic_model:: NotAvailableResolver - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autopydantic_model:: HookSDistResolver :inherited-members: AbstractHookResolver, CooldownMixin - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autopydantic_model:: HookPrebuiltResolver :inherited-members: AbstractHookResolver, CooldownMixin - :members: download_kind, supports_override_hooks + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel .. autoclass:: BuildSDist diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index 879ecabe..213cf073 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -4,12 +4,13 @@ import enum import inspect import logging +import pathlib import re import typing import pydantic -from .. import resolver +from .. import downloads, resolver from ..candidate import Cooldown from ._typedefs import MODEL_CONFIG @@ -17,6 +18,7 @@ from packaging.requirements import Requirement from .. import context, requirements_file + from ..candidate import Candidate logger = logging.getLogger(__name__) @@ -42,6 +44,9 @@ def __bool__(self) -> bool: return self is not DownloadKind.not_available +DownloadKindSet = frozenset[DownloadKind] + + class AbstractResolver(pydantic.BaseModel): """Abstract base class for resolvers""" @@ -52,13 +57,11 @@ class AbstractResolver(pydantic.BaseModel): supports_override_hooks: typing.ClassVar[bool] = False """Does resolver support override hooks?""" - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.not_available - """Kind of artifact that the resolver downloads.""" + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset() + """Set of download kinds this resolver can return.""" - @property - def resolves_prebuilt_wheel(self) -> bool: - """Does resolver return pre-built wheels?""" - return self.download_kind is DownloadKind.prebuilt_wheel + resolves_prebuilt_wheel: typing.ClassVar[bool] = False + """Does resolver return pre-built wheels?""" def resolver_provider( self, @@ -69,6 +72,78 @@ def resolver_provider( """Return a resolver provider for the given requirement.""" raise NotImplementedError + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Download the resolved artifact for *candidate*.""" + raise NotImplementedError + + def _download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + download_kind: DownloadKind, + ) -> tuple[pathlib.Path, DownloadKind]: + """Download the resolved artifact for *candidate*. + + Returns the local path and the kind of artifact that was + downloaded. Dispatches to the appropriate download helper + based on *download_kind*. + """ + if download_kind not in self.download_kinds: + raise ValueError( + f"provider {self.provider!r} does not support download kind " + f"{download_kind!r}, expected one of {self.download_kinds!r}" + ) + match download_kind: + case DownloadKind.sdist: + destination_dir = ctx.sdists_downloads + path = downloads.download_sdist( + destination_dir=destination_dir, + url=candidate.url, + ) + case DownloadKind.tarball: + # TODO: A tarball is not a proper sdist; it should not + # live in sdists_downloads. Using it here for now until + # a dedicated tarballs directory is introduced. + destination_dir = ctx.sdists_downloads + # GitHub tarball URLs have no usable filename in the + # path, so always provide a destination filename. + path = downloads.download_url( + destination_dir=destination_dir, + url=candidate.url, + destination_filename=( + f"{candidate.name}-{candidate.version}.tar.gz" + ), + ) + case DownloadKind.prebuilt_wheel: + destination_dir = ctx.wheels_prebuilt + path = downloads.download_wheel( + destination_dir=destination_dir, + url=candidate.url, + ) + case DownloadKind.git_checkout: + # TODO: Maybe use dedicated directory like + # 'work_dir / f"{name}-{version}" / "checkout"' for + # checkout, generate a proper sdist, then unpack to + # final destination? + destination_dir = ( + ctx.work_dir + / f"{req.name}-{candidate.version}" + / f"{req.name}-{candidate.version}" + ) + path = downloads.download_git_source( + destination_dir=destination_dir, + vcs_url=candidate.url, + ) + case _: # includes any_source and not_available + typing.assert_never(download_kind) # type: ignore[arg-type] + return path, download_kind + class CooldownMixin(pydantic.BaseModel): min_release_age: int | None = pydantic.Field(default=None, ge=0) @@ -117,7 +192,7 @@ class PyPISDistResolver(AbstractPyPIResolver): # It is not safe to use PEP 517 to re-generate a source distribution. # Some PEP 517 backends require VCS to generate correct sdist. build_sdist: typing.ClassVar[BuildSDist | None] = BuildSDist.tarball - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.sdist + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset({DownloadKind.sdist}) def resolver_provider( self, @@ -135,6 +210,15 @@ def resolver_provider( cooldown=self._cooldown, ) + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Download an sdist from PyPI.""" + return self._download(ctx, req, candidate, DownloadKind.sdist) + class PyPIPrebuiltResolver(AbstractPyPIResolver): """Resolve version with PyPI, download pre-built wheel from PyPI @@ -156,7 +240,10 @@ class PyPIPrebuiltResolver(AbstractPyPIResolver): provider: typing.Literal["pypi-prebuilt"] build_sdist: typing.ClassVar[BuildSDist | None] = None - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.prebuilt_wheel + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( + {DownloadKind.prebuilt_wheel} + ) + resolves_prebuilt_wheel: typing.ClassVar[bool] = True def resolver_provider( self, @@ -174,6 +261,15 @@ def resolver_provider( cooldown=self._cooldown, ) + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Download a pre-built wheel from PyPI.""" + return self._download(ctx, req, candidate, DownloadKind.prebuilt_wheel) + class PyPIDownloadResolver(AbstractPyPIResolver): """Resolve version with PyPI, download sdist from arbitrary URL @@ -194,6 +290,7 @@ class PyPIDownloadResolver(AbstractPyPIResolver): provider: pypi-download index_url: https://pypi.test/simple download_url: https://download.test/test_pypidownload-{version}.tar.gz + download_kind: tarball """ provider: typing.Literal["pypi-download"] @@ -205,7 +302,12 @@ class PyPIDownloadResolver(AbstractPyPIResolver): """ build_sdist: typing.ClassVar[BuildSDist | None] = BuildSDist.tarball - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.tarball + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( + {DownloadKind.sdist, DownloadKind.tarball} + ) + + download_kind: typing.Literal[DownloadKind.sdist, DownloadKind.tarball] + """Kind of artifact that the resolver downloads.""" @pydantic.field_validator("download_url", mode="after") @classmethod @@ -235,6 +337,15 @@ def resolver_provider( cooldown=self._cooldown, ) + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Download an sdist or tarball from a custom URL.""" + return self._download(ctx, req, candidate, self.download_kind) + class PyPIGitResolver(AbstractPyPIResolver): """Resolve version with PyPI, build sdist from git clone @@ -260,7 +371,9 @@ class PyPIGitResolver(AbstractPyPIResolver): provider: typing.Literal["pypi-git"] - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.git_checkout + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( + {DownloadKind.git_checkout} + ) clone_url: pydantic.AnyUrl """git clone URL @@ -305,6 +418,15 @@ def resolver_provider( cooldown=self._cooldown, ) + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Clone a git repository at a specific tag.""" + return self._download(ctx, req, candidate, DownloadKind.git_checkout) + _PEP440_TAG_PATTERN = re.compile(r"^(v?\d.*)$") @@ -443,7 +565,7 @@ class GitHubTagDownloadResolver(AbstractGitSourceResolver): provider: typing.Literal["github-tag-download"] - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.tarball + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset({DownloadKind.tarball}) def resolver_provider( self, @@ -457,6 +579,15 @@ def resolver_provider( override_download_url=None, ) + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Download a tarball from a GitHub tag.""" + return self._download(ctx, req, candidate, DownloadKind.tarball) + class GitHubTagCloneResolver(AbstractGitSourceResolver): """Resolve version from GitHub tags, build sdist from a git clone @@ -473,7 +604,9 @@ class GitHubTagCloneResolver(AbstractGitSourceResolver): provider: typing.Literal["github-tag-git"] - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.git_checkout + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( + {DownloadKind.git_checkout} + ) def resolver_provider( self, @@ -487,6 +620,15 @@ def resolver_provider( override_download_url=f"git+{self.project_url}@{{tagname}}", ) + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Clone a git repository from a GitHub tag.""" + return self._download(ctx, req, candidate, DownloadKind.git_checkout) + class GitLabTagDownloadResolver(AbstractGitSourceResolver): """Resolve version from GitLab tags, build sdist from a git tarball download @@ -503,7 +645,7 @@ class GitLabTagDownloadResolver(AbstractGitSourceResolver): provider: typing.Literal["gitlab-tag-download"] - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.tarball + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset({DownloadKind.tarball}) def resolver_provider( self, @@ -517,6 +659,15 @@ def resolver_provider( override_download_url=None, ) + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Download a tarball from a GitLab tag.""" + return self._download(ctx, req, candidate, DownloadKind.tarball) + class GitLabTagCloneResolver(AbstractGitSourceResolver): """Resolve version from GitLab tags, build sdist from a git clone @@ -533,7 +684,9 @@ class GitLabTagCloneResolver(AbstractGitSourceResolver): provider: typing.Literal["gitlab-tag-git"] - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.git_checkout + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( + {DownloadKind.git_checkout} + ) def resolver_provider( self, @@ -547,14 +700,21 @@ def resolver_provider( override_download_url=f"git+{self.project_url}@{{tagname}}", ) + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Clone a git repository from a GitLab tag.""" + return self._download(ctx, req, candidate, DownloadKind.git_checkout) + class NotAvailableResolver(AbstractResolver): """Prevent resolve and download""" provider: typing.Literal["not-available"] - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.not_available - def resolver_provider( self, ctx: context.WorkContext, @@ -563,6 +723,15 @@ def resolver_provider( ) -> resolver.BaseProvider: raise ValueError(f"package {req.name} is not available") + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Raise because package is not available.""" + raise ValueError(f"package {req.name} is not available") + class AbstractHookResolver(AbstractResolver, CooldownMixin): """Abstract base class for hook-based resolvers""" @@ -579,6 +748,15 @@ def resolver_provider( # TODO raise NotImplementedError("Hook resolver needs a hook") + def download( + self, + ctx: context.WorkContext, + req: Requirement, + candidate: Candidate, + ) -> tuple[pathlib.Path, DownloadKind]: + """Raise because hook-based download is not yet implemented.""" + raise NotImplementedError("Hook resolver download needs a hook") + class HookSDistResolver(AbstractHookResolver): """Call resolver_provider and download_source hook, build from source @@ -594,8 +772,9 @@ class HookSDistResolver(AbstractHookResolver): provider: typing.Literal["hook-sdist"] - # Hooks can return any source artifact (sdist, tarball, or git checkout). - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.any_source + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( + {DownloadKind.sdist, DownloadKind.tarball, DownloadKind.git_checkout} + ) class HookPrebuiltResolver(AbstractHookResolver): @@ -611,7 +790,10 @@ class HookPrebuiltResolver(AbstractHookResolver): provider: typing.Literal["hook-prebuilt"] - download_kind: typing.ClassVar[DownloadKind] = DownloadKind.prebuilt_wheel + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( + {DownloadKind.prebuilt_wheel} + ) + resolves_prebuilt_wheel: typing.ClassVar[bool] = True SourceResolver = typing.Annotated[ diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index cea1890d..8132b4a4 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -4,14 +4,16 @@ import re import textwrap import typing +from unittest import mock import pydantic import pytest import yaml from packaging.requirements import Requirement +from packaging.version import Version from fromager import resolver -from fromager.candidate import Cooldown +from fromager.candidate import Candidate, Cooldown from fromager.context import WorkContext from fromager.packagesettings._resolver import ( BuildSDist, @@ -59,6 +61,53 @@ def bad_func(x: str) -> None: _REQ = Requirement("test-pkg") _REQ_TYPE = RequirementType.INSTALL +_VERSION = Version("1.2.3") +_CANDIDATE_SDIST = Candidate( + name="test-pkg", + version=_VERSION, + url="https://pypi.test/packages/test-pkg-1.2.3.tar.gz", + is_sdist=True, +) +_CANDIDATE_WHEEL = Candidate( + name="test-pkg", + version=_VERSION, + url="https://pypi.test/packages/test_pkg-1.2.3-py3-none-any.whl", + is_sdist=False, +) +_CANDIDATE_TARBALL = Candidate( + name="test-pkg", + version=_VERSION, + url="https://download.test/test-pkg-1.2.3.tar.gz", +) +_CANDIDATE_GIT = Candidate( + name="test-pkg", + version=_VERSION, + url="git+https://code.test/project/repo.git@refs/tags/v1.2.3", +) +_CANDIDATE_GITHUB_TARBALL = Candidate( + name="test-pkg", + version=_VERSION, + url="https://api.github.com/repos/org/repo/tarball/v1.2.3", + remote_tag="v1.2.3", +) +_CANDIDATE_GITHUB_CLONE = Candidate( + name="test-pkg", + version=_VERSION, + url="git+https://github.com/python-wheel-build/fromager@v1.2.3", + remote_tag="v1.2.3", +) +_CANDIDATE_GITLAB_TARBALL = Candidate( + name="test-pkg", + version=_VERSION, + url="https://gitlab.test/group/project/-/archive/v1.2.3/project-v1.2.3.tar.gz", + remote_tag="v1.2.3", +) +_CANDIDATE_GITLAB_CLONE = Candidate( + name="test-pkg", + version=_VERSION, + url="git+https://gitlab.test/python-wheel-build/fromager@v1.2.3", + remote_tag="v1.2.3", +) class _SourceWrapper(pydantic.BaseModel): @@ -94,7 +143,7 @@ def test_parse(self) -> None: assert r.min_release_age is None assert r.supports_override_hooks is False assert r.resolves_prebuilt_wheel is False - assert r.download_kind == DownloadKind.sdist + assert r.download_kinds == frozenset({DownloadKind.sdist}) def test_default_index_url(self) -> None: r = PyPISDistResolver(provider="pypi-sdist") @@ -136,6 +185,19 @@ def test_cooldown_negative(self) -> None: with pytest.raises(pydantic.ValidationError, match="min_release_age"): PyPISDistResolver(provider="pypi-sdist", min_release_age=-1) + @mock.patch("fromager.downloads.download_sdist") + def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> None: + expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" + mock_dl.return_value = expected + r = _parse(self.YAML) + path, kind = r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + assert path == expected + assert kind is DownloadKind.sdist + mock_dl.assert_called_once_with( + destination_dir=tmp_context.sdists_downloads, + url=_CANDIDATE_SDIST.url, + ) + class TestPyPIPrebuiltResolver: YAML = """\ @@ -153,7 +215,7 @@ def test_parse(self) -> None: assert r.min_release_age is None assert r.supports_override_hooks is False assert r.resolves_prebuilt_wheel is True - assert r.download_kind == DownloadKind.prebuilt_wheel + assert r.download_kinds == frozenset({DownloadKind.prebuilt_wheel}) def test_resolver_provider(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) @@ -164,6 +226,19 @@ def test_resolver_provider(self, tmp_context: WorkContext) -> None: assert p.sdist_server_url == "https://pypi.test/simple" assert p.ignore_platform is False + @mock.patch("fromager.downloads.download_wheel") + def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> None: + expected = tmp_context.wheels_prebuilt / "test_pkg-1.2.3-py3-none-any.whl" + mock_dl.return_value = expected + r = _parse(self.YAML) + path, kind = r.download(tmp_context, _REQ, _CANDIDATE_WHEEL) + assert path == expected + assert kind is DownloadKind.prebuilt_wheel + mock_dl.assert_called_once_with( + destination_dir=tmp_context.wheels_prebuilt, + url=_CANDIDATE_WHEEL.url, + ) + class TestPyPIDownloadResolver: YAML = """\ @@ -171,6 +246,7 @@ class TestPyPIDownloadResolver: provider: pypi-download index_url: https://pypi.test/simple download_url: https://download.test/pkg-{version}.tar.gz + download_kind: tarball """ def test_parse(self) -> None: @@ -184,6 +260,26 @@ def test_parse(self) -> None: assert r.supports_override_hooks is False assert r.resolves_prebuilt_wheel is False assert r.download_kind == DownloadKind.tarball + assert r.download_kinds == frozenset({DownloadKind.sdist, DownloadKind.tarball}) + + def test_download_kind_sdist(self) -> None: + r = _parse("""\ + source: + provider: pypi-download + download_url: https://download.test/pkg-{version}.tar.gz + download_kind: sdist + """) + assert isinstance(r, PyPIDownloadResolver) + assert r.download_kind == DownloadKind.sdist + + def test_download_kind_rejects_invalid(self) -> None: + with pytest.raises(pydantic.ValidationError, match="download_kind"): + _parse("""\ + source: + provider: pypi-download + download_url: https://download.test/pkg-{version}.tar.gz + download_kind: prebuilt_wheel + """) def test_resolver_provider(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) @@ -200,8 +296,23 @@ def test_missing_version_template(self) -> None: PyPIDownloadResolver( provider="pypi-download", download_url="https://download.test/pkg-latest.tar.gz", # type: ignore[arg-type] + download_kind=DownloadKind.tarball, ) + @mock.patch("fromager.downloads.download_url") + def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> None: + expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" + mock_dl.return_value = expected + r = _parse(self.YAML) + path, kind = r.download(tmp_context, _REQ, _CANDIDATE_TARBALL) + assert path == expected + assert kind is DownloadKind.tarball + mock_dl.assert_called_once_with( + destination_dir=tmp_context.sdists_downloads, + url=_CANDIDATE_TARBALL.url, + destination_filename="test-pkg-1.2.3.tar.gz", + ) + class TestPyPIGitResolver: YAML = """\ @@ -225,7 +336,7 @@ def test_parse(self) -> None: assert r.min_release_age is None assert r.supports_override_hooks is False assert r.resolves_prebuilt_wheel is False - assert r.download_kind == DownloadKind.git_checkout + assert r.download_kinds == frozenset({DownloadKind.git_checkout}) def test_resolver_provider(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) @@ -256,6 +367,21 @@ def test_clone_url_rejects_http(self) -> None: tag="v{version}", ) + @mock.patch("fromager.downloads.download_git_source") + def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> None: + expected_dest = ( + tmp_context.work_dir / f"{_REQ.name}-{_VERSION}" / f"{_REQ.name}-{_VERSION}" + ) + mock_dl.return_value = expected_dest + r = _parse(self.YAML) + path, kind = r.download(tmp_context, _REQ, _CANDIDATE_GIT) + assert path == expected_dest + assert kind is DownloadKind.git_checkout + mock_dl.assert_called_once_with( + destination_dir=expected_dest, + vcs_url=_CANDIDATE_GIT.url, + ) + # -- Git source resolvers ---------------------------------------------------- @@ -277,7 +403,7 @@ def test_parse(self) -> None: assert r.min_release_age is None assert r.supports_override_hooks is False assert r.resolves_prebuilt_wheel is False - assert r.download_kind == DownloadKind.tarball + assert r.download_kinds == frozenset({DownloadKind.tarball}) def test_explicit_matcher_factory(self) -> None: r = _parse("""\ @@ -362,6 +488,20 @@ def test_project_url_rejects_dot_git(self) -> None: project_url="https://github.com/org/repo.git", # type: ignore[arg-type] ) + @mock.patch("fromager.downloads.download_url") + def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> None: + expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" + mock_dl.return_value = expected + r = _parse(self.YAML) + path, kind = r.download(tmp_context, _REQ, _CANDIDATE_GITHUB_TARBALL) + assert path == expected + assert kind is DownloadKind.tarball + mock_dl.assert_called_once_with( + destination_dir=tmp_context.sdists_downloads, + url=_CANDIDATE_GITHUB_TARBALL.url, + destination_filename="test-pkg-1.2.3.tar.gz", + ) + class TestGitHubTagCloneResolver: YAML = """\ @@ -380,7 +520,7 @@ def test_parse(self) -> None: assert r.min_release_age is None assert r.supports_override_hooks is False assert r.resolves_prebuilt_wheel is False - assert r.download_kind == DownloadKind.git_checkout + assert r.download_kinds == frozenset({DownloadKind.git_checkout}) def test_resolver_provider(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) @@ -392,6 +532,21 @@ def test_resolver_provider(self, tmp_context: WorkContext) -> None: "git+https://github.com/python-wheel-build/fromager@{tagname}" ) + @mock.patch("fromager.downloads.download_git_source") + def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> None: + expected_dest = ( + tmp_context.work_dir / f"{_REQ.name}-{_VERSION}" / f"{_REQ.name}-{_VERSION}" + ) + mock_dl.return_value = expected_dest + r = _parse(self.YAML) + path, kind = r.download(tmp_context, _REQ, _CANDIDATE_GITHUB_CLONE) + assert path == expected_dest + assert kind is DownloadKind.git_checkout + mock_dl.assert_called_once_with( + destination_dir=expected_dest, + vcs_url=_CANDIDATE_GITHUB_CLONE.url, + ) + class TestGitLabTagDownloadResolver: YAML = """\ @@ -410,7 +565,7 @@ def test_parse(self) -> None: assert r.min_release_age is None assert r.supports_override_hooks is False assert r.resolves_prebuilt_wheel is False - assert r.download_kind == DownloadKind.tarball + assert r.download_kinds == frozenset({DownloadKind.tarball}) def test_resolver_provider(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) @@ -420,6 +575,20 @@ def test_resolver_provider(self, tmp_context: WorkContext) -> None: assert "gitlab.test" in p.server_url assert p.override_download_url is None + @mock.patch("fromager.downloads.download_url") + def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> None: + expected = tmp_context.sdists_downloads / "test-pkg-1.2.3.tar.gz" + mock_dl.return_value = expected + r = _parse(self.YAML) + path, kind = r.download(tmp_context, _REQ, _CANDIDATE_GITLAB_TARBALL) + assert path == expected + assert kind is DownloadKind.tarball + mock_dl.assert_called_once_with( + destination_dir=tmp_context.sdists_downloads, + url=_CANDIDATE_GITLAB_TARBALL.url, + destination_filename="test-pkg-1.2.3.tar.gz", + ) + class TestGitLabTagCloneResolver: YAML = """\ @@ -438,7 +607,7 @@ def test_parse(self) -> None: assert r.min_release_age is None assert r.supports_override_hooks is False assert r.resolves_prebuilt_wheel is False - assert r.download_kind == DownloadKind.git_checkout + assert r.download_kinds == frozenset({DownloadKind.git_checkout}) def test_resolver_provider(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) @@ -450,6 +619,21 @@ def test_resolver_provider(self, tmp_context: WorkContext) -> None: "git+https://gitlab.test/python-wheel-build/fromager@{tagname}" ) + @mock.patch("fromager.downloads.download_git_source") + def test_download(self, mock_dl: mock.MagicMock, tmp_context: WorkContext) -> None: + expected_dest = ( + tmp_context.work_dir / f"{_REQ.name}-{_VERSION}" / f"{_REQ.name}-{_VERSION}" + ) + mock_dl.return_value = expected_dest + r = _parse(self.YAML) + path, kind = r.download(tmp_context, _REQ, _CANDIDATE_GITLAB_CLONE) + assert path == expected_dest + assert kind is DownloadKind.git_checkout + mock_dl.assert_called_once_with( + destination_dir=expected_dest, + vcs_url=_CANDIDATE_GITLAB_CLONE.url, + ) + # -- Special resolvers -------------------------------------------------------- @@ -466,14 +650,18 @@ def test_parse(self) -> None: assert r.provider == "not-available" assert r.supports_override_hooks is False assert r.resolves_prebuilt_wheel is False - assert r.download_kind == DownloadKind.not_available - assert not r.download_kind + assert r.download_kinds == frozenset() def test_resolver_provider(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) with pytest.raises(ValueError, match="not available"): r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + def test_download(self, tmp_context: WorkContext) -> None: + r = _parse(self.YAML) + with pytest.raises(ValueError, match="not available"): + r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + class TestHookSDistResolver: YAML = """\ @@ -487,14 +675,20 @@ def test_parse(self) -> None: assert r.provider == "hook-sdist" assert r.supports_override_hooks is True assert r.resolves_prebuilt_wheel is False - assert r.download_kind == DownloadKind.any_source - assert r.download_kind + assert r.download_kinds == frozenset( + {DownloadKind.sdist, DownloadKind.tarball, DownloadKind.git_checkout} + ) def test_resolver_provider(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) with pytest.raises(NotImplementedError): r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + def test_download(self, tmp_context: WorkContext) -> None: + r = _parse(self.YAML) + with pytest.raises(NotImplementedError, match="hook"): + r.download(tmp_context, _REQ, _CANDIDATE_SDIST) + class TestHookPrebuiltResolver: YAML = """\ @@ -508,13 +702,18 @@ def test_parse(self) -> None: assert r.provider == "hook-prebuilt" assert r.supports_override_hooks is True assert r.resolves_prebuilt_wheel is True - assert r.download_kind == DownloadKind.prebuilt_wheel + assert r.download_kinds == frozenset({DownloadKind.prebuilt_wheel}) def test_resolver_provider(self, tmp_context: WorkContext) -> None: r = _parse(self.YAML) with pytest.raises(NotImplementedError): r.resolver_provider(tmp_context, _REQ, _REQ_TYPE) + def test_download(self, tmp_context: WorkContext) -> None: + r = _parse(self.YAML) + with pytest.raises(NotImplementedError, match="hook"): + r.download(tmp_context, _REQ, _CANDIDATE_WHEEL) + # -- Discriminated union validation -------------------------------------------