diff --git a/WORKSPACE b/WORKSPACE index 46b26b8a9a..016a9a9d7e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -77,6 +77,14 @@ python_register_multi_toolchains( python_versions = PYTHON_VERSIONS, ) +load("//python/uv:repositories.bzl", "uv_toolchains_manifest") + +uv_toolchains_manifest(name = "uv_manifest") + +load("@uv_manifest//:dist_manifest.bzl", "uv_register_toolchains") + +uv_register_toolchains(name = "uv") + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # Used for Bazel CI diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index b0217c121c..4eb0f926f7 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -155,6 +155,7 @@ sphinx_stardocs( "//python/private/zipapp:py_zipapp_rule", "//python/uv", "//python/uv:lock", + "//python/uv:repositories", "//python/uv:uv_toolchain", "//python/uv:uv_toolchain_info", "//python/zipapp:py_zipapp_binary", diff --git a/docs/pypi/lock.md b/docs/pypi/lock.md index 78c88c5f46..727795c1ef 100644 --- a/docs/pypi/lock.md +++ b/docs/pypi/lock.md @@ -67,12 +67,121 @@ compile_pip_requirements( ) ``` -### uv pip compile (bzlmod only) +### uv pip compile We also have a setup for the `uv pip compile` way of generating lock files. This is well tested with the public PyPI index, but you may hit some rough edges with private mirrors. +#### Setup + +A `uv` toolchain has to be registered, as well as a Python toolchain, which +{obj}`lock` uses to run `uv` with a hermetic interpreter. + +##### Bzlmod + +`rules_python` registers the `uv` toolchains for you; `MODULE.bazel` only has +to configure the version to download with the {obj}`uv` module extension: + +```starlark +uv = use_extension( + "@rules_python//python/uv:uv.bzl", + "uv", + dev_dependency = True, +) +uv.configure(version = "0.13.5") +``` + +Without `version`, the default version specified by `rules_python` is used. +See {obj}`uv.configure` for using a mirror, explicit URLs or extra platforms. + +(uv-workspace-setup)= +##### WORKSPACE + +The setup has two steps: {obj}`uv_toolchains_manifest` fetches the release +manifests of the `uv` versions you want, and the {obj}`uv_register_toolchains` +it exposes creates the toolchains for the platforms those releases have +archives for and registers them: + +```starlark +load("@rules_python//python/uv:repositories.bzl", "uv_toolchains_manifest") + +uv_toolchains_manifest( + name = "uv_manifest", + versions = ["0.13.5"], +) + +load("@uv_manifest//:dist_manifest.bzl", "uv_register_toolchains") + +uv_register_toolchains(name = "uv") +``` + +Any released `uv` version can be fetched this way; the download URLs and +checksums are read from the release manifests. The `configure` argument of +{obj}`uv_register_toolchains` takes the same entries as {obj}`uv.configure`. + +Bazel prefers toolchains in registration order, so to replace one of the +registered toolchains with your own, register yours before calling +`uv_register_toolchains()` and remove ours; to add one without configuring it +here, register yours after: + +```starlark +uv_toolchains_manifest( + name = "uv_manifest", + versions = ["0.13.5", "0.6.5"], +) + +load("@uv_manifest//:dist_manifest.bzl", "uv_register_toolchains") + +# Preferred over the toolchains registered below. +register_toolchains("//tools/uv:patched_linux_toolchain") + +uv_register_toolchains( + name = "uv", + configure = [ + # Replaced by the toolchain registered above. + {"platform": "x86_64-unknown-linux-gnu"}, + # An additional platform for 0.13.5. + { + "platform": "aarch64-unknown-linux-musl", + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + "target_settings": [Label("//tools/uv:is_musl")], + }, + # The older version with the default platforms. Its toolchains + # come after those of 0.13.5 in `@uv//:all`, so they are only + # used where no earlier toolchain matches. + {"version": "0.6.5"}, + ], +) + +# Used only when none of the toolchains above match. +register_toolchains("//tools/uv:freebsd_toolchain") +``` + +Using a manifest is optional. Without {obj}`uv_toolchains_manifest`, load +{obj}`uv_register_toolchains` directly from +`@rules_python//python/uv:repositories.bzl` and specify `urls` for every +platform: + +```starlark +load("@rules_python//python/uv:repositories.bzl", "uv_register_toolchains") + +uv_register_toolchains( + name = "uv", + version = "0.13.5", + configure = [ + { + "platform": "x86_64-unknown-linux-gnu", + "urls": ["https://mirror.example.com/uv/uv-0.13.5-x86_64-unknown-linux-gnu.tar.gz"], + "sha256": "...", + }, + ], +) +``` + #### Example usage ```starlark diff --git a/news/1975.added.md b/news/1975.added.md new file mode 100644 index 0000000000..36487e6d08 --- /dev/null +++ b/news/1975.added.md @@ -0,0 +1,4 @@ +(uv) The {obj}`lock` macro now works in WORKSPACE builds; set up the `uv` +toolchains with the new {obj}`uv_toolchains_manifest` repository rule and +{obj}`uv_register_toolchains` macro. +([#1975](https://github.com/bazel-contrib/rules_python/issues/1975)) diff --git a/python/uv/BUILD.bazel b/python/uv/BUILD.bazel index 40c489a57f..9551051bd2 100644 --- a/python/uv/BUILD.bazel +++ b/python/uv/BUILD.bazel @@ -45,6 +45,15 @@ bzl_library( deps = ["//python/uv/private:lock"], ) +bzl_library( + name = "repositories", + srcs = ["repositories.bzl"], + deps = [ + "//python/uv/private:uv_register_toolchains", + "//python/uv/private:uv_toolchains_manifest", + ], +) + bzl_library( name = "uv", srcs = ["uv.bzl"], diff --git a/python/uv/private/BUILD.bazel b/python/uv/private/BUILD.bazel index e10741948c..e4e652ae78 100644 --- a/python/uv/private/BUILD.bazel +++ b/python/uv/private/BUILD.bazel @@ -13,7 +13,6 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") package(default_visibility = ["//:__subpackages__"]) @@ -28,7 +27,6 @@ distribution_filegroup( filegroup( name = "lock_copier_template", srcs = ["template/lock_copier.py"], - target_compatible_with = [] if BZLMOD_ENABLED else ["@platforms//:incompatible"], visibility = _NOT_REALLY_PUBLIC, ) @@ -38,7 +36,6 @@ filegroup( "@platforms//os:windows": ["template/uv_lock.bat"], "//conditions:default": ["template/uv_lock.sh"], }), - target_compatible_with = [] if BZLMOD_ENABLED else ["@platforms//:incompatible"], visibility = _NOT_REALLY_PUBLIC, ) @@ -48,7 +45,6 @@ filegroup( "@platforms//os:windows": ["template/uv_pip_compile.bat"], "//conditions:default": ["template/uv_pip_compile.sh"], }), - target_compatible_with = [] if BZLMOD_ENABLED else ["@platforms//:incompatible"], visibility = _NOT_REALLY_PUBLIC, ) @@ -59,6 +55,13 @@ bzl_library( deps = [":toolchain_types"], ) +bzl_library( + name = "dist_manifest", + srcs = ["dist_manifest.bzl"], + visibility = ["//python/uv:__subpackages__"], + deps = ["//python/private:auth"], +) + # keep bzl_library( name = "lock", @@ -69,7 +72,6 @@ bzl_library( deps = [ ":toolchain_types", "//python:py_binary", - "//python/private:bzlmod_enabled", "//python/private:common_labels", "//python/private:toolchain_types", "@bazel_skylib//lib:paths", @@ -89,11 +91,37 @@ bzl_library( srcs = ["uv.bzl"], visibility = ["//python/uv:__subpackages__"], deps = [ + ":dist_manifest", ":toolchain_types", ":uv_repository", + ":uv_toolchain_config", ":uv_toolchains_repo", "//python/private:auth", - "//python/private:common_labels", + ], +) + +bzl_library( + name = "uv_register_toolchains", + srcs = ["uv_register_toolchains.bzl"], + visibility = ["//python/uv:__subpackages__"], + deps = [ + ":toolchain_types", + ":uv_defaults", + ":uv_repository", + ":uv_toolchain_config", + ":uv_toolchains_repo", + "@bazel_skylib//lib:types", + ], +) + +bzl_library( + name = "uv_toolchains_manifest", + srcs = ["uv_toolchains_manifest.bzl"], + visibility = ["//python/uv:__subpackages__"], + deps = [ + ":dist_manifest", + ":uv_defaults", + "//python/private:auth", ], ) @@ -128,11 +156,23 @@ bzl_library( ], ) +bzl_library( + name = "uv_toolchain_config", + srcs = ["uv_toolchain_config.bzl"], + visibility = ["//python/uv:__subpackages__"], + deps = ["//python/private:common_labels"], +) + bzl_library( name = "toolchain_types", srcs = ["toolchain_types.bzl"], ) +bzl_library( + name = "uv_defaults", + srcs = ["uv_defaults.bzl"], +) + bzl_library( name = "uv_toolchain_info", srcs = ["uv_toolchain_info.bzl"], diff --git a/python/uv/private/dist_manifest.bzl b/python/uv/private/dist_manifest.bzl new file mode 100644 index 0000000000..a03cf7961f --- /dev/null +++ b/python/uv/private/dist_manifest.bzl @@ -0,0 +1,254 @@ +"""Resolve `uv` release artifact URLs and checksums from a `dist-manifest.json`. + +`uv` releases are packaged with cargo-dist, which publishes a +`dist-manifest.json` next to the release artifacts. This file is shared by the +`uv` module extension (bzlmod) and the {obj}`uv_toolchains_manifest` repository +rule (WORKSPACE), so it must not load either of them. +""" + +load("//python/private:auth.bzl", "get_auth") + +# The manifest file name cargo-dist publishes with each release. +DEFAULT_MANIFEST_FILENAME = "dist-manifest.json" + +# See https://github.com/astral-sh/setup-uv/pull/809. +GITHUB_RELEASES_PREFIX = "https://github.com/astral-sh/uv/releases/download/" +ASTRAL_MIRROR_PREFIX = "https://releases.astral.sh/github/uv/releases/download/" + +def _overlap(first_collection, second_collection): + for x in first_collection: + if x in second_collection: + return True + + return False + +def dist_manifest_resolver(mrctx, *, get_auth = get_auth): + """Return a `resolve_sources` callback that reads the release manifest. + + Args: + mrctx: {type}`module_ctx | repository_ctx` the context used to download + and read the manifests. + get_auth: the auth function to use. + + Returns: + {type}`callable` accepting `version`, `config` and `auth` keyword + arguments, as `define_toolchains` expects, that returns the download + `urls` and `sha256` keyed by platform for the version. + """ + + def _resolve(*, version, config, auth): + return get_tool_urls_from_dist_manifest( + mrctx, + base_url = "{base_url}/{version}".format( + version = version, + base_url = config["base_url"], + ), + manifest_filename = config["manifest_filename"], + platforms = sorted(config["platforms"]), + get_auth = get_auth, + **auth + ) + + return _resolve + +def get_tool_urls_from_dist_manifest( + mrctx, + *, + base_url, + manifest_filename, + platforms, + get_auth = get_auth, + **auth_attrs): + """Download the results about remote tool sources. + + This relies on the tools using the cargo packaging to infer the actual + sha256 values for each binary. + + Example manifest url: https://github.com/astral-sh/uv/releases/download/0.6.5/dist-manifest.json + + The example format is as below + + dist_version "0.28.0" + announcement_tag "0.6.5" + announcement_tag_is_implicit false + announcement_is_prerelease false + announcement_title "0.6.5" + announcement_changelog "text" + announcement_github_body "MD text" + releases [ + { + app_name "uv" + app_version "0.6.5" + env + install_dir_env_var "UV_INSTALL_DIR" + unmanaged_dir_env_var "UV_UNMANAGED_INSTALL" + disable_update_env_var "UV_DISABLE_UPDATE" + no_modify_path_env_var "UV_NO_MODIFY_PATH" + github_base_url_env_var "UV_INSTALLER_GITHUB_BASE_URL" + ghe_base_url_env_var "UV_INSTALLER_GHE_BASE_URL" + display_name "uv" + display true + artifacts [ + "source.tar.gz" + "source.tar.gz.sha256" + "uv-installer.sh" + "uv-installer.ps1" + "sha256.sum" + "uv-aarch64-apple-darwin.tar.gz" + "uv-aarch64-apple-darwin.tar.gz.sha256" + "... + ] + hosting + order + 0 "simple" + 1 "github" + github + artifact_base_url "https://github.com" + artifact_download_path "/astral-sh/uv/releases/download/0.11.2" + owner "astral-sh" + repo "uv" + simple + download_url "https://releases.astral.sh/github/uv/releases/download/0.11.2" + artifacts + uv-aarch64-apple-darwin.tar.gz + name "uv-aarch64-apple-darwin.tar.gz" + kind "executable-zip" + target_triples [ + "aarch64-apple-darwin" + assets [ + { + id "uv-aarch64-apple-darwin-exe-uv" + name "uv" + path "uv" + kind "executable" + }, + { + id "uv-aarch64-apple-darwin-exe-uvx" + name "uvx" + path "uvx" + kind "executable" + } + ] + checksum "uv-aarch64-apple-darwin.tar.gz.sha256" + checksums + sha256 "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + uv-aarch64-apple-darwin.tar.gz.sha256 + name "uv-aarch64-apple-darwin.tar.gz.sha256" + kind "checksum" + target_triples [ + "aarch64-apple-darwin" + ] + + Args: + mrctx: {type}`module_ctx | repository_ctx` the context used to download + and read files. + base_url: {type}`str` the URL of the release directory containing the + manifest and the artifacts, e.g. + `https://github.com/astral-sh/uv/releases/download/0.6.5`. + manifest_filename: {type}`str` the manifest file name in `base_url`. + platforms: {type}`list[str] | None` the platform target triples to + resolve, or `None` to resolve every platform in the manifest. + get_auth: the auth function to use. + **auth_attrs: `netrc` and `auth_patterns` values passed to `get_auth`. + + Returns: + {type}`dict[str, struct]` the download `urls` and `sha256` keyed by + platform. Platforms without an artifact in the manifest are absent. + """ + auth_attr = struct(**auth_attrs) + dist_manifest = mrctx.path(manifest_filename) + urls = [base_url + "/" + manifest_filename] + result = mrctx.download( + url = urls, + output = dist_manifest, + auth = get_auth(mrctx, urls, ctx_attr = auth_attr), + ) + if not result.success: + fail(result) + dist_manifest = json.decode(mrctx.read(dist_manifest)) + + # Use the simple download_url from the manifest, when available. + dist_base_url = ( + dist_manifest + .get("releases", [{}])[0] + .get("hosting", {}) + .get("simple", {}) + .get("download_url", base_url) + ) + + # For official releases, add the astral mirror to improve availability. + # See https://github.com/astral-sh/setup-uv/pull/809. + if dist_base_url.startswith(GITHUB_RELEASES_PREFIX): + astral_base_url = ASTRAL_MIRROR_PREFIX + dist_base_url[len(GITHUB_RELEASES_PREFIX):] + base_urls = [ + astral_base_url, + dist_base_url, + ] + else: + base_urls = [dist_base_url] + + artifacts = dist_manifest["artifacts"] + tool_sources = {} + downloads = {} + for fname, artifact in artifacts.items(): + if artifact.get("kind") != "executable-zip": + continue + target_triples = artifact["target_triples"] + if platforms != None and not _overlap(target_triples, platforms): + # We are not interested in this platform, so skip. + continue + + # Releases of uv >= 0.11.0 have the sha256 directly inline. + sha256 = artifact.get("checksums", {}).get("sha256", "") + if len(sha256) > 0: + for platform in target_triples: + tool_sources[platform] = struct( + urls = [ + "{}/{}".format(base, fname) + for base in base_urls + ], + sha256 = sha256, + ) + continue + + # For uv < 0.11.0, we'll need to fetch the individual sha256 files. + checksum = artifacts[artifact["checksum"]] + checksum_fname = checksum["name"] + checksum_path = mrctx.path(checksum_fname) + urls = ["{}/{}".format(dist_base_url, checksum_fname)] + downloads[checksum_path] = struct( + download = mrctx.download( + url = urls, + output = checksum_path, + block = False, + auth = get_auth(mrctx, urls, ctx_attr = auth_attr), + ), + archive_fname = fname, + platforms = target_triples, + ) + + for checksum_path, download in downloads.items(): + result = download.download.wait() + if not result.success: + fail(result) + + archive_fname = download.archive_fname + + sha256, _, checksummed_fname = mrctx.read(checksum_path).partition(" ") + checksummed_fname = checksummed_fname.strip(" *\n") + if checksummed_fname and archive_fname != checksummed_fname: + fail("The checksum is for a different file, expected '{}' but got '{}'".format( + archive_fname, + checksummed_fname, + )) + + for platform in download.platforms: + tool_sources[platform] = struct( + urls = [ + "{}/{}".format(base, archive_fname) + for base in base_urls + ], + sha256 = sha256, + ) + + return tool_sources diff --git a/python/uv/private/lock.bzl b/python/uv/private/lock.bzl index c0ad8bf8d3..84e4cbbc12 100644 --- a/python/uv/private/lock.bzl +++ b/python/uv/private/lock.bzl @@ -18,7 +18,6 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//lib:shell.bzl", "shell") load("//python:py_binary.bzl", "py_binary") -load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility load("//python/private:common_labels.bzl", "labels") load("//python/private:toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE") # buildifier: disable=bzl-visibility load(":toolchain_types.bzl", "UV_TOOLCHAIN_TYPE") @@ -643,23 +642,33 @@ def lock( used by `name.update`, disable the `uv` cache. - `name.update`: a target that can be run to update the source-tree version of the lock file. The output can be fed to the {obj}`pip.parse` bzlmod - extension tag class via {attr}`pip.parse.uv_lock` or - {attr}`pip.parse.requirements_lock`. Note, you can use `native_test` to - wrap this target to make a test. You can't customize the - args via command line, but you can use RBE to generate requirements - (offload execution and run for different platforms). Note, that for RBE - to be usable, one needs to ensure that the nodes running the action have - internet connectivity or the indexes are provided in a different way for - a fully offline operation. + extension tag class (via {attr}`pip.parse.uv_lock` or + {attr}`pip.parse.requirements_lock`) or the {obj}`pip_parse` WORKSPACE + macro. Note, you can use `native_test` to wrap this target to make a test. + You can't customize the args via command line, but you can use RBE to + generate requirements (offload execution and run for different + platforms). Note, that for RBE to be usable, one needs to ensure that the + nodes running the action have internet connectivity or the indexes are + provided in a different way for a fully offline operation. :::{note} All of the targets have `manual` tags as locking results cannot be cached. ::: + :::{note} + A `uv` toolchain must be registered: under bzlmod via the {obj}`uv` module + extension, under WORKSPACE via {obj}`uv_register_toolchains`. + ::: + :::{versionchanged} VERSION_NEXT_PATCH The `name.run` target uses the `uv` cache by default. ::: + :::{versionchanged} VERSION_NEXT_FEATURE + WORKSPACE builds are supported; previously the targets were marked + incompatible unless bzlmod was enabled. + ::: + Args: name: {type}`str` The prefix of all targets created by this macro. srcs: {type}`list[Label]` The sources that will be used. Add all of the @@ -699,7 +708,8 @@ def lock( {versionadded} 2.1.0 python_version: {type}`str | None` the python_version to transition to when locking the requirements. Defaults to the default python version - configured by the {obj}`python` module extension. + configured by the {obj}`python` module extension or, under + WORKSPACE, by the registered Python toolchains. **kwargs: common kwargs passed to rules. """ update_target = "{}.update".format(name) @@ -711,8 +721,6 @@ def lock( maybe_out = _maybe_file(out) tags = ["manual"] + kwargs.pop("tags", []) - if not BZLMOD_ENABLED: - kwargs["target_compatible_with"] = ["@platforms//:incompatible"] uv_kwargs = { "is_windows": select({ diff --git a/python/uv/private/uv.bzl b/python/uv/private/uv.bzl index bd35624c4c..1ed9794316 100644 --- a/python/uv/private/uv.bzl +++ b/python/uv/private/uv.bzl @@ -15,9 +15,10 @@ """A module extension for working with uv.""" load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") -load("//python/private:common_labels.bzl", "labels") +load(":dist_manifest.bzl", "DEFAULT_MANIFEST_FILENAME", "dist_manifest_resolver") load(":toolchain_types.bzl", "UV_TOOLCHAIN_TYPE") load(":uv_repository.bzl", "uv_repository") +load(":uv_toolchain_config.bzl", "apply_configure", "define_toolchains", "set_config") load(":uv_toolchains_repo.bzl", "uv_toolchains_repo") _DOC = """\ @@ -56,7 +57,7 @@ The distribution manifest filename to use for the metadata fetching from GH. The defaults for this are set in `rules_python` MODULE.bazel file that one can override for a specific version. """, - default = "dist-manifest.json", + default = DEFAULT_MANIFEST_FILENAME, ), "platform": attr.string( doc = """\ @@ -78,8 +79,13 @@ last version used in the module or the default version set by `rules_python`. } | AUTH_ATTRS default = tag_class( - doc = """\ + doc = """ Set the uv configuration defaults. + +:::{versionchanged} VERSION_NEXT_PATCH +`netrc` and `auth_patterns` now apply to the versions configured with +{obj}`uv.configure`; previously they were accepted but ignored. +::: """, attrs = _DEFAULT_ATTRS, ) @@ -131,42 +137,6 @@ for a particular version. }, ) -def _configure(config, *, platform, compatible_with, target_settings, auth_patterns, urls = [], sha256 = "", override = False, **values): - """Set the value in the config if the value is provided""" - for key, value in values.items(): - if not value: - continue - - if not override and config.get(key): - continue - - config[key] = value - - config.setdefault("auth_patterns", {}).update(auth_patterns) - config.setdefault("platforms", {}) - if not platform: - if compatible_with or target_settings or urls: - fail("`platform` name must be specified when specifying `compatible_with`, `target_settings` or `urls`") - elif compatible_with or target_settings: - if not override and config.get("platforms", {}).get(platform): - return - - config["platforms"][platform] = struct( - name = platform.replace("-", "_").lower(), - compatible_with = compatible_with, - target_settings = target_settings, - ) - elif urls: - if not override and config.get("urls", {}).get(platform): - return - - config.setdefault("urls", {})[platform] = struct( - sha256 = sha256, - urls = urls, - ) - else: - config["platforms"].pop(platform) - def process_modules( module_ctx, hub_name = "uv", @@ -208,7 +178,7 @@ def process_modules( continue for tag in mod.tags.default: - _configure( + set_config( defaults, version = tag.version, base_url = tag.base_url, @@ -229,138 +199,26 @@ def process_modules( if not defaults.get(key, None): fail("defaults need to be set for '{}'".format(key)) - # resolved per-version configuration. The shape is something like: - # versions = { - # "1.0.0": { - # "base_url": "", - # "manifest_filename": "", - # "platforms": { - # "platform_name": struct( - # compatible_with = [], - # target_settings = [], - # urls = [], # can be unset - # sha256 = "", # can be unset - # ), - # }, - # }, - # } + # resolved per-version configuration; see `apply_configure`. versions = {} for mod in module_ctx.modules: if not (mod.is_root or mod.name == "rules_python"): continue - # last_version is the last version used in the MODULE.bazel or the default - last_version = None - for tag in mod.tags.configure: - last_version = tag.version or last_version or defaults["version"] - specific_config = versions.setdefault( - last_version, - { - "base_url": defaults["base_url"], - "manifest_filename": defaults["manifest_filename"], - # shallow copy is enough as the values are structs and will - # be replaced on modification - "platforms": dict(defaults["platforms"]), - }, - ) - - _configure( - specific_config, - base_url = tag.base_url, - manifest_filename = tag.manifest_filename, - platform = tag.platform, - compatible_with = tag.compatible_with, - target_settings = tag.target_settings, - sha256 = tag.sha256, - urls = tag.urls, - override = mod.is_root, - netrc = tag.netrc, - auth_patterns = tag.auth_patterns, - ) - - if not versions: - return hub_repo( - name = hub_name, - toolchain_type = toolchain_type, - toolchain_names = ["none"], - toolchain_implementations = { - # NOTE @aignas 2025-02-24: the label to the toolchain can be anything - "none": labels.NONE, - }, - toolchain_compatible_with = { - "none": ["@platforms//:incompatible"], - }, - toolchain_target_settings = {}, + apply_configure( + versions, + mod.tags.configure, + defaults = defaults, + override = mod.is_root, ) - toolchain_names = [] - toolchain_implementations = {} - toolchain_compatible_with_by_toolchain = {} - toolchain_target_settings = {} - for version, config in versions.items(): - platforms = config["platforms"] - - # Use the manually specified urls - urls = { - platform: src - for platform, src in config.get("urls", {}).items() - if src.urls - } - auth = { - "auth_patterns": config.get("auth_patterns"), - "netrc": config.get("netrc"), - } - auth = {k: v for k, v in auth.items() if v} - - # Or fallback to fetching them from GH manifest file - # Example file: https://github.com/astral-sh/uv/releases/download/0.6.3/dist-manifest.json - if not urls: - urls = _get_tool_urls_from_dist_manifest( - module_ctx, - base_url = "{base_url}/{version}".format( - version = version, - base_url = config["base_url"], - ), - manifest_filename = config["manifest_filename"], - platforms = sorted(platforms), - get_auth = get_auth, - **auth - ) - - for platform_name, platform in platforms.items(): - if platform_name not in urls: - continue - - toolchain_name = "{}_{}".format(version.replace(".", "_"), platform_name.lower().replace("-", "_")) - uv_repository_name = "{}_{}".format(hub_name, toolchain_name) - uv_repository( - name = uv_repository_name, - version = version, - platform = platform_name, - urls = urls[platform_name].urls, - sha256 = urls[platform_name].sha256, - **auth - ) - - toolchain_names.append(toolchain_name) - toolchain_implementations[toolchain_name] = "@{}//:uv_toolchain".format(uv_repository_name) - toolchain_compatible_with_by_toolchain[toolchain_name] = [ - str(label) - for label in platform.compatible_with - ] - if platform.target_settings: - toolchain_target_settings[toolchain_name] = [ - str(label) - for label in platform.target_settings - ] - - return hub_repo( - name = hub_name, + return define_toolchains( + hub_name = hub_name, + versions = versions, + uv_repository = uv_repository, + hub_repo = hub_repo, toolchain_type = toolchain_type, - toolchain_names = toolchain_names, - toolchain_implementations = toolchain_implementations, - toolchain_compatible_with = toolchain_compatible_with_by_toolchain, - toolchain_target_settings = toolchain_target_settings, + resolve_sources = dist_manifest_resolver(module_ctx, get_auth = get_auth), ) def _uv_toolchain_extension(module_ctx): @@ -369,196 +227,6 @@ def _uv_toolchain_extension(module_ctx): hub_name = "uv", ) -def _overlap(first_collection, second_collection): - for x in first_collection: - if x in second_collection: - return True - - return False - -# See https://github.com/astral-sh/setup-uv/pull/809. -GITHUB_RELEASES_PREFIX = "https://github.com/astral-sh/uv/releases/download/" -ASTRAL_MIRROR_PREFIX = "https://releases.astral.sh/github/uv/releases/download/" - -def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename, platforms, get_auth = get_auth, **auth_attrs): - """Download the results about remote tool sources. - - This relies on the tools using the cargo packaging to infer the actual - sha256 values for each binary. - - Example manifest url: https://github.com/astral-sh/uv/releases/download/0.6.5/dist-manifest.json - - The example format is as below - - dist_version "0.28.0" - announcement_tag "0.6.5" - announcement_tag_is_implicit false - announcement_is_prerelease false - announcement_title "0.6.5" - announcement_changelog "text" - announcement_github_body "MD text" - releases [ - { - app_name "uv" - app_version "0.6.5" - env - install_dir_env_var "UV_INSTALL_DIR" - unmanaged_dir_env_var "UV_UNMANAGED_INSTALL" - disable_update_env_var "UV_DISABLE_UPDATE" - no_modify_path_env_var "UV_NO_MODIFY_PATH" - github_base_url_env_var "UV_INSTALLER_GITHUB_BASE_URL" - ghe_base_url_env_var "UV_INSTALLER_GHE_BASE_URL" - display_name "uv" - display true - artifacts [ - "source.tar.gz" - "source.tar.gz.sha256" - "uv-installer.sh" - "uv-installer.ps1" - "sha256.sum" - "uv-aarch64-apple-darwin.tar.gz" - "uv-aarch64-apple-darwin.tar.gz.sha256" - "... - ] - hosting - order - 0 "simple" - 1 "github" - github - artifact_base_url "https://github.com" - artifact_download_path "/astral-sh/uv/releases/download/0.11.2" - owner "astral-sh" - repo "uv" - simple - download_url "https://releases.astral.sh/github/uv/releases/download/0.11.2" - artifacts - uv-aarch64-apple-darwin.tar.gz - name "uv-aarch64-apple-darwin.tar.gz" - kind "executable-zip" - target_triples [ - "aarch64-apple-darwin" - assets [ - { - id "uv-aarch64-apple-darwin-exe-uv" - name "uv" - path "uv" - kind "executable" - }, - { - id "uv-aarch64-apple-darwin-exe-uvx" - name "uvx" - path "uvx" - kind "executable" - } - ] - checksum "uv-aarch64-apple-darwin.tar.gz.sha256" - checksums - sha256 "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" - uv-aarch64-apple-darwin.tar.gz.sha256 - name "uv-aarch64-apple-darwin.tar.gz.sha256" - kind "checksum" - target_triples [ - "aarch64-apple-darwin" - ] - """ - auth_attr = struct(**auth_attrs) - dist_manifest = module_ctx.path(manifest_filename) - urls = [base_url + "/" + manifest_filename] - result = module_ctx.download( - url = urls, - output = dist_manifest, - auth = get_auth(module_ctx, urls, ctx_attr = auth_attr), - ) - if not result.success: - fail(result) - dist_manifest = json.decode(module_ctx.read(dist_manifest)) - - # Use the simple download_url from the manifest, when available. - dist_base_url = ( - dist_manifest - .get("releases", [{}])[0] - .get("hosting", {}) - .get("simple", {}) - .get("download_url", base_url) - ) - - # For official releases, add the astral mirror to improve availability. - # See https://github.com/astral-sh/setup-uv/pull/809. - if dist_base_url.startswith(GITHUB_RELEASES_PREFIX): - astral_base_url = ASTRAL_MIRROR_PREFIX + dist_base_url[len(GITHUB_RELEASES_PREFIX):] - base_urls = [ - astral_base_url, - dist_base_url, - ] - else: - base_urls = [dist_base_url] - - artifacts = dist_manifest["artifacts"] - tool_sources = {} - downloads = {} - for fname, artifact in artifacts.items(): - if artifact.get("kind") != "executable-zip": - continue - target_triples = artifact["target_triples"] - if not _overlap(target_triples, platforms): - # We are not interested in this platform, so skip. - continue - - # Releases of uv >= 0.11.0 have the sha256 directly inline. - sha256 = artifact.get("checksums", {}).get("sha256", "") - if len(sha256) > 0: - for platform in target_triples: - tool_sources[platform] = struct( - urls = [ - "{}/{}".format(base, fname) - for base in base_urls - ], - sha256 = sha256, - ) - continue - - # For uv < 0.11.0, we'll need to fetch the individual sha256 files. - checksum = artifacts[artifact["checksum"]] - checksum_fname = checksum["name"] - checksum_path = module_ctx.path(checksum_fname) - urls = ["{}/{}".format(dist_base_url, checksum_fname)] - downloads[checksum_path] = struct( - download = module_ctx.download( - url = urls, - output = checksum_path, - block = False, - auth = get_auth(module_ctx, urls, ctx_attr = auth_attr), - ), - archive_fname = fname, - platforms = target_triples, - ) - - for checksum_path, download in downloads.items(): - result = download.download.wait() - if not result.success: - fail(result) - - archive_fname = download.archive_fname - - sha256, _, checksummed_fname = module_ctx.read(checksum_path).partition(" ") - checksummed_fname = checksummed_fname.strip(" *\n") - if checksummed_fname and archive_fname != checksummed_fname: - fail("The checksum is for a different file, expected '{}' but got '{}'".format( - archive_fname, - checksummed_fname, - )) - - for platform in download.platforms: - tool_sources[platform] = struct( - urls = [ - "{}/{}".format(base, archive_fname) - for base in base_urls - ], - sha256 = sha256, - ) - - return tool_sources - uv = module_extension( doc = _DOC, implementation = _uv_toolchain_extension, diff --git a/python/uv/private/uv_defaults.bzl b/python/uv/private/uv_defaults.bzl new file mode 100644 index 0000000000..d50cec92c1 --- /dev/null +++ b/python/uv/private/uv_defaults.bzl @@ -0,0 +1,37 @@ +"""Default `uv` toolchain configuration for WORKSPACE. + +Under bzlmod, the equivalent defaults are the `uv.default()` calls in +`MODULE.bazel`. Keep the values here in sync with them. +""" + +# Keep in sync with `uv.default(version = ...)` in MODULE.bazel. +DEFAULT_VERSION = "0.13.5" + +# Keep in sync with `uv.default(base_url = ...)` in MODULE.bazel. +DEFAULT_BASE_URL = "https://github.com/astral-sh/uv/releases/download" + +def _platform(platform, *, os, cpu): + # Same shape as the platform structs `set_config` creates. + return struct( + name = platform.replace("-", "_").lower(), + compatible_with = [ + str(Label("@platforms//os:" + os)), + str(Label("@platforms//cpu:" + cpu)), + ], + target_settings = [], + ) + +# Keep in sync with the `uv.default(platform = ...)` calls in MODULE.bazel. +DEFAULT_PLATFORMS = { + platform: _platform(platform, os = os, cpu = cpu) + for platform, os, cpu in [ + ("aarch64-apple-darwin", "macos", "aarch64"), + ("aarch64-unknown-linux-gnu", "linux", "aarch64"), + ("powerpc64le-unknown-linux-gnu", "linux", "ppc64le"), + ("s390x-unknown-linux-gnu", "linux", "s390x"), + ("riscv64-unknown-linux-gnu", "linux", "riscv64"), + ("x86_64-apple-darwin", "macos", "x86_64"), + ("x86_64-pc-windows-msvc", "windows", "x86_64"), + ("x86_64-unknown-linux-gnu", "linux", "x86_64"), + ] +} diff --git a/python/uv/private/uv_register_toolchains.bzl b/python/uv/private/uv_register_toolchains.bzl new file mode 100644 index 0000000000..1da301159e --- /dev/null +++ b/python/uv/private/uv_register_toolchains.bzl @@ -0,0 +1,205 @@ +"""Create and register `uv` toolchains from a WORKSPACE file.""" + +load("@bazel_skylib//lib:types.bzl", "types") +load(":toolchain_types.bzl", "UV_TOOLCHAIN_TYPE") +load(":uv_defaults.bzl", "DEFAULT_PLATFORMS", "DEFAULT_VERSION") +load(":uv_repository.bzl", "uv_repository") +load(":uv_toolchain_config.bzl", "apply_configure", "define_toolchains") +load(":uv_toolchains_repo.bzl", "uv_toolchains_repo") + +# The fields of a `configure` entry, with their empty values. They are the +# attributes of the `uv.configure` tag class except `base_url` and +# `manifest_filename`, which belong to `uv_toolchains_manifest`. +_ENTRY_FIELDS = { + "auth_patterns": {}, + "compatible_with": [], + "netrc": "", + "platform": "", + "sha256": "", + "target_settings": [], + "urls": [], + "version": "", +} + +# `apply_configure` expects every `uv.configure` attribute on an entry. +_MANIFEST_ENTRY_FIELDS = { + "base_url": "", + "manifest_filename": "", +} + +# The fields that hold labels; they are passed on as strings. +_LABEL_LIST_FIELDS = ["compatible_with", "target_settings"] + +def _normalize_entry(entry): + """Validate a `configure` dict and return it as an entry struct.""" + if not types.is_dict(entry): + fail("`configure` entries must be dicts, got {}: {}".format( + type(entry), + entry, + )) + unknown = [key for key in entry if key not in _ENTRY_FIELDS] + if unknown: + fail( + "`configure` entry has unknown keys {}; ".format(unknown) + + "expected a subset of {}: {}".format(sorted(_ENTRY_FIELDS), entry), + ) + normalized = dict(_MANIFEST_ENTRY_FIELDS | _ENTRY_FIELDS) + for key, value in entry.items(): + empty = _ENTRY_FIELDS[key] + if types.is_dict(empty): + ok = types.is_dict(value) + elif types.is_list(empty): + ok = types.is_list(value) + else: + ok = types.is_string(value) + if not ok: + fail("`configure` entry key `{}` expects a {}, got {}: {}".format( + key, + type(empty), + type(value), + entry, + )) + if key in _LABEL_LIST_FIELDS: + value = [str(label) for label in value] + normalized[key] = value + return struct(**normalized) + +def _uv_register_toolchains( + *, + name, + manifests, + version, + configure, + netrc, + auth_patterns, + uv_repository, + hub_repo, + register_toolchains): + """The testable core of {obj}`uv_register_toolchains`. + + The repository rules and `register_toolchains` are injected. + """ + manifests = manifests or {} + version = version or (manifests.keys()[0] if manifests else DEFAULT_VERSION) + + # No entries behaves like one `uv.configure()` per fetched version, or + # for the default version without manifests. + if not configure: + configure = [{"version": v} for v in manifests] or [{}] + entries = [_normalize_entry(entry) for entry in configure] + + # The same role as the `uv.default()` tags under bzlmod. + defaults = { + "auth_patterns": auth_patterns or {}, + "base_url": "", + "manifest_filename": "", + "netrc": netrc or "", + "platforms": DEFAULT_PLATFORMS, + "version": version, + } + versions = apply_configure({}, entries, defaults = defaults, override = True) + + def _resolve_sources(*, version, **_kwargs): + if version not in manifests: + fail(( + "uv_register_toolchains(name = {name}): no release manifest " + + "for uv {version}. Load `uv_register_toolchains` from the " + + "`dist_manifest.bzl` of a `uv_toolchains_manifest` repository " + + "that has {version} in its `versions`, or give `urls` for its " + + "platforms in `configure`." + ).format(name = repr(name), version = version)) + return { + platform: struct( + sha256 = source["sha256"], + urls = source["urls"], + ) + for platform, source in manifests[version].items() + } + + define_toolchains( + hub_name = name, + versions = versions, + uv_repository = uv_repository, + hub_repo = hub_repo, + toolchain_type = str(UV_TOOLCHAIN_TYPE), + resolve_sources = _resolve_sources, + ) + register_toolchains("@{}//:all".format(name)) + +def uv_register_toolchains( + *, + name = "uv", + manifests = None, + version = None, + configure = None, + netrc = None, + auth_patterns = None): + """Create the `uv` toolchain repositories and register them in a WORKSPACE file. + + This is the WORKSPACE equivalent of the {obj}`uv` module extension. It + creates one download repository per `uv` version and platform, and the + toolchain hub repository `name` with a `toolchain` for each of them, which + it registers as `@{name}//:all`; {obj}`lock` can then be used. See the + {ref}`WORKSPACE setup guide ` for usage examples. + + Platforms without an archive in a manifest get no toolchain. The + `configure` entries are dicts with the attributes of {obj}`uv.configure`; + refer to the {obj}`uv.configure` documentation for how entries are applied. + + :::{note} + A Python toolchain must be registered as well, e.g. via + {obj}`python_register_toolchains`, because {obj}`lock` uses the Python + exec tools toolchain to run `uv` with a hermetic interpreter. + ::: + + :::{note} + `compatible_with` and `target_settings` are written into the hub + repository's `BUILD` file as-is, so plain strings resolve relative to + `@{name}`; pass {type}`Label` objects or `@//`-qualified strings for + targets in your own repository. + ::: + + :::{versionadded} VERSION_NEXT_FEATURE + ::: + + Args: + name: {type}`str` the name of the toolchain hub repository. The + download repositories are named `{name}_{version}_{platform}` + with `.` and `-` replaced by `_`. + manifests: {type}`dict[str, dict] | None` the download `urls` and + `sha256` of the archives, keyed by version and then by platform + (`{version: {platform: {"urls": [...], "sha256": "..."}}}`), as + written to `dist_manifest.bzl` by {obj}`uv_toolchains_manifest`, + which binds its own when `uv_register_toolchains` is loaded from + there. + version: {type}`str | None` the `uv` version for `configure` entries + that do not name one. Defaults to the first version of + `manifests`, or to the version the {obj}`uv` extension uses by + default when there are none. + configure: {type}`list[dict] | None` the entries to apply, each a + dict with a subset of the {obj}`uv.configure` attributes: + `version`, `platform`, `compatible_with`, `target_settings`, + `urls`, `sha256`, `netrc` and `auth_patterns`. Defaults to one + empty entry per version of `manifests`, i.e. every version with + the default platforms, or to a single entry for `version`. + netrc: {type}`str | None` the default location of the `.netrc` file + to use for authenticating the archive downloads. + auth_patterns: {type}`dict[str, str] | None` the default dict mapping + host names to custom authorization patterns for the archive + downloads, as understood by `http_archive`. + """ + _uv_register_toolchains( + name = name, + manifests = manifests, + version = version, + configure = configure, + netrc = netrc, + auth_patterns = auth_patterns, + uv_repository = uv_repository, + hub_repo = uv_toolchains_repo, + register_toolchains = native.register_toolchains, + ) + +testing = struct( + uv_register_toolchains = _uv_register_toolchains, +) diff --git a/python/uv/private/uv_toolchain_config.bzl b/python/uv/private/uv_toolchain_config.bzl new file mode 100644 index 0000000000..e8f7f5b8f7 --- /dev/null +++ b/python/uv/private/uv_toolchain_config.bzl @@ -0,0 +1,267 @@ +"""Toolchain configuration shared by the `uv` module extension and WORKSPACE. + +The {obj}`uv.configure` tag class and the `configure` argument of +{obj}`uv_register_toolchains` accept the same fields with the same meaning; +both are turned into per-version configurations and toolchain definitions by +the functions in this file. +""" + +load("//python/private:common_labels.bzl", "labels") + +def uv_toolchain_name(version, platform): + """Return the hub toolchain name for a `uv` version and platform. + + Args: + version: {type}`str` the `uv` version, e.g. `0.13.5`. + platform: {type}`str` the platform target triple, e.g. + `x86_64-unknown-linux-gnu`. + + Returns: + {type}`str` a name usable as a repository or target name suffix. + """ + return "{}_{}".format( + version.replace(".", "_"), + platform.lower().replace("-", "_"), + ) + +def set_config( + config, + *, + platform, + compatible_with, + target_settings, + auth_patterns, + urls = [], + sha256 = "", + override = False, + **values): + """Apply one `uv.default` or `uv.configure` entry to a configuration. + + When `platform` is given without `compatible_with`, `target_settings` or + `urls`, the platform is removed from the configuration. + + Args: + config: {type}`dict` the configuration to update in place; either the + defaults or a per-version configuration. + platform: {type}`str` the platform target triple the entry applies + to, or empty for version-wide values. + compatible_with: {type}`list` the constraints of `platform`. Together + with `target_settings`, adds or replaces the platform. + target_settings: {type}`list` the config settings of `platform`. + auth_patterns: {type}`dict[str, str]` auth patterns merged into the + configuration. + urls: {type}`list[str]` the archive URLs of `platform`, which then + bypass the release manifest. + sha256: {type}`str` the checksum of the archive at `urls`. + override: {type}`bool` whether values already set may be replaced. + **values: version-wide string values (`version`, `base_url`, + `manifest_filename`, `netrc`); empty values are ignored. + """ + for key, value in values.items(): + if not value: + continue + + if not override and config.get(key): + continue + + config[key] = value + + config.setdefault("auth_patterns", {}).update(auth_patterns) + config.setdefault("platforms", {}) + if not platform: + if compatible_with or target_settings or urls: + fail("`platform` name must be specified when specifying `compatible_with`, `target_settings` or `urls`") + elif compatible_with or target_settings: + if not override and config.get("platforms", {}).get(platform): + return + + config["platforms"][platform] = struct( + name = platform.replace("-", "_").lower(), + compatible_with = compatible_with, + target_settings = target_settings, + ) + elif urls: + if not override and config.get("urls", {}).get(platform): + return + + config.setdefault("urls", {})[platform] = struct( + sha256 = sha256, + urls = urls, + ) + else: + config["platforms"].pop(platform) + +def _new_version_config(defaults): + return { + "auth_patterns": dict(defaults.get("auth_patterns", {})), + "base_url": defaults["base_url"], + "manifest_filename": defaults["manifest_filename"], + "netrc": defaults.get("netrc", ""), + # shallow copy is enough as the values are structs and will be + # replaced on modification + "platforms": dict(defaults["platforms"]), + } + +def apply_configure(versions, entries, *, defaults, override): + """Apply `uv.configure`-like entries, in order, to the per-version configs. + + Args: + versions: {type}`dict[str, dict]` the per-version configurations to + update, keyed by version. The shape is something like: + ``` + { + "1.0.0": { + "base_url": "", + "manifest_filename": "", + "platforms": { + "platform_name": struct( + compatible_with = [], + target_settings = [], + ), + }, + "urls": { + "platform_name": struct(urls = [], sha256 = ""), + }, + }, + } + ``` + entries: {type}`list[struct]` the entries to apply. Each has the + fields of the {obj}`uv.configure` tag class: `version`, + `base_url`, `manifest_filename`, `platform`, `compatible_with`, + `target_settings`, `urls`, `sha256`, `netrc` and `auth_patterns`. + An entry without `version` applies to the version of the previous + entry, or to the default version. + defaults: {type}`dict` the defaults for new versions: `version`, + `base_url`, `manifest_filename`, `platforms` and optionally + `netrc` and `auth_patterns`. + override: {type}`bool` whether the entries may replace values that + are already set. + + Returns: + {type}`dict[str, dict]` the updated `versions`. + """ + last_version = None + for tag in entries: + last_version = tag.version or last_version or defaults["version"] + specific_config = versions.setdefault( + last_version, + _new_version_config(defaults), + ) + + set_config( + specific_config, + base_url = tag.base_url, + manifest_filename = tag.manifest_filename, + platform = tag.platform, + compatible_with = tag.compatible_with, + target_settings = tag.target_settings, + sha256 = tag.sha256, + urls = tag.urls, + override = override, + netrc = tag.netrc, + auth_patterns = tag.auth_patterns, + ) + return versions + +def define_toolchains( + *, + hub_name, + versions, + uv_repository, + hub_repo, + toolchain_type, + resolve_sources): + """Create the `uv` repositories and the toolchain hub for `versions`. + + Args: + hub_name: {type}`str` the name of the hub repository. + versions: {type}`dict[str, dict]` the per-version configurations; see + {obj}`apply_configure`. + uv_repository: the repository rule used to download `uv`. + hub_repo: the hub repo factory function to use. + toolchain_type: {type}`str` the toolchain type of the toolchains. + resolve_sources: {type}`callable` called with `version`, `config` and + `auth` keyword arguments for a version without explicit `urls`. + It returns the download `urls` and `sha256` keyed by platform, + e.g. from the release manifest; platforms it omits get no + toolchain. + + Returns: + The result of `hub_repo`. Mainly used for tests. + """ + if not versions: + return hub_repo( + name = hub_name, + toolchain_type = toolchain_type, + toolchain_names = ["none"], + toolchain_implementations = { + # NOTE @aignas 2025-02-24: the label to the toolchain can be anything + "none": labels.NONE, + }, + toolchain_compatible_with = { + "none": ["@platforms//:incompatible"], + }, + toolchain_target_settings = {}, + ) + + toolchain_names = [] + toolchain_implementations = {} + toolchain_compatible_with_by_toolchain = {} + toolchain_target_settings = {} + for version, config in versions.items(): + platforms = config["platforms"] + + # Use the manually specified urls + sources = { + platform: src + for platform, src in config.get("urls", {}).items() + if src.urls + } + auth = { + "auth_patterns": config.get("auth_patterns"), + "netrc": config.get("netrc"), + } + auth = {k: v for k, v in auth.items() if v} + + # Or fallback to fetching them from GH manifest file + # Example file: https://github.com/astral-sh/uv/releases/download/0.6.3/dist-manifest.json + if not sources: + sources = resolve_sources(version = version, config = config, auth = auth) + + for platform_name, platform in platforms.items(): + if platform_name not in sources: + continue + + toolchain_name = uv_toolchain_name(version, platform_name) + uv_repository_name = "{}_{}".format(hub_name, toolchain_name) + uv_repository( + name = uv_repository_name, + version = version, + platform = platform_name, + urls = sources[platform_name].urls, + sha256 = sources[platform_name].sha256, + **auth + ) + + toolchain_names.append(toolchain_name) + toolchain_implementations[toolchain_name] = "@{}//:uv_toolchain".format( + uv_repository_name, + ) + toolchain_compatible_with_by_toolchain[toolchain_name] = [ + str(label) + for label in platform.compatible_with + ] + if platform.target_settings: + toolchain_target_settings[toolchain_name] = [ + str(label) + for label in platform.target_settings + ] + + return hub_repo( + name = hub_name, + toolchain_type = toolchain_type, + toolchain_names = toolchain_names, + toolchain_implementations = toolchain_implementations, + toolchain_compatible_with = toolchain_compatible_with_by_toolchain, + toolchain_target_settings = toolchain_target_settings, + ) diff --git a/python/uv/private/uv_toolchains_manifest.bzl b/python/uv/private/uv_toolchains_manifest.bzl new file mode 100644 index 0000000000..b55d32f5a9 --- /dev/null +++ b/python/uv/private/uv_toolchains_manifest.bzl @@ -0,0 +1,152 @@ +"""Fetch `uv` release manifests from a WORKSPACE file.""" + +load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load(":dist_manifest.bzl", "DEFAULT_MANIFEST_FILENAME", "get_tool_urls_from_dist_manifest") +load(":uv_defaults.bzl", "DEFAULT_BASE_URL", "DEFAULT_VERSION") + +_DIST_MANIFEST_BZL_TEMPLATE = """\ +# Generated by @rules_python//python/uv/private:uv_toolchains_manifest.bzl +\"\"\"The `uv` release manifests fetched by `uv_toolchains_manifest(name = "{name}")`.\"\"\" + +load("@rules_python//python/uv:repositories.bzl", _uv_register_toolchains = "uv_register_toolchains") + +# The download `urls` and `sha256` of every `uv` archive in the manifests, +# keyed by version and then by platform triple. +DIST_MANIFESTS = {manifests} + +def uv_register_toolchains(**kwargs): + \"\"\"`uv_register_toolchains` of `@rules_python` with `manifests = DIST_MANIFESTS`. + + Args: + **kwargs: the arguments of `uv_register_toolchains`; `manifests` given + here are added to `DIST_MANIFESTS`. + \"\"\" + kwargs["manifests"] = DIST_MANIFESTS | (kwargs.get("manifests") or {{}}) + _uv_register_toolchains(**kwargs) +""" + +def _generate_dist_manifest_bzl( + rctx, + *, + name, + versions, + base_url, + manifest_filename, + netrc, + auth_patterns, + get_auth = get_auth): + """Download the release manifests of `versions` and render `dist_manifest.bzl`. + + Args: + rctx: {type}`repository_ctx` the context used to download and read the + manifests. + name: {type}`str` the repository name, for the generated docstring. + versions: {type}`list[str]` the `uv` versions to fetch manifests for. + base_url: {type}`str` the base URL of the `uv` releases. + manifest_filename: {type}`str` the release manifest file name. + netrc: {type}`str` the `.netrc` file for the downloads. + auth_patterns: {type}`dict[str, str]` the auth patterns for the + downloads. + get_auth: the auth function to use. + + Returns: + {type}`str` the `dist_manifest.bzl` content. + """ + auth = { + "auth_patterns": auth_patterns, + "netrc": netrc, + } + auth = {k: v for k, v in auth.items() if v} + + manifests = {} + for version in versions: + sources = get_tool_urls_from_dist_manifest( + rctx, + base_url = "{}/{}".format(base_url, version), + manifest_filename = manifest_filename, + platforms = None, + get_auth = get_auth, + **auth + ) + manifests[version] = { + platform: { + "sha256": source.sha256, + "urls": source.urls, + } + for platform, source in sorted(sources.items()) + } + + return _DIST_MANIFEST_BZL_TEMPLATE.format( + name = name, + manifests = json.encode_indent(manifests, indent = " "), + ) + +def _uv_toolchains_manifest_impl(repository_ctx): + repository_ctx.file("BUILD.bazel", "") + repository_ctx.file("dist_manifest.bzl", _generate_dist_manifest_bzl( + repository_ctx, + name = repository_ctx.attr.name, + versions = repository_ctx.attr.versions, + base_url = repository_ctx.attr.base_url, + manifest_filename = repository_ctx.attr.manifest_filename, + netrc = repository_ctx.attr.netrc, + auth_patterns = repository_ctx.attr.auth_patterns, + )) + +uv_toolchains_manifest = repository_rule( + _uv_toolchains_manifest_impl, + doc = """ +Fetch the release manifests of `uv` versions in a WORKSPACE file. + +The repository downloads the manifest of every version in `versions` and +writes `dist_manifest.bzl`, which has the download URLs and checksums of every +archive in them as `DIST_MANIFESTS`, and {obj}`uv_register_toolchains` bound to +them; load that to create and register the toolchains: + +```starlark +load("@rules_python//python/uv:repositories.bzl", "uv_toolchains_manifest") + +uv_toolchains_manifest( + name = "uv_manifest", + versions = ["0.13.5"], +) + +load("@uv_manifest//:dist_manifest.bzl", "uv_register_toolchains") + +uv_register_toolchains(name = "uv") +``` + +The manifests are read as-is: which platforms get a toolchain, and from which +archives, is decided by {obj}`uv_register_toolchains`. Versions released from +different places can be fetched with one repository each and combined with the +`manifests` argument of {obj}`uv_register_toolchains`. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + attrs = { + "base_url": attr.string( + doc = """ +The base URL of the `uv` releases; the manifest of a version is downloaded from +`{base_url}/{version}/{manifest_filename}`. Defaults to the GitHub releases of +`uv`. +""", + default = DEFAULT_BASE_URL, + ), + "manifest_filename": attr.string( + doc = "The release manifest file name.", + default = DEFAULT_MANIFEST_FILENAME, + ), + "versions": attr.string_list( + doc = """ +The `uv` versions to fetch the manifests of. Defaults to the version the +{obj}`uv` extension uses by default. +""", + default = [DEFAULT_VERSION], + ), + } | AUTH_ATTRS, +) + +testing = struct( + generate_dist_manifest_bzl = _generate_dist_manifest_bzl, +) diff --git a/python/uv/repositories.bzl b/python/uv/repositories.bzl new file mode 100644 index 0000000000..e68678e11a --- /dev/null +++ b/python/uv/repositories.bzl @@ -0,0 +1,7 @@ +"""Repository rules and macros to set up `uv` in a WORKSPACE file.""" + +load("//python/uv/private:uv_register_toolchains.bzl", _uv_register_toolchains = "uv_register_toolchains") +load("//python/uv/private:uv_toolchains_manifest.bzl", _uv_toolchains_manifest = "uv_toolchains_manifest") + +uv_register_toolchains = _uv_register_toolchains +uv_toolchains_manifest = _uv_toolchains_manifest diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 21ef8fad16..9e909dc45b 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -147,6 +147,23 @@ rules_python_integration_test( py_main = "uv_lock_test.py", ) +rules_python_integration_test( + name = "uv_lock_workspace_test", + # Bazel 9 ignores WORKSPACE files, so only test versions that evaluate + # them. + bazel_versions = [ + "7.7.0", + "8.5.1", + ], + bzlmod = False, + py_deps = [ + "@pypiserver//pypiserver", + ":uv_lock_pypi_server_lib", + ], + py_main = "uv_lock_test.py", + workspace_path = "uv_lock", +) + rules_python_integration_test( name = "whl_library_test", py_main = "whl_library_test.py", diff --git a/tests/integration/bzlmod_lockfile/MODULE.bazel.lock b/tests/integration/bzlmod_lockfile/MODULE.bazel.lock index ce430569b8..f86bf246f1 100644 --- a/tests/integration/bzlmod_lockfile/MODULE.bazel.lock +++ b/tests/integration/bzlmod_lockfile/MODULE.bazel.lock @@ -259,7 +259,7 @@ }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "ftw8bFfcFDRCGiiWG42Oac7rJYWWcYAqsP4rkMPfpnI=", + "bzlTransitiveDigest": "a11TI1DxtsKzJJnFDIAhAoVNhub4iqECM+h0qTqOT68=", "usagesDigest": "UgEeJTkNLce2eABymcbLBebdHpF0D8qrT6A6PSESotA=", "recordedInputs": [ "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", @@ -823,8 +823,8 @@ "pycparser": { "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl": "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz": "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", - "https://files.pythonhosted.org/packages/90/11/0e6f11117525ff0eec40ebac3d313376f102df93ca44ad9e893ee85e4f89/pycparser-3.11-py3-none-any.whl": "sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80", - "https://files.pythonhosted.org/packages/da/a8/c5fdbeee588bb8ada9458774f43adf1bdd30bd59157055142183e769a024/pycparser-3.11.tar.gz": "sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc" + "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl": "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", + "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz": "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6" }, "pygments": { "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz": "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", @@ -871,8 +871,8 @@ "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz": "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63" }, "zipp": { - "https://files.pythonhosted.org/packages/b5/98/df615823cd9419131ce19fba00de53a663794369e198aade064a244b385d/zipp-4.1.1-py3-none-any.whl": "sha256:8979f52d874162f485ff2981e3891f3a3317b7a3dd43ff1e1775b9304f307a9c", - "https://files.pythonhosted.org/packages/dc/23/655a1802fe8041302c959774ca7c80b53bc24737ff3ef45cb50ef11bd96c/zipp-4.1.1.tar.gz": "sha256:7ebb7a44c021b29fd8dbd7cce6812d0d7b5b454521f93cc71af6ccd155aaa70b" + "https://files.pythonhosted.org/packages/18/05/ba1458dee0ac485b2a01071c1b88f5199fe878a485df482e10e1ef2205cc/zipp-4.1.2.tar.gz": "sha256:cf474b90b07ea932637e17c6b9479de2b2882900b0d311bdaf4f5c000a15a97f", + "https://files.pythonhosted.org/packages/b8/44/7bd2007260da11cccca29ba2d4c345f1287aa663db38ee7cefeabf6386a5/zipp-4.1.2-py3-none-any.whl": "sha256:b30cb6cd253122169c47de245ca24a86bac2a5d3802ba15bf7ba6414eb9e144d" } } }, diff --git a/tests/integration/integration_test.bzl b/tests/integration/integration_test.bzl index fec4a8962f..f8e8814a7e 100644 --- a/tests/integration/integration_test.bzl +++ b/tests/integration/integration_test.bzl @@ -50,6 +50,7 @@ def rules_python_integration_test( py_deps = None, py_data = None, bazel_versions = None, + env = None, **kwargs): """Runs a bazel-in-bazel integration test. @@ -58,7 +59,9 @@ def rules_python_integration_test( workspace_path: The directory name. Defaults to `name` without the `_test` suffix. bzlmod: bool, default True. If true, run with bzlmod enabled, otherwise - disable bzlmod. + disable bzlmod. Python test runners (see `py_main`) are told the + mode via the `RULES_PYTHON_INTEGRATION_TEST_BZLMOD` environment + variable, which `runner.TestCase` turns into Bazel flags. tags: Test tags. py_main: Optional `.py` file to run tests using. When specified, a python based test runner is used, and this source file is the main @@ -67,12 +70,20 @@ def rules_python_integration_test( py_data: Optional test runner data files to use for setup. bazel_versions: `list[str] | None`, the bazel versions to test. I not specified, defaults to all configured bazel versions. + env: `dict[str, str] | None`, additional environment variables for + the test. **kwargs: Passed to the upstream `bazel_integration_tests` rule. """ workspace_path = workspace_path or name.removesuffix("_test") py_deps = py_deps or [] py_data = py_data or [] tags = tags or [] + env = dict(env or {}) + if py_main: + env.setdefault( + "RULES_PYTHON_INTEGRATION_TEST_BZLMOD", + "1" if bzlmod else "0", + ) # Because glob expansion happens at loading time, the bazel-* symlinks # in the workspaces can recursively expand to tens-of-thousands of entries, @@ -109,6 +120,7 @@ def rules_python_integration_test( test_runner = test_runner, bazel_version = bazel_version, workspace_files = [name + "_workspace_files"], + env = env, # Override the tags so that the `manual` tag isn't applied. tags = tags + [ # These tests are very heavy weight, so much so that only a couple diff --git a/tests/integration/runner.py b/tests/integration/runner.py index 23e41f6cf1..fc85edfc91 100644 --- a/tests/integration/runner.py +++ b/tests/integration/runner.py @@ -101,19 +101,28 @@ def setUp(self): # If not present, it can't find some bash helpers in @bazel_tools "RUNFILES_DIR": os.environ["TEST_SRCDIR"], } + # Always set by `rules_python_integration_test()` for Python runners; + # "0" runs the inner Bazel in WORKSPACE mode. + self.bzlmod = os.environ["RULES_PYTHON_INTEGRATION_TEST_BZLMOD"] == "1" + # Flags inserted after the command of every Bazel invocation. + self.bazel_flags = ( + [] if self.bzlmod else ["--noenable_bzlmod", "--enable_workspace"] + ) def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult: """Run a bazel invocation. Args: *args: The args to pass to bazel; the leading `bazel` command is - added automatically + added automatically. The first value must be the Bazel command + (e.g. `build`), after which `self.bazel_flags` are inserted. check: True if the execution must succeed, False if failure - should raise an error. + should not raise an error. Returns: An `ExecuteResult` from running Bazel """ - cmd_args = [*self.bazel_cmd, *args] + command, *command_args = args + cmd_args = [*self.bazel_cmd, command, *self.bazel_flags, *command_args] env = self.bazel_env _logger.info("executing: %s", shlex.join(cmd_args)) cwd = self.repo_root diff --git a/tests/integration/uv_lock/MODULE.bazel b/tests/integration/uv_lock/MODULE.bazel index de44afb7f3..741b55bf86 100644 --- a/tests/integration/uv_lock/MODULE.bazel +++ b/tests/integration/uv_lock/MODULE.bazel @@ -11,8 +11,11 @@ local_path_override( python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.13") -uv = use_extension("@rules_python//python/uv:uv.bzl", "uv") +# The minimal setup: `rules_python` registers `@uv//:all` itself, so the root +# module only has to configure which version to download. +uv = use_extension( + "@rules_python//python/uv:uv.bzl", + "uv", + dev_dependency = True, +) uv.configure(version = "0.13.5") -use_repo(uv, "uv") - -register_toolchains("@uv//:all") diff --git a/tests/integration/uv_lock/README.md b/tests/integration/uv_lock/README.md index cce142852c..e4e130ba6a 100644 --- a/tests/integration/uv_lock/README.md +++ b/tests/integration/uv_lock/README.md @@ -1,10 +1,15 @@ # uv_lock integration test workspace This directory is a self-contained Bazel workspace used by the -`//tests/integration:uv_lock_test` integration test. +`//tests/integration:uv_lock_test` (bzlmod) and +`//tests/integration:uv_lock_workspace_test` (WORKSPACE) integration tests. It demonstrates how to use the `lock()` macro from `@rules_python//python/uv` -to pin requirements with `uv pip compile`. +to pin requirements with `uv pip compile`. `MODULE.bazel` configures the `uv` +toolchains via the `uv` module extension; `WORKSPACE` does the same via +`uv_toolchains_manifest()` and `uv_register_toolchains()`. The empty +`WORKSPACE.bzlmod` keeps Bazel 7 from evaluating `WORKSPACE` when bzlmod is +enabled. ## Targets @@ -62,3 +67,8 @@ bazel test //tests/integration:uv_lock_test_bazel_self \ --test_output=streamed \ --test_filter= ``` + +The WORKSPACE variant runs the same Python test cases with +`--noenable_bzlmod --enable_workspace`, e.g. +`//tests/integration:uv_lock_workspace_test_bazel_8.5.1`. It only exists for +Bazel 7.x and 8.x because Bazel 9 ignores `WORKSPACE` files. diff --git a/tests/integration/uv_lock/WORKSPACE b/tests/integration/uv_lock/WORKSPACE index e69de29bb2..abc8ce8483 100644 --- a/tests/integration/uv_lock/WORKSPACE +++ b/tests/integration/uv_lock/WORKSPACE @@ -0,0 +1,30 @@ +workspace(name = "uv_lock") + +local_repository( + name = "rules_python", + path = "../../..", +) + +load( + "@rules_python//python:repositories.bzl", + "py_repositories", + "python_register_toolchains", +) + +py_repositories() + +python_register_toolchains( + name = "python_3_13", + python_version = "3.13", +) + +load("@rules_python//python/uv:repositories.bzl", "uv_toolchains_manifest") + +uv_toolchains_manifest( + name = "uv_manifest", + versions = ["0.13.5"], +) + +load("@uv_manifest//:dist_manifest.bzl", "uv_register_toolchains") + +uv_register_toolchains(name = "uv") diff --git a/tests/integration/uv_lock/WORKSPACE.bzlmod b/tests/integration/uv_lock/WORKSPACE.bzlmod new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/uv/lock/lock_run_test.py b/tests/uv/lock/lock_run_test.py index d2d18e69f7..5e077ae709 100644 --- a/tests/uv/lock/lock_run_test.py +++ b/tests/uv/lock/lock_run_test.py @@ -9,6 +9,10 @@ rfiles = runfiles.Create() assert rfiles is not None, "Failed to create runfiles" +# The runfiles directory of the main repository: `_main` under bzlmod, the +# `workspace(name = ...)` value under WORKSPACE. Always set by `bazel test`. +_MAIN_REPO_DIR = os.environ["TEST_WORKSPACE"] + def _relative_rpath(path: str) -> Path: """Find file in runfiles, handling Windows .bat/.exe wrappers.""" @@ -17,7 +21,9 @@ def _relative_rpath(path: str) -> Path: # (e.g. a Python source file instead of the .exe launcher). exts = (".exe", ".bat", "") if os.name == "nt" else ("", ".exe", ".bat") for ext in exts: - p = (Path("_main") / "tests" / "uv" / "lock" / (path + ext)).as_posix() + # Refer to the main repository by its apparent name so the lookup + # works under both bzlmod and WORKSPACE. + p = (Path("rules_python") / "tests" / "uv" / "lock" / (path + ext)).as_posix() rpath = rfiles.Rlocation(p) if rpath: rp = Path(rpath) @@ -30,7 +36,7 @@ def _relative_rpath(path: str) -> Path: if runfiles_dir: exts = (".exe", ".bat", "") if os.name == "nt" else ("", ".bat", ".exe") for ext in exts: - rp = Path(runfiles_dir, "_main", "tests", "uv", "lock", path + ext) + rp = Path(runfiles_dir, _MAIN_REPO_DIR, "tests", "uv", "lock", path + ext) if rp.exists(): return rp diff --git a/tests/uv/lock/uv_with_runfiles.py b/tests/uv/lock/uv_with_runfiles.py index 4e38f3fdbb..1543db4f42 100644 --- a/tests/uv/lock/uv_with_runfiles.py +++ b/tests/uv/lock/uv_with_runfiles.py @@ -9,12 +9,14 @@ def main(): parser.add_argument("--output-file", type=Path, required=True) args, _ = parser.parse_known_args() root = runfiles.CreateOrRaise().root() + # Refer to the main repository by its apparent name so the paths resolve + # under both bzlmod (`_main`) and WORKSPACE (`rules_python`). for location, expected in [ - ("_main/uv_wrapper/symlink_payload.txt", "symlink payload\n"), + ("rules_python/uv_wrapper/symlink_payload.txt", "symlink payload\n"), ("uv_wrapper/root_symlink_payload.txt", "root symlink payload\n"), ]: assert (root / location).read_text() == expected, location - payload = root / "_main/tests/uv/lock/testdata/toolchain_payload.txt" + payload = root / "rules_python/tests/uv/lock/testdata/toolchain_payload.txt" args.output_file.write_bytes(payload.read_bytes()) diff --git a/tests/uv/toolchain/BUILD.bazel b/tests/uv/toolchain/BUILD.bazel index 137b4e041f..9017396ffe 100644 --- a/tests/uv/toolchain/BUILD.bazel +++ b/tests/uv/toolchain/BUILD.bazel @@ -1,14 +1,9 @@ load("//python:py_test.bzl", "py_test") -load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility - -# We only test this feature when `bzlmod` is enabled. -_TARGET_COMPATIBLE_WITH = [] if BZLMOD_ENABLED else ["@platforms//:incompatible"] genrule( name = "uv_help", outs = ["uv_help.txt"], cmd = "$(UV_BIN) --python-fetch manual --help >$@", - target_compatible_with = _TARGET_COMPATIBLE_WITH, toolchains = ["//python/uv:current_toolchain"], ) @@ -17,6 +12,5 @@ py_test( srcs = ["uv_help_test.py"], data = [":uv_help"], env = {"DATA": "$(rlocationpath :uv_help)"}, - target_compatible_with = _TARGET_COMPATIBLE_WITH, deps = ["//python/runfiles"], ) diff --git a/tests/uv/uv/uv_tests.bzl b/tests/uv/uv/uv_tests.bzl index d11bf065d8..e6250a4024 100644 --- a/tests/uv/uv/uv_tests.bzl +++ b/tests/uv/uv/uv_tests.bzl @@ -329,6 +329,50 @@ def _test_default_building(env): _tests.append(_test_default_building) +def _test_default_auth_applies_to_versions(env): + calls = [] + uv = _process_modules( + env, + module_ctx = _uv_mock_mctx( + _mod( + default = [ + _default( + base_url = "https://example.org", + manifest_filename = "manifest.json", + version = "1.0.0", + netrc = "~/.my_netrc", + auth_patterns = {"foo": "bar"}, + ), + _default( + platform = "linux", + compatible_with = ["@platforms//os:linux"], + ), + ], + configure = [ + _configure(), # use defaults, including auth + ], + ), + ), + uv_repository = lambda **kwargs: calls.append(kwargs), + ) + + uv.names().contains_exactly([ + "1_0_0_linux", + ]) + env.expect.that_collection(calls).contains_exactly([ + { + "auth_patterns": {"foo": "bar"}, + "name": "uv_1_0_0_linux", + "netrc": "~/.my_netrc", + "platform": "linux", + "sha256": "deadbeef", + "urls": ["https://example.org/1.0.0/linux"], + "version": "1.0.0", + }, + ]) + +_tests.append(_test_default_auth_applies_to_versions) + def _test_complex_configuring(env): calls = [] uv = _process_modules( diff --git a/tests/uv/uv_register_toolchains/BUILD.bazel b/tests/uv/uv_register_toolchains/BUILD.bazel new file mode 100644 index 0000000000..aac709120f --- /dev/null +++ b/tests/uv/uv_register_toolchains/BUILD.bazel @@ -0,0 +1,3 @@ +load(":uv_register_toolchains_tests.bzl", "uv_register_toolchains_test_suite") + +uv_register_toolchains_test_suite(name = "uv_register_toolchains_tests") diff --git a/tests/uv/uv_register_toolchains/uv_register_toolchains_tests.bzl b/tests/uv/uv_register_toolchains/uv_register_toolchains_tests.bzl new file mode 100644 index 0000000000..9dca59b5c7 --- /dev/null +++ b/tests/uv/uv_register_toolchains/uv_register_toolchains_tests.bzl @@ -0,0 +1,270 @@ +"""Tests for the WORKSPACE `uv_register_toolchains` macro.""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/uv/private:toolchain_types.bzl", "UV_TOOLCHAIN_TYPE") # buildifier: disable=bzl-visibility +load("//python/uv/private:uv_defaults.bzl", "DEFAULT_PLATFORMS", "DEFAULT_VERSION") # buildifier: disable=bzl-visibility +load("//python/uv/private:uv_register_toolchains.bzl", "testing") # buildifier: disable=bzl-visibility + +_tests = [] + +def _manifest(version, platforms): + """The `dist_manifest.bzl` data of one version with archives for `platforms`.""" + return { + platform: { + "sha256": "sha256-of-" + platform, + "urls": ["https://mirror.example.com/{}/uv-{}.tar.gz".format(version, platform)], + } + for platform in platforms + } + +def _manifests(versions): + """The `dist_manifest.bzl` data of `(version, platforms)` pairs, in that order. + + A list rather than a dict literal, because buildifier sorts the keys of + dict literals and the order of the manifests is significant. + """ + return {version: _manifest(version, platforms) for version, platforms in versions} + +def _register(**kwargs): + """Call `uv_register_toolchains` and record what it would create. + + Args: + **kwargs: overrides of the `uv_register_toolchains` arguments. + + Returns: + {type}`struct` with the recorded `uv_repository` calls as + `repositories`, the hub repository call as `hub` and the patterns + passed to `register_toolchains` as `registered`. + """ + repositories = [] + hubs = [] + registered = [] + defaults = dict( + name = "uv", + manifests = None, + version = None, + configure = None, + netrc = None, + auth_patterns = None, + uv_repository = lambda **kwargs: repositories.append(kwargs), + hub_repo = lambda **kwargs: hubs.append(kwargs), + register_toolchains = lambda pattern: registered.append(pattern), + ) + testing.uv_register_toolchains(**(defaults | kwargs)) + return struct( + repositories = repositories, + hub = hubs[0], + registered = registered, + ) + +def _test_defaults(env): + # The manifest has an archive for a platform that is not a default one, + # and lacks the other default platforms. + result = _register( + manifests = { + DEFAULT_VERSION: _manifest(DEFAULT_VERSION, [ + "aarch64-apple-darwin", + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-gnu", + ]), + }, + ) + + # Only the default platforms in the manifest get toolchains, in platform + # order, with the archives the manifest advertises. + env.expect.that_collection(result.repositories).contains_exactly([ + { + "name": "uv_0_13_5_aarch64_apple_darwin", + "platform": "aarch64-apple-darwin", + "sha256": "sha256-of-aarch64-apple-darwin", + "urls": ["https://mirror.example.com/0.13.5/uv-aarch64-apple-darwin.tar.gz"], + "version": "0.13.5", + }, + { + "name": "uv_0_13_5_x86_64_unknown_linux_gnu", + "platform": "x86_64-unknown-linux-gnu", + "sha256": "sha256-of-x86_64-unknown-linux-gnu", + "urls": ["https://mirror.example.com/0.13.5/uv-x86_64-unknown-linux-gnu.tar.gz"], + "version": "0.13.5", + }, + ]).in_order() + env.expect.that_dict(result.hub).contains_exactly({ + "name": "uv", + "toolchain_compatible_with": { + "0_13_5_aarch64_apple_darwin": DEFAULT_PLATFORMS["aarch64-apple-darwin"].compatible_with, + "0_13_5_x86_64_unknown_linux_gnu": DEFAULT_PLATFORMS["x86_64-unknown-linux-gnu"].compatible_with, + }, + "toolchain_implementations": { + "0_13_5_aarch64_apple_darwin": "@uv_0_13_5_aarch64_apple_darwin//:uv_toolchain", + "0_13_5_x86_64_unknown_linux_gnu": "@uv_0_13_5_x86_64_unknown_linux_gnu//:uv_toolchain", + }, + "toolchain_names": [ + "0_13_5_aarch64_apple_darwin", + "0_13_5_x86_64_unknown_linux_gnu", + ], + "toolchain_target_settings": {}, + "toolchain_type": str(UV_TOOLCHAIN_TYPE), + }) + env.expect.that_collection(result.registered).contains_exactly(["@uv//:all"]) + +_tests.append(_test_defaults) + +def _test_every_manifest_version_by_default(env): + # One hub holds the toolchains of all the fetched versions, in manifest + # order. + result = _register( + name = "my_uv", + manifests = _manifests([ + ("0.6.5", ["x86_64-unknown-linux-gnu"]), + ("0.13.5", ["aarch64-apple-darwin", "x86_64-unknown-linux-gnu"]), + ]), + ) + + env.expect.that_collection([r["name"] for r in result.repositories]).contains_exactly([ + "my_uv_0_6_5_x86_64_unknown_linux_gnu", + "my_uv_0_13_5_aarch64_apple_darwin", + "my_uv_0_13_5_x86_64_unknown_linux_gnu", + ]).in_order() + env.expect.that_collection(result.hub["toolchain_names"]).contains_exactly([ + "0_6_5_x86_64_unknown_linux_gnu", + "0_13_5_aarch64_apple_darwin", + "0_13_5_x86_64_unknown_linux_gnu", + ]).in_order() + env.expect.that_collection(result.registered).contains_exactly(["@my_uv//:all"]) + +_tests.append(_test_every_manifest_version_by_default) + +def _test_version_defaults_to_first_manifest(env): + result = _register( + manifests = _manifests([ + ("0.6.5", ["x86_64-unknown-linux-gnu"]), + ("0.13.5", ["x86_64-unknown-linux-gnu"]), + ]), + configure = [ + # Applies to 0.6.5, the first manifest version. + {"platform": "x86_64-unknown-linux-gnu", "target_settings": ["@//:use_old_uv"]}, + {"version": "0.13.5"}, + ], + ) + + env.expect.that_collection(result.hub["toolchain_names"]).contains_exactly([ + "0_6_5_x86_64_unknown_linux_gnu", + "0_13_5_x86_64_unknown_linux_gnu", + ]).in_order() + env.expect.that_dict(result.hub["toolchain_target_settings"]).contains_exactly({ + "0_6_5_x86_64_unknown_linux_gnu": ["@//:use_old_uv"], + }) + +_tests.append(_test_version_defaults_to_first_manifest) + +def _test_configure(env): + result = _register( + manifests = { + DEFAULT_VERSION: _manifest(DEFAULT_VERSION, [ + "aarch64-apple-darwin", + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-gnu", + ]), + }, + netrc = "/home/user/.netrc", + configure = [ + # Removes a default platform that the manifest has. + {"platform": "aarch64-apple-darwin"}, + # Adds a platform the manifest has; labels are passed on as + # strings. + { + "compatible_with": [Label("@platforms//os:linux"), "@platforms//cpu:aarch64"], + "platform": "aarch64-unknown-linux-musl", + "target_settings": [Label("//:is_musl")], + }, + # A version without a manifest, with explicit archives: only the + # listed platform, with its own auth. + { + "auth_patterns": {"mirror.example.com": "Bearer "}, + "platform": "x86_64-apple-darwin", + "sha256": "deadbeef", + "urls": ["https://mirror.example.com/uv-0.6.5-x86_64-apple-darwin.tar.gz"], + "version": "0.6.5", + }, + ], + ) + + # The default auth applies to every version, like `uv.default`; added + # platforms come after the default ones. + env.expect.that_collection(result.repositories).contains_exactly([ + { + "name": "uv_0_13_5_x86_64_unknown_linux_gnu", + "netrc": "/home/user/.netrc", + "platform": "x86_64-unknown-linux-gnu", + "sha256": "sha256-of-x86_64-unknown-linux-gnu", + "urls": ["https://mirror.example.com/0.13.5/uv-x86_64-unknown-linux-gnu.tar.gz"], + "version": "0.13.5", + }, + { + "name": "uv_0_13_5_aarch64_unknown_linux_musl", + "netrc": "/home/user/.netrc", + "platform": "aarch64-unknown-linux-musl", + "sha256": "sha256-of-aarch64-unknown-linux-musl", + "urls": ["https://mirror.example.com/0.13.5/uv-aarch64-unknown-linux-musl.tar.gz"], + "version": "0.13.5", + }, + { + "auth_patterns": {"mirror.example.com": "Bearer "}, + "name": "uv_0_6_5_x86_64_apple_darwin", + "netrc": "/home/user/.netrc", + "platform": "x86_64-apple-darwin", + "sha256": "deadbeef", + "urls": ["https://mirror.example.com/uv-0.6.5-x86_64-apple-darwin.tar.gz"], + "version": "0.6.5", + }, + ]).in_order() + env.expect.that_dict(result.hub["toolchain_compatible_with"]).contains_exactly({ + "0_13_5_aarch64_unknown_linux_musl": [ + str(Label("@platforms//os:linux")), + "@platforms//cpu:aarch64", + ], + "0_13_5_x86_64_unknown_linux_gnu": DEFAULT_PLATFORMS["x86_64-unknown-linux-gnu"].compatible_with, + "0_6_5_x86_64_apple_darwin": DEFAULT_PLATFORMS["x86_64-apple-darwin"].compatible_with, + }) + env.expect.that_dict(result.hub["toolchain_target_settings"]).contains_exactly({ + "0_13_5_aarch64_unknown_linux_musl": [str(Label("//:is_musl"))], + }) + +_tests.append(_test_configure) + +def _test_without_manifests(env): + result = _register( + configure = [ + { + "platform": "x86_64-unknown-linux-gnu", + "sha256": "deadbeef", + "urls": ["https://mirror.example.com/uv.tar.gz"], + }, + ], + ) + + env.expect.that_collection(result.repositories).contains_exactly([ + { + "name": "uv_0_13_5_x86_64_unknown_linux_gnu", + "platform": "x86_64-unknown-linux-gnu", + "sha256": "deadbeef", + "urls": ["https://mirror.example.com/uv.tar.gz"], + "version": DEFAULT_VERSION, + }, + ]) + env.expect.that_collection(result.hub["toolchain_names"]).contains_exactly([ + "0_13_5_x86_64_unknown_linux_gnu", + ]) + +_tests.append(_test_without_manifests) + +def uv_register_toolchains_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite( + name = name, + basic_tests = _tests, + ) diff --git a/tests/uv/uv_toolchains_manifest/BUILD.bazel b/tests/uv/uv_toolchains_manifest/BUILD.bazel new file mode 100644 index 0000000000..34096b22f6 --- /dev/null +++ b/tests/uv/uv_toolchains_manifest/BUILD.bazel @@ -0,0 +1,3 @@ +load(":uv_toolchains_manifest_tests.bzl", "uv_toolchains_manifest_test_suite") + +uv_toolchains_manifest_test_suite(name = "uv_toolchains_manifest_tests") diff --git a/tests/uv/uv_toolchains_manifest/uv_toolchains_manifest_tests.bzl b/tests/uv/uv_toolchains_manifest/uv_toolchains_manifest_tests.bzl new file mode 100644 index 0000000000..00cbe9b2ef --- /dev/null +++ b/tests/uv/uv_toolchains_manifest/uv_toolchains_manifest_tests.bzl @@ -0,0 +1,142 @@ +"""Tests for the WORKSPACE `uv_toolchains_manifest` repository rule.""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/uv/private:dist_manifest.bzl", "DEFAULT_MANIFEST_FILENAME") # buildifier: disable=bzl-visibility +load("//python/uv/private:uv_defaults.bzl", "DEFAULT_BASE_URL", "DEFAULT_VERSION") # buildifier: disable=bzl-visibility +load("//python/uv/private:uv_toolchains_manifest.bzl", "testing") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +# The platforms the fake release manifest has archives for. +_RELEASED = ["aarch64-apple-darwin", "x86_64-unknown-linux-gnu"] + +def _manifest(): + """A fake cargo-dist manifest with inline checksums (uv >= 0.11).""" + return json.encode({ + "artifacts": { + "uv-{}.tar.gz".format(platform): { + "checksums": {"sha256": "sha256-of-" + platform}, + "kind": "executable-zip", + "target_triples": [platform], + } + for platform in _RELEASED + }, + "releases": [{ + "hosting": { + "simple": { + "download_url": "https://mirror.example.com/archives", + }, + }, + }], + }) + +def _generate(*, manifest_urls, **kwargs): + """Call `generate_dist_manifest_bzl` with a mocked repository context. + + Args: + manifest_urls: {type}`list[str]` the manifest URLs the mock serves; + any other download fails. + **kwargs: overrides of the `generate_dist_manifest_bzl` arguments. + + Returns: + {type}`struct` with the generated `content` and the `downloads` that + were requested. + """ + downloads = [] + + def _download(rctx, url, *_): + _ = rctx # @unused + downloads.append(url) + return struct(success = True, wait = lambda: struct(success = True)) + + rctx = mocks.rctx( + mock_downloads = {url: _download for url in manifest_urls}, + mock_files = { + kwargs.get("manifest_filename", DEFAULT_MANIFEST_FILENAME): _manifest(), + }, + ) + defaults = dict( + name = "uv_manifest", + versions = [DEFAULT_VERSION], + base_url = DEFAULT_BASE_URL, + manifest_filename = DEFAULT_MANIFEST_FILENAME, + netrc = "", + auth_patterns = {}, + get_auth = lambda *_, **__: None, + ) + content = testing.generate_dist_manifest_bzl(rctx, **(defaults | kwargs)) + return struct( + content = content, + downloads = downloads, + ) + +def _test_defaults(env): + manifest_url = "{}/{}/{}".format(DEFAULT_BASE_URL, DEFAULT_VERSION, DEFAULT_MANIFEST_FILENAME) + result = _generate(manifest_urls = [manifest_url]) + + env.expect.that_collection(result.downloads).contains_exactly([manifest_url]) + + # Every archive of the manifest is written out, with the download URL the + # manifest advertises, and `uv_register_toolchains` is bound to them. + env.expect.that_str(result.content).contains("""\ +DIST_MANIFESTS = { + "0.13.5": { + "aarch64-apple-darwin": { + "sha256": "sha256-of-aarch64-apple-darwin", + "urls": [ + "https://mirror.example.com/archives/uv-aarch64-apple-darwin.tar.gz" + ] + }, + "x86_64-unknown-linux-gnu": { + "sha256": "sha256-of-x86_64-unknown-linux-gnu", + "urls": [ + "https://mirror.example.com/archives/uv-x86_64-unknown-linux-gnu.tar.gz" + ] + } + } +} +""") + env.expect.that_str(result.content).contains( + 'load("@rules_python//python/uv:repositories.bzl", _uv_register_toolchains = "uv_register_toolchains")', + ) + env.expect.that_str(result.content).contains("def uv_register_toolchains(**kwargs):") + env.expect.that_str(result.content).contains( + 'kwargs["manifests"] = DIST_MANIFESTS | (kwargs.get("manifests") or {})', + ) + env.expect.that_str(result.content).contains('uv_toolchains_manifest(name = "uv_manifest")') + +_tests.append(_test_defaults) + +def _test_versions_from_a_mirror(env): + manifest_urls = [ + "https://mirror.example.com/uv/{}/{}".format(version, "manifest.json") + for version in ["0.13.5", "0.6.5"] + ] + result = _generate( + manifest_urls = manifest_urls, + versions = ["0.13.5", "0.6.5"], + base_url = "https://mirror.example.com/uv", + manifest_filename = "manifest.json", + ) + + # One manifest per version, in `versions` order. + env.expect.that_collection(result.downloads).contains_exactly(manifest_urls).in_order() + env.expect.that_str(result.content).contains('\n "0.13.5": {\n') + env.expect.that_str(result.content).contains('\n "0.6.5": {\n') + env.expect.that_bool( + result.content.find('"0.13.5"') < result.content.find('"0.6.5"'), + ).equals(True) + +_tests.append(_test_versions_from_a_mirror) + +def uv_toolchains_manifest_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite( + name = name, + basic_tests = _tests, + )