From a00311a63053a3aaaa1f4c7eb0278eda285ed37c Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Fri, 2 Oct 2026 10:14:21 +0200 Subject: [PATCH 1/4] =?UTF-8?q?chore:=20support=20Python=203.13=E2=80=933.?= =?UTF-8?q?15?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci-gpu.yml | 9 ++++---- .github/workflows/ci.yml | 4 ++++ .github/workflows/publish.yml | 4 ++++ .gitignore | 1 + ci/min-constraints.py | 14 ++++++++++++ pyproject.toml | 25 +++++++++++---------- src/testing/fast_array_utils/_array_type.py | 12 +++------- src/testing/fast_array_utils/pytest.py | 2 +- tests/test_stats.py | 21 ++--------------- 9 files changed, 47 insertions(+), 45 deletions(-) create mode 100644 ci/min-constraints.py diff --git a/.github/workflows/ci-gpu.yml b/.github/workflows/ci-gpu.yml index debd039a..1d3a1f59 100644 --- a/.github/workflows/ci-gpu.yml +++ b/.github/workflows/ci-gpu.yml @@ -9,6 +9,10 @@ on: - opened - synchronize +defaults: + run: + shell: bash + env: PYTEST_ADDOPTS: "-v --color=yes" FORCE_COLOR: "1" @@ -35,9 +39,6 @@ jobs: pull-requests: write # for removing the label runs-on: "cirun-aws-gpu--${{ github.run_id }}" timeout-minutes: 30 - defaults: - run: - shell: bash -el {0} steps: - uses: actions/checkout@v7 with: @@ -46,7 +47,7 @@ jobs: run: nvidia-smi - uses: actions/setup-python@v7.0.0 with: - python-version: "3.12" + python-version: "3.13" - uses: astral-sh/setup-uv@v10.1.0 with: enable-cache: true # "auto" is `false` on non-GitHub runners diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba04d644..573ce34d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,10 @@ on: branches: [main] pull_request: +defaults: + run: + shell: bash + env: PYTEST_ADDOPTS: "-v --color=yes" FORCE_COLOR: "1" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 513b11cd..d75e04cf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,6 +4,10 @@ on: release: types: [published] +defaults: + run: + shell: bash + jobs: publish: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 09ee0929..33a894a1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ _version.py # IDEs /.idea/ +/ci/min-constraints.txt diff --git a/ci/min-constraints.py b/ci/min-constraints.py new file mode 100644 index 00000000..d64f1b28 --- /dev/null +++ b/ci/min-constraints.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: MPL-2.0 +"""Pin every `>=` lower bound of the dependencies and extras (not test deps) to `==`.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + + +HERE = Path(__file__).parent + +project = tomllib.loads((HERE.parent / "pyproject.toml").read_text())["project"] +deps: list[str] = [*project["dependencies"], *(d for ds in project["optional-dependencies"].values() for d in ds)] +(HERE / "min-constraints.txt").write_text("".join(f"{d.replace('>=', '==')}\n" for d in deps if ">=" in d)) diff --git a/pyproject.toml b/pyproject.toml index 8b5fb1a6..15d20a71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,22 +13,21 @@ license = "MPL-2.0" authors = [ { name = "Philipp A.", email = "flying-sheep@web.de" }, ] -requires-python = ">=3.12" +requires-python = ">=3.13" classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3.15", ] dynamic = [ "description", "readme", "version" ] -dependencies = [ "array-api-compat", "numpy>=2" ] -optional-dependencies.accel = [ "numba>=0.57" ] +dependencies = [ "array-api-compat", "numpy>=2.1" ] +optional-dependencies.accel = [ "numba>=0.61" ] optional-dependencies.dask = [ "dask>=2023.6.1" ] optional-dependencies.full = [ "fast-array-utils[accel,dask,sparse]", "h5py", "zarr" ] -optional-dependencies.sparse = [ "scipy>=1.13" ] +optional-dependencies.sparse = [ "scipy>=1.15" ] optional-dependencies.testing = [ "packaging" ] urls."Issue Tracker" = "https://github.com/scverse/fast-array-utils/issues" urls."Source Code" = "https://github.com/scverse/fast-array-utils" @@ -92,16 +91,17 @@ envs.hatch-check-types.dependency-groups = [ "typecheck", "test" ] envs.hatch-test.extra-dependencies = [ "ipycytoscape", "ipykernel" ] envs.hatch-test.env-vars.CODSPEED_PROFILE_FOLDER = "test-data/codspeed" envs.hatch-test.matrix = [ - { python = [ "3.14", "3.12" ], extras = [ "full", "min" ] }, - { python = [ "3.12" ], extras = [ "full" ], resolution = [ "lowest" ] }, + { python = [ "3.15", "3.13" ], extras = [ "full", "min" ] }, + { python = [ "3.13" ], extras = [ "full" ], resolution = [ "lowest" ] }, ] envs.hatch-test.overrides.matrix.extras.dependency-groups = [ { value = "test", if = [ "full" ] }, ] -envs.hatch-test.overrides.matrix.resolution.dependencies = [ - { value = "numpy==2", if = [ "lowest" ] }, - { value = "dask==2023.6.1", if = [ "lowest" ] }, - { value = "scipy==1.13.0", if = [ "lowest" ] }, +envs.hatch-test.overrides.matrix.resolution.pre-install-commands = [ + { value = "python ci/min-constraints.py", if = [ "lowest" ] }, +] +envs.hatch-test.overrides.matrix.resolution.env-vars = [ + { key = "UV_CONSTRAINT", value = "ci/min-constraints.txt", if = [ "lowest" ] }, ] envs.hatch-test.default-args = [] envs.hatch-test.dependency-groups = [ "test-min" ] @@ -129,7 +129,8 @@ lint.ignore = [ "S101", # asserts are fine "TID252", # relative imports are fine ] -# No __init__.py in docs +# No __init__.py in scripts and docs +lint.per-file-ignores."ci/**/*.py" = [ "INP001" ] lint.per-file-ignores."docs/**/*.py" = [ "INP001" ] # Shadows builtins like `sum` lint.per-file-ignores."src/**/stats/*.py" = [ "A001", "A004" ] diff --git a/src/testing/fast_array_utils/_array_type.py b/src/testing/fast_array_utils/_array_type.py index 7105c7dd..66fdaf2e 100644 --- a/src/testing/fast_array_utils/_array_type.py +++ b/src/testing/fast_array_utils/_array_type.py @@ -4,7 +4,6 @@ from __future__ import annotations import enum -import sys from dataclasses import KW_ONLY, dataclass, field from functools import cached_property, partial from importlib.metadata import version @@ -74,17 +73,12 @@ class ConversionContext: hdf5_file: h5py.File # TODO(flying-sheep): ReadOnly -if TYPE_CHECKING or sys.version_info >= (3, 13): - # TODO(flying-sheep): move vars into type parameter syntax # noqa: TD003 - Arr = TypeVar("Arr", bound="ExtendedArray", default="Array") - Inner = TypeVar("Inner", bound="ArrayType[InnerArray, None] | None", default="Any") -else: - Arr = TypeVar("Arr") - Inner = TypeVar("Inner") +Arr = TypeVar("Arr", bound="ExtendedArray", default="Array") +Inner = TypeVar("Inner", bound="ArrayType[InnerArray, None] | None", default="Any") @dataclass(frozen=True) -class ArrayType(Generic[Arr, Inner]): # noqa: UP046 +class ArrayType(Generic[Arr, Inner]): """Supported array type with methods for conversion and random generation. Examples diff --git a/src/testing/fast_array_utils/pytest.py b/src/testing/fast_array_utils/pytest.py index 2500922e..6e93a223 100644 --- a/src/testing/fast_array_utils/pytest.py +++ b/src/testing/fast_array_utils/pytest.py @@ -159,7 +159,7 @@ def hdf5_file(self) -> h5py.File: # type: ignore[override] @pytest.fixture -def tmp_hdf5_file(tmp_path: Path) -> Generator[h5py.File, None, None]: +def tmp_hdf5_file(tmp_path: Path) -> Generator[h5py.File]: import h5py with h5py.File(tmp_path / "test.h5", "w") as f: diff --git a/tests/test_stats.py b/tests/test_stats.py index 4d5afc68..d039fa3f 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -3,7 +3,6 @@ from concurrent.futures import ThreadPoolExecutor from functools import partial -from importlib.metadata import version from importlib.util import find_spec from pathlib import Path from typing import TYPE_CHECKING, cast @@ -12,7 +11,6 @@ import pytest import scipy.sparse as sps from numpy.exceptions import AxisError -from packaging.version import Version from fast_array_utils import stats, types from testing.fast_array_utils import SUPPORTED_TYPES, Flags @@ -43,11 +41,6 @@ ATS_CUPY_SPARSE = {at for at in SUPPORTED_TYPES if "cupyx.scipy" in str(at)} -def _xfail_if_old_scipy(array_type: ArrayType[Any], ndim: Literal[1, 2]) -> pytest.MarkDecorator: - cond = ndim == 1 and bool(array_type.flags & Flags.Sparse) and Version(version("scipy")) < Version("1.14") - return pytest.mark.xfail(cond, reason="Sparse matrices don’t support 1d arrays") - - @pytest.fixture( scope="session", params=[ @@ -145,10 +138,7 @@ def pbmc64k_reduced_raw() -> sps.csr_array[np.float32]: @pytest.mark.array_type(skip={*ATS_SPARSE_DS, Flags.Matrix}) @pytest.mark.parametrize("func", STAT_FUNCS) @pytest.mark.parametrize(("ndim", "axis"), [(1, 0), (2, 3), (2, -1)], ids=["1d-ax0", "2d-ax3", "2d-axneg"]) -def test_ndim_error( - request: pytest.FixtureRequest, array_type: ArrayType[Array], func: StatFunNoDtype, ndim: Literal[1, 2], axis: Literal[0, 1] | None -) -> None: - request.applymarker(_xfail_if_old_scipy(array_type, ndim)) +def test_ndim_error(array_type: ArrayType[Array], func: StatFunNoDtype, ndim: Literal[1, 2], axis: Literal[0, 1] | None) -> None: check_ndim(array_type, ndim) # not using the fixture because we don’t need to test multiple dtypes np_arr = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32) @@ -163,15 +153,12 @@ def test_ndim_error( @pytest.mark.array_type(skip=ATS_SPARSE_DS) def test_sum( *, - request: pytest.FixtureRequest, array_type: ArrayType[CpuArray | GpuArray | DiskArray | types.DaskArray], dtype_in: type[DTypeIn], dtype_arg: type[DTypeOut] | None, axis: Literal[0, 1] | None, np_arr: NDArray[DTypeIn], - ndim: Literal[1, 2], ) -> None: - request.applymarker(_xfail_if_old_scipy(array_type, ndim)) if np.dtype(dtype_arg).kind in "iu" and (array_type.flags & Flags.Gpu) and (array_type.flags & Flags.Sparse): pytest.skip("GPU sparse matrices don’t support int dtypes") arr = array_type(np_arr.copy()) @@ -242,8 +229,7 @@ def test_dask_shapes(array_type: ArrayType[types.DaskArray], axis: Literal[0, 1] @pytest.mark.array_type(skip=ATS_SPARSE_DS) -def test_mean(request: pytest.FixtureRequest, array_type: ArrayType[Array], axis: Literal[0, 1] | None, np_arr: NDArray[DTypeIn], ndim: Literal[1, 2]) -> None: - request.applymarker(_xfail_if_old_scipy(array_type, ndim)) +def test_mean(array_type: ArrayType[Array], axis: Literal[0, 1] | None, np_arr: NDArray[DTypeIn]) -> None: arr = array_type(np_arr) result = stats.mean(arr, axis=axis) # type: ignore[arg-type] # https://github.com/python/mypy/issues/16777 @@ -258,13 +244,10 @@ def test_mean(request: pytest.FixtureRequest, array_type: ArrayType[Array], axis @pytest.mark.array_type(skip=Flags.Disk) def test_mean_var( - request: pytest.FixtureRequest, array_type: ArrayType[CpuArray | GpuArray | types.DaskArray], axis: Literal[0, 1] | None, np_arr: NDArray[DTypeIn], - ndim: Literal[1, 2], ) -> None: - request.applymarker(_xfail_if_old_scipy(array_type, ndim)) arr = array_type(np_arr) mean, var = stats.mean_var(arr, axis=axis, correction=1) From e6a86fa82acc51d66f934f21dc039c21733ebe10 Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Fri, 2 Oct 2026 10:23:55 +0200 Subject: [PATCH 2/4] syntax --- src/testing/fast_array_utils/__init__.py | 28 ++++++++++----------- src/testing/fast_array_utils/_array_type.py | 8 ++---- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/testing/fast_array_utils/__init__.py b/src/testing/fast_array_utils/__init__.py index e2b78761..86e2466f 100644 --- a/src/testing/fast_array_utils/__init__.py +++ b/src/testing/fast_array_utils/__init__.py @@ -6,7 +6,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING from ._array_type import ArrayType, ConversionContext, Flags, random_mat @@ -28,25 +28,23 @@ ] -_TP_MEM = ( +_TP_MEM: tuple[ArrayType[CpuArray | GpuArray, None], ...] = ( ArrayType("numpy", "ndarray", Flags.Any), ArrayType("cupy", "ndarray", Flags.Any | Flags.Gpu), - *(ArrayType("scipy.sparse", n, Flags.Any | Flags.Sparse) for n in ["csr_array", "csc_array"]), - *( - ArrayType(mod, n, Flags.Any | Flags.Sparse | Flags.Matrix | flags) - for n in ["csr_matrix", "csc_matrix"] - for (mod, flags) in [("scipy.sparse", Flags(0)), ("cupyx.scipy.sparse", Flags.Gpu)] - ), + ArrayType("scipy.sparse", "csr_array", Flags.Any | Flags.Sparse), + ArrayType("scipy.sparse", "csc_array", Flags.Any | Flags.Sparse), + ArrayType("scipy.sparse", "csr_matrix", Flags.Any | Flags.Sparse | Flags.Matrix), + ArrayType("scipy.sparse", "csc_matrix", Flags.Any | Flags.Sparse | Flags.Matrix), + ArrayType("cupyx.scipy.sparse", "csr_matrix", Flags.Any | Flags.Sparse | Flags.Matrix | Flags.Gpu), + ArrayType("cupyx.scipy.sparse", "csc_matrix", Flags.Any | Flags.Sparse | Flags.Matrix | Flags.Gpu), ) -_TP_DASK = tuple( - ArrayType("dask.array", "Array", Flags.Dask | t.flags, inner=t) # type: ignore[type-var] - for t in cast("tuple[ArrayType[CpuArray | GpuArray, None], ...]", _TP_MEM) +_TP_DASK = tuple(ArrayType("dask.array", "Array", Flags.Dask | t.flags, inner=t) for t in _TP_MEM) +_TP_DISK_DENSE: tuple[ArrayType[DiskArray, None], ...] = ( + ArrayType("h5py", "Dataset", Flags.Any | Flags.Disk), + ArrayType("zarr", "Array", Flags.Any | Flags.Disk), ) -_TP_DISK_DENSE = tuple(ArrayType(m, n, Flags.Any | Flags.Disk) for m, n in [("h5py", "Dataset"), ("zarr", "Array")]) _TP_DISK_SPARSE = tuple( - ArrayType("anndata.abc", n, Flags.Any | Flags.Disk | Flags.Sparse, inner=t) # type: ignore[type-var] - for t in cast("tuple[ArrayType[DiskArray, None], ...]", _TP_DISK_DENSE) - for n in ["CSRDataset", "CSCDataset"] + ArrayType("anndata.abc", n, Flags.Any | Flags.Disk | Flags.Sparse, inner=t) for t in _TP_DISK_DENSE for n in ["CSRDataset", "CSCDataset"] ) SUPPORTED_TYPES: tuple[ArrayType, ...] = (*_TP_MEM, *_TP_DASK, *_TP_DISK_DENSE, *_TP_DISK_SPARSE) diff --git a/src/testing/fast_array_utils/_array_type.py b/src/testing/fast_array_utils/_array_type.py index 66fdaf2e..aa52f1b9 100644 --- a/src/testing/fast_array_utils/_array_type.py +++ b/src/testing/fast_array_utils/_array_type.py @@ -7,7 +7,7 @@ from dataclasses import KW_ONLY, dataclass, field from functools import cached_property, partial from importlib.metadata import version -from typing import TYPE_CHECKING, Generic, TypedDict, TypeVar, cast +from typing import TYPE_CHECKING, TypedDict, cast import numpy as np from packaging.version import Version @@ -73,12 +73,8 @@ class ConversionContext: hdf5_file: h5py.File # TODO(flying-sheep): ReadOnly -Arr = TypeVar("Arr", bound="ExtendedArray", default="Array") -Inner = TypeVar("Inner", bound="ArrayType[InnerArray, None] | None", default="Any") - - @dataclass(frozen=True) -class ArrayType(Generic[Arr, Inner]): +class ArrayType[Arr: ExtendedArray = Array, Inner: ArrayType[InnerArray, None] | None = ArrayType[InnerArray, None] | None]: """Supported array type with methods for conversion and random generation. Examples From 4a00e35fc1bb1fb02bd38a9cc3ef41915be67922 Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Fri, 2 Oct 2026 10:28:59 +0200 Subject: [PATCH 3/4] latest --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2adeb83..22f3bba3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ jobs: outputs: envs: ${{ steps.get-envs.outputs.envs }} pythons: ${{ steps.get-pythons.outputs.pythons }} + latest-full: ${{ steps.get-pythons.outputs.latest-full }} steps: - uses: actions/checkout@v7 with: { fetch-depth: 0, filter: "blob:none" } @@ -40,6 +41,8 @@ jobs: run: | PYTHONS_JSON=$(echo "$ENVS_JSON" | jq -c 'map(.python) | unique') echo "pythons=$PYTHONS_JSON" | tee $GITHUB_OUTPUT + LATEST_FULL_JSON=$(echo "$ENVS_JSON" | jq -c 'map(select(.name | endswith("-full"))) | max_by(.python | split(".") | map(tonumber))') + echo "latest-full=$LATEST_FULL_JSON" | tee -a $GITHUB_OUTPUT test: name: Tests needs: get-environments @@ -51,9 +54,7 @@ jobs: env: ${{ fromJSON(needs.get-environments.outputs.envs) }} os: [ubuntu-latest] include: - - env: - name: hatch-test.py3.14-full - python: "3.14" + - env: ${{ fromJSON(needs.get-environments.outputs.latest-full) }} os: macos-latest steps: - uses: actions/checkout@v7 From 9fc2cfec33af31787f5403b7db119824fd622cd4 Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Fri, 2 Oct 2026 11:07:53 +0200 Subject: [PATCH 4/4] h5py hack --- .vscode/settings.json | 5 +---- ci/h5py-nightly.txt | 1 + pyproject.toml | 7 +++++++ 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 ci/h5py-nightly.txt diff --git a/.vscode/settings.json b/.vscode/settings.json index f1e49819..fa0dd075 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,10 +1,7 @@ { - "[toml][json][jsonc][python]": { + "[json][jsonc][python]": { "editor.formatOnSave": true, }, - "[toml]": { - "editor.defaultFormatter": "tamasfe.even-better-toml", - }, "[json][jsonc]": { "editor.defaultFormatter": "biomejs.biome", }, diff --git a/ci/h5py-nightly.txt b/ci/h5py-nightly.txt new file mode 100644 index 00000000..1be41295 --- /dev/null +++ b/ci/h5py-nightly.txt @@ -0,0 +1 @@ +h5py>=3.17.0.dev0 diff --git a/pyproject.toml b/pyproject.toml index 15d20a71..328439a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,13 @@ envs.hatch-test.overrides.matrix.resolution.pre-install-commands = [ envs.hatch-test.overrides.matrix.resolution.env-vars = [ { key = "UV_CONSTRAINT", value = "ci/min-constraints.txt", if = [ "lowest" ] }, ] +# TODO: remove once h5py ships cp315 wheels: https://github.com/h5py/h5py/issues/2972 +envs.hatch-test.overrides.matrix.python.env-vars = [ + { key = "UV_FIND_LINKS", value = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/h5py/", if = [ + "3.15" + ] }, + { key = "UV_CONSTRAINT", value = "ci/h5py-nightly.txt", if = [ "3.15" ] }, +] envs.hatch-test.default-args = [] envs.hatch-test.dependency-groups = [ "test-min" ]