diff --git a/.github/workflows/test-models.yml b/.github/workflows/test-models.yml index 421a232d..d2c2edd4 100644 --- a/.github/workflows/test-models.yml +++ b/.github/workflows/test-models.yml @@ -79,6 +79,9 @@ jobs: pixi remove linopy pixi add --pypi --git https://github.com/${{ github.repository }}.git ${{ github.event.repository.name }} --rev ${{ github.sha }} pixi add --pypi pypsa + # linopy[polars] is not requestable from the git ref above; the default + # solve() writes an LP file and needs polars installed alongside it. + pixi add --pypi polars - name: Run snakemake test workflows if: env.pinned == 'false' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 08aca3d0..83078676 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,3 +125,63 @@ jobs: - name: Run type checker (mypy) run: | mypy . + + no-extras: + # The base install must work without polars, dask, tqdm and deprecation. + # This job catches a hard import reintroduced into the core path. + name: Core install without optional dependencies + needs: [build] + runs-on: ubuntu-latest + + steps: + - name: Set up Python 3.12 + uses: actions/setup-python@v7 + with: + python-version: 3.12 + + - name: Download package + uses: actions/download-artifact@v8 + with: + name: Packages + path: dist + + - name: Install the wheel with no extras + run: | + python -m pip install uv + uv pip install --system "$(ls dist/*.whl)" + uv pip install --system highspy + + - name: Smoke test the core install + run: | + python - <<'PY' + import importlib.util + + # The wheel with no extras must not pull in any of the optional extras. + for optional in ("polars", "dask", "tqdm"): + assert importlib.util.find_spec(optional) is None, optional + + import pandas as pd + + from linopy import Model + + m = Model() + t = pd.RangeIndex(6, name="t") + x = m.add_variables(lower=0, coords=[t], name="x") + y = m.add_variables(lower=0, coords=[t], name="y") + m.add_constraints(x + y >= 1) + m.add_objective(2 * x + 3 * y) + + # xarray's ffill/bfill import bottleneck directly, with no numpy + # fallback, so bottleneck has to stay a declared dependency. + x.ffill("t") + x.bfill("t") + + status, condition = m.solve(io_api="direct") + assert status == "ok", (status, condition) + # One x per time step at cost 2, since x + y >= 1 and 2x + 3y. + assert abs(float(m.objective.value) - 2.0 * len(t)) < 1e-9 + + m.to_file("smoke.mps", io_api="mps") + + print("core install smoke test passed") + PY diff --git a/doc/prerequisites.rst b/doc/prerequisites.rst index b8f1ea82..f50e918a 100644 --- a/doc/prerequisites.rst +++ b/doc/prerequisites.rst @@ -25,6 +25,37 @@ or conda install -c conda-forge linopy +Optional dependencies +--------------------- + +The base install pulls numpy, scipy, xarray, pandas, packaging and bottleneck. +Three features need an extra: + +``linopy[polars]`` + The LP file writer and the ``to_polars()`` export methods. This is required + by the default ``Model.solve()``, which writes an LP file. Install it if you + want the default solve path or any polars export: + + .. code:: bash + + uv pip install "linopy[polars]" + + Without it, ``Model.solve()`` raises ``ImportError``. Pass + ``io_api="direct"`` to solve without writing a file, or ``io_api="mps"`` to + write an MPS file instead (the MPS writer uses highspy). + +``linopy[dask]`` + Chunked (dask-backed) arrays, for example via ``Variable.chunk()``. Plain + numpy and xarray models do not need it. + +``linopy[progress]`` + Progress bars while writing LP files. Without it, LP export runs silently. + +.. code:: bash + + uv pip install "linopy[polars,dask,progress]" + + Install a solver ---------------- diff --git a/linopy/_polars.py b/linopy/_polars.py new file mode 100644 index 00000000..113c3cb6 --- /dev/null +++ b/linopy/_polars.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +""" +Placeholder used in place of :mod:`polars` when it is not installed. + +Polars backs the ``to_polars()`` export methods and the LP file writer, which +the default ``Model.solve()`` uses. It is an optional dependency, so every +module that touches it imports it as:: + + try: + import polars as pl + except ImportError: + from linopy._polars import missing_polars as pl + +Attribute access on the placeholder raises :class:`ImportError` with an install +hint, so the failure surfaces at the first real use rather than at import time. +""" + +from typing import Any + +_INSTALL_HINT = ( + "polars is required for this feature. Install it with `pip install linopy[polars]`." +) + + +class _MissingPolars: + """Stand-in for the ``polars`` module; every attribute access raises.""" + + def __getattr__(self, name: str) -> Any: + # Tools probe dunders such as ``__wrapped__`` and expect AttributeError. + # Raising ImportError there breaks pytest's doctest collection. + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + raise ImportError(_INSTALL_HINT) + + +missing_polars = _MissingPolars() diff --git a/linopy/alignment.py b/linopy/alignment.py index 597e4a05..f5f4d672 100644 --- a/linopy/alignment.py +++ b/linopy/alignment.py @@ -20,11 +20,10 @@ from collections import Counter from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence from functools import partial -from typing import TYPE_CHECKING, Any, Literal, overload +from typing import TYPE_CHECKING, Any, Literal, cast, overload import numpy as np import pandas as pd -import polars as pl import scipy.sparse from numpy import arange from xarray import Coordinates, DataArray, Dataset @@ -48,7 +47,12 @@ enforce_no_multiindex, first_mismatched_dim, ) -from linopy.types import UNLABELED_TYPES, CoordsLike, DimsLike +from linopy.types import ( + POLARS_SERIES_TYPES, + UNLABELED_TYPES, + CoordsLike, + DimsLike, +) def as_constant(other: Any) -> Any: @@ -384,8 +388,11 @@ def as_dataarray( arr = pandas_to_dataarray(arr, dims=dims, **kwargs) elif isinstance(arr, np.ndarray): arr = numpy_to_dataarray(arr, coords=coords, dims=dims, **kwargs) - elif isinstance(arr, pl.Series): - arr = numpy_to_dataarray(arr.to_numpy(), coords=coords, dims=dims, **kwargs) + elif isinstance(arr, POLARS_SERIES_TYPES): + # mypy cannot narrow `arr` through the runtime polars type tuple. + arr = numpy_to_dataarray( + cast(Any, arr).to_numpy(), coords=coords, dims=dims, **kwargs + ) elif isinstance(arr, np.number | int | float | str | bool | list): if isinstance(arr, np.number): arr = float(arr) @@ -406,7 +413,7 @@ def as_dataarray( pd.DataFrame, np.ndarray, DataArray, - pl.Series, + *POLARS_SERIES_TYPES, ] supported_types_str = ", ".join([t.__name__ for t in supported_types]) raise TypeError( diff --git a/linopy/common.py b/linopy/common.py index 3fbe8247..9294b541 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -16,7 +16,6 @@ import numpy as np import pandas as pd -import polars as pl from numpy import nan, signedinteger from numpy.typing import DTypeLike from xarray import DataArray, Dataset, apply_ufunc, broadcast @@ -24,6 +23,11 @@ from xarray.core import indexing from xarray.namedarray.utils import is_dict_like +try: + import polars as pl +except ImportError: # pragma: no cover + from linopy._polars import missing_polars as pl # type: ignore[no-redef] + from linopy.alignment import _as_index from linopy.config import options from linopy.constants import ( diff --git a/linopy/constraints.py b/linopy/constraints.py index 9762d468..607af9f8 100644 --- a/linopy/constraints.py +++ b/linopy/constraints.py @@ -25,7 +25,6 @@ import numpy as np import pandas as pd -import polars as pl import scipy.sparse import xarray as xr from numpy import array @@ -34,6 +33,11 @@ from xarray.core.indexes import Indexes from xarray.core.utils import Frozen +try: + import polars as pl +except ImportError: # pragma: no cover + from linopy._polars import missing_polars as pl # type: ignore[no-redef] + from linopy import expressions, variables from linopy.alignment import as_dataarray, broadcast_to_coords from linopy.common import ( diff --git a/linopy/expressions.py b/linopy/expressions.py index 6bbdcf47..f74ac764 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -36,7 +36,6 @@ import numpy as np import pandas as pd -import polars as pl import scipy import xarray as xr import xarray.core.groupby @@ -60,6 +59,11 @@ from types import EllipsisType, NotImplementedType +try: + import polars as pl +except ImportError: # pragma: no cover + from linopy._polars import missing_polars as pl # type: ignore[no-redef] + from linopy import constraints, variables from linopy.alignment import ( _matmul_operand_to_dataarray, diff --git a/linopy/io.py b/linopy/io.py index 9ee5ecd1..7cf29a8f 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -20,9 +20,17 @@ import numpy as np import pandas as pd -import polars as pl import xarray as xr -from tqdm import tqdm + +try: + import polars as pl +except ImportError: # pragma: no cover + from linopy._polars import missing_polars as pl # type: ignore[no-redef] + +try: + from tqdm import tqdm as _tqdm +except ImportError: # pragma: no cover + _tqdm = None from linopy import solvers from linopy.common import ( @@ -56,6 +64,13 @@ TQDM_COLOR = "#80bfff" +def tqdm(iterable: Any, **kwargs: Any) -> Any: + """Wrap in a progress bar when tqdm is installed, otherwise pass through.""" + if _tqdm is None: + return iterable + return _tqdm(iterable, **kwargs) + + name_sanitizer = str.maketrans("-+*^[] ", "_______") @@ -187,7 +202,7 @@ def get_printers_scalar( m: Model, explicit_coordinate_names: bool = False ) -> tuple[Callable, Callable]: """ - Get batch printer functions for numpy label arrays (non-polars). + Get batch printer functions for numpy label arrays. Returns two callables that take an int64 numpy array of labels and return a list of name strings. @@ -214,10 +229,10 @@ def print_constraints(labels: np.ndarray) -> list[str]: else: def print_variables(labels: np.ndarray) -> list[str]: - return ("x" + pl.Series(labels).cast(pl.String)).to_list() + return np.char.add("x", labels.astype(str)).tolist() def print_constraints(labels: np.ndarray) -> list[str]: - return ("c" + pl.Series(labels).cast(pl.String)).to_list() + return np.char.add("c", labels.astype(str)).tolist() return print_variables, print_constraints @@ -707,6 +722,20 @@ def to_lp_file( progress: bool = True, explicit_coordinate_names: bool = False, ) -> None: + """ + Write out a model to an LP file. + + Requires the optional ``polars`` dependency. Progress bars require ``tqdm``; + without it the export runs silently. + """ + try: + import polars # noqa: F401 + except ImportError as exc: + raise ImportError( + "LP file export requires polars. Install it with " + "`pip install linopy[polars]`, or pass io_api='direct' to solve()." + ) from exc + with open(fn, mode="wb") as f: start = time.time() variable_scaling = variable_scaling_lookup(m) diff --git a/linopy/objective.py b/linopy/objective.py index 46b698b2..538ccf0b 100644 --- a/linopy/objective.py +++ b/linopy/objective.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any import numpy as np -import polars as pl from numpy import float64 from pandas.core.frame import DataFrame from scipy.sparse._csc import csc_matrix @@ -20,6 +19,11 @@ from xarray.core.dataarray import DataArray from xarray.core.dataset import Dataset from xarray.core.indexes import Indexes + +try: + import polars as pl +except ImportError: # pragma: no cover + from linopy._polars import missing_polars as pl # type: ignore[no-redef] from xarray.core.utils import Frozen from linopy import expressions diff --git a/linopy/types.py b/linopy/types.py index 6ca69efb..e62c8fbe 100644 --- a/linopy/types.py +++ b/linopy/types.py @@ -4,11 +4,15 @@ from typing import TYPE_CHECKING, TypeAlias, Union, get_args import numpy -import polars as pl from pandas import DataFrame, Index, Series from xarray import DataArray from xarray.core.coordinates import DataArrayCoordinates, DatasetCoordinates +try: + import polars as pl +except ImportError: # pragma: no cover + pl = None # type: ignore[assignment] + if TYPE_CHECKING: from linopy.constraints import AnonymousScalarConstraint, Constraint from linopy.expressions import ( @@ -23,20 +27,38 @@ ) DimsLike: TypeAlias = str | Iterable[Hashable] -ConstantLike: TypeAlias = ( - int - | float - | numpy.floating - | numpy.integer - | numpy.ndarray - | DataArray - | Series - | DataFrame - | pl.Series -) -CONSTANT_TYPES: tuple[type, ...] = get_args(ConstantLike) +# polars is optional. When it is missing, `pl` is None and the polars entries +# drop out of the aliases and the runtime type tuples below, so `isinstance` +# checks keep working and simply never match a polars input. +POLARS_SERIES_TYPES: tuple[type, ...] = (pl.Series,) if pl is not None else () -UnlabeledLike: TypeAlias = numpy.ndarray | list | pl.Series +if pl is None: # pragma: no cover + ConstantLike: TypeAlias = ( + int + | float + | numpy.floating + | numpy.integer + | numpy.ndarray + | DataArray + | Series + | DataFrame + ) + UnlabeledLike: TypeAlias = numpy.ndarray | list +else: + ConstantLike: TypeAlias = ( # type: ignore[misc,no-redef] + int + | float + | numpy.floating + | numpy.integer + | numpy.ndarray + | DataArray + | Series + | DataFrame + | pl.Series + ) + UnlabeledLike: TypeAlias = numpy.ndarray | list | pl.Series # type: ignore[misc,no-redef] + +CONSTANT_TYPES: tuple[type, ...] = get_args(ConstantLike) UNLABELED_TYPES = get_args(UnlabeledLike) SignLike: TypeAlias = str | numpy.ndarray | DataArray | Series | DataFrame diff --git a/linopy/variables.py b/linopy/variables.py index 0e93ae28..dbbbf377 100644 --- a/linopy/variables.py +++ b/linopy/variables.py @@ -21,7 +21,6 @@ import numpy as np import pandas as pd -import polars as pl from numpy import floating, int64, issubdtype, ndarray, str_ from pandas.core.frame import DataFrame from xarray import DataArray, Dataset, broadcast @@ -30,6 +29,11 @@ from xarray.core.types import JoinOptions from xarray.core.utils import Frozen +try: + import polars as pl +except ImportError: # pragma: no cover + from linopy._polars import missing_polars as pl # type: ignore[no-redef] + import linopy.expressions as expressions from linopy.alignment import as_dataarray, broadcast_to_coords from linopy.common import ( diff --git a/pyproject.toml b/pyproject.toml index 32365905..4570d558 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,15 +30,12 @@ requires-python = ">=3.11" dependencies = [ "numpy", "scipy", - "bottleneck", - "toolz", - "numexpr", "xarray>=2024.2.0", - "dask>=0.18.0", - "polars>=1.31.1", - "tqdm", - "deprecation", + "pandas", "packaging", + # Variable.ffill/bfill go through xarray's push, which imports bottleneck + # directly with no numpy fallback. + "bottleneck", ] [project.urls] @@ -46,6 +43,19 @@ Homepage = "https://github.com/PyPSA/linopy" Source = "https://github.com/PyPSA/linopy" [project.optional-dependencies] +# Needed for the LP file writer (the default `Model.solve()` export path) and +# for the `to_polars()` export methods. +polars = [ + "polars>=1.31.1", +] +# Needed only for chunked (dask-backed) arrays, e.g. `Variable.chunk()`. +dask = [ + "dask>=0.18.0", +] +# Progress bars while writing LP files. +progress = [ + "tqdm", +] oetc = [ "google-cloud-storage", "requests", @@ -67,6 +77,8 @@ docs = [ "ipykernel==7.3.0", "matplotlib==3.11.0", "highspy>=1.7.1", + # The example notebooks call solve()/to_file(), which need the LP writer. + "linopy[polars]", ] dev = [ "pytest", @@ -81,6 +93,7 @@ dev = [ "highspy", "jupyter", "matplotlib", + "linopy[polars,dask,progress]", ] # Perf-relevant deps pinned exactly so run-to-run deltas reflect linopy # changes, not dependency bumps. diff --git a/test/test_io.py b/test/test_io.py index d6524ebc..b3350302 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -8,6 +8,7 @@ import importlib.util import json import pickle +import sys import warnings from collections.abc import Callable from pathlib import Path @@ -19,6 +20,7 @@ import pytest import xarray as xr +import linopy from linopy import LESS_EQUAL, Model, available_solvers, licensed_solvers, read_netcdf from linopy.constants import FACTOR_DIM from linopy.constraints import Constraint @@ -619,6 +621,58 @@ def test_to_file_invalid(model: Model, tmp_path: Path) -> None: model.to_file(fn) +def test_to_file_lp_progress_with_tqdm(model: Model, tmp_path: Path) -> None: + """The installed progress bar runs and leaves the LP file unchanged.""" + quiet = tmp_path / "quiet.lp" + loud = tmp_path / "loud.lp" + model.to_file(quiet, io_api="lp", progress=False) + model.to_file(loud, io_api="lp", progress=True) + + assert loud.read_bytes() == quiet.read_bytes() + + +def test_to_file_lp_progress_without_tqdm( + model: Model, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The progress bars are cosmetic: without tqdm the LP file is unchanged.""" + monkeypatch.setattr(linopy.io, "_tqdm", None) + + quiet = tmp_path / "quiet.lp" + loud = tmp_path / "loud.lp" + model.to_file(quiet, io_api="lp", progress=False) + model.to_file(loud, io_api="lp", progress=True) + + assert loud.read_bytes() == quiet.read_bytes() + + +def test_to_file_lp_without_polars( + model: Model, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """LP export names the missing extra instead of failing inside polars.""" + monkeypatch.setitem(sys.modules, "polars", None) + + with pytest.raises(ImportError, match=r"linopy\[polars\]"): + model.to_file(tmp_path / "test.lp", io_api="lp") + + +def test_missing_polars_placeholder() -> None: + """The polars stand-in raises on first attribute access, not on import.""" + from linopy._polars import missing_polars + + with pytest.raises(ImportError, match=r"linopy\[polars\]"): + missing_polars.DataFrame + + +def test_missing_polars_placeholder_dunder_probe() -> None: + """Dunder probes must raise AttributeError, not the install hint.""" + from linopy._polars import missing_polars + + with pytest.raises(AttributeError): + missing_polars.__wrapped__ + + assert hasattr(missing_polars, "__class__") + + @pytest.mark.skipif("gurobi" not in licensed_solvers, reason="Gurobipy not installed") def test_to_gurobipy(model: Model) -> None: gm = model.to_gurobipy()