From 7c9378833b092fff61a4146c1f1949d96359a00c Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 23 Oct 2026 09:19:06 +0200 Subject: [PATCH 1/8] refactor: Model.store_cls construction seam, Store.to_sparse(reason) for mixed merges (#1045) --- linopy/csr.py | 34 +++++++--- linopy/expressions.py | 22 ++++--- linopy/io.py | 6 +- linopy/model.py | 11 +++- linopy/store.py | 109 ++++++++++++++++++++------------- linopy/variables.py | 37 +++++------ test/test_csr.py | 14 ++--- test/test_io.py | 17 ++++- test/test_store_conformance.py | 36 +++++++++-- test/test_store_protocol.py | 48 +++++++++++++-- test/test_variables.py | 5 ++ 11 files changed, 239 insertions(+), 100 deletions(-) diff --git a/linopy/csr.py b/linopy/csr.py index 63939ad54..f7badb4a5 100644 --- a/linopy/csr.py +++ b/linopy/csr.py @@ -735,6 +735,16 @@ def add(self, *others: CSRStore) -> CSRStore: csr = _interleaved([p.csr for p in parts], shape, self.model) return replace(self, csr=csr, const=const, grid=merged_grid(parts)).absorbed() + @classmethod + def from_matmul( + cls, + source: CSRStore, + matrix: scipy.sparse.csr_array, + contracted_dims: Iterable[str], + new_indexes: Iterable[pd.Index], + ) -> CSRStore: + return source.dot(matrix, contracted_dims, new_indexes) + def dot( self, matrix: scipy.sparse.csr_array, @@ -983,18 +993,26 @@ def csr_to_term_arrays( return vars_, coeffs -def _densify_notice(reason: str, model: Model, explicit: bool = False) -> None: +def _conversion_notice(message: str, model: Model, explicit: bool) -> None: """ - Emit a :class:`~linopy.constants.PerformanceWarning` naming why a sparse - (CSR) backing is dropped, if ``options["warn_on_densify"]`` is set, or - left at its default ``None``, ``model`` is sparse and the conversion is - an implicit fallback rather than an ``explicit`` user request. + Emit ``message`` as a :class:`~linopy.constants.PerformanceWarning` if + ``options["warn_on_densify"]`` is set, or left at its default ``None``, + ``model`` is sparse and the conversion is an implicit fallback rather + than an ``explicit`` user request. """ enabled = options["warn_on_densify"] if model.sparse and not explicit if enabled is None else enabled: - warn_outside_linopy( - f"Sparse (CSR) backing densified: {reason}.", PerformanceWarning - ) + warn_outside_linopy(message, PerformanceWarning) + + +def _densify_notice(reason: str, model: Model, explicit: bool = False) -> None: + """The conversion notice naming why a sparse (CSR) backing is dropped.""" + _conversion_notice(f"Sparse (CSR) backing densified: {reason}.", model, explicit) + + +def _sparsify_notice(reason: str, model: Model, explicit: bool = False) -> None: + """The conversion notice naming why a dense backing is converted to sparse.""" + _conversion_notice(f"Dense backing sparsified: {reason}.", model, explicit) def _readonly(values: np.ndarray, copy: bool = True) -> np.ndarray: diff --git a/linopy/expressions.py b/linopy/expressions.py index 1a60bdd33..41b860410 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -556,11 +556,11 @@ def sum( multikey_frame = None if use_fallback else _multikey_value_frame(group, labels) store = self._store - sparse = self.model.sparse or isinstance(store, CSRStore) + store_cls = result_store_cls(store, self.model) if multikey_frame is not None and not observed: _warn_dense_grid(multikey_frame) - if sparse: + if store_cls is CSRStore: grouper = multikey_frame if grouper is None: is_1d = isinstance(group, DataArray) and group.ndim == 1 @@ -578,7 +578,7 @@ def sum( if isinstance(grouper, pd.DataFrame) else str(grouper.name or "group") ) - res = CSRStore.from_grouper( + res = store_cls.from_grouper( store, self.model, grouper, group_name, stacked, coord_dims ) return LinearExpression._from_store(res, self.model) @@ -2571,7 +2571,7 @@ def __mul__( try: if isinstance(other, variables.Variable): - other = other.to_linexpr(_dense=True) + other = other._linexpr(DenseStore) elif isinstance(other, variables.ScalarVariable): other = other.to_linexpr() @@ -2659,6 +2659,8 @@ def _sparse_matmul(self, other: ConstantLike) -> LinearExpression | Reason: if not self.coord_dims: return "matmul(zero-dimensional expression)" csr = self._store.to_sparse() + if isinstance(csr, str): + return f"matmul({csr})" if not csr.grid.is_unique: return "matmul(non-unique labels)" if csr.grid.size == 0: @@ -2673,10 +2675,9 @@ def _sparse_matmul(self, other: ConstantLike) -> LinearExpression | Reason: da, contracted, new_dims, csr.grid.indexes, csr.grid.aux ) new_indexes = [da.indexes[d].rename(d) for d in new_dims] - res = csr.dot(matrix, contracted, new_indexes) - if self.is_sparse or self.model.sparse: - return type(self)._from_store(res, self.model) - return type(self)._from_store(res.to_dense(), self.model) + store_cls = result_store_cls(self._store, self.model) + res = store_cls.from_matmul(csr, matrix, contracted, new_indexes) + return type(self)._from_store(res, self.model) @property def flat(self) -> pd.DataFrame: @@ -3407,6 +3408,11 @@ def as_expression( return LinearExpression(obj, model) +def result_store_cls(store: Store, model: Model) -> type[Store]: + """The store class an operation on ``store`` builds its result on.""" + return CSRStore if isinstance(store, CSRStore) else model.store_cls + + def _dense_copies(exprs: Sequence[Any], reason: Reason) -> list[Any]: """ ``exprs`` with every CSR-backed LinearExpression replaced by a dense diff --git a/linopy/io.py b/linopy/io.py index 7bd2f8603..5edbb7933 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -33,6 +33,7 @@ from linopy.constants import FACTOR_DIM, SOS_DIM_ATTR, SOS_TYPE_ATTR from linopy.objective import Objective, linear_part from linopy.scaling import constraint_scaling_lookup, variable_scaling_lookup +from linopy.store import expression_dataset if TYPE_CHECKING: from cuopt.linear_programming import DataModel as cuoptDataModel @@ -1270,7 +1271,10 @@ def owner(k: str) -> str | None: ): expr = QuadraticExpression(expr_ds, m) else: - expr = LinearExpression(expr_ds, m) + store = m.store_cls.from_dense(expression_dataset(expr_ds, m), m) + if isinstance(store, str): + raise ValueError(f"Cannot read expression {name!r}: {store}.") + expr = LinearExpression._from_store(store, m) expr.attrs["name"] = name expressions[name] = expr diff --git a/linopy/model.py b/linopy/model.py index 2781460c1..b65f935c1 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -60,7 +60,7 @@ Constraints, CSRConstraint, ) -from linopy.csr import _densify_notice +from linopy.csr import CSRStore, _densify_notice from linopy.dualization import dualize from linopy.expressions import ( Expressions, @@ -107,6 +107,7 @@ sos_reformulation_context, undo_sos_reformulation, ) +from linopy.store import DenseStore, Store from linopy.types import ( ConstantLike, ConstraintLike, @@ -521,6 +522,14 @@ def sparse(self) -> bool: """ return self._sparse + @property + def store_cls(self) -> type[Store]: + """ + The store class new expressions are built on: :class:`~linopy.csr.CSRStore` + in a sparse model, :class:`~linopy.store.DenseStore` otherwise. + """ + return CSRStore if self._sparse else DenseStore + @property def set_names_in_solver_io(self) -> bool: """Whether direct solver exports include names by default.""" diff --git a/linopy/store.py b/linopy/store.py index e863ffa9e..2f132ef2d 100644 --- a/linopy/store.py +++ b/linopy/store.py @@ -92,13 +92,27 @@ def from_grouper( ) -> Self: """The grouped sum of ``source``, see :meth:`linopy.csr.CSRStore.from_grouper`.""" + @classmethod + @abstractmethod + def from_matmul( + cls, + source: CSRStore, + matrix: scipy.sparse.csr_array, + contracted_dims: Iterable[str], + new_indexes: Iterable[pd.Index], + ) -> Self: + """The contraction ``source @ matrix``, see :meth:`linopy.csr.CSRStore.dot`.""" + @abstractmethod def to_dense(self, reason: Reason | None = None) -> DenseStore: """The dense equivalent, with a densify notice naming ``reason`` if given.""" @abstractmethod - def to_sparse(self, reason: Reason | None = None) -> CSRStore: - """The sparse equivalent.""" + def to_sparse(self, reason: Reason | None = None) -> CSRStore | Reason: + """ + The sparse equivalent, with a sparsify notice naming ``reason`` if + given; the reason a dense store cannot convert instead. + """ @abstractmethod def to_dataset(self) -> Dataset: @@ -233,15 +247,6 @@ def add(self, *others: Self) -> Self: def concat(self, others: Iterable[Self], dim: str, grid: Grid) -> Self | Reason: """Stack along the grid dim ``dim`` into the cells of ``grid``.""" - @abstractmethod - def dot( - self, - matrix: scipy.sparse.csr_array, - contracted_dims: Iterable[str], - new_indexes: Iterable[pd.Index], - ) -> Self: - """Contract grid dims against a sparse constant, see :meth:`linopy.csr.CSRStore.dot`.""" - def expression_dataset(data: Dataset | Any | None, model: Model) -> Dataset: """Normalize ``data`` to the dataset layout of an expression.""" @@ -324,10 +329,18 @@ class DenseStore(Store): def grid(self) -> Grid: from linopy.csr import Grid - grid = Grid.from_dataset(self.ds, self.dims) - if lost := set(map(str, self.ds.coords)) - {*grid.dims, *grid.aux}: - raise ValueError(f"The grid cannot hold the coordinates {sorted(lost)}.") - return grid + if lost := self._off_grid(): + raise ValueError(f"The grid cannot hold the coordinates {lost}.") + return Grid.from_dataset(self.ds, self.dims) + + def _off_grid(self) -> list[str]: + """The coordinates the grid cannot hold, e.g. two-dimensional ones.""" + from linopy.csr import _aux_coords + + own = set(self.dims) + return sorted( + set(map(str, self.ds.coords)) - own - set(_aux_coords(self.ds, own)) + ) @property def dims(self) -> tuple[str, ...]: @@ -410,7 +423,7 @@ def from_dense(cls, ds: Dataset, model: Model) -> DenseStore: def from_labels(cls, ds: Dataset, model: Model) -> DenseStore: labels, coeffs = xr.broadcast(ds.vars, ds.coeffs) const = xr.where(labels == -1, np.nan, 0.0) - terms = Dataset({"vars": labels, "coeffs": coeffs.astype(float)}) + terms = Dataset({"vars": labels, "coeffs": coeffs}) dense = terms.expand_dims(TERM_DIM, -1).assign(const=const) return cls(expression_dataset(absorb_absence(dense), model), model) @@ -429,12 +442,29 @@ def from_grouper( args = (grouper, group_dim, stacked, coord_dims) return CSRStore.from_grouper(source, model, *args).to_dense() + @classmethod + def from_matmul( + cls, + source: CSRStore, + matrix: scipy.sparse.csr_array, + contracted_dims: Iterable[str], + new_indexes: Iterable[pd.Index], + ) -> DenseStore: + from linopy.csr import CSRStore + + args = (matrix, contracted_dims, new_indexes) + return CSRStore.from_matmul(source, *args).to_dense() + def to_dense(self, reason: Reason | None = None) -> DenseStore: return self - def to_sparse(self, reason: Reason | None = None) -> CSRStore: - from linopy.csr import CSRStore + def to_sparse(self, reason: Reason | None = None) -> CSRStore | Reason: + from linopy.csr import CSRStore, _sparsify_notice + if lost := self._off_grid(): + return f"coordinates the Grid cannot hold: {', '.join(lost)}" + if reason is not None: + _sparsify_notice(reason, self.model) return CSRStore.from_dense(self.ds, self.model) def to_dataset(self) -> Dataset: @@ -536,7 +566,7 @@ def sum(self, dims: Iterable[str] | None = None) -> DenseStore: return self._derived(summed(self.ds, None if dims is None else list(dims))) def aggregate(self, grid: Grid, rows: np.ndarray) -> DenseStore: - return self.to_sparse().aggregate(grid, rows).to_dense() + return as_sparse(self).aggregate(grid, rows).to_dense() def add(self, *others: DenseStore) -> DenseStore: return self.merged((self, *others), TERM_DIM, "override", 0.0, {}) @@ -544,13 +574,13 @@ def add(self, *others: DenseStore) -> DenseStore: def concat(self, others: Iterable[DenseStore], dim: str, grid: Grid) -> DenseStore: return self.merged((self, *others), dim, "override", np.nan, {}) - def dot( - self, - matrix: scipy.sparse.csr_array, - contracted_dims: Iterable[str], - new_indexes: Iterable[pd.Index], - ) -> DenseStore: - return self.to_sparse().dot(matrix, contracted_dims, new_indexes).to_dense() + +def as_sparse(store: Store) -> CSRStore: + """``store`` as a sparse store, raising where it cannot convert.""" + res = store.to_sparse() + if isinstance(res, str): + raise ValueError(f"Cannot convert to a sparse store: {res}.") + return res def _stack_into_term_dim(ds: Dataset, dims: list[Hashable]) -> Dataset: @@ -727,16 +757,6 @@ def _merge_dims(store: Store, union: set[str]) -> list[str]: return [d for d in dims if d in union] + list(store.dims) -def _off_grid(store: Store) -> bool: - """Whether a dense store carries coordinates its grid cannot hold.""" - from linopy.csr import _aux_coords - - if not isinstance(store, DenseStore): - return False - own = set(store.dims) - return set(store.ds.coords) - own != set(_aux_coords(store.ds, own)) - - def merge_stores( stores: Sequence[Store], dim: str, @@ -751,8 +771,9 @@ def merge_stores( store is sparse, all are merged as sparse stores: along the term dimension as sparse matrix addition on the joined grid (:func:`align_stores`), along a grid dimension as a row stack - (:func:`concat_stores`). Returns the reason where the dense merge owns - the semantics instead. + (:func:`concat_stores`); the dense stores convert up through + :meth:`Store.to_sparse`, with one sparsify notice for all. Returns the + reason where the dense merge owns the semantics instead. """ dense = [s for s in stores if isinstance(s, DenseStore)] if len(dense) == len(stores): @@ -765,9 +786,15 @@ def merge_stores( return "merge(operands over different dimensions)" if dim != TERM_DIM and dim not in grid_dims: return "merge(new dimension)" - if any(_off_grid(s) for s in stores): - return "merge(dense operand with non-grid coordinates)" - csrs = [s.to_sparse() for s in stores] + last = max( + (i for i, s in enumerate(stores) if isinstance(s, DenseStore)), default=-1 + ) + csrs: list[CSRStore] = [] + for i, s in enumerate(stores): + csr = s.to_sparse("merge with a sparse operand" if i == last else None) + if isinstance(csr, str): + return f"merge({csr})" + csrs.append(csr) if dim != TERM_DIM: order = csrs[0].grid.dims reordered = [ diff --git a/linopy/variables.py b/linopy/variables.py index 392b99135..31ba7e7e3 100644 --- a/linopy/variables.py +++ b/linopy/variables.py @@ -59,10 +59,8 @@ STASHED_ATTRS, STASHED_LOWER, STASHED_UPPER, - TERM_DIM, ) from linopy.containers import LabelContainer -from linopy.csr import CSRStore from linopy.scaling import ensure_scaling, validate_scaling from linopy.semantics import ( FillValueLike, @@ -73,6 +71,7 @@ first_mismatched_dim, reindex_like_if_needed, ) +from linopy.store import DenseStore, Store from linopy.types import ( ConstantLike, DimsLike, @@ -353,8 +352,6 @@ def solver_scaling(self) -> DataArray: def to_linexpr( self, coefficient: ConstantLike = 1, - *, - _dense: bool = False, ) -> expressions.LinearExpression: """ Create a linear expression from the variables. @@ -376,10 +373,15 @@ def to_linexpr( broadcast aligns it away — the reindex below only fills absence and broadcasts non-shared dims. A NaN coefficient is user data, so it raises (§5). The expression carries the variable's absence as NaN so - that §6 propagates it through downstream arithmetic. In a sparse - model the expression is CSR-backed; ``_dense`` builds it dense for - callers that continue on a path without a sparse branch. + that §6 propagates it through downstream arithmetic. The expression + is built on the model's :attr:`~linopy.model.Model.store_cls`. """ + return self._linexpr(self.model.store_cls, coefficient) + + def _linexpr( + self, store_cls: type[Store], coefficient: ConstantLike = 1 + ) -> expressions.LinearExpression: + """:meth:`to_linexpr` built on ``store_cls``.""" if not np.isscalar(coefficient): coeff_da = as_dataarray(coefficient) enforce_aux_conflict([self.labels, coeff_da]) @@ -390,19 +392,12 @@ def to_linexpr( if coefficient.isnull().any(): check_user_nan() absent = self.labels == -1 - has_absence = bool(absent.any()) coefficient = reindex_like_if_needed(coefficient, self.labels, np.nan) - if has_absence: + if absent.any(): coefficient = coefficient.where(~absent) ds = Dataset({"coeffs": coefficient, "vars": self.labels}) - if self.model.sparse and not _dense: - csr = CSRStore.from_labels(ds, self.model) - return expressions.LinearExpression._from_store(csr, self.model) - ds = ds.expand_dims(TERM_DIM, -1) - if has_absence: - const = DataArray(np.where(absent, np.nan, 0.0), coords=self.labels.coords) - ds = ds.assign(const=const) - return expressions.LinearExpression(ds, self.model) + store = store_cls.from_labels(ds, self.model) + return expressions.LinearExpression._from_store(store, self.model) def __repr__(self) -> str: """ @@ -481,7 +476,7 @@ def __mul__(self, other: SideLike) -> ExpressionLike: """ try: if isinstance(other, Variable | ScalarVariable): - return self.to_linexpr(_dense=True) * other + return self._linexpr(DenseStore) * other return self.to_linexpr(other) except TypeError: return NotImplemented @@ -502,7 +497,7 @@ def __pow__(self, other: int) -> QuadraticExpression: if not isinstance(other, int): return NotImplemented if other == 2: - expr = self.to_linexpr(_dense=True) + expr = self._linexpr(DenseStore) return cast( "QuadraticExpression", expr._multiply_by_linear_expression(expr) ) @@ -831,7 +826,7 @@ def rolling( ------- linopy.expression.LinearExpressionRolling """ - return self.to_linexpr(_dense=True).rolling( + return self._linexpr(DenseStore).rolling( dim=dim, min_periods=min_periods, center=center, **window_kwargs ) @@ -871,7 +866,7 @@ def cumsum( ------- linopy.expression.LinearExpression """ - return self.to_linexpr(_dense=True).cumsum( + return self._linexpr(DenseStore).cumsum( dim=dim, skipna=skipna, keep_attrs=keep_attrs, **kwargs ) diff --git a/test/test_csr.py b/test/test_csr.py index 559f11aae..8aae7b5ea 100644 --- a/test/test_csr.py +++ b/test/test_csr.py @@ -20,7 +20,7 @@ import pytest import scipy.sparse import xarray as xr -from test_store_conformance import no_densify +from test_store_conformance import no_densify, sparsified from xarray.core.types import JoinOptions import linopy @@ -990,7 +990,7 @@ def test_cross_grid_merge_with_aux_coord_operand_stays_csr(join: JoinOptions) -> sparse, dense = twin_cross_grid_parts() tag = xr.DataArray(["x", "y"], coords=[dense[1].indexes["bus"]]) tagged = densified(sparse[1]).assign_coords(tag=tag) - with no_densify(): + with sparsified(): res = linopy.merge([sparse[0], tagged], join=join, cls=LinearExpression) assert res.is_sparse labels = res.indexes["bus"] @@ -1896,7 +1896,7 @@ def test_merge_with_dense_expression_stays_csr_and_matches_dense( c = base_model(sparse=True) sparse, dense = sparse_and_dense(build, c) other = DENSE_ADDENDS[addend](c, dense) - with no_densify(): + with sparsified(): res = other + sparse if swap else sparse + other assert_sparse_matches(res, other + dense if swap else dense + other) @@ -1918,7 +1918,7 @@ def test_merge_over_different_dims_joins_like_dense( sparse, dense = sparse_and_dense("grouped", c) other = unaligned(DENSE_ADDENDS[addend](c, dense)) first, second = (other, sparse) if swap else (sparse, other) - with no_densify(): + with sparsified(): res = first.add(second, join=join, fill_value=fill_value) want_first, want_second = (other, dense) if swap else (dense, other) want = want_first.add(want_second, join=join, fill_value=fill_value) @@ -2333,7 +2333,7 @@ def test_scalar_selection_coords_merge_like_dense() -> None: c = base_model(sparse=True) sparse, dense = sparse_and_dense("grouped", c) flow = densified((1.0 * c.flow).groupby(c.bus0).sum()) - with no_densify(): + with sparsified(): res = sparse.sel(snapshot=1) + flow.sel(snapshot=1) assert_sparse_matches(res, dense.sel(snapshot=1) + flow.sel(snapshot=1)) with pytest.raises(ValueError) as want: @@ -2503,7 +2503,7 @@ def test_expression_rhs_freezes_sparse_and_matches_dense(rhs: str, form: str) -> ) lhs2 = c2.gen_sum() rhs2 = EXPR_RHS[rhs](c2) - with no_densify(): + with sparsified() if rhs == "dense-expr-const" else no_densify(): if form == "operator": con2 = c2.m.add_constraints(lhs2 <= rhs2, name="c") else: @@ -2821,7 +2821,7 @@ def test_concat_with_dense_operand_stays_csr_and_matches_dense( tail = LinearExpression( tail.data.transpose(*reversed(tail.coord_dims), ...), tail.model ) - with no_densify(): + with sparsified(): res = concat([sparse.isel(snapshot=[0]), tail]) assert_sparse_matches(res, concat([dense.isel(snapshot=[0]), tail])) diff --git a/test/test_io.py b/test/test_io.py index 7a4ab8f9d..a7bf32f07 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -24,7 +24,7 @@ from linopy.constraints import Constraint from linopy.expressions import LinearExpression, QuadraticExpression from linopy.io import CONTAINER_ORDER_ATTR, signed_number -from linopy.testing import assert_exprequal, assert_model_equal +from linopy.testing import assert_equivalent, assert_exprequal, assert_model_equal HAS_NETCDF4 = importlib.util.find_spec("netCDF4") is not None @@ -1154,3 +1154,18 @@ def test_to_file_lp_unsorted_constraint_labels(tmp_path: Path) -> None: m.constraints.data["a"] = Constraint(con.data.isel(i=slice(None, None, -1)), m, "a") assert not m.constraints["a"].to_polars()["labels"].is_sorted() assert _lp_constraint_section(m, tmp_path / "unsorted.lp") == expected + + +def test_named_linear_expression_reads_back_on_the_model_store( + backend: Model, tmp_path: Path +) -> None: + x = backend.add_variables(coords=[pd.RangeIndex(3, name="i")], name="x") + want = backend.add_expressions(2 * x + 1, name="e") + fn = tmp_path / "e.nc" + backend.to_netcdf(fn) + e = read_netcdf(fn).expressions["e"] + assert isinstance(e, LinearExpression) + assert e.is_sparse == backend.sparse + assert e.attrs["name"] == "e" + assert isinstance(want, LinearExpression) + assert_equivalent(e, want) diff --git a/test/test_store_conformance.py b/test/test_store_conformance.py index 5230e05a6..7448e2313 100644 --- a/test/test_store_conformance.py +++ b/test/test_store_conformance.py @@ -22,6 +22,7 @@ from linopy import LinearExpression, Model, QuadraticExpression, Variable from linopy.constants import TERM_DIM from linopy.constraints import ConstraintBase, CSRConstraint +from linopy.store import DenseStore from linopy.testing import assert_equivalent KEPT = "kept" @@ -32,6 +33,7 @@ DROP_VARS = "drop_vars(dimension or data variable)" ASSIGNMENT = "assignment(const)" ASSIGN_COEFFS = "assignment(coeffs)" +SPARSIFIED = "sparsified: merge with a sparse operand" A = pd.Index([0, 1, 2], name="a") B = pd.Index(["p", "q"], name="b") @@ -53,6 +55,16 @@ def var(e: LinearExpression, name: str) -> Variable: return e.model.variables[name] +def dense_linexpr(v: Variable) -> LinearExpression: + return v._linexpr(DenseStore) + + +def dense_merged(e: LinearExpression) -> pd.DataFrame: + res = dense_linexpr(var(e, "x")) + dense_linexpr(var(e, "y")) + assert not res.is_sparse + return frame(res.flat) + + def grid(m: Model) -> LinearExpression: x = m.add_variables(coords=[A, B], name="x").assign_coords(tag=TAG) y = m.add_variables(coords=[A], name="y").assign_coords(tag=TAG) @@ -277,8 +289,9 @@ class Case(NamedTuple): KEPT, ), "add-dense-operand": Case( - grid, lambda e: e + var(e, "y").to_linexpr(_dense=True), KEPT + grid, lambda e: e + dense_linexpr(var(e, "y")), SPARSIFIED ), + "add-dense-operands": Case(grid, dense_merged, KEPT), "merge-along-grid-dim": Case( grid, lambda e: linopy.merge([e.sel(a=[0]), e.sel(a=[1, 2])], dim="a"), KEPT ), @@ -289,7 +302,7 @@ class Case(NamedTuple): ), "merge-dataset-operand": Case( grid, - lambda e: linopy.merge([e, var(e, "y").to_linexpr(_dense=True).data]), + lambda e: linopy.merge([e, dense_linexpr(var(e, "y")).data]), "merge(operand not a LinearExpression)", ), "merge-spread-along-dim": Case( @@ -315,8 +328,8 @@ class Case(NamedTuple): ), "merge-dense-2d-coord": Case( grid, - lambda e: e + with_2d_coord(var(e, "x").to_linexpr(_dense=True)), - "merge(dense operand with non-grid coordinates)", + lambda e: e + with_2d_coord(dense_linexpr(var(e, "x"))), + "merge(coordinates the Grid cannot hold: two)", ), "merge-factor-dim": Case( grid, @@ -398,13 +411,24 @@ class Case(NamedTuple): @contextmanager def no_densify() -> Iterator[None]: - """Fail on any densify notice inside the block.""" + """Fail on any conversion notice, densify or sparsify, inside the block.""" with linopy.options as opts, warnings.catch_warnings(): warnings.simplefilter("error", linopy.PerformanceWarning) opts.set_value(warn_on_densify=True) yield +@contextmanager +def sparsified() -> Iterator[None]: + """Expect exactly one conversion notice inside the block, the sparsify one.""" + with linopy.options as opts, warnings.catch_warnings(record=True) as rec: + warnings.simplefilter("always") + opts.set_value(warn_on_densify=True) + yield + notices = [str(w.message) for w in rec if w.category is linopy.PerformanceWarning] + assert notices == [f"Dense backing {SPARSIFIED}."] + + def assert_same(res: Any, want: Any) -> None: if isinstance(res, QuadraticExpression): pd.testing.assert_frame_equal(res.flat, want.flat) @@ -443,7 +467,7 @@ def test_operation(backend: Model, case: str, request: pytest.FixtureRequest) -> assert_same(res, want) if backend.sparse and expected not in (ASSIGNMENT, ASSIGN_COEFFS): assert expr.is_sparse - if backend.sparse and expected == KEPT: + if backend.sparse and expected in (KEPT, SPARSIFIED): if isinstance(res, LinearExpression): assert res.is_sparse elif isinstance(res, ConstraintBase): diff --git a/test/test_store_protocol.py b/test/test_store_protocol.py index 47779922c..34122603b 100644 --- a/test/test_store_protocol.py +++ b/test/test_store_protocol.py @@ -6,6 +6,7 @@ from __future__ import annotations import operator +import warnings from collections.abc import Callable from dataclasses import replace @@ -14,11 +15,21 @@ import pytest import scipy.sparse import xarray as xr -from test_store_conformance import COND, OVER_AB, SERIES, B, C, grid, no_densify - +from test_store_conformance import ( + COND, + OVER_AB, + SERIES, + B, + C, + grid, + no_densify, + with_2d_coord, +) + +import linopy from linopy import LinearExpression, Model, PerformanceWarning from linopy.csr import CSRStore, Grid -from linopy.store import DenseStore, Store +from linopy.store import DenseStore, Store, as_sparse from linopy.testing import assert_equivalent BUILDS: dict[str, Callable[[Model], LinearExpression]] = { @@ -114,7 +125,8 @@ def taken(s: Store) -> Store: "combine-reindex": lambda s: combined(s, True), "aggregate": lambda s: s.aggregate(*collapsed(s, "b")), "concat": concatenated, - "dot": lambda s: s.dot( + "from-matmul": lambda s: type(s).from_matmul( + as_sparse(s), scipy.sparse.csr_array(np.ones((len(B), 1))), ["b"], [pd.Index(["z"], name="z")], @@ -139,10 +151,34 @@ def test_round_trip_through_dataset(build: str, cls: type[Store]) -> None: def test_conversions_agree(build: str) -> None: dense, sparse = (build_store(build, cls) for cls in STORE_CLASSES) assert isinstance(sparse.to_dense(), DenseStore) - assert isinstance(dense.to_sparse(), CSRStore) + converted = dense.to_sparse() + assert isinstance(converted, CSRStore) assert dense.to_dense() is dense and sparse.to_sparse() is sparse assert_equivalent(as_expr(sparse.to_dense()), as_expr(dense)) - assert_equivalent(as_expr(dense.to_sparse()), as_expr(sparse)) + assert_equivalent(as_expr(converted), as_expr(sparse)) + + +def test_model_store_cls(backend: Model) -> None: + assert backend.store_cls is (CSRStore if backend.sparse else DenseStore) + + +def test_dense_to_sparse_names_off_grid_coords() -> None: + expr = with_2d_coord(as_expr(build_store("aux", DenseStore))) + with no_densify(): + assert expr._store.to_sparse("why") == "coordinates the Grid cannot hold: two" + + +@pytest.mark.parametrize("reason", [None, "why"]) +def test_dense_to_sparse_notice(reason: str | None) -> None: + dense = build_store("aux", DenseStore) + with linopy.options as opts, warnings.catch_warnings(record=True) as rec: + warnings.simplefilter("always") + opts.set_value(warn_on_densify=True) + assert isinstance(dense.to_sparse(reason), CSRStore) + notices = [str(w.message) for w in rec] + assert notices == ( + [] if reason is None else [f"Dense backing sparsified: {reason}."] + ) @pytest.mark.parametrize("op", list(OPS)) diff --git a/test/test_variables.py b/test/test_variables.py index e55ca680f..8fbc474d9 100644 --- a/test/test_variables.py +++ b/test/test_variables.py @@ -164,3 +164,8 @@ def test_scalar_variable(m: Model) -> None: x = ScalarVariable(label=0, model=m) assert isinstance(x, ScalarVariable) assert x.__rmul__(x) is NotImplemented # type: ignore + + +def test_to_linexpr_builds_on_the_model_store_cls(backend: Model) -> None: + x = backend.add_variables(coords=[pd.RangeIndex(3, name="i")], name="x") + assert type(x.to_linexpr(2)._store) is backend.store_cls From 6307d3a4f61bb6f1a5cb364ee4f2e3b4ed10a072 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 23 Oct 2026 09:19:07 +0200 Subject: [PATCH 2/8] docs: store_cls, sparsify notice and sparse netcdf read-back (#1045) --- doc/api.rst | 9 +++++---- doc/release_notes.rst | 3 +++ examples/sparse-models.ipynb | 6 ++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 181ab6afb..db873545d 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -691,10 +691,11 @@ Utilities ``options`` holds these keys: - ``display_max_rows``, ``display_max_terms``: size of the printed repr. -- ``warn_on_densify``: emit a ``PerformanceWarning`` when a sparse (CSR) - backing is converted to dense. ``None`` (default) warns in a sparse model - on implicit conversions only, ``True`` always warns, ``False`` never - warns. See :doc:`sparse-models`. +- ``warn_on_densify``: the conversion notice, a ``PerformanceWarning`` when + a sparse (CSR) backing is converted to dense or a dense operand is + converted to sparse for a merge with a sparse one. ``None`` (default) + warns in a sparse model on implicit conversions only, ``True`` always + warns, ``False`` never warns. See :doc:`sparse-models`. Warnings diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 2b1855746..36ed8a00c 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -34,6 +34,9 @@ Version 1.0.0 * ``linopy.csr.CSRLinearExpression`` is renamed to ``CSRStore`` without an alias. A ``LinearExpression`` now always holds a store from the new ``linopy.store`` module, a ``DenseStore`` or a ``CSRStore``, in place of the private ``_csr`` attribute. ``CSRStore.to_dense()`` returns a ``DenseStore`` instead of a ``LinearExpression``. Writing a model with a sparse objective to netcdf or copying the model no longer converts the objective to dense, and emits no densify notice. On both backends, a cell whose constant becomes NaN, e.g. by ``where``, ``reindex``, a join with ``fill_value=linopy.ABSENT`` or a constant operation yielding NaN such as ``0 / 0``, drops its terms, so its variables no longer reach ``sum()``, a constraint or the objective. The ``const`` setter and ``assign(const=...)`` of ``LinearExpression`` and ``QuadraticExpression`` raise ``ValueError`` when the assigned value holds a NaN or does not cover every coordinate of the expression. (`#1020 `__) * An operation without a sparse path on a CSR-backed ``LinearExpression``, including reading ``.data``, now works on a dense copy and no longer converts the expression itself, which stays sparse; only assigning ``vars``, ``coeffs`` or an off-grid ``const`` replaces its store. The densify notice names the operation and the case as ``op(case)``, e.g. ``Sparse (CSR) backing densified: reindex(method=...)``, so filters matching the former reason texts need updating. A named sparse expression stays sparse through ``Model.add_expressions`` and ``Model.copy`` and keeps its name through assignments. ``densify_terms``, ``reset_const`` (so also ``Constraint.update(rhs=...)``) and ``Model.assign_coords`` keep a sparse expression sparse. ``LinearExpression.where`` and ``QuadraticExpression.where`` take ``drop`` as their only keyword argument instead of forwarding ``**kwargs`` to ``xarray.Dataset.where``, which accepts no other. (`#1020 `__) +* ``Model.store_cls`` names the store class a model builds expressions on, ``CSRStore`` in a sparse model and ``DenseStore`` otherwise. ``Variable.to_linexpr`` always returns an expression on it; the private ``_dense`` keyword is removed. (`#1045 `__) +* A dense operand in a merge with a CSR-backed one is converted up through the new ``Store.to_sparse(reason)`` and emits one ``Dense backing sparsified: merge with a sparse operand`` notice per merge, under the same ``warn_on_densify`` option as the densify notice. A dense operand with coordinates the sparse grid cannot hold, e.g. two-dimensional auxiliary ones, now falls the whole merge back to dense with the reason ``merge(coordinates the Grid cannot hold: )``. A merge of dense operands stays dense, also in a sparse model. (`#1045 `__) +* A named linear expression of a sparse model reads back from netcdf as a CSR-backed expression. (`#1045 `__) **Deprecations** diff --git a/examples/sparse-models.ipynb b/examples/sparse-models.ipynb index f9015202f..9ef2e2548 100644 --- a/examples/sparse-models.ipynb +++ b/examples/sparse-models.ipynb @@ -242,6 +242,12 @@ "Explicit conversions are `CSRConstraint.mutable()`, `CSRConstraint.to_dense()`\n", "and `add_constraints(..., freeze=False)`. They stay quiet under `None`.\n", "\n", + "The option also governs the opposite conversion: a dense operand merged with\n", + "a sparse one, e.g. `sparse_expr + dense_expr`, is converted to sparse with one\n", + "`Dense backing sparsified: merge with a sparse operand` notice per merge. A\n", + "dense operand with coordinates the sparse grid cannot hold, such as\n", + "two-dimensional auxiliary ones, makes the whole merge dense instead.\n", + "\n", "To find the line that triggers a warning, turn it into an error with\n", "`warnings.simplefilter(\"error\", linopy.PerformanceWarning)`; the traceback\n", "then points to it.\n", From a3263ff45f95257a240248b46e4e439fb3a24210 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 23 Oct 2026 09:34:11 +0200 Subject: [PATCH 3/8] fix: netcdf read keeps off-grid coords dense in a sparse model --- linopy/io.py | 12 +++++++----- test/test_io.py | 13 +++++++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/linopy/io.py b/linopy/io.py index 5edbb7933..e478d5f04 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -31,9 +31,10 @@ to_polars, ) from linopy.constants import FACTOR_DIM, SOS_DIM_ATTR, SOS_TYPE_ATTR +from linopy.csr import CSRStore from linopy.objective import Objective, linear_part from linopy.scaling import constraint_scaling_lookup, variable_scaling_lookup -from linopy.store import expression_dataset +from linopy.store import DenseStore, expression_dataset if TYPE_CHECKING: from cuopt.linear_programming import DataModel as cuoptDataModel @@ -1271,10 +1272,11 @@ def owner(k: str) -> str | None: ): expr = QuadraticExpression(expr_ds, m) else: - store = m.store_cls.from_dense(expression_dataset(expr_ds, m), m) - if isinstance(store, str): - raise ValueError(f"Cannot read expression {name!r}: {store}.") - expr = LinearExpression._from_store(store, m) + dense = DenseStore(expression_dataset(expr_ds, m), m) + store = dense.to_sparse() if m.store_cls is CSRStore else dense + expr = LinearExpression._from_store( + dense if isinstance(store, str) else store, m + ) expr.attrs["name"] = name expressions[name] = expr diff --git a/test/test_io.py b/test/test_io.py index a7bf32f07..34a0ff78c 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -1156,16 +1156,21 @@ def test_to_file_lp_unsorted_constraint_labels(tmp_path: Path) -> None: assert _lp_constraint_section(m, tmp_path / "unsorted.lp") == expected +@pytest.mark.parametrize("two_d", [False, True], ids=["grid", "2d-coord"]) def test_named_linear_expression_reads_back_on_the_model_store( - backend: Model, tmp_path: Path + backend: Model, tmp_path: Path, two_d: bool ) -> None: - x = backend.add_variables(coords=[pd.RangeIndex(3, name="i")], name="x") - want = backend.add_expressions(2 * x + 1, name="e") + i, j = pd.RangeIndex(3, name="i"), pd.RangeIndex(2, name="j") + expr = 2 * backend.add_variables(coords=[i, j], name="x") + 1 + if two_d: + expr = expr.assign_coords(two=(("i", "j"), np.arange(6).reshape(3, 2))) + want = backend.add_expressions(expr, name="e") fn = tmp_path / "e.nc" backend.to_netcdf(fn) e = read_netcdf(fn).expressions["e"] assert isinstance(e, LinearExpression) - assert e.is_sparse == backend.sparse + assert e.is_sparse == (backend.sparse and not two_d) + assert ("two" in e.coords) == two_d assert e.attrs["name"] == "e" assert isinstance(want, LinearExpression) assert_equivalent(e, want) From 4a1fb19e7eeda6659e112b7176722d36a75b4809 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 23 Oct 2026 09:34:12 +0200 Subject: [PATCH 4/8] fix: sparsify notice only once the sparse merge succeeds --- linopy/store.py | 30 ++++++++++++++++++++++-------- test/test_store_conformance.py | 12 ++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/linopy/store.py b/linopy/store.py index 2f132ef2d..91da05160 100644 --- a/linopy/store.py +++ b/linopy/store.py @@ -52,7 +52,7 @@ class Store(ABC): Terms and constant of a linear expression on a fixed coordinate grid. A method returning ``Self | Reason`` may refuse a case with the reason; - :class:`DenseStore` never does. + :class:`DenseStore` does so only in :meth:`to_sparse`. """ model: Model @@ -772,8 +772,9 @@ def merge_stores( dimension as sparse matrix addition on the joined grid (:func:`align_stores`), along a grid dimension as a row stack (:func:`concat_stores`); the dense stores convert up through - :meth:`Store.to_sparse`, with one sparsify notice for all. Returns the - reason where the dense merge owns the semantics instead. + :meth:`Store.to_sparse`, with one sparsify notice once the sparse merge + succeeds. Returns the reason where the dense merge owns the semantics + instead. """ dense = [s for s in stores if isinstance(s, DenseStore)] if len(dense) == len(stores): @@ -786,15 +787,28 @@ def merge_stores( return "merge(operands over different dimensions)" if dim != TERM_DIM and dim not in grid_dims: return "merge(new dimension)" - last = max( - (i for i, s in enumerate(stores) if isinstance(s, DenseStore)), default=-1 - ) csrs: list[CSRStore] = [] - for i, s in enumerate(stores): - csr = s.to_sparse("merge with a sparse operand" if i == last else None) + for s in stores: + csr = s.to_sparse() if isinstance(csr, str): return f"merge({csr})" csrs.append(csr) + res = _merged_sparse(csrs, dim, join, fill, grid_dims) + if dense and not isinstance(res, str): + from linopy.csr import _sparsify_notice + + _sparsify_notice("merge with a sparse operand", stores[0].model) + return res + + +def _merged_sparse( + csrs: list[CSRStore], + dim: str, + join: JoinOptions | None, + fill: float, + grid_dims: tuple[str, ...], +) -> CSRStore | Reason: + """The sparse merge of :func:`merge_stores` on the converted stores.""" if dim != TERM_DIM: order = csrs[0].grid.dims reordered = [ diff --git a/test/test_store_conformance.py b/test/test_store_conformance.py index 7448e2313..54b893839 100644 --- a/test/test_store_conformance.py +++ b/test/test_store_conformance.py @@ -316,6 +316,18 @@ class Case(NamedTuple): "merge-non-unique": Case( non_unique, lambda e: e + 1 * var(e, "w"), "merge(non-unique labels)" ), + "merge-non-unique-dense-operand": Case( + non_unique, + lambda e: e + dense_linexpr(var(e, "w")), + "merge(non-unique labels)", + ), + "merge-dense-operands-around-sparse": Case( + grid, + lambda e: linopy.merge( + [dense_linexpr(var(e, "y")), e, dense_linexpr(var(e, "x"))] + ), + SPARSIFIED, + ), "merge-differing-grids": Case( lambda m: without_tag(grid(m)), lambda e: linopy.merge([e.sum("a"), e.sel(a=[0, 1]), e.sel(a=[1, 2])]), From 7a4d734ace9e661a6d1324f027b4826ea1b5827d Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 23 Oct 2026 09:34:13 +0200 Subject: [PATCH 5/8] test: matmul on a dense operand with a 2-D coord on kept dims --- test/test_csr.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/test/test_csr.py b/test/test_csr.py index 8aae7b5ea..8d5cc4a56 100644 --- a/test/test_csr.py +++ b/test/test_csr.py @@ -20,7 +20,15 @@ import pytest import scipy.sparse import xarray as xr -from test_store_conformance import no_densify, sparsified +from test_store_conformance import ( + OVER_B, + A, + B, + C, + dense_linexpr, + no_densify, + sparsified, +) from xarray.core.types import JoinOptions import linopy @@ -1427,6 +1435,17 @@ def test_matmul_keeps_aux_coords_on_kept_dims_only() -> None: assert not {"bus", "tag"} & set(dropped.coords) +def test_matmul_keeps_2d_coord_on_kept_dims(backend: Model) -> None: + x = backend.add_variables(coords=[A, B, C], name="x") + two = (("a", "c"), np.arange(6).reshape(3, 2)) + expr = dense_linexpr(x).assign_coords(two=two) + with no_densify(): + res = expr @ OVER_B + assert not res.is_sparse + assert "two" in res.coords + assert_equivalent(res, (expr * OVER_B).sum("b")) + + def test_matmul_output_dim_order_matches_dense() -> None: c = base_model() expr = c.eff * c.gen_p From 8add12ae72c2f9e006a949de2916c661a4bcb773 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 23 Oct 2026 09:34:14 +0200 Subject: [PATCH 6/8] refactor: drop unused sparsify explicit flag and label masking, fix docstrings --- doc/release_notes.rst | 4 ++-- linopy/csr.py | 4 ++-- linopy/expressions.py | 7 ++++--- linopy/model.py | 8 ++++++-- linopy/variables.py | 3 --- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 36ed8a00c..855a2da77 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -35,8 +35,8 @@ Version 1.0.0 * ``linopy.csr.CSRLinearExpression`` is renamed to ``CSRStore`` without an alias. A ``LinearExpression`` now always holds a store from the new ``linopy.store`` module, a ``DenseStore`` or a ``CSRStore``, in place of the private ``_csr`` attribute. ``CSRStore.to_dense()`` returns a ``DenseStore`` instead of a ``LinearExpression``. Writing a model with a sparse objective to netcdf or copying the model no longer converts the objective to dense, and emits no densify notice. On both backends, a cell whose constant becomes NaN, e.g. by ``where``, ``reindex``, a join with ``fill_value=linopy.ABSENT`` or a constant operation yielding NaN such as ``0 / 0``, drops its terms, so its variables no longer reach ``sum()``, a constraint or the objective. The ``const`` setter and ``assign(const=...)`` of ``LinearExpression`` and ``QuadraticExpression`` raise ``ValueError`` when the assigned value holds a NaN or does not cover every coordinate of the expression. (`#1020 `__) * An operation without a sparse path on a CSR-backed ``LinearExpression``, including reading ``.data``, now works on a dense copy and no longer converts the expression itself, which stays sparse; only assigning ``vars``, ``coeffs`` or an off-grid ``const`` replaces its store. The densify notice names the operation and the case as ``op(case)``, e.g. ``Sparse (CSR) backing densified: reindex(method=...)``, so filters matching the former reason texts need updating. A named sparse expression stays sparse through ``Model.add_expressions`` and ``Model.copy`` and keeps its name through assignments. ``densify_terms``, ``reset_const`` (so also ``Constraint.update(rhs=...)``) and ``Model.assign_coords`` keep a sparse expression sparse. ``LinearExpression.where`` and ``QuadraticExpression.where`` take ``drop`` as their only keyword argument instead of forwarding ``**kwargs`` to ``xarray.Dataset.where``, which accepts no other. (`#1020 `__) * ``Model.store_cls`` names the store class a model builds expressions on, ``CSRStore`` in a sparse model and ``DenseStore`` otherwise. ``Variable.to_linexpr`` always returns an expression on it; the private ``_dense`` keyword is removed. (`#1045 `__) -* A dense operand in a merge with a CSR-backed one is converted up through the new ``Store.to_sparse(reason)`` and emits one ``Dense backing sparsified: merge with a sparse operand`` notice per merge, under the same ``warn_on_densify`` option as the densify notice. A dense operand with coordinates the sparse grid cannot hold, e.g. two-dimensional auxiliary ones, now falls the whole merge back to dense with the reason ``merge(coordinates the Grid cannot hold: )``. A merge of dense operands stays dense, also in a sparse model. (`#1045 `__) -* A named linear expression of a sparse model reads back from netcdf as a CSR-backed expression. (`#1045 `__) +* A dense operand in a merge with a CSR-backed one is converted up through the new ``Store.to_sparse(reason)``; a merge that stays sparse emits one ``Dense backing sparsified: merge with a sparse operand`` notice, under the same ``warn_on_densify`` option as the densify notice. A dense operand with coordinates the sparse grid cannot hold, e.g. two-dimensional auxiliary ones, now falls the whole merge back to dense with the reason ``merge(coordinates the Grid cannot hold: )``. A merge of dense operands stays dense, also in a sparse model. (`#1045 `__) +* A named linear expression of a sparse model reads back from netcdf as a CSR-backed expression, unless it carries coordinates the sparse grid cannot hold, e.g. two-dimensional auxiliary ones; then it reads back dense with those coordinates. (`#1045 `__) **Deprecations** diff --git a/linopy/csr.py b/linopy/csr.py index f7badb4a5..9ab0b082e 100644 --- a/linopy/csr.py +++ b/linopy/csr.py @@ -1010,9 +1010,9 @@ def _densify_notice(reason: str, model: Model, explicit: bool = False) -> None: _conversion_notice(f"Sparse (CSR) backing densified: {reason}.", model, explicit) -def _sparsify_notice(reason: str, model: Model, explicit: bool = False) -> None: +def _sparsify_notice(reason: str, model: Model) -> None: """The conversion notice naming why a dense backing is converted to sparse.""" - _conversion_notice(f"Dense backing sparsified: {reason}.", model, explicit) + _conversion_notice(f"Dense backing sparsified: {reason}.", model, False) def _readonly(values: np.ndarray, copy: bool = True) -> np.ndarray: diff --git a/linopy/expressions.py b/linopy/expressions.py index 41b860410..fc1822339 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -2647,9 +2647,10 @@ def _sparse_matmul(self, other: ConstantLike) -> LinearExpression | Reason: Contract against a constant as one sparse matrix product, skipping the dense broadcast intermediate of ``(self * other).sum(dim)``. - Returns the reason where the dense path owns the semantics: non-unique - grid labels, a zero-size grid, an operand sharing no dimension with - the grid, and an unlabelled output dimension. The result is the compact + Returns the reason where the dense path owns the semantics: a dense + expression with coordinates the grid cannot hold, non-unique grid + labels, a zero-size grid, an operand sharing no dimension with the + grid, and an unlabelled output dimension. The result is the compact canonical form of :meth:`CSRStore.dot`, so its term count may differ from the dense path's while the values agree; the result is CSR-backed when the input was or the model is sparse. diff --git a/linopy/model.py b/linopy/model.py index b65f935c1..c28a20f99 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -525,8 +525,12 @@ def sparse(self) -> bool: @property def store_cls(self) -> type[Store]: """ - The store class new expressions are built on: :class:`~linopy.csr.CSRStore` - in a sparse model, :class:`~linopy.store.DenseStore` otherwise. + The store class the construction seam builds on: ``to_linexpr``, + ``groupby(...).sum()``, ``@`` and the netcdf reader yield a + :class:`~linopy.csr.CSRStore` in a sparse model and a + :class:`~linopy.store.DenseStore` otherwise. Quadratic products, + ``rolling``, ``cumsum`` and the ``LinearExpression`` constructor + build dense. """ return CSRStore if self._sparse else DenseStore diff --git a/linopy/variables.py b/linopy/variables.py index 31ba7e7e3..08c23e32a 100644 --- a/linopy/variables.py +++ b/linopy/variables.py @@ -391,10 +391,7 @@ def _linexpr( coefficient = broadcast_to_coords(coefficient, coords=self.coords, strict=False) if coefficient.isnull().any(): check_user_nan() - absent = self.labels == -1 coefficient = reindex_like_if_needed(coefficient, self.labels, np.nan) - if absent.any(): - coefficient = coefficient.where(~absent) ds = Dataset({"coeffs": coefficient, "vars": self.labels}) store = store_cls.from_labels(ds, self.model) return expressions.LinearExpression._from_store(store, self.model) From 60c6a53d8ec32809ee883b9ddd6cf201ba07188c Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 23 Oct 2026 11:39:35 +0200 Subject: [PATCH 7/8] perf: DenseStore.from_labels builds const and masks coeffs only on absence Restores the peak memory of to_linexpr on dense models to master: no broadcast, no always-present const array, no absorb_absence pass. --- linopy/store.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/linopy/store.py b/linopy/store.py index 91da05160..cca1c9991 100644 --- a/linopy/store.py +++ b/linopy/store.py @@ -421,11 +421,12 @@ def from_dense(cls, ds: Dataset, model: Model) -> DenseStore: @classmethod def from_labels(cls, ds: Dataset, model: Model) -> DenseStore: - labels, coeffs = xr.broadcast(ds.vars, ds.coeffs) - const = xr.where(labels == -1, np.nan, 0.0) - terms = Dataset({"vars": labels, "coeffs": coeffs}) - dense = terms.expand_dims(TERM_DIM, -1).assign(const=const) - return cls(expression_dataset(absorb_absence(dense), model), model) + terms = ds[["vars", "coeffs"]].expand_dims(TERM_DIM, -1) + absent = ds.vars == -1 + if absent.any(): + const = xr.where(absent, np.nan, 0.0) + terms = terms.assign(coeffs=terms.coeffs.where(~absent), const=const) + return cls(expression_dataset(terms, model), model) @classmethod def from_grouper( From c9c3bd34e1b5e76ff859d72f28c4777824a03bf4 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 23 Oct 2026 12:32:01 +0200 Subject: [PATCH 8/8] feat: NaN from constant arithmetic raises; absent cells drop terms on construction A constant op producing NaN (0*inf, 0/0, inf-inf) at a live cell raises ValueError on both stores instead of reading it as absence; sparse /0 emits no numpy warnings. _set_data absorbs absence so a dataset with terms under a NaN const or a vars assignment cannot revive a cell. --- doc/release_notes.rst | 4 +- linopy/csr.py | 28 ++++++++++---- linopy/expressions.py | 11 ++++-- linopy/semantics.py | 62 ++++++++++++++++++++++++++++++- linopy/store.py | 14 +++++-- test/test_linear_expression.py | 16 ++++++++ test/test_quadratic_expression.py | 17 +++++++++ test/test_store_conformance.py | 42 +++++++++++++-------- test/test_store_protocol.py | 2 +- 9 files changed, 162 insertions(+), 34 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 855a2da77..46ee3004b 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -32,11 +32,13 @@ Version 1.0.0 * ``Variables.print_labels``, ``Constraints.print_labels``, ``Model.print_infeasibilities`` and ``Model.compute_set_of_infeasible_constraints`` are removed; use ``format_labels``, ``Model.compute_infeasibilities`` and ``Model.format_infeasibilities``. * The unused ``Variables.dataset_names``, ``Constraints.dataset_names`` and ``LinearExpression.sanitize`` are removed. -* ``linopy.csr.CSRLinearExpression`` is renamed to ``CSRStore`` without an alias. A ``LinearExpression`` now always holds a store from the new ``linopy.store`` module, a ``DenseStore`` or a ``CSRStore``, in place of the private ``_csr`` attribute. ``CSRStore.to_dense()`` returns a ``DenseStore`` instead of a ``LinearExpression``. Writing a model with a sparse objective to netcdf or copying the model no longer converts the objective to dense, and emits no densify notice. On both backends, a cell whose constant becomes NaN, e.g. by ``where``, ``reindex``, a join with ``fill_value=linopy.ABSENT`` or a constant operation yielding NaN such as ``0 / 0``, drops its terms, so its variables no longer reach ``sum()``, a constraint or the objective. The ``const`` setter and ``assign(const=...)`` of ``LinearExpression`` and ``QuadraticExpression`` raise ``ValueError`` when the assigned value holds a NaN or does not cover every coordinate of the expression. (`#1020 `__) +* ``linopy.csr.CSRLinearExpression`` is renamed to ``CSRStore`` without an alias. A ``LinearExpression`` now always holds a store from the new ``linopy.store`` module, a ``DenseStore`` or a ``CSRStore``, in place of the private ``_csr`` attribute. ``CSRStore.to_dense()`` returns a ``DenseStore`` instead of a ``LinearExpression``. Writing a model with a sparse objective to netcdf or copying the model no longer converts the objective to dense, and emits no densify notice. On both backends, a cell whose constant becomes NaN, e.g. by ``where``, ``reindex`` or a join with ``fill_value=linopy.ABSENT``, drops its terms, so its variables no longer reach ``sum()``, a constraint or the objective. The ``const`` setter and ``assign(const=...)`` of ``LinearExpression`` and ``QuadraticExpression`` raise ``ValueError`` when the assigned value holds a NaN or does not cover every coordinate of the expression. (`#1020 `__) * An operation without a sparse path on a CSR-backed ``LinearExpression``, including reading ``.data``, now works on a dense copy and no longer converts the expression itself, which stays sparse; only assigning ``vars``, ``coeffs`` or an off-grid ``const`` replaces its store. The densify notice names the operation and the case as ``op(case)``, e.g. ``Sparse (CSR) backing densified: reindex(method=...)``, so filters matching the former reason texts need updating. A named sparse expression stays sparse through ``Model.add_expressions`` and ``Model.copy`` and keeps its name through assignments. ``densify_terms``, ``reset_const`` (so also ``Constraint.update(rhs=...)``) and ``Model.assign_coords`` keep a sparse expression sparse. ``LinearExpression.where`` and ``QuadraticExpression.where`` take ``drop`` as their only keyword argument instead of forwarding ``**kwargs`` to ``xarray.Dataset.where``, which accepts no other. (`#1020 `__) * ``Model.store_cls`` names the store class a model builds expressions on, ``CSRStore`` in a sparse model and ``DenseStore`` otherwise. ``Variable.to_linexpr`` always returns an expression on it; the private ``_dense`` keyword is removed. (`#1045 `__) * A dense operand in a merge with a CSR-backed one is converted up through the new ``Store.to_sparse(reason)``; a merge that stays sparse emits one ``Dense backing sparsified: merge with a sparse operand`` notice, under the same ``warn_on_densify`` option as the densify notice. A dense operand with coordinates the sparse grid cannot hold, e.g. two-dimensional auxiliary ones, now falls the whole merge back to dense with the reason ``merge(coordinates the Grid cannot hold: )``. A merge of dense operands stays dense, also in a sparse model. (`#1045 `__) * A named linear expression of a sparse model reads back from netcdf as a CSR-backed expression, unless it carries coordinates the sparse grid cannot hold, e.g. two-dimensional auxiliary ones; then it reads back dense with those coordinates. (`#1045 `__) +* A constant operation that produces NaN in a live cell, e.g. ``0 * inf``, ``0 / 0`` or ``inf - inf`` from multiplying, dividing, adding or subtracting a constant, now raises ``ValueError`` on both backends. Before, the cell kept its terms under a NaN constant. linopy trusts NaN only as absence, so mark cells to drop with ``.where(...)`` or ``mask=`` instead. Cells that were absent before stay absent. A sparse division by zero no longer emits numpy ``RuntimeWarning``\ s. (`#1045 `__) +* Cells with a NaN constant now drop their terms when a ``LinearExpression`` or ``QuadraticExpression`` is built from a dataset or its ``vars`` or ``coeffs`` are assigned, so neither a dataset with terms under a NaN constant nor a ``vars`` assignment can revive an absent cell. (`#1045 `__) **Deprecations** diff --git a/linopy/csr.py b/linopy/csr.py index 9ab0b082e..830603917 100644 --- a/linopy/csr.py +++ b/linopy/csr.py @@ -53,7 +53,7 @@ from linopy.common import coords_from_dataset, coords_to_dataset_vars from linopy.config import options from linopy.constants import HELPER_DIMS, TERM_DIM, PerformanceWarning -from linopy.semantics import warn_outside_linopy +from linopy.semantics import check_arithmetic_nan, warn_outside_linopy from linopy.store import DenseStore, Reason, Store, merged_grid if TYPE_CHECKING: @@ -576,15 +576,27 @@ def scale( """ Apply ``op`` with ``factor`` -- a scalar or one value per cell -- to every coefficient and to the constant. Explicit zeros are kept, as by - :meth:`add`; a cell made absent drops its terms. + :meth:`add`; a NaN it produces in a live cell raises. """ - factor = np.broadcast_to(factor, (self.n_cells,)) - per_term = np.repeat(factor, np.diff(self.csr.indptr)) + with np.errstate(divide="ignore", invalid="ignore"): + const = op(self.const, factor) + check_arithmetic_nan(const, op, factor, self.const) + return self._scaled_terms(factor, op).assign_const(const) + + def _scaled_terms( + self, factor: float | np.ndarray, op: Callable[[Any, Any], Any] + ) -> CSRStore: + """Apply ``op`` with a scalar or per-cell ``factor`` to every coefficient.""" + per_term = factor + if np.ndim(factor): + per_term = np.repeat(factor, np.diff(self.csr.indptr)) + with np.errstate(divide="ignore", invalid="ignore"): + data = op(self.csr.data, per_term) + check_arithmetic_nan(data, op, per_term) csr = scipy.sparse.csr_array( - (op(self.csr.data, per_term), self.csr.indices, self.csr.indptr), - shape=self.csr.shape, + (data, self.csr.indices, self.csr.indptr), shape=self.csr.shape ) - return replace(self, csr=csr, const=op(self.const, factor)).absorbed() + return replace(self, csr=csr) def assign_const(self, const: np.ndarray) -> CSRStore: """ @@ -612,7 +624,7 @@ def combine( store = self.reindex(grid) if reindex else self if factor is not None: factor = factor.broadcast_like(const).transpose(*dims) - store = store.scale(factor.to_numpy().reshape(-1), op) + store = store._scaled_terms(factor.to_numpy().reshape(-1), op) return store.assign_const(const.to_numpy().reshape(-1)).with_grid(grid) def absorbed(self) -> CSRStore: diff --git a/linopy/expressions.py b/linopy/expressions.py index fc1822339..fd72660a4 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -111,6 +111,7 @@ FillValueLike, _shared_dim_mismatch_message, absorb_absence, + check_arithmetic_nan, check_join_fill_value, check_user_nan, enforce_aux_conflict, @@ -1100,10 +1101,11 @@ def _combined_with_constant( """ expr = self._reindexed_to(self_const, needs_data_reindex) fields = {"const": op(self_const, operand)} + check_arithmetic_nan(fields["const"], op, operand, self_const) if scale: fields["coeffs"] = op(expr.coeffs, operand) - data = absorb_absence(assign_fields(expr.data, **fields)) - return self.__class__(data, self.model) + check_arithmetic_nan(fields["coeffs"], op, operand, labels=expr.vars) + return self.__class__(assign_fields(expr.data, **fields), self.model) def _apply_constant_op( self, @@ -2118,7 +2120,7 @@ def _from_store(cls, store: Store, model: Model) -> Self: return obj def _set_data(self, data: Dataset) -> None: - self._store = DenseStore(data, self._model) + self._store = DenseStore(absorb_absence(data), self._model) def _dense_store(self, reason: Reason | None) -> DenseStore: """ @@ -2279,6 +2281,7 @@ def _combined_with_constant( scale: bool, ) -> Self: const = op(self_const, operand) + check_arithmetic_nan(const, op, operand, self_const) factor = operand if scale else None store = self._store.combine(const, op, factor, needs_data_reindex) if isinstance(store, str): @@ -3159,7 +3162,7 @@ def __init__(self, data: Dataset | None, model: Model) -> None: self._data = data def _set_data(self, data: Dataset) -> None: - self._data = data + self._data = absorb_absence(data) @property def data(self) -> Dataset: diff --git a/linopy/semantics.py b/linopy/semantics.py index 6ddbe6c2c..81a07041d 100644 --- a/linopy/semantics.py +++ b/linopy/semantics.py @@ -11,9 +11,10 @@ from __future__ import annotations +import operator import os import sys -from collections.abc import Sequence +from collections.abc import Callable, Sequence from typing import Any, NoReturn, TypeAlias from warnings import warn @@ -173,6 +174,50 @@ def check_user_nan(context: str = "constant") -> NoReturn: raise ValueError(_user_nan_message(context)) +def _arithmetic_nan_message() -> str: + """Arithmetic-NaN error text — a live cell turned NaN by an operation.""" + return ( + "A constant operation produced NaN (0*inf, 0/0 or inf-inf) in a live " + "cell. linopy trusts NaN only as absence, so it does not silently drop " + "the cell; to drop cells on purpose, mark them with .where(cond) or " + "mask= instead." + ) + + +def may_produce_nan(op: Callable[[Any, Any], Any], operand: Any) -> bool: + """ + Whether ``op`` with ``operand`` can turn a non-NaN value into NaN: only + with a non-finite operand, or a zero one for a product or quotient. + """ + values = np.asarray(operand) + if not np.isfinite(values.sum()): + return True + return op not in (operator.add, operator.sub) and not values.all() + + +def check_arithmetic_nan( + result: Any, + op: Callable[[Any, Any], Any], + operand: Any, + *inputs: Any, + labels: Any = None, +) -> None: + """ + Enforce §5 for ``result``, ``op`` applied with ``operand``: raise where it + is NaN at a live term, one whose ``labels`` are not ``-1``, although + neither ``operand`` nor any of the ``inputs`` is NaN there. + """ + if not may_produce_nan(op, operand): + return + produced = np.isnan(result) + if labels is not None: + produced = produced & (labels != -1) + for value in (operand, *inputs): + produced = produced & ~np.isnan(value) + if produced.any(): + raise ValueError(_arithmetic_nan_message()) + + class AbsentType: """ Sentinel for ``fill_value=``: create the label, leave the slot absent. @@ -365,6 +410,17 @@ def _aux_values_equal(a: np.ndarray, b: np.ndarray) -> bool: return bool(pd.Series(a.ravel()).equals(pd.Series(b.ravel()))) +def _reduced_at( + values: DataArray, cells: DataArray, reduce: Callable[..., Any], initial: float +) -> Any: + """Reduce ``values`` at the ``cells`` to a scalar, ``initial`` if none.""" + extra = [d for d in cells.dims if d not in values.dims] + if extra: + cells = cells.any(extra) + where = cells.broadcast_like(values).transpose(*values.dims).values + return reduce(values.values, axis=None, initial=initial, where=where) + + def absorb_absence(ds: Dataset) -> Dataset: """ Enforce the dead-term invariant on a merged dataset. @@ -380,6 +436,10 @@ def absorb_absence(ds: Dataset) -> Dataset: if not np.isnan(ds["const"].values.sum()): return ds mask = ds["const"].isnull() + labels = _reduced_at(ds["vars"], mask, np.max, -1) + coeffs = _reduced_at(ds["coeffs"], mask, np.fmax.reduce, np.nan) + if labels == -1 and np.isnan(coeffs): + return ds coeffs = ds["coeffs"].where(~mask, np.nan) vars_ = ds["vars"].where(~mask, -1) return ds.assign(coeffs=coeffs, vars=vars_) diff --git a/linopy/store.py b/linopy/store.py index cca1c9991..d70323bef 100644 --- a/linopy/store.py +++ b/linopy/store.py @@ -31,7 +31,12 @@ from linopy.alignment import as_dataarray, fill_missing_coords from linopy.common import assign_fields, check_common_keys_values from linopy.constants import FACTOR_DIM, HELPER_DIMS, STACKED_TERM_DIM, TERM_DIM -from linopy.semantics import absorb_absence, enforce_aux_conflict, enforce_merge_dims +from linopy.semantics import ( + absorb_absence, + check_arithmetic_nan, + enforce_aux_conflict, + enforce_merge_dims, +) from linopy.types import CONSTANT_TYPES, DimsLike if TYPE_CHECKING: @@ -162,7 +167,7 @@ def scale( ) -> Self: """ Apply ``op`` with a scalar or per-cell ``factor`` to coefficients and - constant; a cell made absent drops its terms. + constant; a NaN it produces in a live cell raises. """ @abstractmethod @@ -186,7 +191,7 @@ def combine( Set the constant to ``const``, an aligned constant whose labels the cells are first remapped onto if ``reindex``, the new cells without terms; with a ``factor`` on those labels, apply ``op`` with it to the - coefficients too. + coefficients too, a NaN it produces at a live term raising. """ @abstractmethod @@ -508,6 +513,8 @@ def scale( ) -> DenseStore: ds, per_cell = self.ds, self._on_cells(factor) coeffs, const = op(ds.coeffs, per_cell), op(ds.const, per_cell) + check_arithmetic_nan(const, op, per_cell, ds.const) + check_arithmetic_nan(coeffs, op, per_cell, labels=ds.vars) return self._derived( absorb_absence(assign_fields(ds, coeffs=coeffs, const=const)) ) @@ -530,6 +537,7 @@ def combine( fields = {"const": const} if factor is not None: fields["coeffs"] = op(ds.coeffs, factor) + check_arithmetic_nan(fields["coeffs"], op, factor, labels=ds.vars) return self._derived(absorb_absence(assign_fields(ds, **fields))) def absorbed(self) -> DenseStore: diff --git a/test/test_linear_expression.py b/test/test_linear_expression.py index bb06499bb..fd0024a60 100644 --- a/test/test_linear_expression.py +++ b/test/test_linear_expression.py @@ -917,6 +917,22 @@ def test_ignores_const(self, v: Variable) -> None: assert not filled.isnull().any() assert_equal(filled.has_terms, filter.rename("has_terms")) + @pytest.mark.parametrize("build", ["linear", "quadratic"]) + @pytest.mark.parametrize("revive", ["construct", "vars-setter"]) + def test_absent_cell_drops_terms( + self, x: Variable, build: str, revive: str + ) -> None: + keep = xr.DataArray([True, False], coords=[x.indexes["dim_0"]]) + expr = 2 * x if build == "linear" else x * x + if revive == "construct": + data = expr.data.assign(const=expr.const.where(keep)) + res = type(expr)(data, expr.model) + else: + res = expr.where(keep) + res.vars = expr.vars + assert res.has_terms.values.tolist() == [True, False] + assert (res.vars.sel(dim_0=1) == -1).all() + def test_merge_reindex(self, x: Variable, y: Variable) -> None: # the nodal-balance pattern: outer merge, then reindex to a superset of # coordinates; slots beyond the original coordinates carry no terms diff --git a/test/test_quadratic_expression.py b/test/test_quadratic_expression.py index 1c851cb86..30054b7e7 100644 --- a/test/test_quadratic_expression.py +++ b/test/test_quadratic_expression.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +from collections.abc import Callable + import numpy as np import pandas as pd import polars as pl @@ -369,3 +371,18 @@ def test_variable_names(x: Variable, y: Variable) -> None: expr = 2 * (x * x) + 1 assert expr.variable_names == {"x"} + + +@pytest.mark.filterwarnings("error::RuntimeWarning") +@pytest.mark.parametrize( + "op", + [lambda q: q / 0, lambda q: (0 * q + 1) * np.inf], + ids=["div-zero", "mul-inf-zero-coeff"], +) +def test_nonfinite_constant_op_raises( + x: Variable, op: Callable[[QuadraticExpression], QuadraticExpression] +) -> None: + quad = x * x + assert isinstance(quad, QuadraticExpression) + with pytest.raises(ValueError, match="produced NaN"): + op(quad) diff --git a/test/test_store_conformance.py b/test/test_store_conformance.py index 54b893839..92a2527a1 100644 --- a/test/test_store_conformance.py +++ b/test/test_store_conformance.py @@ -509,25 +509,35 @@ def test_nan_constant_operand_rejects(backend: Model) -> None: assert expr.is_sparse == backend.sparse -@pytest.mark.parametrize( - "op", - [ - lambda e: e * OVER_A.where(OVER_A != 2.0, np.inf), - lambda e: e / OVER_A.where(OVER_A != 2.0, 0.0), - ], - ids=["mul-inf", "div-zero"], -) -def test_nonfinite_constant_op_drops_cell( - backend: Model, op: Callable[[LinearExpression], LinearExpression] -) -> None: +NONFINITE_OPS: dict[str, Callable[[LinearExpression], LinearExpression]] = { + "mul-inf": lambda e: e * OVER_A.where(OVER_A != 2.0, np.inf), + "div-zero": lambda e: e / OVER_A.where(OVER_A != 2.0, 0.0), + "add-inf-then-sub-inf": lambda e: (e + np.inf) - np.inf, + "mul-inf-zero-coeff": lambda e: (0 * e + 1) * np.inf, + "mul-array-inf-zero-coeff": lambda e: ( + (0 * e + 1) * OVER_A.where(OVER_A != 2.0, np.inf) + ), +} + + +@pytest.mark.filterwarnings("error::RuntimeWarning") +@pytest.mark.parametrize("op", list(NONFINITE_OPS)) +def test_nonfinite_constant_op_raises(backend: Model, op: str) -> None: expr = grid(backend) - res = op(expr) - absent = np.repeat([[False], [True], [False]], 2, axis=1) + with no_densify(), pytest.raises(ValueError, match="produced NaN"): + NONFINITE_OPS[op](expr) + assert expr.is_sparse == backend.sparse + + +@pytest.mark.filterwarnings("error::RuntimeWarning") +def test_nonfinite_constant_op_keeps_absent_cell(backend: Model) -> None: + expr = (grid(backend) + 1).where(COND) + with no_densify(): + res = expr * np.inf + assert res.is_sparse == backend.sparse + absent = ~COND.broadcast_like(res.const) np.testing.assert_array_equal(res.const.isnull(), absent) np.testing.assert_array_equal(res.has_terms, ~absent) - summed = res.sum().vars.values - live = expr.vars.sel(a=[0, 2]).values - np.testing.assert_array_equal(np.sort(summed[summed != -1]), np.sort(live, None)) REASON_SOURCES = [linopy.expressions, linopy.store, linopy.csr, linopy.constraints] diff --git a/test/test_store_protocol.py b/test/test_store_protocol.py index 34122603b..1fafcc9bd 100644 --- a/test/test_store_protocol.py +++ b/test/test_store_protocol.py @@ -93,7 +93,7 @@ def taken(s: Store) -> Store: "scale": lambda s: s.scale(2.0), "scale-cells": lambda s: s.scale(np.arange(s.n_cells, dtype=float) % 3), "scale-div": lambda s: s.scale( - np.arange(s.n_cells, dtype=float) % 3, operator.truediv + np.arange(s.n_cells, dtype=float) % 3 + 1, operator.truediv ), "assign-const": lambda s: s.assign_const( np.where(np.arange(s.n_cells) % 2, np.nan, 1.0)