feat: long-form inputs via from_long and relation group-by via Relation (#1019 step 5)

doc/api.rst

@@ -644,6 +644,8 @@ Utilities
    align
    options
    ABSENT
+   from_long
+   Relation
 
 ``options`` holds these keys:
 

doc/release_notes.rst

@@ -58,6 +58,15 @@ Version 1.0.0
 * ``linopy.testing.assert_conequal(strict=False)`` compares the lhs with ``assert_equivalent``, independently of its store and term layout, instead of comparing the dense datasets. (`#1047 <https://github.com/PyPSA/linopy/issues/1047>`__)
 * ``Constraint.update`` raises a ``ValueError`` when the new ``lhs`` or ``rhs`` expression adds dimensions to the constraint; broadcasting a constraint to a larger grid is not supported. Before, the update was accepted and the labels stayed on the old dimensions, which left an inconsistent constraint.
 
+**Long-form inputs**
+
+.. note::
+
+   The following content was generated by AI.
+
+* ``linopy.from_long(frame, value, coords, default=nan)`` pivots a long-form pandas or polars frame, key columns plus one value column, into a DataArray to pass wherever a constant is accepted, e.g. bounds, coefficients, right-hand sides and penalties. Each key column names a dimension of ``coords``; the result spans the key dimensions only, with their full coordinates, so it is the full cube over the key dimensions: key combinations not listed take ``default``, and linopy broadcasts the result over the dimensions without a key column. Missing value or key columns, key columns not naming a dimension, null or duplicate keys, keys outside ``coords``, a non-numeric value column and key dtypes (numeric, temporal, string) differing from the coordinates raise. (`#1049 <https://github.com/PyPSA/linopy/issues/1049>`__)
+* ``expr.groupby(linopy.Relation(frame, member, group)).sum()`` sums over a many-to-many relation, a pandas or polars frame of member and group columns whose rows may repeat a member: every row adds its member's terms to its group, e.g. links to the buses of both their ports, without an incidence matrix and ``@``. Members not listed drop out; null values and members outside the dimension raise. It stays sparse on a sparse expression; a quadratic expression raises ``NotImplementedError``. (`#1049 <https://github.com/PyPSA/linopy/issues/1049>`__)
+
 **Deprecations**
 
 * The ``active_fill`` parameter of ``add_piecewise_formulation`` is deprecated and emits a ``FutureWarning``; it is removed in 1.1. Pad a partial gate explicitly instead, e.g. ``active=status.reindex(gen=gens).fillna(1)``, which gives the same model. A partial ``active`` without padding still raises.

linopy/__init__.py

@@ -24,6 +24,7 @@
 from linopy.constraints import Constraint, Constraints
 from linopy.expressions import LinearExpression, QuadraticExpression, merge
 from linopy.io import read_netcdf
+from linopy.long import Relation, from_long
 from linopy.model import Model, Variable, Variables
 from linopy.objective import Objective
 from linopy.piecewise import (
@@ -60,13 +61,15 @@
     "QuadraticExpression",
     "RemoteHandler",
     "Slopes",
+    "Relation",
     "SolverFeature",
     "Variable",
     "Variables",
     "align",
     "available_solvers",
     "licensed_solvers",
     "breakpoints",
+    "from_long",
     "merge",
     "options",
     "read_netcdf",

linopy/expressions.py

@@ -106,6 +106,7 @@
     TERM_DIM,
 )
 from linopy.csr import CSRStore, Grid, _densify_notice
+from linopy.long import Relation
 from linopy.semantics import (
     AbsentType,
     FillValueLike,
@@ -218,6 +219,13 @@ def _resolve_group(group: Any, data: Dataset) -> Any:
     like ``groupby(data["name"])``, mirroring xarray. Other inputs (Series,
     DataFrame, DataArray, multi-key lists) are returned unchanged.
     """
+    if isinstance(group, pl.DataFrame) or (
+        isinstance(group, pd.DataFrame) and group.index.name is None
+    ):
+        raise TypeError(
+            "Grouping by a frame needs a pandas index named after a dimension, "
+            "use `linopy.Relation(frame, member, group)` for member-group pairs."
+        )
     if isinstance(group, (list, tuple)) and len(group) == 1:
         group = group[0]
     if isinstance(group, str) and group in data.coords:
@@ -433,7 +441,7 @@ class LinearExpressionGroupby:
     """
 
     obj: xr.Dataset | BaseExpression
-    group: Hashable | DataArray | IndexVariable | pd.Series | pd.DataFrame
+    group: Hashable | DataArray | IndexVariable | pd.Series | pd.DataFrame | Relation
     model: Any
     kwargs: Mapping[str, Any] = field(default_factory=dict)
 
@@ -550,13 +558,26 @@ def sum(
                 "`observed=True` is not supported with `use_fallback=True`."
             )
 
+        if isinstance(self.group, Relation):
+            if isinstance(self.obj, QuadraticExpression):
+                raise NotImplementedError(
+                    "Grouping a quadratic expression by a relation is not supported."
+                )
+            if use_fallback or observed:
+                raise ValueError(
+                    "`use_fallback` and `observed` are not supported when "
+                    "grouping by a relation."
+                )
+            return self._relation_sum(self._store, self.group)
+
+        store = self._store
+
         labels = self._labels
         group = _resolve_group(self.group, labels)
         _check_grouper_alignment(group, labels)
 
         multikey_frame = None if use_fallback else _multikey_value_frame(group, labels)
 
-        store = self._store
         store_cls = result_store_cls(store, self.model)
         if multikey_frame is not None and not observed:
             _warn_dense_grid(multikey_frame)
@@ -649,6 +670,37 @@ def func(ds: Dataset) -> Dataset:
         ds = _restore_group_dim_position(ds, original_dims, grouped_dims)
         return LinearExpression(ds, self.model)
 
+    def _relation_sum(self, store: Store, relation: Relation) -> LinearExpression:
+        """
+        Sum over a relation: every row adds its member's terms to its group,
+        so a member listed in several rows lands in several groups and one
+        not listed in none. The member cells are gathered once per row onto a
+        positional member dim, then scattered by
+        :meth:`~linopy.store.Store.from_grouper`.
+        """
+        member, group = relation.member, relation.group
+        if group in store.dims:
+            raise ValueError(
+                f"relation group column {group!r} must not name an "
+                f"existing dimension {list(store.dims)}."
+            )
+        if member not in store.dims:
+            raise ValueError(
+                f"relation member column {member!r} must name a dimension "
+                f"of {list(store.dims)}."
+            )
+        positions = relation.positions(store.grid.indexes[member])
+        rows = pd.RangeIndex(len(positions), name=member)
+        cells = store.grid.dataarray(np.arange(store.grid.size))
+        cells = cells.isel({member: positions}).assign_coords({member: rows})
+        gathered = store.take_positions(cells)
+        grouper = pd.Series(relation.groups(), index=rows, name=group)
+        store_cls = result_store_cls(store, self.model)
+        res = store_cls.from_grouper(
+            gathered, self.model, grouper, group, True, store.dims
+        )
+        return LinearExpression._from_store(res, self.model)
+
     def _grouped_sum(self, group: pd.Series, data: Dataset | None = None) -> Dataset:
         """
         Sum groups by scattering all terms directly into the final padded arrays.
@@ -1830,7 +1882,7 @@ def diff(self, dim: str, n: int = 1) -> Self:
 
     def groupby(
         self,
-        group: DataFrame | Series | DataArray,
+        group: DataFrame | Series | DataArray | Relation,
         restore_coord_dims: bool | None = None,
         **kwargs: Any,
     ) -> LinearExpressionGroupby:
@@ -1842,9 +1894,12 @@ def groupby(
 
         Parameters
         ----------
-        group : str, DataArray or IndexVariable
+        group : str, DataArray, IndexVariable, pandas.Series, pandas.DataFrame or linopy.Relation
             Array whose unique values should be used to group this array. If a
             string, must be the name of a variable contained in this dataset.
+            A :class:`~linopy.long.Relation` maps members to groups, possibly
+            several groups per member: ``sum`` adds each member to each of
+            its groups.
         restore_coord_dims : bool, optional
             If True, also restore the dimension order of multi-dimensional
             coordinates.