From ba80b65cfd1b27f934b20c6703ef65d62ec64fc8 Mon Sep 17 00:00:00 2001 From: Lucas Morin Date: Sun, 18 Oct 2026 16:20:02 +0200 Subject: [PATCH 1/2] Add grouped lag, rolling and expanding features --- .../forecasting/ExpandingWindowFeatures.rst | 28 +++- .../timeseries/forecasting/LagFeatures.rst | 28 +++- .../timeseries/forecasting/WindowFeatures.rst | 28 +++- .../forecasting/base_forecast_transformers.py | 105 +++++++++++- .../forecasting/expanding_window_features.py | 23 ++- .../timeseries/forecasting/lag_features.py | 23 ++- .../timeseries/forecasting/window_features.py | 23 ++- .../test_forecasting/test_grouped_features.py | 149 ++++++++++++++++++ 8 files changed, 400 insertions(+), 7 deletions(-) create mode 100644 tests/test_time_series/test_forecasting/test_grouped_features.py diff --git a/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst b/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst index 3d8714a67..02d4e7ffd 100644 --- a/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst +++ b/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst @@ -471,4 +471,30 @@ forecast time series using traditional machine learning models, like linear regr or gradient boosting machines. By purchasing them you are supporting `Sole `_, -the main developer of feature-engine. \ No newline at end of file +the main developer of feature-engine. + +Independent series in one dataframe +----------------------------------- + +Use ``groupby`` to identify independent series by one column or a combination of +columns. ``sort_by`` orders observations using a column, overriding ``sort_index``:: + + transformer = ExpandingWindowFeatures( + variables=["sales"], groupby=["store", "product"], sort_by="date", + min_periods=1 + ) + X_transformed = transformer.fit_transform(X) + +No lag, rolling aggregate, or expanding aggregate crosses a group boundary. +Grouping and sorting columns are retained and excluded from automatic feature +selection. The dataframe must retain a unique, non-missing row index; dates may +repeat across groups when stored in the ``sort_by`` column. Within each group, +``sort_by`` values must be unique and non-missing. Group labels must be complete. +Use a datetime sorting column or index for time-based windows and frequency shifts. + +The output preserves row labels and follows the requested sorting order. With +``sort_by=None`` and ``sort_index=False``, calculations follow input order within +each group. ``drop_na`` removes rows with incomplete generated features, and +``transform_x_y`` aligns the target with the retained rows. The transformer uses +only the rows passed to ``transform``; it does not carry training history into a +later batch. With ``groupby=None``, existing behaviour is unchanged. diff --git a/docs/user_guide/timeseries/forecasting/LagFeatures.rst b/docs/user_guide/timeseries/forecasting/LagFeatures.rst index 2e9941b0c..9372aa7b3 100644 --- a/docs/user_guide/timeseries/forecasting/LagFeatures.rst +++ b/docs/user_guide/timeseries/forecasting/LagFeatures.rst @@ -674,4 +674,30 @@ forecast time series using traditional machine learning models, like linear regr or gradient boosting machines. By purchasing them you are supporting `Sole `_, -the main developer of feature-engine. \ No newline at end of file +the main developer of feature-engine. + +Independent series in one dataframe +----------------------------------- + +Use ``groupby`` to identify independent series by one column or a combination of +columns. ``sort_by`` orders observations using a column, overriding ``sort_index``:: + + transformer = LagFeatures( + variables=["sales"], groupby=["store", "product"], sort_by="date", + periods=[1, 2] + ) + X_transformed = transformer.fit_transform(X) + +No lag, rolling aggregate, or expanding aggregate crosses a group boundary. +Grouping and sorting columns are retained and excluded from automatic feature +selection. The dataframe must retain a unique, non-missing row index; dates may +repeat across groups when stored in the ``sort_by`` column. Within each group, +``sort_by`` values must be unique and non-missing. Group labels must be complete. +Use a datetime sorting column or index for time-based windows and frequency shifts. + +The output preserves row labels and follows the requested sorting order. With +``sort_by=None`` and ``sort_index=False``, calculations follow input order within +each group. ``drop_na`` removes rows with incomplete generated features, and +``transform_x_y`` aligns the target with the retained rows. The transformer uses +only the rows passed to ``transform``; it does not carry training history into a +later batch. With ``groupby=None``, existing behaviour is unchanged. diff --git a/docs/user_guide/timeseries/forecasting/WindowFeatures.rst b/docs/user_guide/timeseries/forecasting/WindowFeatures.rst index f601afcb6..0b19dbb79 100644 --- a/docs/user_guide/timeseries/forecasting/WindowFeatures.rst +++ b/docs/user_guide/timeseries/forecasting/WindowFeatures.rst @@ -657,4 +657,30 @@ forecast time series using traditional machine learning models, like linear regr or gradient boosting machines. By purchasing them you are supporting `Sole `_, -the main developer of feature-engine. \ No newline at end of file +the main developer of feature-engine. + +Independent series in one dataframe +----------------------------------- + +Use ``groupby`` to identify independent series by one column or a combination of +columns. ``sort_by`` orders observations using a column, overriding ``sort_index``:: + + transformer = WindowFeatures( + variables=["sales"], groupby=["store", "product"], sort_by="date", + window=3, min_periods=1 + ) + X_transformed = transformer.fit_transform(X) + +No lag, rolling aggregate, or expanding aggregate crosses a group boundary. +Grouping and sorting columns are retained and excluded from automatic feature +selection. The dataframe must retain a unique, non-missing row index; dates may +repeat across groups when stored in the ``sort_by`` column. Within each group, +``sort_by`` values must be unique and non-missing. Group labels must be complete. +Use a datetime sorting column or index for time-based windows and frequency shifts. + +The output preserves row labels and follows the requested sorting order. With +``sort_by=None`` and ``sort_index=False``, calculations follow input order within +each group. ``drop_na`` removes rows with incomplete generated features, and +``transform_x_y`` aligns the target with the retained rows. The transformer uses +only the rows passed to ``transform``; it does not carry training history into a +later batch. With ``groupby=None``, existing behaviour is unchanged. diff --git a/feature_engine/timeseries/forecasting/base_forecast_transformers.py b/feature_engine/timeseries/forecasting/base_forecast_transformers.py index f7ec468bf..ad8e85267 100644 --- a/feature_engine/timeseries/forecasting/base_forecast_transformers.py +++ b/feature_engine/timeseries/forecasting/base_forecast_transformers.py @@ -77,6 +77,8 @@ def __init__( missing_values: str = "raise", drop_original: bool = False, drop_na: bool = False, + groupby=None, + sort_by=None, ) -> None: if missing_values not in ["raise", "ignore"]: @@ -104,6 +106,12 @@ def __init__( self.missing_values = missing_values self.drop_original = drop_original self.drop_na = drop_na + self.groupby = _check_variables_input_value(groupby) + if sort_by is not None and not isinstance(sort_by, (str, int)): + raise ValueError("sort_by must be a column name or None.") + if sort_by is not None and groupby is None: + raise ValueError("sort_by requires groupby.") + self.sort_by = sort_by def _check_index(self, X: pd.DataFrame): """ @@ -178,6 +186,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # Otherwise, when we merge the new features we will duplicate rows. self._check_index(X) + self._check_grouping(X) + # find or check for numerical variables if self.variables is None: self.variables_ = find_numerical_variables( @@ -186,6 +196,18 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): else: self.variables_ = check_numerical_variables(X, self.variables) + if self.groupby is not None: + excluded = self._group_columns() + ( + [] if self.sort_by is None else [self.sort_by] + ) + if self.variables is not None and set(self.variables_) & set(excluded): + raise ValueError("Grouping and sorting columns cannot be variables.") + self.variables_ = [v for v in self.variables_ if v not in excluded] + if not self.variables_ and not self.return_empty: + raise ValueError( + "No numerical variables remain after excluding group columns." + ) + # check if dataset contains na if self.missing_values == "raise": self._check_na_and_inf(X) @@ -221,6 +243,7 @@ def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: # Dataframes must have unique values in the index and no missing data. # Otherwise, when we merge the created features we will duplicate rows. self._check_index(X) + self._check_grouping(X) # check if dataset contains na if self.missing_values == "raise": @@ -229,11 +252,91 @@ def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: # reorder variables to match train set X = X[self.feature_names_in_] - if self.sort_index is True: + if self.sort_by is not None: + X = X.sort_values(self.sort_by, kind="stable") + elif self.sort_index is True: X.sort_index(inplace=True) return X + def _group_columns(self): + return self.groupby if isinstance(self.groupby, list) else [self.groupby] + + def _check_grouping(self, X): + if self.groupby is None: + return + columns = self._group_columns() + required = columns + ([] if self.sort_by is None else [self.sort_by]) + if any(c not in X.columns for c in required): + raise ValueError("Grouping and sorting columns must be present in X.") + if X[required].isna().any().any(): + raise ValueError( + "Grouping and sorting columns cannot contain missing values." + ) + if self.sort_by is not None and X.duplicated(required).any(): + raise ValueError("sort_by must be unique within each group.") + + def _transform_grouped(self, X, kind): + """Compute features within groups, then align on group and time together.""" + if not self.variables_: + return X + data = X[self.variables_].copy() + if self.sort_by is not None: + data.index = pd.Index(X[self.sort_by]) + keys = [X[c].to_numpy() for c in self._group_columns()] + target_index = pd.MultiIndex.from_arrays(keys + [data.index]) + grouped = data.groupby(keys, sort=False, observed=True) + levels = list(range(len(keys))) + parts = [] + if kind == "lag": + shifts = self.freq if self.freq is not None else self.periods + shifts = shifts if isinstance(shifts, list) else [shifts] + for shift in shifts: + if self.freq is None: + part = grouped.shift(shift, fill_value=self.fill_value) + part.index = X.index + else: + part = data.copy() + part.index = pd.MultiIndex.from_arrays( + keys + [data.index.shift(1, freq=shift)] + ) + part = part.reindex(target_index) + part.index = X.index + if self.fill_value is not None: + part = part.fillna(self.fill_value) + parts.append(part) + else: + windows = self.window if kind == "window" else [None] + windows = windows if isinstance(windows, list) else [windows] + for window in windows: + if kind == "window": + part = grouped.rolling( + window=window, min_periods=self.min_periods + ).agg(self.functions) + else: + part = grouped.expanding(min_periods=self.min_periods).agg( + self.functions + ) + if self.freq is None: + part = part.groupby(level=levels, sort=False).shift(self.periods) + else: + arrays = [part.index.get_level_values(i) for i in levels] + time = part.index.get_level_values(-1) + part.index = pd.MultiIndex.from_arrays( + arrays + [time.shift(self.periods, freq=self.freq)] + ) + part = part.reindex(target_index) + part.index = X.index + parts.append(part) + features = pd.concat(parts, axis=1) + features.columns = self._get_new_features_name() + result = pd.concat([X, features], axis=1) + if self.drop_original: + result = result.drop(columns=self.variables_) + if self.drop_na: + result = result.dropna(subset=features.columns) + return result + def _more_tags(self): tags_dict = _return_tags() tags_dict["allow_nan"] = True diff --git a/feature_engine/timeseries/forecasting/expanding_window_features.py b/feature_engine/timeseries/forecasting/expanding_window_features.py index c121734e1..2c9f07932 100644 --- a/feature_engine/timeseries/forecasting/expanding_window_features.py +++ b/feature_engine/timeseries/forecasting/expanding_window_features.py @@ -101,6 +101,16 @@ class ExpandingWindowFeatures(BaseForecastTransformer): Whether the NAN introduced in the created features should be removed. + groupby: str, int or list, default=None + Columns whose combined values identify each independent time series. + Grouping columns are excluded from automatic variable selection. + + sort_by: str or int, default=None + Column used to order observations before grouped transformations. Overrides + ``sort_index``. Required to be non-missing, and unique within each group. + Use a datetime column for time-based windows or frequency shifts. Requires + ``groupby``. The output follows this sorted order and preserves row indexes. + Attributes ---------- variables_: @@ -164,6 +174,8 @@ def __init__( missing_values: str = "raise", drop_original: bool = False, drop_na: bool = False, + groupby=None, + sort_by=None, ) -> None: if not isinstance(functions, (str, list)) or not all( @@ -182,7 +194,13 @@ def __init__( ) super().__init__( - variables, return_empty, missing_values, drop_original, drop_na + variables, + return_empty, + missing_values, + drop_original, + drop_na, + groupby=groupby, + sort_by=sort_by, ) self.min_periods = min_periods @@ -208,6 +226,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # Common dataframe checks and setting up. X = self._check_transform_input_and_state(X) + if self.groupby is not None: + return self._transform_grouped(X, "expanding") + tmp = ( X[self.variables_] .expanding(min_periods=self.min_periods) diff --git a/feature_engine/timeseries/forecasting/lag_features.py b/feature_engine/timeseries/forecasting/lag_features.py index 578802d43..16cdf332b 100644 --- a/feature_engine/timeseries/forecasting/lag_features.py +++ b/feature_engine/timeseries/forecasting/lag_features.py @@ -88,6 +88,16 @@ class LagFeatures(BaseForecastTransformer): drop_na: bool, default=False. Whether the NAN introduced in the lag features should be removed. + groupby: str, int or list, default=None + Columns whose combined values identify each independent time series. + Grouping columns are excluded from automatic variable selection. + + sort_by: str or int, default=None + Column used to order observations before grouped transformations. Overrides + ``sort_index``. Required to be non-missing, and unique within each group. + Use a datetime column for time-based windows or frequency shifts. Requires + ``groupby``. The output follows this sorted order and preserves row indexes. + Attributes ---------- variables_: @@ -147,6 +157,8 @@ def __init__( missing_values: str = "raise", drop_original: bool = False, drop_na: bool = False, + groupby=None, + sort_by=None, ) -> None: if not ( @@ -172,7 +184,13 @@ def __init__( ) super().__init__( - variables, return_empty, missing_values, drop_original, drop_na + variables, + return_empty, + missing_values, + drop_original, + drop_na, + groupby=groupby, + sort_by=sort_by, ) self.periods = periods @@ -197,6 +215,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # Common dataframe checks and setting up. X = self._check_transform_input_and_state(X) + if self.groupby is not None: + return self._transform_grouped(X, "lag") + # if freq is not None, it overrides periods. if self.freq is not None: diff --git a/feature_engine/timeseries/forecasting/window_features.py b/feature_engine/timeseries/forecasting/window_features.py index f5f73c7f7..84d5ccc13 100644 --- a/feature_engine/timeseries/forecasting/window_features.py +++ b/feature_engine/timeseries/forecasting/window_features.py @@ -105,6 +105,16 @@ class WindowFeatures(BaseForecastTransformer): drop_na: bool, default=False. Whether the NAN introduced in the lag features should be removed. + groupby: str, int or list, default=None + Columns whose combined values identify each independent time series. + Grouping columns are excluded from automatic variable selection. + + sort_by: str or int, default=None + Column used to order observations before grouped transformations. Overrides + ``sort_index``. Required to be non-missing, and unique within each group. + Use a datetime column for time-based windows or frequency shifts. Requires + ``groupby``. The output follows this sorted order and preserves row indexes. + Attributes ---------- variables_: @@ -168,6 +178,8 @@ def __init__( missing_values: str = "raise", drop_original: bool = False, drop_na: bool = False, + groupby=None, + sort_by=None, ) -> None: if isinstance(window, list) and len(window) != len(set(window)): @@ -189,7 +201,13 @@ def __init__( ) super().__init__( - variables, return_empty, missing_values, drop_original, drop_na + variables, + return_empty, + missing_values, + drop_original, + drop_na, + groupby=groupby, + sort_by=sort_by, ) self.window = window @@ -216,6 +234,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # Common dataframe checks and setting up. X = self._check_transform_input_and_state(X) + if self.groupby is not None: + return self._transform_grouped(X, "window") + if isinstance(self.window, list): df_ls = [] for win in self.window: diff --git a/tests/test_time_series/test_forecasting/test_grouped_features.py b/tests/test_time_series/test_forecasting/test_grouped_features.py new file mode 100644 index 000000000..e1f6f973d --- /dev/null +++ b/tests/test_time_series/test_forecasting/test_grouped_features.py @@ -0,0 +1,149 @@ +import pandas as pd +import pytest +from sklearn.base import clone + +from feature_engine.timeseries.forecasting import ( + ExpandingWindowFeatures, + LagFeatures, + WindowFeatures, +) + + +def panel(): + return pd.DataFrame( + { + "store": [1, 2, 1, 2, 1, 2], + "product": ["a"] * 6, + "time": pd.to_datetime( + [ + "2024-01-01", + "2024-01-01", + "2024-01-02", + "2024-01-02", + "2024-01-03", + "2024-01-03", + ] + ), + "sales": [10.0, 100.0, 20.0, 200.0, 30.0, 300.0], + }, + index=[9, 7, 5, 3, 1, 0], + ) + + +@pytest.mark.parametrize( + "cls,kwargs", + [ + (LagFeatures, {"periods": [1, 2]}), + (LagFeatures, {"freq": ["1D", "2D"]}), + ( + WindowFeatures, + {"window": [2, 3], "min_periods": 1, "functions": ["mean", "max"]}, + ), + (WindowFeatures, {"window": "2D", "min_periods": 1}), + (WindowFeatures, {"window": 2, "min_periods": 1, "freq": "1D"}), + (ExpandingWindowFeatures, {"min_periods": 1}), + (ExpandingWindowFeatures, {"min_periods": 1, "freq": "1D"}), + ], +) +@pytest.mark.parametrize("drop_na", [False, True]) +def test_grouped_output_matches_independent_series(cls, kwargs, drop_na): + X = panel() + original = X.copy(deep=True) + transformer = cls( + variables=["sales"], + groupby=["store", "product"], + sort_by="time", + drop_na=drop_na, + **kwargs, + ) + actual = transformer.fit_transform(X) + expected = [] + for _, group in X.groupby(["store", "product"], sort=False): + subset = group.sort_values("time").set_index("time") + result = cls(variables=["sales"], drop_na=drop_na, **kwargs).fit_transform( + subset + ) + result["time"] = result.index + result.index = ( + group.set_index("time") + .loc[result.index] + .index.map(dict(zip(group.time, group.index))) + ) + expected.append(result) + expected = pd.concat(expected).reindex(actual.index)[actual.columns] + expected.index.name = actual.index.name + pd.testing.assert_frame_equal(actual, expected) + pd.testing.assert_frame_equal(X, original) + assert list(actual.columns) == list(transformer.get_feature_names_out()) + clone(transformer).fit(X) + + +@pytest.mark.parametrize("cls", [LagFeatures, WindowFeatures, ExpandingWindowFeatures]) +def test_group_columns_excluded_and_target_aligned(cls): + X = panel() + transformer = cls(groupby="store", sort_by="time", drop_na=True) + transformer.fit(X) + assert transformer.variables_ == ["sales"] + actual, target = transformer.transform_x_y(X, pd.Series(range(6), index=X.index)) + pd.testing.assert_series_equal( + target, pd.Series(range(6), index=X.index).loc[actual.index] + ) + + +def test_positional_lags_do_not_cross_groups(): + X = panel() + result = LagFeatures( + groupby="store", sort_index=False, fill_value=-1 + ).fit_transform(X) + assert result.sales_lag_1.tolist() == [-1.0, -1.0, 10.0, 100.0, 20.0, 200.0] + + +def test_invalid_group_data(): + X = panel() + with pytest.raises(ValueError, match="present"): + LagFeatures(groupby="unknown").fit(X) + with pytest.raises(ValueError, match="cannot be variables"): + LagFeatures(groupby="store", variables="store").fit(X) + X.loc[9, "store"] = float("nan") + with pytest.raises(ValueError, match="missing"): + LagFeatures(groupby="store").fit(X) + + +def test_combined_keys_define_groups_not_individual_columns(): + X = pd.DataFrame( + { + "store": [1, 1, 2, 2] * 2, + "product": ["a", "b", "a", "b"] * 2, + "sales": [1.0, 10.0, 100.0, 1000.0, 2.0, 20.0, 200.0, 2000.0], + } + ) + result = LagFeatures(groupby=["store", "product"], fill_value=-1).fit_transform(X) + assert result.sales_lag_1.tolist() == [ + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 10.0, + 100.0, + 1000.0, + ] + + +def test_grouped_window_drop_original_and_unseen_group(): + X = panel() + transformer = WindowFeatures( + groupby="store", sort_by="time", window=2, min_periods=1, drop_original=True + ).fit(X) + X["store"] += 100 + result = transformer.transform(X) + assert "sales" not in result + assert result.sales_window_2_mean.iloc[:2].isna().all() + assert result.sales_window_2_mean.iloc[2:].tolist() == [10.0, 100.0, 15.0, 150.0] + + +def test_duplicate_time_within_group_is_rejected(): + X = panel() + X.loc[5, "time"] = X.loc[9, "time"] + with pytest.raises(ValueError, match="unique within each group"): + LagFeatures(groupby="store", sort_by="time").fit(X) From 89702c8a52336ba96407f67c90d9f24a8002b042 Mon Sep 17 00:00:00 2001 From: Lucas Morin Date: Sun, 18 Oct 2026 17:07:58 +0200 Subject: [PATCH 2/2] Fix sparse input validation in sklearn test adapter --- tests/estimator_checks/sklearn_check_wrapper.py | 4 ++++ .../test_check_input_wrapper.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/test_sklearn_compatible/test_check_input_wrapper.py diff --git a/tests/estimator_checks/sklearn_check_wrapper.py b/tests/estimator_checks/sklearn_check_wrapper.py index 5e5deb459..ac4511f07 100644 --- a/tests/estimator_checks/sklearn_check_wrapper.py +++ b/tests/estimator_checks/sklearn_check_wrapper.py @@ -20,6 +20,7 @@ import numpy as np import pandas as pd +from scipy.sparse import issparse from sklearn.base import BaseEstimator, TransformerMixin, clone from sklearn.utils.validation import check_is_fitted @@ -62,6 +63,9 @@ def _to_df(X): if hasattr(X, "iloc"): return X + if issparse(X): + raise TypeError("Sparse input is not supported by this transformer.") + arr = np.asarray(X) if arr.ndim == 0: raise ValueError( diff --git a/tests/test_sklearn_compatible/test_check_input_wrapper.py b/tests/test_sklearn_compatible/test_check_input_wrapper.py new file mode 100644 index 000000000..4da6fd393 --- /dev/null +++ b/tests/test_sklearn_compatible/test_check_input_wrapper.py @@ -0,0 +1,17 @@ +import numpy as np +import pytest +from scipy import sparse + +from tests.estimator_checks.sklearn_check_wrapper import _SklearnCheckInputWrapper + + +@pytest.mark.parametrize("format", ["bsr", "coo", "csc", "csr", "dia", "dok", "lil"]) +@pytest.mark.parametrize("kind", ["matrix", "array"]) +def test_sparse_input_raises_explicit_error(format, kind): + constructor = getattr(sparse, f"{format}_{kind}", None) + if constructor is None: + pytest.skip("Sparse array format unavailable in this SciPy version.") + X = constructor(np.eye(3)) + + with pytest.raises(TypeError, match="[Ss]parse.*not supported"): + _SklearnCheckInputWrapper._to_df(X)