From c4aee8450a6f142efa1f081f62d28047cf4590a7 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 22 Jul 2026 07:00:34 +0200 Subject: [PATCH 1/9] update dataframe checks --- feature_engine/dataframe_checks.py | 289 +++++++++++++---------------- tests/test_dataframe_checks.py | 120 +++++++++++- 2 files changed, 241 insertions(+), 168 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index fe313a627..e826f3a49 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -2,120 +2,105 @@ transform(). """ -from typing import List, Tuple, Union +from typing import List, Union +import narwhals as nw +import narwhals.dependencies as nwd import numpy as np import pandas as pd from scipy.sparse import issparse -from sklearn.utils.validation import _check_y, check_consistent_length, column_or_1d +from sklearn.utils.validation import ( + _check_y, + check_array, + check_consistent_length, + column_or_1d, +) -from feature_engine.variable_handling._variable_type_checks import is_object - -def check_X(X: Union[np.generic, np.ndarray, pd.DataFrame]) -> pd.DataFrame: +def check_X(X): """ - Checks if the input is a DataFrame and then creates a copy. This is an important - step not to accidentally transform the original dataset entered by the user. - - If the input is a numpy array, it converts it to a pandas Dataframe. The column - names are strings representing the column index starting at 0. + Checks that X is a pandas or polars dataframe, or a numpy array, and returns a + copy. Copying is an important step so that we don't accidentally modify the + original dataset entered by the user. - Feature-engine was originally designed to work with pandas dataframes. However, - allowing numpy arrays as input allows 2 things: + Numpy arrays are converted to a pandas DataFrame, with column names that are + strings representing the column index starting at 0. This, together with + accepting numpy arrays at all, allows Feature-engine transformers to: - We can use the Scikit-learn tests for transformers provided by the - `check_estimator` function to test the compatibility of our transformers with - sklearn functionality. - - Feature-engine transformers can be used within a Scikit-learn Pipeline together - with Scikit-learn transformers like the `SimpleImputer`, which return by default - Numpy arrays. + - be evaluated with Scikit-learn's `check_estimator`, which passes numpy arrays + to `fit()` and `transform()`. + - be used within a Scikit-learn Pipeline, next to Scikit-learn transformers like + the `SimpleImputer`, which return numpy arrays by default. Parameters ---------- - X : pandas Dataframe or numpy array. + X : pandas or polars DataFrame, or numpy array. The input to check and copy or transform. Raises ------ TypeError - If the input is not a Pandas DataFrame or a numpy array. + If the input is not a pandas or polars DataFrame, or a numpy array. ValueError - If the input is an empty dataframe. + If the input has duplicated column names, or 0 columns or rows. Returns ------- - X : pandas Dataframe. - A copy of original DataFrame or a converted Numpy array. + X : pandas or polars DataFrame. + A copy of the original dataframe, or the numpy array converted to a pandas + DataFrame. """ - if isinstance(X, pd.DataFrame): - if not X.columns.is_unique: - raise ValueError("Input data contains duplicated variable names.") - X = X.copy() - - elif isinstance(X, (np.generic, np.ndarray)): - # If input is scalar raise error - if X.ndim == 0: - raise ValueError( - "Expected 2D array, got scalar array instead:\narray={}.\n" - "Reshape your data either using array.reshape(-1, 1) if " - "your data has a single feature or array.reshape(1, -1) " - "if it contains a single sample.".format(X) - ) - # If input is 1D raise error - if X.ndim == 1: + if nwd.is_into_dataframe(X): + # from_native() raises narwhals.exceptions.DuplicateError, a ValueError + # subclass, when the dataframe has duplicated column names. + nw_X = nw.from_native(X, eager_only=True) + if nw_X.is_empty(): raise ValueError( - "Expected 2D array, got 1D array instead:\narray={}.\n" - "Reshape your data either using array.reshape(-1, 1) if " - "your data has a single feature or array.reshape(1, -1) " - "if it contains a single sample.".format(X) + f"Found array with 0 feature(s) (shape={nw_X.shape}) while a " + "minimum of 1 is required." ) + return nw_X.clone().to_native() - if np.any(np.iscomplex(X)): - raise TypeError("Complex data not supported by this transformer.") - - X = pd.DataFrame(X) - X.columns = [f"x{i}" for i in range(X.shape[1])] - - elif issparse(X): - raise TypeError("This transformer does not support sparse matrices.") - - else: - raise TypeError( - f"X must be a numpy array or pandas dataframe. Got {type(X)} instead." + if isinstance(X, (np.generic, np.ndarray)) or issparse(X): + X = check_array( + X, accept_sparse=False, dtype=None, ensure_all_finite=False, ensure_2d=True ) + return pd.DataFrame(X, columns=[f"x{i}" for i in range(X.shape[1])]) - if X.empty: - raise ValueError( - "0 feature(s) (shape=%s) while a minimum of %d is required." % (X.shape, 1) - ) + raise TypeError( + "X must be a numpy array or a pandas or polars dataframe. " + f"Got {type(X)} instead." + ) - return X + +def _copy_series(y: nw.Series): + """Returns a copy of a narwhals Series, in its native library format.""" + name = y.name + frame = y.to_frame().clone() + return frame.get_column(frame.columns[0]).alias(name).to_native() def check_y( y: Union[np.generic, np.ndarray, pd.Series, pd.DataFrame, List], y_numeric: bool = False, -) -> pd.Series: +): """ - Checks that y is a series or a dataframe, or alternatively, if it can be converted - to a series or dataframe. + Checks that y is a pandas or polars Series or DataFrame, or alternatively, if it + can be converted to a pandas Series or DataFrame. Parameters ---------- - y : pd.Series, pd.DataFrame, np.array, list + y : pandas or polars Series or DataFrame, np.array, list The input to check and copy or transform. y_numeric : bool, default=False - Whether to ensure that y has a numeric type. If dtype of y is object, - it is converted to float64. Should only be used for regression - algorithms. + Whether to ensure that y has a numeric type. If dtype of y is not numeric, + it is cast to float64. Should only be used for regression algorithms. Returns ------- - y: pd.Series or pd.DataFrame + y: pandas or polars Series or DataFrame """ - if y is None: raise ValueError( "requires y to be passed, but the target y is None", @@ -123,109 +108,98 @@ def check_y( "y should be a 1d array", ) - elif isinstance(y, pd.Series): - if y.isnull().any(): + if nwd.is_into_series(y): + nw_y = nw.from_native(y, series_only=True) + if nw_y.is_null().any(): raise ValueError("y contains NaN values.") - if not is_object(y) and not np.isfinite(y).all(): - raise ValueError("y contains infinity values.") - if y_numeric and is_object(y): - y = y.astype("float64") - y = y.copy() - - elif isinstance(y, pd.DataFrame): - if y.isnull().any().any(): + if nw_y.dtype.is_numeric(): + if not np.isfinite(nw_y.to_numpy()).all(): + raise ValueError("y contains infinity values.") + elif y_numeric: + nw_y = nw_y.cast(nw.Float64()) + return _copy_series(nw_y) + + if nwd.is_into_dataframe(y): + nw_y = nw.from_native(y, eager_only=True) + if nw_y.select(nw.all().is_null().any()).to_numpy().any(): raise ValueError("y contains NaN values.") - if not np.isfinite(y).all().all(): + if not np.isfinite(nw_y.to_numpy()).all(): raise ValueError("y contains infinity values.") - y = y.copy() - - else: - try: - y = column_or_1d(y) - y = _check_y(y, multi_output=False, y_numeric=y_numeric) - y = pd.Series(y).copy() - except ValueError: - y = _check_y(y, multi_output=True, y_numeric=y_numeric) - y = pd.DataFrame(y).copy() + return nw_y.clone().to_native() + + try: + y = column_or_1d(y) + y = _check_y(y, multi_output=False, y_numeric=y_numeric) + y = pd.Series(y) + except ValueError: + y = _check_y(y, multi_output=True, y_numeric=y_numeric) + y = pd.DataFrame(y) return y def check_X_y( - X: Union[np.generic, np.ndarray, pd.DataFrame], + X, y: Union[np.generic, np.ndarray, pd.Series, List], y_numeric: bool = False, -) -> Tuple[pd.DataFrame, pd.Series]: +): """ - Ensures X and y are compatible pandas DataFrame and Series. If both are pandas - objects, checks that their indexes match. If any is a numpy array, converts to - pandas object with compatible index. - - This transformer ensures that we can concatenate X and y using `pandas.concat`, - functionality needed in the encoders. + Ensures X and y are compatible dataframe/array-like objects with a consistent + number of rows. If both are pandas objects, also checks that their indexes match. Parameters ---------- - X: Pandas DataFrame or numpy ndarray + X: pandas or polars DataFrame, or numpy ndarray The input to check and copy or transform. - y: pd.Series, np.array, list + y: pandas or polars Series, np.array, list The input to check and copy or transform. y_numeric : bool, default=False - Whether to ensure that y has a numeric type. If dtype of y is object, - it is converted to float64. Should only be used for regression - algorithms. + Whether to ensure that y has a numeric type. If dtype of y is not numeric, + it is cast to float64. Should only be used for regression algorithms. Raises ------ - ValueError: if X and y are pandas objects with inconsistent indexes. - TypeError: if X is sparse matrix, empty dataframe or not a dataframe. - TypeError: if y can't be parsed as pandas Series. + ValueError: if X and y have a different number of rows, or if they are pandas + objects with inconsistent indexes. + TypeError: if X is a sparse matrix, an empty dataframe, or not a recognised type. Returns ------- - X: Pandas DataFrame - y: Pandas Series + X: pandas or polars DataFrame + y: pandas or polars Series """ - - def _check_X_y(X, y): - X = check_X(X) - y = check_y(y, y_numeric=y_numeric) - check_consistent_length(X, y) - return X, y - - # case 1: both are pandas objects - if isinstance(X, pd.DataFrame) and isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) - # Check that their indexes match. - if X.index.equals(y.index) is False: + # Whether the raw inputs already carried a meaningful pandas index, before + # check_X/check_y potentially convert them (e.g. a numpy array gets a default + # index, which should be overridden by the other input's index, if it has one). + x_had_index = isinstance(X, pd.DataFrame) + y_had_index = isinstance(y, (pd.Series, pd.DataFrame)) + + X = check_X(X) + y = check_y(y, y_numeric=y_numeric) + check_consistent_length(X, y) + + if x_had_index and y_had_index: + if not X.index.equals(y.index): raise ValueError("The indexes of X and y do not match.") - - # case 2: X is dataframe and y is something else - if isinstance(X, pd.DataFrame) and not isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) + elif x_had_index and isinstance(y, (pd.Series, pd.DataFrame)): y.index = X.index - - # case 3: X is not a dataframe and y is a series - elif not isinstance(X, pd.DataFrame) and isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) + elif y_had_index and isinstance(X, pd.DataFrame): X.index = y.index - - # all other cases - else: - X, y = _check_X_y(X, y) + # else: neither raw input carried a pandas index to reconcile (e.g. polars + # objects), so check_consistent_length above is the only check that applies. return X, y -def _check_X_matches_training_df(X: pd.DataFrame, reference: int) -> None: +def _check_X_matches_training_df(X, reference: int) -> None: """ - Checks that DataFrame to transform has the same number of columns that the - DataFrame used with the fit() method. + Checks that the dataframe to transform has the same number of columns as the + dataframe used with the fit() method. Parameters ---------- - X : Pandas DataFrame + X : pandas or polars DataFrame The df to be checked reference : int The number of columns in the dataframe that was used with the fit() method. @@ -234,31 +208,24 @@ def _check_X_matches_training_df(X: pd.DataFrame, reference: int) -> None: ------ ValueError If the number of columns does not match. - - Returns - ------- - None """ - if X.shape[1] != reference: raise ValueError( "The number of columns in this dataset is different from the one used to " "fit this transformer (when using the fit() method)." ) - return None - def _check_contains_na( - X: pd.DataFrame, + X, variables: List[Union[str, int]], ) -> None: """ - Checks if DataFrame contains null values in the selected columns. + Checks if the dataframe contains null values in the selected columns. Parameters ---------- - X : Pandas DataFrame + X : pandas or polars DataFrame variables : List The selected group of variables in which null values will be examined. @@ -268,23 +235,21 @@ def _check_contains_na( ValueError If the variable(s) contain null values. """ - - if X[variables].isnull().any().any(): + nw_X = nw.from_native(X, eager_only=True) + if nw_X.select(nw.col(variables).is_null().any()).to_numpy().any(): raise ValueError( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer." ) -def _check_optional_contains_na( - X: pd.DataFrame, variables: List[Union[str, int]] -) -> None: +def _check_optional_contains_na(X, variables: List[Union[str, int]]) -> None: """ - Checks if DataFrame contains null values in the selected columns. + Checks if the dataframe contains null values in the selected columns. Parameters ---------- - X : Pandas DataFrame + X : pandas or polars DataFrame variables : List The selected group of variables in which null values will be examined. @@ -294,8 +259,8 @@ def _check_optional_contains_na( ValueError If the variable(s) contain null values. """ - - if X[variables].isnull().any().any(): + nw_X = nw.from_native(X, eager_only=True) + if nw_X.select(nw.col(variables).is_null().any()).to_numpy().any(): raise ValueError( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer or set the parameter " @@ -303,23 +268,23 @@ def _check_optional_contains_na( ) -def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> None: +def _check_contains_inf(X, variables: List[Union[str, int]]) -> None: """ - Checks if DataFrame contains inf values in the selected columns. + Checks if the dataframe contains inf values in the selected columns. Parameters ---------- - X : Pandas DataFrame + X : pandas or polars DataFrame variables : List - The selected group of variables in which null values will be examined. + The selected group of variables in which infinite values will be examined. Raises ------ ValueError If the variable(s) contain np.inf values """ - - if np.isinf(X[variables]).any().any(): + values = nw.from_native(X, eager_only=True).select(nw.col(variables)).to_numpy() + if np.isinf(values.astype(float)).any(): raise ValueError( "Some of the variables to transform contain inf values. Check and " "remove those before using this transformer." diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 711b1aea7..c27311990 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -1,7 +1,10 @@ import numpy as np import pandas as pd +import polars as pl import pytest from pandas.testing import assert_frame_equal, assert_series_equal +from polars.testing import assert_frame_equal as pl_assert_frame_equal +from polars.testing import assert_series_equal as pl_assert_series_equal from scipy.sparse import csr_matrix from feature_engine.dataframe_checks import ( @@ -44,7 +47,7 @@ def test_check_X_raises_error_with_complex_data(): rng = np.random.RandomState(0) X = rng.uniform(size=10) + 1j * rng.uniform(size=10) X = X.reshape(-1, 1) - with pytest.raises(TypeError, match=msg): + with pytest.raises(ValueError, match=msg): assert check_X(X) @@ -284,21 +287,126 @@ def test_check_X_raises_error_on_duplicated_column_names(): df.columns = ["var_A", "var_A", "var_B", "var_C"] with pytest.raises(ValueError) as err_txt: check_X(df) - assert err_txt.match("Input data contains duplicated variable names.") + assert err_txt.match("Expected unique column names") def test_check_X_errors(): - # Test scalar array error (line 58) + # Test scalar array error with pytest.raises(ValueError) as record: check_X(np.array(1)) assert record.match("Expected 2D array, got scalar array instead") - # Test 1D array error (line 65) + # Test 1D array error with pytest.raises(ValueError) as record: check_X(np.array([1, 2, 3])) assert record.match("Expected 2D array, got 1D array instead") - # Test incorrect type error (line 80) + # Test incorrect type error with pytest.raises(TypeError) as record: check_X("not a dataframe") - assert record.match("X must be a numpy array or pandas dataframe") + assert record.match("X must be a numpy array or a pandas or polars dataframe") + + +# ------------------------ +# polars support +# ------------------------ + + +def test_check_X_accepts_polars_and_returns_a_copy(): + df = pl.DataFrame({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]}) + X = check_X(df) + assert isinstance(X, pl.DataFrame) + pl_assert_frame_equal(X, df) + assert X is not df + + +def test_check_X_polars_raises_error_if_empty(): + with pytest.raises(ValueError): + check_X(pl.DataFrame({"a": []})) + + +def test_check_y_accepts_polars_series_and_returns_a_copy(): + s = pl.Series("target", [0, 1, 2, 3, 4]) + y = check_y(s) + assert isinstance(y, pl.Series) + pl_assert_series_equal(y, s) + assert y is not s + + +def test_check_y_polars_raises_nan_error(): + s = pl.Series("target", [0.0, None, 2.0]) + with pytest.raises(ValueError) as record: + check_y(s) + assert str(record.value) == "y contains NaN values." + + +def test_check_y_polars_raises_inf_error(): + s = pl.Series("target", [0.0, float("inf"), 2.0]) + with pytest.raises(ValueError) as record: + check_y(s) + assert str(record.value) == "y contains infinity values." + + +def test_check_y_polars_converts_string_to_number(): + s = pl.Series("target", ["0", "1", "2"]) + y = check_y(s, y_numeric=True) + assert y.dtype.is_numeric() + pl_assert_series_equal(y, pl.Series("target", [0.0, 1.0, 2.0])) + + +def test_check_x_y_polars_returns_polars(): + df = pl.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + s = pl.Series("target", [0, 1, 2]) + X, y = check_X_y(df, s) + assert isinstance(X, pl.DataFrame) and isinstance(y, pl.Series) + pl_assert_frame_equal(X, df) + pl_assert_series_equal(y, s) + + +def test_check_x_y_polars_raises_error_when_inconsistent_length(): + df = pl.DataFrame({"a": [1, 2, 3]}) + s = pl.Series([0, 1]) + with pytest.raises(ValueError): + check_X_y(df, s) + + +def test_check_X_matches_training_df_with_polars(): + df = pl.DataFrame({"a": [1, 2], "b": [3, 4]}) + with pytest.raises(ValueError): + _check_X_matches_training_df(df, 3) + + +def test_contains_na_with_polars(): + df = pl.DataFrame({"Name": ["tom", None], "City": ["London", "Manchester"]}) + msg = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer." + ) + with pytest.raises(ValueError) as record: + _check_contains_na(df, ["Name", "City"]) + assert str(record.value) == msg + + +def test_optional_contains_na_with_polars(): + df = pl.DataFrame({"Name": ["tom", None], "City": ["London", "Manchester"]}) + msg = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer or set the parameter " + "`missing_values='ignore'` when initialising this transformer." + ) + with pytest.raises(ValueError) as record: + _check_optional_contains_na(df, ["Name", "City"]) + assert str(record.value) == msg + + +def test_contains_inf_with_polars(): + msg = ( + "Some of the variables to transform contain inf values. Check and " + "remove those before using this transformer." + ) + df = pl.DataFrame({"A": [1.1, float("inf"), 3.3]}) + with pytest.raises(ValueError, match=msg): + _check_contains_inf(df, ["A"]) + + df_ok = pl.DataFrame({"A": [1.1, 2.2, 3.3]}) + assert _check_contains_inf(df_ok, ["A"]) is None From 67d9850b4a1f9d3e33038745deb373599ced1b22 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 22 Jul 2026 07:35:01 +0200 Subject: [PATCH 2/9] update dataframe checks take 2 --- feature_engine/dataframe_checks.py | 80 +++++++++++++++++++----------- tests/test_dataframe_checks.py | 2 +- 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index e826f3a49..10355ea7d 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -7,7 +7,6 @@ import narwhals as nw import narwhals.dependencies as nwd import numpy as np -import pandas as pd from scipy.sparse import issparse from sklearn.utils.validation import ( _check_y, @@ -19,9 +18,10 @@ def check_X(X): """ - Checks that X is a pandas or polars dataframe, or a numpy array, and returns a - copy. Copying is an important step so that we don't accidentally modify the - original dataset entered by the user. + Checks that X is a dataframe from any library supported by narwhals (for example + pandas, polars, modin, cuDF or PyArrow), or a numpy array, and returns a copy. + Copying is an important step so that we don't accidentally modify the original + dataset entered by the user. Numpy arrays are converted to a pandas DataFrame, with column names that are strings representing the column index starting at 0. This, together with @@ -32,21 +32,25 @@ def check_X(X): - be used within a Scikit-learn Pipeline, next to Scikit-learn transformers like the `SimpleImputer`, which return numpy arrays by default. + Pandas is only imported, lazily, when X is a numpy array. If you pass a dataframe + from any other narwhals-supported library, pandas does not need to be installed. + Parameters ---------- - X : pandas or polars DataFrame, or numpy array. + X : dataframe (pandas, polars, or any other library supported by narwhals), or + numpy array. The input to check and copy or transform. Raises ------ TypeError - If the input is not a pandas or polars DataFrame, or a numpy array. + If the input is not a recognised dataframe, or a numpy array. ValueError If the input has duplicated column names, or 0 columns or rows. Returns ------- - X : pandas or polars DataFrame. + X : dataframe. A copy of the original dataframe, or the numpy array converted to a pandas DataFrame. """ @@ -62,14 +66,16 @@ def check_X(X): return nw_X.clone().to_native() if isinstance(X, (np.generic, np.ndarray)) or issparse(X): + import pandas as pd + X = check_array( X, accept_sparse=False, dtype=None, ensure_all_finite=False, ensure_2d=True ) return pd.DataFrame(X, columns=[f"x{i}" for i in range(X.shape[1])]) raise TypeError( - "X must be a numpy array or a pandas or polars dataframe. " - f"Got {type(X)} instead." + "X must be a numpy array or a dataframe from a library supported by " + f"narwhals (e.g. pandas, polars). Got {type(X)} instead." ) @@ -81,16 +87,21 @@ def _copy_series(y: nw.Series): def check_y( - y: Union[np.generic, np.ndarray, pd.Series, pd.DataFrame, List], + y: Union[np.generic, np.ndarray, List], y_numeric: bool = False, ): """ - Checks that y is a pandas or polars Series or DataFrame, or alternatively, if it - can be converted to a pandas Series or DataFrame. + Checks that y is a Series or DataFrame from a library supported by narwhals (for + example pandas or polars), or alternatively, if it can be converted to a pandas + Series or DataFrame. + + Pandas is only imported, lazily, when y is not already a narwhals-recognised + Series or DataFrame (e.g. when it is a numpy array or a list). Parameters ---------- - y : pandas or polars Series or DataFrame, np.array, list + y : Series or DataFrame (pandas, polars, or any other library supported by + narwhals), np.array, list The input to check and copy or transform. y_numeric : bool, default=False @@ -99,7 +110,7 @@ def check_y( Returns ------- - y: pandas or polars Series or DataFrame + y: Series or DataFrame """ if y is None: raise ValueError( @@ -127,6 +138,8 @@ def check_y( raise ValueError("y contains infinity values.") return nw_y.clone().to_native() + import pandas as pd + try: y = column_or_1d(y) y = _check_y(y, multi_output=False, y_numeric=y_numeric) @@ -139,7 +152,7 @@ def check_y( def check_X_y( X, - y: Union[np.generic, np.ndarray, pd.Series, List], + y: Union[np.generic, np.ndarray, List], y_numeric: bool = False, ): """ @@ -148,10 +161,12 @@ def check_X_y( Parameters ---------- - X: pandas or polars DataFrame, or numpy ndarray + X: dataframe (pandas, polars, or any other library supported by narwhals), or + numpy ndarray The input to check and copy or transform. - y: pandas or polars Series, np.array, list + y: Series (pandas, polars, or any other library supported by narwhals), np.array, + list The input to check and copy or transform. y_numeric : bool, default=False @@ -166,26 +181,31 @@ def check_X_y( Returns ------- - X: pandas or polars DataFrame - y: pandas or polars Series + X: dataframe + y: Series """ # Whether the raw inputs already carried a meaningful pandas index, before # check_X/check_y potentially convert them (e.g. a numpy array gets a default # index, which should be overridden by the other input's index, if it has one). - x_had_index = isinstance(X, pd.DataFrame) - y_had_index = isinstance(y, (pd.Series, pd.DataFrame)) + # These checks, unlike `isinstance(X, pd.DataFrame)`, don't require pandas to be + # installed. + x_had_index = nwd.is_pandas_dataframe(X) + y_had_index = nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y) X = check_X(X) y = check_y(y, y_numeric=y_numeric) check_consistent_length(X, y) + x_has_index = nwd.is_pandas_dataframe(X) + y_has_index = nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y) + if x_had_index and y_had_index: - if not X.index.equals(y.index): + if not X.index.equals(y.index): # type: ignore[union-attr] raise ValueError("The indexes of X and y do not match.") - elif x_had_index and isinstance(y, (pd.Series, pd.DataFrame)): - y.index = X.index - elif y_had_index and isinstance(X, pd.DataFrame): - X.index = y.index + elif x_had_index and y_has_index: + y.index = X.index # type: ignore[union-attr,method-assign] + elif y_had_index and x_has_index: + X.index = y.index # type: ignore[union-attr] # else: neither raw input carried a pandas index to reconcile (e.g. polars # objects), so check_consistent_length above is the only check that applies. @@ -199,7 +219,7 @@ def _check_X_matches_training_df(X, reference: int) -> None: Parameters ---------- - X : pandas or polars DataFrame + X : dataframe The df to be checked reference : int The number of columns in the dataframe that was used with the fit() method. @@ -225,7 +245,7 @@ def _check_contains_na( Parameters ---------- - X : pandas or polars DataFrame + X : dataframe variables : List The selected group of variables in which null values will be examined. @@ -249,7 +269,7 @@ def _check_optional_contains_na(X, variables: List[Union[str, int]]) -> None: Parameters ---------- - X : pandas or polars DataFrame + X : dataframe variables : List The selected group of variables in which null values will be examined. @@ -274,7 +294,7 @@ def _check_contains_inf(X, variables: List[Union[str, int]]) -> None: Parameters ---------- - X : pandas or polars DataFrame + X : dataframe variables : List The selected group of variables in which infinite values will be examined. diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index c27311990..c7d33d4eb 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -304,7 +304,7 @@ def test_check_X_errors(): # Test incorrect type error with pytest.raises(TypeError) as record: check_X("not a dataframe") - assert record.match("X must be a numpy array or a pandas or polars dataframe") + assert record.match("X must be a numpy array or a dataframe from a library") # ------------------------ From 24ba905af239da79ed5db55c8a5fac0c1d3ecd4d Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 22 Jul 2026 07:50:36 +0200 Subject: [PATCH 3/9] update dataframe checks take 3 --- feature_engine/dataframe_checks.py | 74 +++++++++---------------- tests/test_dataframe_checks.py | 88 ++++++++++++++---------------- 2 files changed, 66 insertions(+), 96 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 10355ea7d..da8d6871d 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -23,17 +23,17 @@ def check_X(X): Copying is an important step so that we don't accidentally modify the original dataset entered by the user. - Numpy arrays are converted to a pandas DataFrame, with column names that are - strings representing the column index starting at 0. This, together with - accepting numpy arrays at all, allows Feature-engine transformers to: + Numpy arrays are validated but otherwise returned as numpy arrays, unchanged. + Feature-engine accepts numpy arrays, in addition to dataframes, so that: - - be evaluated with Scikit-learn's `check_estimator`, which passes numpy arrays - to `fit()` and `transform()`. - - be used within a Scikit-learn Pipeline, next to Scikit-learn transformers like - the `SimpleImputer`, which return numpy arrays by default. + - transformers can be evaluated with Scikit-learn's `check_estimator`, which + passes numpy arrays to `fit()` and `transform()`. + - transformers can be used within a Scikit-learn Pipeline, next to Scikit-learn + transformers like the `SimpleImputer`, which return numpy arrays by default. - Pandas is only imported, lazily, when X is a numpy array. If you pass a dataframe - from any other narwhals-supported library, pandas does not need to be installed. + This module never imports pandas. Feature-engine is dataframe-library agnostic, + so a user working only with numpy arrays, or with a narwhals-supported dataframe + library other than pandas, is not required to have pandas installed. Parameters ---------- @@ -50,9 +50,8 @@ def check_X(X): Returns ------- - X : dataframe. - A copy of the original dataframe, or the numpy array converted to a pandas - DataFrame. + X : dataframe or numpy array. + A copy of the original dataframe, or the validated numpy array. """ if nwd.is_into_dataframe(X): # from_native() raises narwhals.exceptions.DuplicateError, a ValueError @@ -66,12 +65,9 @@ def check_X(X): return nw_X.clone().to_native() if isinstance(X, (np.generic, np.ndarray)) or issparse(X): - import pandas as pd - - X = check_array( + return check_array( X, accept_sparse=False, dtype=None, ensure_all_finite=False, ensure_2d=True ) - return pd.DataFrame(X, columns=[f"x{i}" for i in range(X.shape[1])]) raise TypeError( "X must be a numpy array or a dataframe from a library supported by " @@ -92,11 +88,8 @@ def check_y( ): """ Checks that y is a Series or DataFrame from a library supported by narwhals (for - example pandas or polars), or alternatively, if it can be converted to a pandas - Series or DataFrame. - - Pandas is only imported, lazily, when y is not already a narwhals-recognised - Series or DataFrame (e.g. when it is a numpy array or a list). + example pandas or polars), or alternatively, if it can be converted to a numpy + array. This module never imports pandas. Parameters ---------- @@ -110,7 +103,7 @@ def check_y( Returns ------- - y: Series or DataFrame + y: Series, DataFrame, or numpy array """ if y is None: raise ValueError( @@ -138,16 +131,11 @@ def check_y( raise ValueError("y contains infinity values.") return nw_y.clone().to_native() - import pandas as pd - try: y = column_or_1d(y) - y = _check_y(y, multi_output=False, y_numeric=y_numeric) - y = pd.Series(y) + return _check_y(y, multi_output=False, y_numeric=y_numeric) except ValueError: - y = _check_y(y, multi_output=True, y_numeric=y_numeric) - y = pd.DataFrame(y) - return y + return _check_y(y, multi_output=True, y_numeric=y_numeric) def check_X_y( @@ -181,33 +169,21 @@ def check_X_y( Returns ------- - X: dataframe - y: Series + X: dataframe or numpy array + y: Series, DataFrame, or numpy array """ - # Whether the raw inputs already carried a meaningful pandas index, before - # check_X/check_y potentially convert them (e.g. a numpy array gets a default - # index, which should be overridden by the other input's index, if it has one). - # These checks, unlike `isinstance(X, pd.DataFrame)`, don't require pandas to be - # installed. - x_had_index = nwd.is_pandas_dataframe(X) - y_had_index = nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y) - X = check_X(X) y = check_y(y, y_numeric=y_numeric) check_consistent_length(X, y) - x_has_index = nwd.is_pandas_dataframe(X) - y_has_index = nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y) - - if x_had_index and y_had_index: + # A numpy array, or a dataframe/Series from a library other than pandas, has no + # index to reconcile. `is_pandas_dataframe`/`is_pandas_series`, unlike + # `isinstance(X, pd.DataFrame)`, don't require pandas to be installed. + if nwd.is_pandas_dataframe(X) and ( + nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y) + ): if not X.index.equals(y.index): # type: ignore[union-attr] raise ValueError("The indexes of X and y do not match.") - elif x_had_index and y_has_index: - y.index = X.index # type: ignore[union-attr,method-assign] - elif y_had_index and x_has_index: - X.index = y.index # type: ignore[union-attr] - # else: neither raw input carried a pandas index to reconcile (e.g. polars - # objects), so check_consistent_length above is the only check that applies. return X, y diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index c7d33d4eb..b4a94dee5 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -22,13 +22,14 @@ def test_check_X_returns_df(df_vartypes): assert_frame_equal(check_X(df_vartypes), df_vartypes) -def test_check_X_converts_numpy_to_pandas(): +def test_check_X_returns_numpy_array_unchanged(): a1D = np.array([1, 2, 3, 4]) a2D = np.array([[1, 2], [3, 4]]) a3D = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) - df_2D = pd.DataFrame(a2D, columns=["x0", "x1"]) - assert_frame_equal(df_2D, check_X(a2D)) + X = check_X(a2D) + assert isinstance(X, np.ndarray) + np.testing.assert_array_equal(a2D, X) with pytest.raises(ValueError): check_X(a3D) @@ -67,16 +68,18 @@ def test_check_y_returns_dataframe(): assert_frame_equal(check_y(d), d) -def test_check_y_converts_np_array(): +def test_check_y_returns_numpy_array_unchanged(): a1D = np.array([1, 2, 3, 4]) - s = pd.Series(a1D) - assert_series_equal(check_y(a1D), s) + y = check_y(a1D) + assert isinstance(y, np.ndarray) + np.testing.assert_array_equal(a1D, y) -def test_check_y_converts_np_array_2D(): +def test_check_y_returns_2D_numpy_array_unchanged(): a2D = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(2, 4) - d = pd.DataFrame(a2D) - assert_frame_equal(check_y(a2D), d) + y = check_y(a2D) + assert isinstance(y, np.ndarray) + np.testing.assert_array_equal(a2D, y) def test_check_y_raises_none_error(): @@ -161,69 +164,50 @@ def test_check_X_y_raises_error_when_pandas_index_dont_match(): assert str(record.value) == msg -def test_check_x_y_reassings_index_when_only_one_input_is_pandas(): - # X is dataframe, y is 1D array +def test_check_x_y_numpy_side_has_no_index_to_reconcile(): + # X is dataframe, y is 1D array: y has no index, so X keeps its own, and y is + # returned as a numpy array, unchanged. df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) s = np.array([1, 2, 3, 4]) - s_exp = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) x, y = check_X_y(df, s) assert_frame_equal(df, x) - assert_series_equal(s_exp.astype(int), y.astype(int)) + assert isinstance(y, np.ndarray) + np.testing.assert_array_equal(s, y) # X is dataframe, y is 2d array s = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2) - s_exp = pd.DataFrame(s, index=[22, 99, 101, 212]) x, y = check_X_y(df, s) assert_frame_equal(df, x) - assert_frame_equal(s_exp.astype(int), y.astype(int)) + assert isinstance(y, np.ndarray) + np.testing.assert_array_equal(s, y) - # X is not a df, y is a series + # X is a numpy array, y is a series: X has no index, so y keeps its own, and X is + # returned as a numpy array, unchanged. df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) - df_exp = pd.DataFrame(df, columns=["x0", "x1"]) - df_exp.index = s.index x, y = check_X_y(df, s) - assert_frame_equal(df_exp, x) + assert isinstance(x, np.ndarray) + np.testing.assert_array_equal(df, x) assert_series_equal(s, y) - # X is not a df, y is a dataframe + # X is a numpy array, y is a dataframe s = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2) s = pd.DataFrame(s, index=[22, 99, 101, 212]) df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T - df_exp = pd.DataFrame(df, columns=["x0", "x1"]) - df_exp.index = s.index x, y = check_X_y(df, s) - assert_frame_equal(df_exp, x) + assert isinstance(x, np.ndarray) + np.testing.assert_array_equal(df, x) assert_frame_equal(s, y) -def test_check_x_y_converts_numpy_to_pandas(): +def test_check_x_y_with_numpy_arrays_on_both_sides(): a2D = np.array([[1, 2], [3, 4], [3, 4], [3, 4]]) - df2D = pd.DataFrame(a2D, columns=["x0", "x1"]) - a1D = np.array([1, 2, 3, 4]) - s1D = pd.Series(a1D) - - # X is df and y is array - x, y = check_X_y(df2D, a1D) - assert_frame_equal(df2D, x) - assert_series_equal(s1D, y) - - # X is array and y is series - x, y = check_X_y(a2D, s1D) - assert_frame_equal(df2D, x) - assert_series_equal(s1D, y) - # X is df and y is 2d array - y2D = pd.DataFrame(a2D, columns=[0, 1]) - x, y = check_X_y(df2D, a2D) - assert_frame_equal(df2D, x) - assert_frame_equal(y2D, y) - - # X is array and y multioutput df - x, y = check_X_y(a2D, df2D) - assert_frame_equal(df2D, x) - assert_frame_equal(df2D, y) + x, y = check_X_y(a2D, a1D) + assert isinstance(x, np.ndarray) and isinstance(y, np.ndarray) + np.testing.assert_array_equal(a2D, x) + np.testing.assert_array_equal(a1D, y) def test_check_x_y_raises_error_when_inconsistent_length(df_vartypes): @@ -370,6 +354,16 @@ def test_check_x_y_polars_raises_error_when_inconsistent_length(): check_X_y(df, s) +def test_check_x_y_mixed_pandas_and_polars_has_no_index_to_reconcile(): + # Polars has no index, so there is nothing to compare or reassign: only the row + # count, checked via check_consistent_length, applies. + df = pd.DataFrame({"a": [1, 2, 3]}, index=[22, 99, 101]) + s = pl.Series("target", [0, 1, 2]) + x, y = check_X_y(df, s) + assert_frame_equal(df, x) + pl_assert_series_equal(s, y) + + def test_check_X_matches_training_df_with_polars(): df = pl.DataFrame({"a": [1, 2], "b": [3, 4]}) with pytest.raises(ValueError): From 7a90384f1559153370010580111c2446423142d9 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 23 Jul 2026 16:35:46 +0200 Subject: [PATCH 4/9] update docstrings --- feature_engine/dataframe_checks.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index da8d6871d..f101c7fba 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -19,9 +19,7 @@ def check_X(X): """ Checks that X is a dataframe from any library supported by narwhals (for example - pandas, polars, modin, cuDF or PyArrow), or a numpy array, and returns a copy. - Copying is an important step so that we don't accidentally modify the original - dataset entered by the user. + pandas, polars, modin, cuDF, or PyArrow), or a numpy array, and returns a copy. Numpy arrays are validated but otherwise returned as numpy arrays, unchanged. Feature-engine accepts numpy arrays, in addition to dataframes, so that: @@ -31,10 +29,6 @@ def check_X(X): - transformers can be used within a Scikit-learn Pipeline, next to Scikit-learn transformers like the `SimpleImputer`, which return numpy arrays by default. - This module never imports pandas. Feature-engine is dataframe-library agnostic, - so a user working only with numpy arrays, or with a narwhals-supported dataframe - library other than pandas, is not required to have pandas installed. - Parameters ---------- X : dataframe (pandas, polars, or any other library supported by narwhals), or @@ -62,7 +56,7 @@ def check_X(X): f"Found array with 0 feature(s) (shape={nw_X.shape}) while a " "minimum of 1 is required." ) - return nw_X.clone().to_native() + return nw_X.to_native() if isinstance(X, (np.generic, np.ndarray)) or issparse(X): return check_array( @@ -89,7 +83,7 @@ def check_y( """ Checks that y is a Series or DataFrame from a library supported by narwhals (for example pandas or polars), or alternatively, if it can be converted to a numpy - array. This module never imports pandas. + array. Parameters ---------- From 4abca061932544fb537b78a203dbed87c4ebdcfd Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 26 Jul 2026 11:54:25 +0200 Subject: [PATCH 5/9] refactor dataframe checks --- feature_engine/dataframe_checks.py | 146 ++--- feature_engine/encoding/base_encoder.py | 6 +- feature_engine/encoding/rare_label.py | 4 +- feature_engine/encoding/similarity_encoder.py | 6 +- .../preprocessing/match_categories.py | 6 +- feature_engine/text/text_features.py | 10 +- tests/test_dataframe_checks.py | 553 ++++++++---------- 7 files changed, 328 insertions(+), 403 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index f101c7fba..f8477f5e3 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -7,77 +7,53 @@ import narwhals as nw import narwhals.dependencies as nwd import numpy as np -from scipy.sparse import issparse -from sklearn.utils.validation import ( - _check_y, - check_array, - check_consistent_length, - column_or_1d, -) +from narwhals.typing import IntoDataFrame, IntoSeries +from sklearn.utils.validation import _check_y, check_consistent_length, column_or_1d def check_X(X): """ Checks that X is a dataframe from any library supported by narwhals (for example - pandas, polars, modin, cuDF, or PyArrow), or a numpy array, and returns a copy. - - Numpy arrays are validated but otherwise returned as numpy arrays, unchanged. - Feature-engine accepts numpy arrays, in addition to dataframes, so that: - - - transformers can be evaluated with Scikit-learn's `check_estimator`, which - passes numpy arrays to `fit()` and `transform()`. - - transformers can be used within a Scikit-learn Pipeline, next to Scikit-learn - transformers like the `SimpleImputer`, which return numpy arrays by default. + pandas, polars, modin, cuDF, or PyArrow). Parameters ---------- - X : dataframe (pandas, polars, or any other library supported by narwhals), or - numpy array. - The input to check and copy or transform. + X : dataframe (pandas, polars, or any other library supported by narwhals). + The input to check and transform. Raises ------ TypeError - If the input is not a recognised dataframe, or a numpy array. + If the input is not a recognised dataframe. ValueError If the input has duplicated column names, or 0 columns or rows. Returns ------- - X : dataframe or numpy array. - A copy of the original dataframe, or the validated numpy array. + X : dataframe. + The validated dataframe in its native format. """ if nwd.is_into_dataframe(X): # from_native() raises narwhals.exceptions.DuplicateError, a ValueError # subclass, when the dataframe has duplicated column names. nw_X = nw.from_native(X, eager_only=True) - if nw_X.is_empty(): + if nw_X.is_empty() or nw_X.shape[1] == 0: raise ValueError( f"Found array with 0 feature(s) (shape={nw_X.shape}) while a " "minimum of 1 is required." ) - return nw_X.to_native() - if isinstance(X, (np.generic, np.ndarray)) or issparse(X): - return check_array( - X, accept_sparse=False, dtype=None, ensure_all_finite=False, ensure_2d=True + else: + raise TypeError( + "X must be a dataframe from a library supported by narwhals " + f"(e.g. pandas, polars, PyArrow). Got {type(X)} instead." ) - raise TypeError( - "X must be a numpy array or a dataframe from a library supported by " - f"narwhals (e.g. pandas, polars). Got {type(X)} instead." - ) - - -def _copy_series(y: nw.Series): - """Returns a copy of a narwhals Series, in its native library format.""" - name = y.name - frame = y.to_frame().clone() - return frame.get_column(frame.columns[0]).alias(name).to_native() + return nw_X.to_native() def check_y( - y: Union[np.generic, np.ndarray, List], + y: Union[IntoSeries, IntoDataFrame, np.generic, np.ndarray, List], y_numeric: bool = False, ): """ @@ -89,7 +65,7 @@ def check_y( ---------- y : Series or DataFrame (pandas, polars, or any other library supported by narwhals), np.array, list - The input to check and copy or transform. + The input to check. y_numeric : bool, default=False Whether to ensure that y has a numeric type. If dtype of y is not numeric, @@ -115,7 +91,7 @@ def check_y( raise ValueError("y contains infinity values.") elif y_numeric: nw_y = nw_y.cast(nw.Float64()) - return _copy_series(nw_y) + return nw_y.to_native() if nwd.is_into_dataframe(y): nw_y = nw.from_native(y, eager_only=True) @@ -123,7 +99,7 @@ def check_y( raise ValueError("y contains NaN values.") if not np.isfinite(nw_y.to_numpy()).all(): raise ValueError("y contains infinity values.") - return nw_y.clone().to_native() + return nw_y.to_native() try: y = column_or_1d(y) @@ -134,22 +110,21 @@ def check_y( def check_X_y( X, - y: Union[np.generic, np.ndarray, List], + y: Union[IntoSeries, IntoDataFrame, np.generic, np.ndarray, List], y_numeric: bool = False, ): """ Ensures X and y are compatible dataframe/array-like objects with a consistent - number of rows. If both are pandas objects, also checks that their indexes match. + number of rows. If both are pandas objects, checks that their indexes match. Parameters ---------- - X: dataframe (pandas, polars, or any other library supported by narwhals), or - numpy ndarray - The input to check and copy or transform. + X: dataframe (pandas, polars, or any other library supported by narwhals) + The input to check. - y: Series (pandas, polars, or any other library supported by narwhals), np.array, - list - The input to check and copy or transform. + y: Series, DataFrame (pandas, polars, or any other library supported by + narwhals), np.array, list + The input to check. y_numeric : bool, default=False Whether to ensure that y has a numeric type. If dtype of y is not numeric, @@ -157,27 +132,26 @@ def check_X_y( Raises ------ - ValueError: if X and y have a different number of rows, or if they are pandas - objects with inconsistent indexes. - TypeError: if X is a sparse matrix, an empty dataframe, or not a recognised type. + TypeError + If X is not a recognised dataframe. + ValueError + If X has duplicated column names, 0 columns, or 0 rows; if y is None, or + contains NaN or infinity values; if X and y have a different number of + rows; or if X and y are pandas objects with mismatched indexes. Returns ------- - X: dataframe or numpy array + X: dataframe y: Series, DataFrame, or numpy array """ X = check_X(X) y = check_y(y, y_numeric=y_numeric) check_consistent_length(X, y) - # A numpy array, or a dataframe/Series from a library other than pandas, has no - # index to reconcile. `is_pandas_dataframe`/`is_pandas_series`, unlike - # `isinstance(X, pd.DataFrame)`, don't require pandas to be installed. - if nwd.is_pandas_dataframe(X) and ( - nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y) - ): - if not X.index.equals(y.index): # type: ignore[union-attr] - raise ValueError("The indexes of X and y do not match.") + if nwd.is_pandas_dataframe(X): + if nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y): + if not X.index.equals(y.index): # type: ignore[union-attr] + raise ValueError("The indexes of X and y do not match.") return X, y @@ -189,8 +163,8 @@ def _check_X_matches_training_df(X, reference: int) -> None: Parameters ---------- - X : dataframe - The df to be checked + X : dataframe (pandas, polars, or any other library supported by narwhals) + The df to be checked. reference : int The number of columns in the dataframe that was used with the fit() method. @@ -209,6 +183,7 @@ def _check_X_matches_training_df(X, reference: int) -> None: def _check_contains_na( X, variables: List[Union[str, int]], + error_msg: str = "simple", ) -> None: """ Checks if the dataframe contains null values in the selected columns. @@ -220,42 +195,29 @@ def _check_contains_na( variables : List The selected group of variables in which null values will be examined. - Raises - ------ - ValueError - If the variable(s) contain null values. - """ - nw_X = nw.from_native(X, eager_only=True) - if nw_X.select(nw.col(variables).is_null().any()).to_numpy().any(): - raise ValueError( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer." - ) - - -def _check_optional_contains_na(X, variables: List[Union[str, int]]) -> None: - """ - Checks if the dataframe contains null values in the selected columns. - - Parameters - ---------- - X : dataframe - - variables : List - The selected group of variables in which null values will be examined. + error_msg : str, default="simple" + The message in the error. Some transformers can ignore null values. Raises ------ ValueError If the variable(s) contain null values. """ + error_msg_simple = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer." + ) + error_msg_ignore = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer or set the parameter " + "`missing_values='ignore'` when initialising this transformer." + ) nw_X = nw.from_native(X, eager_only=True) if nw_X.select(nw.col(variables).is_null().any()).to_numpy().any(): - raise ValueError( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer or set the parameter " - "`missing_values='ignore'` when initialising this transformer." - ) + if error_msg == "simple": + raise ValueError(error_msg_simple) + else: + raise ValueError(error_msg_ignore) def _check_contains_inf(X, variables: List[Union[str, int]]) -> None: diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index 35427f260..53eca3095 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -20,7 +20,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.substitute import Substitution from feature_engine.dataframe_checks import ( - _check_optional_contains_na, + _check_contains_na, _check_X_matches_training_df, check_X, ) @@ -123,7 +123,7 @@ class CategoricalMethodsMixin(TransformerMixin, BaseEstimator, GetFeatureNamesOu def _check_na(self, X: pd.DataFrame, variables): if self.missing_values == "raise": - _check_optional_contains_na(X, variables) + _check_contains_na(X, variables, error_msg="optional") def _check_or_select_variables(self, X: pd.DataFrame): """ @@ -225,7 +225,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") X = self._encode(X) diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index 84c1f6910..2bbd2bf73 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -23,7 +23,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixinNA, CategoricalMethodsMixin, @@ -255,7 +255,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") with_nan = [] else: with_nan = [np.nan] diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index c438487d6..f15f87003 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -17,7 +17,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixin, CategoricalMethodsMixin, @@ -247,7 +247,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # if data contains nan, fail before running any logic if self.missing_values == "raise": - _check_optional_contains_na(X, variables_) + _check_contains_na(X, variables_, error_msg="optional") self.encoder_dict_ = {} @@ -317,7 +317,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: check_is_fitted(self) X = self._check_transform_input_and_state(X) if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") if len(self.variables_) == 0: return X diff --git a/feature_engine/preprocessing/match_categories.py b/feature_engine/preprocessing/match_categories.py index e0e863c1f..9d66c3d6c 100644 --- a/feature_engine/preprocessing/match_categories.py +++ b/feature_engine/preprocessing/match_categories.py @@ -19,7 +19,7 @@ ) from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixinNA, CategoricalMethodsMixin, @@ -148,7 +148,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): variables_ = self._check_or_select_variables(X) if self.missing_values == "raise": - _check_optional_contains_na(X, variables_) + _check_contains_na(X, variables_, error_msg="optional") self.category_dict_ = dict() for var in variables_: @@ -175,7 +175,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: X = self._check_transform_input_and_state(X) if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") for feature, levels in self.category_dict_.items(): X[feature] = pd.Categorical( diff --git a/feature_engine/text/text_features.py b/feature_engine/text/text_features.py index c6f2692d4..659938419 100644 --- a/feature_engine/text/text_features.py +++ b/feature_engine/text/text_features.py @@ -12,7 +12,7 @@ _check_param_missing_values, ) from feature_engine.dataframe_checks import ( - _check_optional_contains_na, + _check_contains_na, _check_X_matches_training_df, check_X, ) @@ -236,7 +236,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, cast(list[Union[str, int]], self.variables_)) + _check_contains_na( + X, cast(list[Union[str, int]], self.variables_), error_msg="optional" + ) # Set features to extract if self.features is None: @@ -278,7 +280,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, cast(list[Union[str, int]], self.variables_)) + _check_contains_na( + X, cast(list[Union[str, int]], self.variables_), error_msg="optional" + ) else: X[self.variables_] = X[self.variables_].fillna("") diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index b4a94dee5..b9ab1b6bc 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -10,135 +10,254 @@ from feature_engine.dataframe_checks import ( _check_contains_inf, _check_contains_na, - _check_optional_contains_na, _check_X_matches_training_df, check_X, check_X_y, check_y, ) - -def test_check_X_returns_df(df_vartypes): - assert_frame_equal(check_X(df_vartypes), df_vartypes) +# ------------------------ +# test check_X +# ------------------------ -def test_check_X_returns_numpy_array_unchanged(): - a1D = np.array([1, 2, 3, 4]) - a2D = np.array([[1, 2], [3, 4]]) - a3D = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_returns_df_unchanged(make_df, assert_equal_fn): + df = make_df({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]}) + X = check_X(df) + assert isinstance(X, type(df)) + assert_equal_fn(X, df) - X = check_X(a2D) - assert isinstance(X, np.ndarray) - np.testing.assert_array_equal(a2D, X) +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_returns_df_with_mixed_dtypes(make_df, assert_equal_fn): + data = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="min"), + } + df = make_df(data) + assert_equal_fn(check_X(df), df) + + +@pytest.mark.parametrize( + "df", + [ + pd.DataFrame([]), + pd.DataFrame({"a": []}), + pl.DataFrame({"a": []}), + ], +) +def test_raises_error_if_empty_df(df): with pytest.raises(ValueError): - check_X(a3D) + check_X(df) + + +def test_check_X_raises_error_if_0_columns(): + # A dataframe with rows but no columns is not caught by `is_empty()`, which + # only looks at the row count, so it needs its own explicit check. Polars has + # no representation for "rows with 0 columns", so this case is pandas-only. + df = pd.DataFrame(index=range(3)) + assert df.shape == (3, 0) with pytest.raises(ValueError): - check_X(a1D) + check_X(df) -def test_check_X_raises_error_sparse_matrix(): - sparse_mx = csr_matrix([[5]]) - with pytest.raises(TypeError): - assert check_X(sparse_mx) +def test_check_X_raises_error_on_duplicated_column_names(): + # only relevant for pandas + df = pd.DataFrame( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + } + ) + df.columns = ["var_A", "var_A", "var_B", "var_C"] + with pytest.raises(ValueError) as err_txt: + check_X(df) + assert err_txt.match("Expected unique column names") -def test_check_X_raises_error_with_complex_data(): - msg = "Complex data not supported" - rng = np.random.RandomState(0) - X = rng.uniform(size=10) + 1j * rng.uniform(size=10) - X = X.reshape(-1, 1) - with pytest.raises(ValueError, match=msg): - assert check_X(X) +@pytest.mark.parametrize( + "X", + [ + np.array([[1, 2], [3, 4]]), + np.array([1, 2, 3]), + np.array(1), + [1, 2, 3], + {"a": [1, 2, 3]}, + "not a dataframe", + None, + csr_matrix([[1, 2], [3, 4]]), + ], +) +def test_check_X_raises_error_on_non_dataframe_input(X): + with pytest.raises(TypeError) as record: + check_X(X) + assert record.match("X must be a dataframe from a library supported by narwhals") -def test_raises_error_if_empty_df(): - df = pd.DataFrame([]) - with pytest.raises(ValueError): - check_X(df) +# ------------------------ +# test check_y +# ------------------------ +# --- series input --- -def test_check_y_returns_series(): - s = pd.Series([0, 1, 2, 3, 4]) - assert_series_equal(check_y(s), s) +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_returns_values_unchanged(make_series, assert_equal_fn): + s = make_series([0, 1, 2, 3, 4]) + assert_equal_fn(check_y(s), s) -def test_check_y_returns_dataframe(): - d = pd.DataFrame({"t1": [0, 1, 2, 3, 4], "t2": [5, 6, 7, 8, 9]}) - assert_frame_equal(check_y(d), d) +@pytest.mark.parametrize( + "make_series", + [pd.Series, pl.Series], +) +def test_check_y_series_raises_nan_error(make_series): + s = make_series([0.0, None, 2.0]) + with pytest.raises(ValueError, match="y contains NaN values."): + check_y(s) -def test_check_y_returns_numpy_array_unchanged(): - a1D = np.array([1, 2, 3, 4]) - y = check_y(a1D) - assert isinstance(y, np.ndarray) - np.testing.assert_array_equal(a1D, y) +@pytest.mark.parametrize( + "make_series", + [pd.Series, pl.Series], +) +def test_check_y_series_raises_inf_error(make_series): + s = make_series([0.0, float("inf"), 2.0]) + with pytest.raises(ValueError, match="y contains infinity values."): + check_y(s) -def test_check_y_returns_2D_numpy_array_unchanged(): - a2D = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(2, 4) - y = check_y(a2D) - assert isinstance(y, np.ndarray) - np.testing.assert_array_equal(a2D, y) +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_converts_string_to_number_when_y_numeric( + make_series, assert_equal_fn +): + s = make_series(["0", "1", "2"]) + y = check_y(s, y_numeric=True) + expected = make_series([0.0, 1.0, 2.0]) + assert_equal_fn(y, expected) -def test_check_y_raises_none_error(): - with pytest.raises(ValueError): - check_y(None) +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_leaves_non_numeric_unchanged_by_default( + make_series, assert_equal_fn +): + # y_numeric defaults to False: a non-numeric series (e.g. classification + # labels) should be returned as-is, without being cast to float. + s = make_series(["a", "b", "c"]) + assert_equal_fn(check_y(s), s) -def test_check_y_raises_nan_error(): - msg = "y contains NaN values." - # y is series - s = pd.Series([0, np.nan, 2, 3, 4]) - with pytest.raises(ValueError) as record: - check_y(s) - assert str(record.value) == msg +# --- dataframe (multioutput) input --- - # y is multioutput - d = pd.DataFrame(np.array([1, np.nan, 3, 4, 5, 6, np.nan, 8]).reshape(2, 4)) - with pytest.raises(ValueError) as record: - check_y(d) - assert str(record.value) == msg +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_y_dataframe_returns_values_unchanged(make_df, assert_equal_fn): + d = make_df({"t1": [0, 1, 2, 3, 4], "t2": [5, 6, 7, 8, 9]}) + assert_equal_fn(check_y(d), d) -def test_check_y_raises_inf_error(): - msg = "y contains infinity values." - # y is series - s = pd.Series([0, np.inf, 2, 3, 4]) - with pytest.raises(ValueError) as record: - check_y(s) - assert str(record.value) == msg +@pytest.mark.parametrize( + "make_df", + [pd.DataFrame, pl.DataFrame], +) +def test_check_y_dataframe_raises_nan_error(make_df): + d = make_df({"t1": [0.0, None, 2.0], "t2": [5.0, 6.0, 7.0]}) + with pytest.raises(ValueError, match="y contains NaN values."): + check_y(d) + - # y is multioutput - d = pd.DataFrame(np.array([1, np.inf, 3, 4, 5, 6, np.inf, 8]).reshape(2, 4)) - with pytest.raises(ValueError) as record: +@pytest.mark.parametrize( + "make_df", + [pd.DataFrame, pl.DataFrame], +) +def test_check_y_dataframe_raises_inf_error(make_df): + d = make_df({"t1": [0.0, 0.4, 2.0], "t2": [5.0, float("inf"), 7.0]}) + with pytest.raises(ValueError, match="y contains infinity values."): check_y(d) - assert str(record.value) == msg -def test_check_y_converts_string_to_number(): - s = pd.Series(["0", "1", "2", "3", "4"]) - assert_series_equal(check_y(s, y_numeric=True), s.astype("float")) +# --- array-like input --- -def test_check_x_y_returns_pandas_from_pandas(df_vartypes): - # when s is series - s = pd.Series([0, 1, 2, 3]) - x, y = check_X_y(df_vartypes, s) - assert_frame_equal(df_vartypes, x) - assert_series_equal(s, y) +@pytest.mark.parametrize( + "a", + [ + np.array([1, 2, 3, 4]), + np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(2, 4), + ], +) +def test_check_y_array_returns_unchanged(a): + y = check_y(a) + assert isinstance(y, np.ndarray) + np.testing.assert_array_equal(a, y) - # when y is multioutput - d = pd.DataFrame(np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2)) - x, y = check_X_y(df_vartypes, d) - assert_frame_equal(df_vartypes, x) - assert_frame_equal(d, y) + +def test_check_y_raises_none_error(): + with pytest.raises(ValueError): + check_y(None) + + +# ------------------------ +# test check_X_y +# ------------------------ + + +@pytest.mark.parametrize( + "make_df, assert_frame_fn, make_series, assert_series_fn", + [ + (pd.DataFrame, assert_frame_equal, pd.Series, assert_series_equal), + (pl.DataFrame, pl_assert_frame_equal, pl.Series, pl_assert_series_equal), + ], +) +def test_check_X_y_returns_df_and_series_unchanged( + make_df, assert_frame_fn, make_series, assert_series_fn +): + df = make_df({"a": [1, 2, 3], "b": [4, 5, 6]}) + s = make_series([0, 1, 2]) + X, y = check_X_y(df, s) + assert isinstance(X, type(df)) and isinstance(y, type(s)) + assert_frame_fn(X, df) + assert_series_fn(y, s) + + +@pytest.mark.parametrize( + "make_df, assert_frame_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_y_returns_df_and_multioutput_y_unchanged(make_df, assert_frame_fn): + df = make_df({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) + d = make_df({"t1": [1, 2, 3, 4], "t2": [5, 6, 7, 8]}) + X, y = check_X_y(df, d) + assert_frame_fn(X, df) + assert_frame_fn(y, d) -def test_check_X_y_returns_pandas_from_pandas_with_non_typical_index(): +def test_check_X_y_returns_pandas_with_non_typical_index(): + # only relevant for pandas: polars has no index to reconcile df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) x, y = check_X_y(df, s) @@ -147,260 +266,100 @@ def test_check_X_y_returns_pandas_from_pandas_with_non_typical_index(): def test_check_X_y_raises_error_when_pandas_index_dont_match(): + # only relevant for pandas: polars has no index to reconcile msg = "The indexes of X and y do not match." df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 999]) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=msg): check_X_y(df, s) - assert str(record.value) == msg # when y is multioutput d = pd.DataFrame( np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2), index=[22, 99, 101, 999] ) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=msg): check_X_y(df, d) - assert str(record.value) == msg - - -def test_check_x_y_numpy_side_has_no_index_to_reconcile(): - # X is dataframe, y is 1D array: y has no index, so X keeps its own, and y is - # returned as a numpy array, unchanged. - df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) - s = np.array([1, 2, 3, 4]) - x, y = check_X_y(df, s) - assert_frame_equal(df, x) - assert isinstance(y, np.ndarray) - np.testing.assert_array_equal(s, y) - # X is dataframe, y is 2d array - s = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2) - x, y = check_X_y(df, s) - assert_frame_equal(df, x) - assert isinstance(y, np.ndarray) - np.testing.assert_array_equal(s, y) - # X is a numpy array, y is a series: X has no index, so y keeps its own, and X is - # returned as a numpy array, unchanged. - df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T - s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) - x, y = check_X_y(df, s) - assert isinstance(x, np.ndarray) - np.testing.assert_array_equal(df, x) - assert_series_equal(s, y) +@pytest.mark.parametrize( + "make_df, make_series", + [(pd.DataFrame, pd.Series), (pl.DataFrame, pl.Series)], +) +def test_check_x_y_raises_error_when_inconsistent_length(make_df, make_series): + df = make_df({"a": [1, 2, 3]}) + s = make_series([0, 1]) + with pytest.raises(ValueError): + check_X_y(df, s) - # X is a numpy array, y is a dataframe - s = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2) - s = pd.DataFrame(s, index=[22, 99, 101, 212]) - df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T - x, y = check_X_y(df, s) - assert isinstance(x, np.ndarray) - np.testing.assert_array_equal(df, x) - assert_frame_equal(s, y) +# ----------------------------------- +# test _check_X_matches_training_df +# ----------------------------------- -def test_check_x_y_with_numpy_arrays_on_both_sides(): - a2D = np.array([[1, 2], [3, 4], [3, 4], [3, 4]]) - a1D = np.array([1, 2, 3, 4]) - x, y = check_X_y(a2D, a1D) - assert isinstance(x, np.ndarray) and isinstance(y, np.ndarray) - np.testing.assert_array_equal(a2D, x) - np.testing.assert_array_equal(a1D, y) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_X_matches_training_df_passes_when_columns_match(make_df): + df = make_df({"a": [1, 2], "b": [3, 4]}) + assert _check_X_matches_training_df(df, 2) is None -def test_check_x_y_raises_error_when_inconsistent_length(df_vartypes): - s = pd.Series([0, 1, 2, 3, 5]) - with pytest.raises(ValueError): - check_X_y(df_vartypes, s) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_X_matches_training_df_raises_error_when_columns_dont_match(make_df): + msg = "The number of columns in this dataset is different from" + df = make_df({"a": [1, 2], "b": [3, 4]}) + with pytest.raises(ValueError, match=msg): + _check_X_matches_training_df(df, 3) -def test_check_X_matches_training_df(df_vartypes): - with pytest.raises(ValueError): - assert _check_X_matches_training_df(df_vartypes, 4) +# ------------------------- +# test _check_contains_na +# ------------------------- -def test_contains_na(df_na): - msg = ( +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_raises_when_nan(make_df): + msg1 = ( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer." ) - - with pytest.raises(ValueError) as record: - assert _check_contains_na(df_na, ["Name", "City"]) - assert str(record.value) == msg - - -def test_optional_contains_na(df_na): - msg = ( + msg2 = ( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer or set the parameter " "`missing_values='ignore'` when initialising this transformer." ) - with pytest.raises(ValueError) as record: - assert _check_optional_contains_na(df_na, ["Name", "City"]) - assert str(record.value) == msg - - -def test_contains_inf_raises_on_inf(): - msg = ( - "Some of the variables to transform contain inf values. Check and " - "remove those before using this transformer." - ) - df = pd.DataFrame({"A": [1.1, np.inf, 3.3]}) - with pytest.raises(ValueError, match=msg): - _check_contains_inf(df, ["A"]) - - -def test_contains_inf_passes_without_inf(): - df = pd.DataFrame({"A": [1.1, 2.2, 3.3]}) - assert _check_contains_inf(df, ["A"]) is None - - -def test_check_X_raises_error_on_duplicated_column_names(): - df = pd.DataFrame( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - } - ) - df.columns = ["var_A", "var_A", "var_B", "var_C"] - with pytest.raises(ValueError) as err_txt: - check_X(df) - assert err_txt.match("Expected unique column names") - - -def test_check_X_errors(): - # Test scalar array error - with pytest.raises(ValueError) as record: - check_X(np.array(1)) - assert record.match("Expected 2D array, got scalar array instead") - - # Test 1D array error - with pytest.raises(ValueError) as record: - check_X(np.array([1, 2, 3])) - assert record.match("Expected 2D array, got 1D array instead") - - # Test incorrect type error - with pytest.raises(TypeError) as record: - check_X("not a dataframe") - assert record.match("X must be a numpy array or a dataframe from a library") - - -# ------------------------ -# polars support -# ------------------------ - - -def test_check_X_accepts_polars_and_returns_a_copy(): - df = pl.DataFrame({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]}) - X = check_X(df) - assert isinstance(X, pl.DataFrame) - pl_assert_frame_equal(X, df) - assert X is not df - - -def test_check_X_polars_raises_error_if_empty(): - with pytest.raises(ValueError): - check_X(pl.DataFrame({"a": []})) - - -def test_check_y_accepts_polars_series_and_returns_a_copy(): - s = pl.Series("target", [0, 1, 2, 3, 4]) - y = check_y(s) - assert isinstance(y, pl.Series) - pl_assert_series_equal(y, s) - assert y is not s - - -def test_check_y_polars_raises_nan_error(): - s = pl.Series("target", [0.0, None, 2.0]) - with pytest.raises(ValueError) as record: - check_y(s) - assert str(record.value) == "y contains NaN values." - - -def test_check_y_polars_raises_inf_error(): - s = pl.Series("target", [0.0, float("inf"), 2.0]) - with pytest.raises(ValueError) as record: - check_y(s) - assert str(record.value) == "y contains infinity values." - - -def test_check_y_polars_converts_string_to_number(): - s = pl.Series("target", ["0", "1", "2"]) - y = check_y(s, y_numeric=True) - assert y.dtype.is_numeric() - pl_assert_series_equal(y, pl.Series("target", [0.0, 1.0, 2.0])) - - -def test_check_x_y_polars_returns_polars(): - df = pl.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) - s = pl.Series("target", [0, 1, 2]) - X, y = check_X_y(df, s) - assert isinstance(X, pl.DataFrame) and isinstance(y, pl.Series) - pl_assert_frame_equal(X, df) - pl_assert_series_equal(y, s) - - -def test_check_x_y_polars_raises_error_when_inconsistent_length(): - df = pl.DataFrame({"a": [1, 2, 3]}) - s = pl.Series([0, 1]) - with pytest.raises(ValueError): - check_X_y(df, s) - - -def test_check_x_y_mixed_pandas_and_polars_has_no_index_to_reconcile(): - # Polars has no index, so there is nothing to compare or reassign: only the row - # count, checked via check_consistent_length, applies. - df = pd.DataFrame({"a": [1, 2, 3]}, index=[22, 99, 101]) - s = pl.Series("target", [0, 1, 2]) - x, y = check_X_y(df, s) - assert_frame_equal(df, x) - pl_assert_series_equal(s, y) + df = make_df({"Name": ["tom", None], "City": ["London", "Manchester"]}) + with pytest.raises(ValueError, match=msg1): + _check_contains_na(df, ["Name", "City"]) + with pytest.raises(ValueError, match=msg2): + _check_contains_na(df, ["Name", "City"], error_msg="other") -def test_check_X_matches_training_df_with_polars(): - df = pl.DataFrame({"a": [1, 2], "b": [3, 4]}) - with pytest.raises(ValueError): - _check_X_matches_training_df(df, 3) - -def test_contains_na_with_polars(): - df = pl.DataFrame({"Name": ["tom", None], "City": ["London", "Manchester"]}) - msg = ( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer." - ) - with pytest.raises(ValueError) as record: - _check_contains_na(df, ["Name", "City"]) - assert str(record.value) == msg +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_passes_when_no_nan(make_df): + df = make_df({"Name": ["tom", "nick"], "City": ["London", "Manchester"]}) + assert _check_contains_na(df, ["Name", "City"]) is None -def test_optional_contains_na_with_polars(): - df = pl.DataFrame({"Name": ["tom", None], "City": ["London", "Manchester"]}) - msg = ( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer or set the parameter " - "`missing_values='ignore'` when initialising this transformer." - ) - with pytest.raises(ValueError) as record: - _check_optional_contains_na(df, ["Name", "City"]) - assert str(record.value) == msg +# -------------------------- +# test _check_contains_inf +# -------------------------- -def test_contains_inf_with_polars(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_raises_on_inf(make_df): msg = ( "Some of the variables to transform contain inf values. Check and " "remove those before using this transformer." ) - df = pl.DataFrame({"A": [1.1, float("inf"), 3.3]}) + df = make_df({"A": [1.1, np.inf, 3.3]}) with pytest.raises(ValueError, match=msg): _check_contains_inf(df, ["A"]) - df_ok = pl.DataFrame({"A": [1.1, 2.2, 3.3]}) - assert _check_contains_inf(df_ok, ["A"]) is None + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_passes_without_inf(make_df): + df = make_df({"A": [1.1, 2.2, 3.3]}) + assert _check_contains_inf(df, ["A"]) is None From edfc196cbc381c332a18326b6bc09b12966d2327 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 26 Jul 2026 11:58:54 +0200 Subject: [PATCH 6/9] fix mypy error --- feature_engine/dataframe_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index f8477f5e3..d0befa420 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -150,7 +150,7 @@ def check_X_y( if nwd.is_pandas_dataframe(X): if nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y): - if not X.index.equals(y.index): # type: ignore[union-attr] + if not X.index.equals(y.index): raise ValueError("The indexes of X and y do not match.") return X, y From 05b17a80f79707573d8e855e2c7bc6ada986e1d2 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 26 Jul 2026 12:04:26 +0200 Subject: [PATCH 7/9] add missing type hints --- feature_engine/dataframe_checks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index d0befa420..ae3be3c7b 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -11,7 +11,7 @@ from sklearn.utils.validation import _check_y, check_consistent_length, column_or_1d -def check_X(X): +def check_X(X: IntoDataFrame): """ Checks that X is a dataframe from any library supported by narwhals (for example pandas, polars, modin, cuDF, or PyArrow). @@ -109,7 +109,7 @@ def check_y( def check_X_y( - X, + X: IntoDataFrame, y: Union[IntoSeries, IntoDataFrame, np.generic, np.ndarray, List], y_numeric: bool = False, ): @@ -156,7 +156,7 @@ def check_X_y( return X, y -def _check_X_matches_training_df(X, reference: int) -> None: +def _check_X_matches_training_df(X: IntoDataFrame, reference: int) -> None: """ Checks that the dataframe to transform has the same number of columns as the dataframe used with the fit() method. @@ -181,7 +181,7 @@ def _check_X_matches_training_df(X, reference: int) -> None: def _check_contains_na( - X, + X: IntoDataFrame, variables: List[Union[str, int]], error_msg: str = "simple", ) -> None: @@ -220,7 +220,7 @@ def _check_contains_na( raise ValueError(error_msg_ignore) -def _check_contains_inf(X, variables: List[Union[str, int]]) -> None: +def _check_contains_inf(X: IntoDataFrame, variables: List[Union[str, int]]) -> None: """ Checks if the dataframe contains inf values in the selected columns. From aaf089910c647e4f7ef0990efeb2ca24204a5295 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 26 Jul 2026 12:16:05 +0200 Subject: [PATCH 8/9] add missing matching error syntax --- tests/test_dataframe_checks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index b9ab1b6bc..98a591880 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -82,9 +82,9 @@ def test_check_X_raises_error_on_duplicated_column_names(): } ) df.columns = ["var_A", "var_A", "var_B", "var_C"] - with pytest.raises(ValueError) as err_txt: + msg = "Expected unique column names" + with pytest.raises(ValueError, match=msg): check_X(df) - assert err_txt.match("Expected unique column names") @pytest.mark.parametrize( @@ -217,7 +217,8 @@ def test_check_y_array_returns_unchanged(a): def test_check_y_raises_none_error(): - with pytest.raises(ValueError): + msg = "requires y to be passed, but the target y" + with pytest.raises(ValueError, match=msg): check_y(None) From 7519a77a94a196ca222e95cd37f489942054f89e Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 26 Jul 2026 18:23:06 +0200 Subject: [PATCH 9/9] finalise tests for df checks' --- tests/test_dataframe_checks.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 98a591880..085a82fad 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -208,6 +208,7 @@ def test_check_y_dataframe_raises_inf_error(make_df): [ np.array([1, 2, 3, 4]), np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(2, 4), + [1, 2, 3, 4], ], ) def test_check_y_array_returns_unchanged(a): @@ -257,6 +258,27 @@ def test_check_X_y_returns_df_and_multioutput_y_unchanged(make_df, assert_frame_ assert_frame_fn(y, d) +@pytest.mark.parametrize( + "make_df, assert_frame_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +@pytest.mark.parametrize( + "y", + [ + np.array([0, 1, 2]), + [0, 1, 2], + np.array([[0, 1], [2, 3], [4, 5]]), + ], +) +def test_check_X_y_with_array_like_y_returns_check_y_output( + make_df, assert_frame_fn, y +): + df = make_df({"a": [1, 2, 3], "b": [4, 5, 6]}) + X, y_out = check_X_y(df, y) + assert_frame_fn(X, df) + np.testing.assert_array_equal(y_out, check_y(y)) + + def test_check_X_y_returns_pandas_with_non_typical_index(): # only relevant for pandas: polars has no index to reconcile df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) @@ -344,6 +366,12 @@ def test_contains_na_passes_when_no_nan(make_df): assert _check_contains_na(df, ["Name", "City"]) is None +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_ignores_columns_not_in_variables(make_df): + df = make_df({"Name": ["tom", None], "City": ["London", "Manchester"]}) + assert _check_contains_na(df, ["City"]) is None + + # -------------------------- # test _check_contains_inf # -------------------------- @@ -364,3 +392,9 @@ def test_contains_inf_raises_on_inf(make_df): def test_contains_inf_passes_without_inf(make_df): df = make_df({"A": [1.1, 2.2, 3.3]}) assert _check_contains_inf(df, ["A"]) is None + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_ignores_columns_not_in_variables(make_df): + df = make_df({"A": [1.1, float("inf"), 3.3], "B": [1.0, 2.0, 3.0]}) + assert _check_contains_inf(df, ["B"]) is None