diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index fe313a627..ae3be3c7b 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -2,120 +2,79 @@ transform(). """ -from typing import List, Tuple, Union +from typing import List, Union +import narwhals as nw +import narwhals.dependencies as nwd import numpy as np -import pandas as pd -from scipy.sparse import issparse +from narwhals.typing import IntoDataFrame, IntoSeries from sklearn.utils.validation import _check_y, check_consistent_length, column_or_1d -from feature_engine.variable_handling._variable_type_checks import is_object - -def check_X(X: Union[np.generic, np.ndarray, pd.DataFrame]) -> pd.DataFrame: +def check_X(X: IntoDataFrame): """ - Checks if the input is a DataFrame and then creates a copy. This is an important - step not to accidentally transform the original dataset entered by the user. - - If the input is a numpy array, it converts it to a pandas Dataframe. The column - names are strings representing the column index starting at 0. - - Feature-engine was originally designed to work with pandas dataframes. However, - allowing numpy arrays as input allows 2 things: - - We can use the Scikit-learn tests for transformers provided by the - `check_estimator` function to test the compatibility of our transformers with - sklearn functionality. - - Feature-engine transformers can be used within a Scikit-learn Pipeline together - with Scikit-learn transformers like the `SimpleImputer`, which return by default - Numpy arrays. + Checks that X is a dataframe from any library supported by narwhals (for example + pandas, polars, modin, cuDF, or PyArrow). Parameters ---------- - X : pandas Dataframe or numpy array. - The input to check and copy or transform. + X : dataframe (pandas, polars, or any other library supported by narwhals). + The input to check and transform. Raises ------ TypeError - If the input is not a Pandas DataFrame or a numpy array. + If the input is not a recognised dataframe. ValueError - If the input is an empty dataframe. + If the input has duplicated column names, or 0 columns or rows. Returns ------- - X : pandas Dataframe. - A copy of original DataFrame or a converted Numpy array. + X : dataframe. + The validated dataframe in its native format. """ - if isinstance(X, pd.DataFrame): - if not X.columns.is_unique: - raise ValueError("Input data contains duplicated variable names.") - X = X.copy() - - elif isinstance(X, (np.generic, np.ndarray)): - # If input is scalar raise error - if X.ndim == 0: - raise ValueError( - "Expected 2D array, got scalar array instead:\narray={}.\n" - "Reshape your data either using array.reshape(-1, 1) if " - "your data has a single feature or array.reshape(1, -1) " - "if it contains a single sample.".format(X) - ) - # If input is 1D raise error - if X.ndim == 1: + if nwd.is_into_dataframe(X): + # from_native() raises narwhals.exceptions.DuplicateError, a ValueError + # subclass, when the dataframe has duplicated column names. + nw_X = nw.from_native(X, eager_only=True) + if nw_X.is_empty() or nw_X.shape[1] == 0: raise ValueError( - "Expected 2D array, got 1D array instead:\narray={}.\n" - "Reshape your data either using array.reshape(-1, 1) if " - "your data has a single feature or array.reshape(1, -1) " - "if it contains a single sample.".format(X) + f"Found array with 0 feature(s) (shape={nw_X.shape}) while a " + "minimum of 1 is required." ) - if np.any(np.iscomplex(X)): - raise TypeError("Complex data not supported by this transformer.") - - X = pd.DataFrame(X) - X.columns = [f"x{i}" for i in range(X.shape[1])] - - elif issparse(X): - raise TypeError("This transformer does not support sparse matrices.") - else: raise TypeError( - f"X must be a numpy array or pandas dataframe. Got {type(X)} instead." - ) - - if X.empty: - raise ValueError( - "0 feature(s) (shape=%s) while a minimum of %d is required." % (X.shape, 1) + "X must be a dataframe from a library supported by narwhals " + f"(e.g. pandas, polars, PyArrow). Got {type(X)} instead." ) - return X + return nw_X.to_native() def check_y( - y: Union[np.generic, np.ndarray, pd.Series, pd.DataFrame, List], + y: Union[IntoSeries, IntoDataFrame, np.generic, np.ndarray, List], y_numeric: bool = False, -) -> pd.Series: +): """ - Checks that y is a series or a dataframe, or alternatively, if it can be converted - to a series or dataframe. + Checks that y is a Series or DataFrame from a library supported by narwhals (for + example pandas or polars), or alternatively, if it can be converted to a numpy + array. Parameters ---------- - y : pd.Series, pd.DataFrame, np.array, list - The input to check and copy or transform. + y : Series or DataFrame (pandas, polars, or any other library supported by + narwhals), np.array, list + The input to check. y_numeric : bool, default=False - Whether to ensure that y has a numeric type. If dtype of y is object, - it is converted to float64. Should only be used for regression - algorithms. + Whether to ensure that y has a numeric type. If dtype of y is not numeric, + it is cast to float64. Should only be used for regression algorithms. Returns ------- - y: pd.Series or pd.DataFrame + y: Series, DataFrame, or numpy array """ - if y is None: raise ValueError( "requires y to be passed, but the target y is None", @@ -123,110 +82,89 @@ def check_y( "y should be a 1d array", ) - elif isinstance(y, pd.Series): - if y.isnull().any(): + if nwd.is_into_series(y): + nw_y = nw.from_native(y, series_only=True) + if nw_y.is_null().any(): raise ValueError("y contains NaN values.") - if not is_object(y) and not np.isfinite(y).all(): - raise ValueError("y contains infinity values.") - if y_numeric and is_object(y): - y = y.astype("float64") - y = y.copy() - - elif isinstance(y, pd.DataFrame): - if y.isnull().any().any(): + if nw_y.dtype.is_numeric(): + if not np.isfinite(nw_y.to_numpy()).all(): + raise ValueError("y contains infinity values.") + elif y_numeric: + nw_y = nw_y.cast(nw.Float64()) + return nw_y.to_native() + + if nwd.is_into_dataframe(y): + nw_y = nw.from_native(y, eager_only=True) + if nw_y.select(nw.all().is_null().any()).to_numpy().any(): raise ValueError("y contains NaN values.") - if not np.isfinite(y).all().all(): + if not np.isfinite(nw_y.to_numpy()).all(): raise ValueError("y contains infinity values.") - y = y.copy() + return nw_y.to_native() - else: - try: - y = column_or_1d(y) - y = _check_y(y, multi_output=False, y_numeric=y_numeric) - y = pd.Series(y).copy() - except ValueError: - y = _check_y(y, multi_output=True, y_numeric=y_numeric) - y = pd.DataFrame(y).copy() - return y + try: + y = column_or_1d(y) + return _check_y(y, multi_output=False, y_numeric=y_numeric) + except ValueError: + return _check_y(y, multi_output=True, y_numeric=y_numeric) def check_X_y( - X: Union[np.generic, np.ndarray, pd.DataFrame], - y: Union[np.generic, np.ndarray, pd.Series, List], + X: IntoDataFrame, + y: Union[IntoSeries, IntoDataFrame, np.generic, np.ndarray, List], y_numeric: bool = False, -) -> Tuple[pd.DataFrame, pd.Series]: +): """ - Ensures X and y are compatible pandas DataFrame and Series. If both are pandas - objects, checks that their indexes match. If any is a numpy array, converts to - pandas object with compatible index. - - This transformer ensures that we can concatenate X and y using `pandas.concat`, - functionality needed in the encoders. + Ensures X and y are compatible dataframe/array-like objects with a consistent + number of rows. If both are pandas objects, checks that their indexes match. Parameters ---------- - X: Pandas DataFrame or numpy ndarray - The input to check and copy or transform. + X: dataframe (pandas, polars, or any other library supported by narwhals) + The input to check. - y: pd.Series, np.array, list - The input to check and copy or transform. + y: Series, DataFrame (pandas, polars, or any other library supported by + narwhals), np.array, list + The input to check. y_numeric : bool, default=False - Whether to ensure that y has a numeric type. If dtype of y is object, - it is converted to float64. Should only be used for regression - algorithms. + Whether to ensure that y has a numeric type. If dtype of y is not numeric, + it is cast to float64. Should only be used for regression algorithms. Raises ------ - ValueError: if X and y are pandas objects with inconsistent indexes. - TypeError: if X is sparse matrix, empty dataframe or not a dataframe. - TypeError: if y can't be parsed as pandas Series. + TypeError + If X is not a recognised dataframe. + ValueError + If X has duplicated column names, 0 columns, or 0 rows; if y is None, or + contains NaN or infinity values; if X and y have a different number of + rows; or if X and y are pandas objects with mismatched indexes. Returns ------- - X: Pandas DataFrame - y: Pandas Series + X: dataframe + y: Series, DataFrame, or numpy array """ + X = check_X(X) + y = check_y(y, y_numeric=y_numeric) + check_consistent_length(X, y) - def _check_X_y(X, y): - X = check_X(X) - y = check_y(y, y_numeric=y_numeric) - check_consistent_length(X, y) - return X, y - - # case 1: both are pandas objects - if isinstance(X, pd.DataFrame) and isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) - # Check that their indexes match. - if X.index.equals(y.index) is False: - raise ValueError("The indexes of X and y do not match.") - - # case 2: X is dataframe and y is something else - if isinstance(X, pd.DataFrame) and not isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) - y.index = X.index - - # case 3: X is not a dataframe and y is a series - elif not isinstance(X, pd.DataFrame) and isinstance(y, (pd.Series, pd.DataFrame)): - X, y = _check_X_y(X, y) - X.index = y.index - - # all other cases - else: - X, y = _check_X_y(X, y) + if nwd.is_pandas_dataframe(X): + if nwd.is_pandas_series(y) or nwd.is_pandas_dataframe(y): + if not X.index.equals(y.index): + raise ValueError("The indexes of X and y do not match.") return X, y -def _check_X_matches_training_df(X: pd.DataFrame, reference: int) -> None: +def _check_X_matches_training_df(X: IntoDataFrame, reference: int) -> None: """ - Checks that DataFrame to transform has the same number of columns that the - DataFrame used with the fit() method. + Checks that the dataframe to transform has the same number of columns as the + dataframe used with the fit() method. Parameters ---------- - X : Pandas DataFrame - The df to be checked + X : dataframe (pandas, polars, or any other library supported by narwhals) + The df to be checked. reference : int The number of columns in the dataframe that was used with the fit() method. @@ -234,92 +172,71 @@ def _check_X_matches_training_df(X: pd.DataFrame, reference: int) -> None: ------ ValueError If the number of columns does not match. - - Returns - ------- - None """ - if X.shape[1] != reference: raise ValueError( "The number of columns in this dataset is different from the one used to " "fit this transformer (when using the fit() method)." ) - return None - def _check_contains_na( - X: pd.DataFrame, + X: IntoDataFrame, variables: List[Union[str, int]], + error_msg: str = "simple", ) -> None: """ - Checks if DataFrame contains null values in the selected columns. + Checks if the dataframe contains null values in the selected columns. Parameters ---------- - X : Pandas DataFrame + X : dataframe variables : List The selected group of variables in which null values will be examined. - Raises - ------ - ValueError - If the variable(s) contain null values. - """ - - if X[variables].isnull().any().any(): - raise ValueError( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer." - ) - - -def _check_optional_contains_na( - X: pd.DataFrame, variables: List[Union[str, int]] -) -> None: - """ - Checks if DataFrame contains null values in the selected columns. - - Parameters - ---------- - X : Pandas DataFrame - - variables : List - The selected group of variables in which null values will be examined. + error_msg : str, default="simple" + The message in the error. Some transformers can ignore null values. Raises ------ ValueError If the variable(s) contain null values. """ - - if X[variables].isnull().any().any(): - raise ValueError( - "Some of the variables in the dataset contain NaN. Check and " - "remove those before using this transformer or set the parameter " - "`missing_values='ignore'` when initialising this transformer." - ) - - -def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> None: + error_msg_simple = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer." + ) + error_msg_ignore = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer or set the parameter " + "`missing_values='ignore'` when initialising this transformer." + ) + nw_X = nw.from_native(X, eager_only=True) + if nw_X.select(nw.col(variables).is_null().any()).to_numpy().any(): + if error_msg == "simple": + raise ValueError(error_msg_simple) + else: + raise ValueError(error_msg_ignore) + + +def _check_contains_inf(X: IntoDataFrame, variables: List[Union[str, int]]) -> None: """ - Checks if DataFrame contains inf values in the selected columns. + Checks if the dataframe contains inf values in the selected columns. Parameters ---------- - X : Pandas DataFrame + X : dataframe variables : List - The selected group of variables in which null values will be examined. + The selected group of variables in which infinite values will be examined. Raises ------ ValueError If the variable(s) contain np.inf values """ - - if np.isinf(X[variables]).any().any(): + values = nw.from_native(X, eager_only=True).select(nw.col(variables)).to_numpy() + if np.isinf(values.astype(float)).any(): raise ValueError( "Some of the variables to transform contain inf values. Check and " "remove those before using this transformer." diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index 35427f260..53eca3095 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -20,7 +20,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.substitute import Substitution from feature_engine.dataframe_checks import ( - _check_optional_contains_na, + _check_contains_na, _check_X_matches_training_df, check_X, ) @@ -123,7 +123,7 @@ class CategoricalMethodsMixin(TransformerMixin, BaseEstimator, GetFeatureNamesOu def _check_na(self, X: pd.DataFrame, variables): if self.missing_values == "raise": - _check_optional_contains_na(X, variables) + _check_contains_na(X, variables, error_msg="optional") def _check_or_select_variables(self, X: pd.DataFrame): """ @@ -225,7 +225,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") X = self._encode(X) diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index 84c1f6910..2bbd2bf73 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -23,7 +23,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixinNA, CategoricalMethodsMixin, @@ -255,7 +255,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") with_nan = [] else: with_nan = [np.nan] diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index c438487d6..f15f87003 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -17,7 +17,7 @@ from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixin, CategoricalMethodsMixin, @@ -247,7 +247,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # if data contains nan, fail before running any logic if self.missing_values == "raise": - _check_optional_contains_na(X, variables_) + _check_contains_na(X, variables_, error_msg="optional") self.encoder_dict_ = {} @@ -317,7 +317,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: check_is_fitted(self) X = self._check_transform_input_and_state(X) if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") if len(self.variables_) == 0: return X diff --git a/feature_engine/preprocessing/match_categories.py b/feature_engine/preprocessing/match_categories.py index e0e863c1f..9d66c3d6c 100644 --- a/feature_engine/preprocessing/match_categories.py +++ b/feature_engine/preprocessing/match_categories.py @@ -19,7 +19,7 @@ ) from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import _check_optional_contains_na, check_X +from feature_engine.dataframe_checks import _check_contains_na, check_X from feature_engine.encoding.base_encoder import ( CategoricalInitMixinNA, CategoricalMethodsMixin, @@ -148,7 +148,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): variables_ = self._check_or_select_variables(X) if self.missing_values == "raise": - _check_optional_contains_na(X, variables_) + _check_contains_na(X, variables_, error_msg="optional") self.category_dict_ = dict() for var in variables_: @@ -175,7 +175,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: X = self._check_transform_input_and_state(X) if self.missing_values == "raise": - _check_optional_contains_na(X, self.variables_) + _check_contains_na(X, self.variables_, error_msg="optional") for feature, levels in self.category_dict_.items(): X[feature] = pd.Categorical( diff --git a/feature_engine/text/text_features.py b/feature_engine/text/text_features.py index c6f2692d4..659938419 100644 --- a/feature_engine/text/text_features.py +++ b/feature_engine/text/text_features.py @@ -12,7 +12,7 @@ _check_param_missing_values, ) from feature_engine.dataframe_checks import ( - _check_optional_contains_na, + _check_contains_na, _check_X_matches_training_df, check_X, ) @@ -236,7 +236,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, cast(list[Union[str, int]], self.variables_)) + _check_contains_na( + X, cast(list[Union[str, int]], self.variables_), error_msg="optional" + ) # Set features to extract if self.features is None: @@ -278,7 +280,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": - _check_optional_contains_na(X, cast(list[Union[str, int]], self.variables_)) + _check_contains_na( + X, cast(list[Union[str, int]], self.variables_), error_msg="optional" + ) else: X[self.variables_] = X[self.variables_].fillna("") diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 711b1aea7..085a82fad 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -1,138 +1,286 @@ import numpy as np import pandas as pd +import polars as pl import pytest from pandas.testing import assert_frame_equal, assert_series_equal +from polars.testing import assert_frame_equal as pl_assert_frame_equal +from polars.testing import assert_series_equal as pl_assert_series_equal from scipy.sparse import csr_matrix from feature_engine.dataframe_checks import ( _check_contains_inf, _check_contains_na, - _check_optional_contains_na, _check_X_matches_training_df, check_X, check_X_y, check_y, ) +# ------------------------ +# test check_X +# ------------------------ -def test_check_X_returns_df(df_vartypes): - assert_frame_equal(check_X(df_vartypes), df_vartypes) +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_returns_df_unchanged(make_df, assert_equal_fn): + df = make_df({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]}) + X = check_X(df) + assert isinstance(X, type(df)) + assert_equal_fn(X, df) -def test_check_X_converts_numpy_to_pandas(): - a1D = np.array([1, 2, 3, 4]) - a2D = np.array([[1, 2], [3, 4]]) - a3D = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) - - df_2D = pd.DataFrame(a2D, columns=["x0", "x1"]) - assert_frame_equal(df_2D, check_X(a2D)) +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_returns_df_with_mixed_dtypes(make_df, assert_equal_fn): + data = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="min"), + } + df = make_df(data) + assert_equal_fn(check_X(df), df) + + +@pytest.mark.parametrize( + "df", + [ + pd.DataFrame([]), + pd.DataFrame({"a": []}), + pl.DataFrame({"a": []}), + ], +) +def test_raises_error_if_empty_df(df): with pytest.raises(ValueError): - check_X(a3D) + check_X(df) + + +def test_check_X_raises_error_if_0_columns(): + # A dataframe with rows but no columns is not caught by `is_empty()`, which + # only looks at the row count, so it needs its own explicit check. Polars has + # no representation for "rows with 0 columns", so this case is pandas-only. + df = pd.DataFrame(index=range(3)) + assert df.shape == (3, 0) with pytest.raises(ValueError): - check_X(a1D) + check_X(df) -def test_check_X_raises_error_sparse_matrix(): - sparse_mx = csr_matrix([[5]]) - with pytest.raises(TypeError): - assert check_X(sparse_mx) +def test_check_X_raises_error_on_duplicated_column_names(): + # only relevant for pandas + df = pd.DataFrame( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + } + ) + df.columns = ["var_A", "var_A", "var_B", "var_C"] + msg = "Expected unique column names" + with pytest.raises(ValueError, match=msg): + check_X(df) -def test_check_X_raises_error_with_complex_data(): - msg = "Complex data not supported" - rng = np.random.RandomState(0) - X = rng.uniform(size=10) + 1j * rng.uniform(size=10) - X = X.reshape(-1, 1) - with pytest.raises(TypeError, match=msg): - assert check_X(X) +@pytest.mark.parametrize( + "X", + [ + np.array([[1, 2], [3, 4]]), + np.array([1, 2, 3]), + np.array(1), + [1, 2, 3], + {"a": [1, 2, 3]}, + "not a dataframe", + None, + csr_matrix([[1, 2], [3, 4]]), + ], +) +def test_check_X_raises_error_on_non_dataframe_input(X): + with pytest.raises(TypeError) as record: + check_X(X) + assert record.match("X must be a dataframe from a library supported by narwhals") -def test_raises_error_if_empty_df(): - df = pd.DataFrame([]) - with pytest.raises(ValueError): - check_X(df) +# ------------------------ +# test check_y +# ------------------------ + +# --- series input --- + + +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_returns_values_unchanged(make_series, assert_equal_fn): + s = make_series([0, 1, 2, 3, 4]) + assert_equal_fn(check_y(s), s) -def test_check_y_returns_series(): - s = pd.Series([0, 1, 2, 3, 4]) - assert_series_equal(check_y(s), s) +@pytest.mark.parametrize( + "make_series", + [pd.Series, pl.Series], +) +def test_check_y_series_raises_nan_error(make_series): + s = make_series([0.0, None, 2.0]) + with pytest.raises(ValueError, match="y contains NaN values."): + check_y(s) -def test_check_y_returns_dataframe(): - d = pd.DataFrame({"t1": [0, 1, 2, 3, 4], "t2": [5, 6, 7, 8, 9]}) - assert_frame_equal(check_y(d), d) +@pytest.mark.parametrize( + "make_series", + [pd.Series, pl.Series], +) +def test_check_y_series_raises_inf_error(make_series): + s = make_series([0.0, float("inf"), 2.0]) + with pytest.raises(ValueError, match="y contains infinity values."): + check_y(s) -def test_check_y_converts_np_array(): - a1D = np.array([1, 2, 3, 4]) - s = pd.Series(a1D) - assert_series_equal(check_y(a1D), s) +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_converts_string_to_number_when_y_numeric( + make_series, assert_equal_fn +): + s = make_series(["0", "1", "2"]) + y = check_y(s, y_numeric=True) + expected = make_series([0.0, 1.0, 2.0]) + assert_equal_fn(y, expected) + + +@pytest.mark.parametrize( + "make_series, assert_equal_fn", + [(pd.Series, assert_series_equal), (pl.Series, pl_assert_series_equal)], +) +def test_check_y_series_leaves_non_numeric_unchanged_by_default( + make_series, assert_equal_fn +): + # y_numeric defaults to False: a non-numeric series (e.g. classification + # labels) should be returned as-is, without being cast to float. + s = make_series(["a", "b", "c"]) + assert_equal_fn(check_y(s), s) -def test_check_y_converts_np_array_2D(): - a2D = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(2, 4) - d = pd.DataFrame(a2D) - assert_frame_equal(check_y(a2D), d) +# --- dataframe (multioutput) input --- -def test_check_y_raises_none_error(): - with pytest.raises(ValueError): - check_y(None) +@pytest.mark.parametrize( + "make_df, assert_equal_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_y_dataframe_returns_values_unchanged(make_df, assert_equal_fn): + d = make_df({"t1": [0, 1, 2, 3, 4], "t2": [5, 6, 7, 8, 9]}) + assert_equal_fn(check_y(d), d) -def test_check_y_raises_nan_error(): - msg = "y contains NaN values." +@pytest.mark.parametrize( + "make_df", + [pd.DataFrame, pl.DataFrame], +) +def test_check_y_dataframe_raises_nan_error(make_df): + d = make_df({"t1": [0.0, None, 2.0], "t2": [5.0, 6.0, 7.0]}) + with pytest.raises(ValueError, match="y contains NaN values."): + check_y(d) - # y is series - s = pd.Series([0, np.nan, 2, 3, 4]) - with pytest.raises(ValueError) as record: - check_y(s) - assert str(record.value) == msg - # y is multioutput - d = pd.DataFrame(np.array([1, np.nan, 3, 4, 5, 6, np.nan, 8]).reshape(2, 4)) - with pytest.raises(ValueError) as record: +@pytest.mark.parametrize( + "make_df", + [pd.DataFrame, pl.DataFrame], +) +def test_check_y_dataframe_raises_inf_error(make_df): + d = make_df({"t1": [0.0, 0.4, 2.0], "t2": [5.0, float("inf"), 7.0]}) + with pytest.raises(ValueError, match="y contains infinity values."): check_y(d) - assert str(record.value) == msg -def test_check_y_raises_inf_error(): - msg = "y contains infinity values." +# --- array-like input --- - # y is series - s = pd.Series([0, np.inf, 2, 3, 4]) - with pytest.raises(ValueError) as record: - check_y(s) - assert str(record.value) == msg - # y is multioutput - d = pd.DataFrame(np.array([1, np.inf, 3, 4, 5, 6, np.inf, 8]).reshape(2, 4)) - with pytest.raises(ValueError) as record: - check_y(d) - assert str(record.value) == msg +@pytest.mark.parametrize( + "a", + [ + np.array([1, 2, 3, 4]), + np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(2, 4), + [1, 2, 3, 4], + ], +) +def test_check_y_array_returns_unchanged(a): + y = check_y(a) + assert isinstance(y, np.ndarray) + np.testing.assert_array_equal(a, y) -def test_check_y_converts_string_to_number(): - s = pd.Series(["0", "1", "2", "3", "4"]) - assert_series_equal(check_y(s, y_numeric=True), s.astype("float")) +def test_check_y_raises_none_error(): + msg = "requires y to be passed, but the target y" + with pytest.raises(ValueError, match=msg): + check_y(None) -def test_check_x_y_returns_pandas_from_pandas(df_vartypes): - # when s is series - s = pd.Series([0, 1, 2, 3]) - x, y = check_X_y(df_vartypes, s) - assert_frame_equal(df_vartypes, x) - assert_series_equal(s, y) +# ------------------------ +# test check_X_y +# ------------------------ + + +@pytest.mark.parametrize( + "make_df, assert_frame_fn, make_series, assert_series_fn", + [ + (pd.DataFrame, assert_frame_equal, pd.Series, assert_series_equal), + (pl.DataFrame, pl_assert_frame_equal, pl.Series, pl_assert_series_equal), + ], +) +def test_check_X_y_returns_df_and_series_unchanged( + make_df, assert_frame_fn, make_series, assert_series_fn +): + df = make_df({"a": [1, 2, 3], "b": [4, 5, 6]}) + s = make_series([0, 1, 2]) + X, y = check_X_y(df, s) + assert isinstance(X, type(df)) and isinstance(y, type(s)) + assert_frame_fn(X, df) + assert_series_fn(y, s) + + +@pytest.mark.parametrize( + "make_df, assert_frame_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +def test_check_X_y_returns_df_and_multioutput_y_unchanged(make_df, assert_frame_fn): + df = make_df({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) + d = make_df({"t1": [1, 2, 3, 4], "t2": [5, 6, 7, 8]}) + X, y = check_X_y(df, d) + assert_frame_fn(X, df) + assert_frame_fn(y, d) - # when y is multioutput - d = pd.DataFrame(np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2)) - x, y = check_X_y(df_vartypes, d) - assert_frame_equal(df_vartypes, x) - assert_frame_equal(d, y) +@pytest.mark.parametrize( + "make_df, assert_frame_fn", + [(pd.DataFrame, assert_frame_equal), (pl.DataFrame, pl_assert_frame_equal)], +) +@pytest.mark.parametrize( + "y", + [ + np.array([0, 1, 2]), + [0, 1, 2], + np.array([[0, 1], [2, 3], [4, 5]]), + ], +) +def test_check_X_y_with_array_like_y_returns_check_y_output( + make_df, assert_frame_fn, y +): + df = make_df({"a": [1, 2, 3], "b": [4, 5, 6]}) + X, y_out = check_X_y(df, y) + assert_frame_fn(X, df) + np.testing.assert_array_equal(y_out, check_y(y)) -def test_check_X_y_returns_pandas_from_pandas_with_non_typical_index(): + +def test_check_X_y_returns_pandas_with_non_typical_index(): + # only relevant for pandas: polars has no index to reconcile df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) x, y = check_X_y(df, s) @@ -141,164 +289,112 @@ def test_check_X_y_returns_pandas_from_pandas_with_non_typical_index(): def test_check_X_y_raises_error_when_pandas_index_dont_match(): + # only relevant for pandas: polars has no index to reconcile msg = "The indexes of X and y do not match." df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 999]) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=msg): check_X_y(df, s) - assert str(record.value) == msg # when y is multioutput d = pd.DataFrame( np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2), index=[22, 99, 101, 999] ) - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=msg): check_X_y(df, d) - assert str(record.value) == msg - - -def test_check_x_y_reassings_index_when_only_one_input_is_pandas(): - # X is dataframe, y is 1D array - df = pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]) - s = np.array([1, 2, 3, 4]) - s_exp = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) - x, y = check_X_y(df, s) - assert_frame_equal(df, x) - assert_series_equal(s_exp.astype(int), y.astype(int)) - # X is dataframe, y is 2d array - s = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2) - s_exp = pd.DataFrame(s, index=[22, 99, 101, 212]) - x, y = check_X_y(df, s) - assert_frame_equal(df, x) - assert_frame_equal(s_exp.astype(int), y.astype(int)) - # X is not a df, y is a series - df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T - s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) - df_exp = pd.DataFrame(df, columns=["x0", "x1"]) - df_exp.index = s.index - x, y = check_X_y(df, s) - assert_frame_equal(df_exp, x) - assert_series_equal(s, y) - - # X is not a df, y is a dataframe - s = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(4, 2) - s = pd.DataFrame(s, index=[22, 99, 101, 212]) - df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T - df_exp = pd.DataFrame(df, columns=["x0", "x1"]) - df_exp.index = s.index - x, y = check_X_y(df, s) - assert_frame_equal(df_exp, x) - assert_frame_equal(s, y) - - -def test_check_x_y_converts_numpy_to_pandas(): - a2D = np.array([[1, 2], [3, 4], [3, 4], [3, 4]]) - df2D = pd.DataFrame(a2D, columns=["x0", "x1"]) - - a1D = np.array([1, 2, 3, 4]) - s1D = pd.Series(a1D) +@pytest.mark.parametrize( + "make_df, make_series", + [(pd.DataFrame, pd.Series), (pl.DataFrame, pl.Series)], +) +def test_check_x_y_raises_error_when_inconsistent_length(make_df, make_series): + df = make_df({"a": [1, 2, 3]}) + s = make_series([0, 1]) + with pytest.raises(ValueError): + check_X_y(df, s) - # X is df and y is array - x, y = check_X_y(df2D, a1D) - assert_frame_equal(df2D, x) - assert_series_equal(s1D, y) - # X is array and y is series - x, y = check_X_y(a2D, s1D) - assert_frame_equal(df2D, x) - assert_series_equal(s1D, y) +# ----------------------------------- +# test _check_X_matches_training_df +# ----------------------------------- - # X is df and y is 2d array - y2D = pd.DataFrame(a2D, columns=[0, 1]) - x, y = check_X_y(df2D, a2D) - assert_frame_equal(df2D, x) - assert_frame_equal(y2D, y) - # X is array and y multioutput df - x, y = check_X_y(a2D, df2D) - assert_frame_equal(df2D, x) - assert_frame_equal(df2D, y) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_X_matches_training_df_passes_when_columns_match(make_df): + df = make_df({"a": [1, 2], "b": [3, 4]}) + assert _check_X_matches_training_df(df, 2) is None -def test_check_x_y_raises_error_when_inconsistent_length(df_vartypes): - s = pd.Series([0, 1, 2, 3, 5]) - with pytest.raises(ValueError): - check_X_y(df_vartypes, s) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_check_X_matches_training_df_raises_error_when_columns_dont_match(make_df): + msg = "The number of columns in this dataset is different from" + df = make_df({"a": [1, 2], "b": [3, 4]}) + with pytest.raises(ValueError, match=msg): + _check_X_matches_training_df(df, 3) -def test_check_X_matches_training_df(df_vartypes): - with pytest.raises(ValueError): - assert _check_X_matches_training_df(df_vartypes, 4) +# ------------------------- +# test _check_contains_na +# ------------------------- -def test_contains_na(df_na): - msg = ( +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_raises_when_nan(make_df): + msg1 = ( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer." ) - - with pytest.raises(ValueError) as record: - assert _check_contains_na(df_na, ["Name", "City"]) - assert str(record.value) == msg - - -def test_optional_contains_na(df_na): - msg = ( + msg2 = ( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer or set the parameter " "`missing_values='ignore'` when initialising this transformer." ) - with pytest.raises(ValueError) as record: - assert _check_optional_contains_na(df_na, ["Name", "City"]) - assert str(record.value) == msg + df = make_df({"Name": ["tom", None], "City": ["London", "Manchester"]}) + with pytest.raises(ValueError, match=msg1): + _check_contains_na(df, ["Name", "City"]) + + with pytest.raises(ValueError, match=msg2): + _check_contains_na(df, ["Name", "City"], error_msg="other") + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_passes_when_no_nan(make_df): + df = make_df({"Name": ["tom", "nick"], "City": ["London", "Manchester"]}) + assert _check_contains_na(df, ["Name", "City"]) is None -def test_contains_inf_raises_on_inf(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_na_ignores_columns_not_in_variables(make_df): + df = make_df({"Name": ["tom", None], "City": ["London", "Manchester"]}) + assert _check_contains_na(df, ["City"]) is None + + +# -------------------------- +# test _check_contains_inf +# -------------------------- + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_raises_on_inf(make_df): msg = ( "Some of the variables to transform contain inf values. Check and " "remove those before using this transformer." ) - df = pd.DataFrame({"A": [1.1, np.inf, 3.3]}) + df = make_df({"A": [1.1, np.inf, 3.3]}) with pytest.raises(ValueError, match=msg): _check_contains_inf(df, ["A"]) -def test_contains_inf_passes_without_inf(): - df = pd.DataFrame({"A": [1.1, 2.2, 3.3]}) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_passes_without_inf(make_df): + df = make_df({"A": [1.1, 2.2, 3.3]}) assert _check_contains_inf(df, ["A"]) is None -def test_check_X_raises_error_on_duplicated_column_names(): - df = pd.DataFrame( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - } - ) - df.columns = ["var_A", "var_A", "var_B", "var_C"] - with pytest.raises(ValueError) as err_txt: - check_X(df) - assert err_txt.match("Input data contains duplicated variable names.") - - -def test_check_X_errors(): - # Test scalar array error (line 58) - with pytest.raises(ValueError) as record: - check_X(np.array(1)) - assert record.match("Expected 2D array, got scalar array instead") - - # Test 1D array error (line 65) - with pytest.raises(ValueError) as record: - check_X(np.array([1, 2, 3])) - assert record.match("Expected 2D array, got 1D array instead") - - # Test incorrect type error (line 80) - with pytest.raises(TypeError) as record: - check_X("not a dataframe") - assert record.match("X must be a numpy array or pandas dataframe") +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_contains_inf_ignores_columns_not_in_variables(make_df): + df = make_df({"A": [1.1, float("inf"), 3.3], "B": [1.0, 2.0, 3.0]}) + assert _check_contains_inf(df, ["B"]) is None