From d1215d2c0dad5a7e9c925ba9bfaff560c270e15e Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 19 Jul 2026 17:05:35 +0200 Subject: [PATCH] add standalone performance benchmark suite with pre-2.0 baseline Measures fit() and transform() wall-clock time for 40 representative transformers across all modules, on synthetic data of configurable size. Supports saving a baseline and comparing later runs against it, failing when a benchmark regresses beyond a threshold. Not collected by pytest and not run in CI; see benchmarks/README.md. Baseline recorded before the 2.0 refactoring (polars support, numpy compute) as the reference to verify that the refactoring speeds up the transformers rather than slowing them down. Co-Authored-By: Claude Fable 5 --- benchmarks/README.md | 72 +++ .../2026-07-19_macos_py3.14_pandas3.0.3.json | 416 ++++++++++++++ benchmarks/run_benchmarks.py | 506 ++++++++++++++++++ 3 files changed, 994 insertions(+) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/baselines/2026-07-19_macos_py3.14_pandas3.0.3.json create mode 100644 benchmarks/run_benchmarks.py diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..711132cf8 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,72 @@ +# Feature-engine performance benchmarks + +This directory contains a standalone benchmark suite that measures the +wall-clock time of `fit()` and `transform()` for a representative set of +feature-engine transformers. + +It was created to establish a performance baseline **before** the version 2.0 +refactoring (polars support and numpy-based computation), so that we can +verify that the refactoring makes the transformers faster, not slower. + +The suite is deliberately self-contained: it needs no extra dependencies, it +is not collected by pytest, and it does not run in CI. + +## Running the benchmarks + +From the repository root: + +```bash +python benchmarks/run_benchmarks.py +``` + +Useful options: + +| option | default | meaning | +|---|---|---| +| `--rows N` | 100000 | number of rows in the synthetic datasets | +| `--repeats N` | 5 | timed repetitions per benchmark (min is reported) | +| `--filter STR` | | run only benchmarks whose name contains STR | +| `--save FILE` | | save the results as a JSON baseline | +| `--compare FILE` | | compare this run against a saved baseline | +| `--threshold F` | 0.10 | regression threshold for `--compare` (0.10 = 10%) | + +## Workflow: checking that a refactor does not slow things down + +1. On the branch **before** the change, save a baseline: + + ```bash + python benchmarks/run_benchmarks.py --save benchmarks/baselines/before.json + ``` + +2. Switch to the branch **with** the change, and compare: + + ```bash + python benchmarks/run_benchmarks.py --compare benchmarks/baselines/before.json + ``` + + The run exits with code 1 and prints `<< REGRESSION` for every benchmark + that is more than the threshold slower than the baseline. + +Timings are machine-specific: only compare runs made on the same machine with +the same versions of python, pandas, numpy and scikit-learn (the versions are +stored in the baseline and printed during comparison). + +## What is covered + +One or more representative transformers per module: imputation, encoding, +discretisation, outliers, transformation, scaling, creation, datetime, +timeseries, selection, preprocessing and text. + +Deliberately **not** covered: transformers whose runtime is dominated by +scikit-learn model training rather than dataframe operations (the decision +tree transformers/encoders and the model-based selectors such as +`SelectByShuffling` or `RecursiveFeatureElimination`). The 2.0 refactoring +targets the dataframe layer, which is noise compared to model training in +those transformers. + +## Baselines + +`baselines/` holds saved runs. The committed baseline +(`2026-07-19_macos_py3.14_pandas3.0.3.json`) is the pre-refactor reference +recorded on an Apple Silicon laptop; treat it as indicative only and record +your own baseline on your own machine before starting performance work. diff --git a/benchmarks/baselines/2026-07-19_macos_py3.14_pandas3.0.3.json b/benchmarks/baselines/2026-07-19_macos_py3.14_pandas3.0.3.json new file mode 100644 index 000000000..2a947da41 --- /dev/null +++ b/benchmarks/baselines/2026-07-19_macos_py3.14_pandas3.0.3.json @@ -0,0 +1,416 @@ +{ + "meta": { + "date": "2026-07-19T17:03:47", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "processor": "arm", + "python": "3.14.4", + "feature_engine": "1.9.4", + "pandas": "3.0.3", + "numpy": "2.5.1", + "sklearn": "1.9.0", + "n_rows": 100000, + "repeats": 5 + }, + "results": { + "MeanMedianImputer": { + "fit": { + "min": 0.009017041011247784, + "median": 0.009051375003764406 + }, + "transform": { + "min": 0.0012765410065185279, + "median": 0.0013295829994603992 + } + }, + "EndTailImputer": { + "fit": { + "min": 0.004566749994410202, + "median": 0.004592625016812235 + }, + "transform": { + "min": 0.001244459010194987, + "median": 0.0012746670108754188 + } + }, + "ArbitraryNumberImputer": { + "fit": { + "min": 0.0003259159857407212, + "median": 0.0003415000101085752 + }, + "transform": { + "min": 0.0012666659895330667, + "median": 0.0012704999826382846 + } + }, + "CategoricalImputer(frequent)": { + "fit": { + "min": 0.023438457981683314, + "median": 0.02387737500248477 + }, + "transform": { + "min": 0.008599541004514322, + "median": 0.008879832981619984 + } + }, + "AddMissingIndicator": { + "fit": { + "min": 0.004801625007530674, + "median": 0.004833666986087337 + }, + "transform": { + "min": 0.004979749995982274, + "median": 0.005044249992351979 + } + }, + "DropMissingData": { + "fit": { + "min": 0.004749917017761618, + "median": 0.0048513330111745745 + }, + "transform": { + "min": 0.005794249998871237, + "median": 0.005804332991829142 + } + }, + "RandomSampleImputer(general)": { + "fit": { + "min": 0.0019095419847872108, + "median": 0.0019631669856607914 + }, + "transform": { + "min": 0.030068000021856278, + "median": 0.03060700002242811 + } + }, + "CountFrequencyEncoder": { + "fit": { + "min": 0.03666441698442213, + "median": 0.03797195898368955 + }, + "transform": { + "min": 0.01947608301998116, + "median": 0.019579582993173972 + } + }, + "OrdinalEncoder(ordered)": { + "fit": { + "min": 0.03482383300433867, + "median": 0.035270624997792765 + }, + "transform": { + "min": 0.018942708004033193, + "median": 0.019339999998919666 + } + }, + "MeanEncoder": { + "fit": { + "min": 0.054782249993877485, + "median": 0.05492441699607298 + }, + "transform": { + "min": 0.01881795801455155, + "median": 0.019292374985525385 + } + }, + "WoEEncoder": { + "fit": { + "min": 0.05166304198792204, + "median": 0.05211866699391976 + }, + "transform": { + "min": 0.019153916015056893, + "median": 0.019283875008113682 + } + }, + "RareLabelEncoder": { + "fit": { + "min": 0.04635304200928658, + "median": 0.04671020800014958 + }, + "transform": { + "min": 0.018958041997393593, + "median": 0.019277416984550655 + } + }, + "OneHotEncoder(top10)": { + "fit": { + "min": 0.044597208994673565, + "median": 0.04565166699467227 + }, + "transform": { + "min": 0.14039945899276063, + "median": 0.14140562500688247 + } + }, + "StringSimilarityEncoder(top5)": { + "fit": { + "min": 0.038264500006334856, + "median": 0.03868825000245124 + }, + "transform": { + "min": 0.16405570899951272, + "median": 0.16565599999739788 + } + }, + "EqualFrequencyDiscretiser": { + "fit": { + "min": 0.03756304198759608, + "median": 0.0385260840121191 + }, + "transform": { + "min": 0.016643582988763228, + "median": 0.016811582987429574 + } + }, + "EqualWidthDiscretiser": { + "fit": { + "min": 0.008530708990292624, + "median": 0.00877179199596867 + }, + "transform": { + "min": 0.005341167008737102, + "median": 0.005530375026864931 + } + }, + "GeometricWidthDiscretiser": { + "fit": { + "min": 0.001970541983610019, + "median": 0.002176957990741357 + }, + "transform": { + "min": 0.008961582992924377, + "median": 0.00909999999566935 + } + }, + "Winsorizer(gaussian)": { + "fit": { + "min": 0.0032734579872339964, + "median": 0.0034566250105854124 + }, + "transform": { + "min": 0.003116375009994954, + "median": 0.003476999991107732 + } + }, + "Winsorizer(quantiles)": { + "fit": { + "min": 0.010074124991660938, + "median": 0.01041312501183711 + }, + "transform": { + "min": 0.003048333979677409, + "median": 0.003127083007711917 + } + }, + "OutlierTrimmer(iqr)": { + "fit": { + "min": 0.011493582976981997, + "median": 0.011667040991596878 + }, + "transform": { + "min": 0.005700000008800998, + "median": 0.005830917012644932 + } + }, + "LogTransformer": { + "fit": { + "min": 0.0013442079944070429, + "median": 0.0013731670042034239 + }, + "transform": { + "min": 0.005749500007368624, + "median": 0.005836375028593466 + } + }, + "YeoJohnsonTransformer(3vars)": { + "fit": { + "min": 0.35511329100700095, + "median": 0.3562482080014888 + }, + "transform": { + "min": 0.004160790995229036, + "median": 0.0041737080027814955 + } + }, + "MeanNormalizationScaler": { + "fit": { + "min": 0.00238716701278463, + "median": 0.002573042002040893 + }, + "transform": { + "min": 0.0023373750154860318, + "median": 0.0024080000002868474 + } + }, + "CyclicalFeatures": { + "fit": { + "min": 0.0015145830111578107, + "median": 0.0015192079881671816 + }, + "transform": { + "min": 0.006116333999671042, + "median": 0.006196250003995374 + } + }, + "MathFeatures(sum,mean,std)": { + "fit": { + "min": 0.0009276250202674419, + "median": 0.0009459999855607748 + }, + "transform": { + "min": 12.777485375001561, + "median": 12.777485375001561 + } + }, + "RelativeFeatures(add,sub)": { + "fit": { + "min": 0.0006364170112647116, + "median": 0.0006545000069309026 + }, + "transform": { + "min": 0.0014168749912641943, + "median": 0.0014533329813275486 + } + }, + "DatetimeFeatures(datetime)": { + "fit": { + "min": 0.0006023749883752316, + "median": 0.0006746659928467125 + }, + "transform": { + "min": 0.07020204199943691, + "median": 0.07492308298242278 + } + }, + "DatetimeFeatures(strings)": { + "fit": { + "min": 0.02922229201067239, + "median": 0.02972170800785534 + }, + "transform": { + "min": 0.07825820898870006, + "median": 0.07934654201380908 + } + }, + "DatetimeSubtraction": { + "fit": { + "min": 0.00048233300913125277, + "median": 0.0005674590065609664 + }, + "transform": { + "min": 0.012257249996764585, + "median": 0.014148040994768962 + } + }, + "LagFeatures": { + "fit": { + "min": 0.0008387920097447932, + "median": 0.0008921249827835709 + }, + "transform": { + "min": 0.0018157919985242188, + "median": 0.0018804999999701977 + } + }, + "WindowFeatures": { + "fit": { + "min": 0.000792292004916817, + "median": 0.0008562910079490393 + }, + "transform": { + "min": 0.020049999991897494, + "median": 0.020350415987195447 + } + }, + "ExpandingWindowFeatures": { + "fit": { + "min": 0.0007813749834895134, + "median": 0.0008767920080572367 + }, + "transform": { + "min": 0.0044607500021811575, + "median": 0.004644375003408641 + } + }, + "DropConstantFeatures": { + "fit": { + "min": 0.03899062500568107, + "median": 0.04023016701103188 + }, + "transform": { + "min": 0.001837707997765392, + "median": 0.0020253330003470182 + } + }, + "DropDuplicateFeatures": { + "fit": { + "min": 0.758810833998723, + "median": 0.7661326465022285 + }, + "transform": { + "min": 0.0009148330136667937, + "median": 0.0011433330073487014 + } + }, + "DropCorrelatedFeatures": { + "fit": { + "min": 0.03283170799841173, + "median": 0.03324908300419338 + }, + "transform": { + "min": 0.0008644159825053066, + "median": 0.0008922499837353826 + } + }, + "DropHighPSIFeatures": { + "fit": { + "min": 0.06104629099718295, + "median": 0.06221291600377299 + }, + "transform": { + "min": 0.000473457999760285, + "median": 0.0004956249904353172 + } + }, + "SelectByInformationValue": { + "fit": { + "min": 0.06349800000316463, + "median": 0.06453079101629555 + }, + "transform": { + "min": 0.0015955840062815696, + "median": 0.001641083013964817 + } + }, + "MatchCategories": { + "fit": { + "min": 0.03359491698211059, + "median": 0.03431370801990852 + }, + "transform": { + "min": 0.036480208014836535, + "median": 0.037048457976197824 + } + }, + "MatchVariables": { + "fit": { + "min": 0.004911583004286513, + "median": 0.005044833000283688 + }, + "transform": { + "min": 0.005132000020239502, + "median": 0.005184457986615598 + } + }, + "TextFeatures(3features)": { + "fit": { + "min": 0.0007148750009946525, + "median": 0.0007334580004680902 + }, + "transform": { + "min": 0.32659166600205936, + "median": 0.33521408299566247 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/run_benchmarks.py b/benchmarks/run_benchmarks.py new file mode 100644 index 000000000..2094220fc --- /dev/null +++ b/benchmarks/run_benchmarks.py @@ -0,0 +1,506 @@ +"""Performance benchmarks for feature-engine transformers. + +This suite measures the wall-clock time of fit() and transform() for a +representative set of transformers, on synthetic data of configurable size. +It exists to establish a performance baseline before the 2.0 refactoring +(polars support + numpy-based compute), and to verify afterwards that the +refactoring makes the transformers faster, not slower. + +The suite is standalone on purpose: it needs no pytest plugins and it is not +collected by the regular test suite or CI. + +Usage +----- +Run all benchmarks and print a table: + + python benchmarks/run_benchmarks.py + +Save a baseline: + + python benchmarks/run_benchmarks.py --save benchmarks/baselines/my_run.json + +Compare a new run against a saved baseline (exit code 1 if any benchmark is +slower than the baseline by more than --threshold, default 10%): + + python benchmarks/run_benchmarks.py --compare benchmarks/baselines/my_run.json + +Other options: --rows N (default 100_000), --repeats N (default 5), +--filter SUBSTRING (run only matching benchmarks). + +Note: timings are machine-specific. Compare runs only on the same machine and +the same versions of python and the dependencies. +""" + +import argparse +import datetime +import gc +import json +import platform +import statistics +import sys +import time + +import numpy as np +import pandas as pd +import sklearn + +import feature_engine +from feature_engine.creation import CyclicalFeatures, MathFeatures, RelativeFeatures +from feature_engine.datetime import DatetimeFeatures, DatetimeSubtraction +from feature_engine.discretisation import ( + EqualFrequencyDiscretiser, + EqualWidthDiscretiser, + GeometricWidthDiscretiser, +) +from feature_engine.encoding import ( + CountFrequencyEncoder, + MeanEncoder, + OneHotEncoder, + OrdinalEncoder, + RareLabelEncoder, + StringSimilarityEncoder, + WoEEncoder, +) +from feature_engine.imputation import ( + AddMissingIndicator, + ArbitraryNumberImputer, + CategoricalImputer, + DropMissingData, + EndTailImputer, + MeanMedianImputer, + RandomSampleImputer, +) +from feature_engine.outliers import OutlierTrimmer, Winsorizer +from feature_engine.preprocessing import MatchCategories, MatchVariables +from feature_engine.scaling import MeanNormalizationScaler +from feature_engine.selection import ( + DropConstantFeatures, + DropCorrelatedFeatures, + DropDuplicateFeatures, + DropHighPSIFeatures, + SelectByInformationValue, +) +from feature_engine.text import TextFeatures +from feature_engine.timeseries.forecasting import ( + ExpandingWindowFeatures, + LagFeatures, + WindowFeatures, +) +from feature_engine.transformation import LogTransformer, YeoJohnsonTransformer + +# -------------------------------------------------------------------------- +# Synthetic datasets +# -------------------------------------------------------------------------- + +WORDS = [ + "feature", "engine", "pandas", "polars", "numpy", "python", "model", + "variable", "transform", "encode", "impute", "select", "data", "frame", +] + + +def make_data(n_rows: int, seed: int = 42) -> dict: + """Create the synthetic datasets used by the benchmarks.""" + rng = np.random.default_rng(seed) + + # strictly positive numerical data (so that log-type transformers work) + num = {f"num_{i}": rng.lognormal(0.0, 1.0, n_rows) for i in range(10)} + df_num = pd.DataFrame(num) + + # numerical data with 10% missing values + df_num_na = df_num.copy() + for col in df_num_na.columns[:5]: + mask = rng.random(n_rows) < 0.10 + df_num_na.loc[mask, col] = np.nan + + # categorical data: 5 string variables with 20 categories each + categories = [f"cat_{i}" for i in range(20)] + cat = { + f"str_{i}": rng.choice(categories, n_rows).astype(object) + for i in range(5) + } + df_cat = pd.DataFrame(cat) + + df_cat_na = df_cat.copy() + for col in df_cat_na.columns[:3]: + mask = rng.random(n_rows) < 0.10 + df_cat_na.loc[mask, col] = np.nan + + # mixed data + df_mixed = pd.concat([df_num.iloc[:, :5], df_cat.iloc[:, :3]], axis=1) + df_mixed_na = pd.concat( + [df_num_na.iloc[:, :5], df_cat_na.iloc[:, :3]], axis=1 + ) + + # quasi-constant and duplicated variables for the selectors + df_const = df_mixed.copy() + df_const["const_1"] = 1.0 + quasi = np.ones(n_rows) + quasi[: max(1, n_rows // 100)] = 0.0 + df_const["quasi_1"] = quasi + + df_dup = df_num.iloc[:, :5].copy() + df_dup["dup_0"] = df_dup["num_0"] + df_dup["dup_1"] = df_dup["num_1"] + + # correlated variables + base = rng.normal(size=(n_rows, 10)) + corr = {f"corr_{i}": base[:, i] for i in range(10)} + for i in range(10): + corr[f"corr_noisy_{i}"] = base[:, i] + rng.normal( + 0.0, 0.1, n_rows + ) + df_corr = pd.DataFrame(corr) + + # datetime data + start = pd.Timestamp("2020-01-01") + dts = { + f"dt_{i}": pd.to_datetime( + rng.integers(0, 4 * 365 * 24, n_rows), unit="h", origin=start + ) + for i in range(3) + } + df_dt = pd.DataFrame(dts) + df_dt_str = df_dt.astype(str) + + # time series data with a sorted datetime index + df_ts = pd.DataFrame( + {f"num_{i}": rng.normal(size=n_rows) for i in range(5)}, + index=pd.date_range("2020-01-01", periods=n_rows, freq="min"), + ) + + # text data + n_words = rng.integers(3, 15, n_rows) + df_text = pd.DataFrame( + {"text": [" ".join(rng.choice(WORDS, k)) for k in n_words]} + ) + + y = pd.Series(rng.integers(0, 2, n_rows), name="target") + + return { + "num": df_num, + "num_na": df_num_na, + "cat": df_cat, + "cat_na": df_cat_na, + "mixed": df_mixed, + "mixed_na": df_mixed_na, + "const": df_const, + "dup": df_dup, + "corr": df_corr, + "dt": df_dt, + "dt_str": df_dt_str, + "ts": df_ts, + "text": df_text, + "y": y, + } + + +# -------------------------------------------------------------------------- +# Benchmark cases: (name, transformer factory, dataset key, needs_y) +# -------------------------------------------------------------------------- + +CASES = [ + # imputation + ("MeanMedianImputer", lambda: MeanMedianImputer(), "num_na", False), + ("EndTailImputer", lambda: EndTailImputer(), "num_na", False), + ("ArbitraryNumberImputer", lambda: ArbitraryNumberImputer(), "num_na", False), + ( + "CategoricalImputer(frequent)", + lambda: CategoricalImputer(imputation_method="frequent"), + "cat_na", + False, + ), + ("AddMissingIndicator", lambda: AddMissingIndicator(), "mixed_na", False), + ("DropMissingData", lambda: DropMissingData(), "mixed_na", False), + ( + "RandomSampleImputer(general)", + lambda: RandomSampleImputer(random_state=0), + "mixed_na", + False, + ), + # encoding + ("CountFrequencyEncoder", lambda: CountFrequencyEncoder(), "cat", False), + ( + "OrdinalEncoder(ordered)", + lambda: OrdinalEncoder(encoding_method="ordered"), + "cat", + True, + ), + ("MeanEncoder", lambda: MeanEncoder(), "cat", True), + ("WoEEncoder", lambda: WoEEncoder(), "cat", True), + ( + "RareLabelEncoder", + lambda: RareLabelEncoder(tol=0.01, n_categories=2), + "cat", + False, + ), + ("OneHotEncoder(top10)", lambda: OneHotEncoder(top_categories=10), "cat", False), + ( + "StringSimilarityEncoder(top5)", + lambda: StringSimilarityEncoder(top_categories=5), + "cat", + False, + ), + # discretisation + ("EqualFrequencyDiscretiser", lambda: EqualFrequencyDiscretiser(), "num", False), + ("EqualWidthDiscretiser", lambda: EqualWidthDiscretiser(), "num", False), + ("GeometricWidthDiscretiser", lambda: GeometricWidthDiscretiser(), "num", False), + # outliers + ("Winsorizer(gaussian)", lambda: Winsorizer(), "num", False), + ( + "Winsorizer(quantiles)", + lambda: Winsorizer(capping_method="quantiles", fold=0.05), + "num", + False, + ), + ( + "OutlierTrimmer(iqr)", + lambda: OutlierTrimmer(capping_method="iqr", tail="both"), + "num", + False, + ), + # transformation + ("LogTransformer", lambda: LogTransformer(), "num", False), + ( + "YeoJohnsonTransformer(3vars)", + lambda: YeoJohnsonTransformer(variables=["num_0", "num_1", "num_2"]), + "num", + False, + ), + # scaling + ("MeanNormalizationScaler", lambda: MeanNormalizationScaler(), "num", False), + # creation + ("CyclicalFeatures", lambda: CyclicalFeatures(), "num", False), + ( + "MathFeatures(sum,mean,std)", + lambda: MathFeatures( + variables=[f"num_{i}" for i in range(5)], + func=["sum", "mean", "std"], + ), + "num", + False, + ), + ( + "RelativeFeatures(add,sub)", + lambda: RelativeFeatures( + variables=["num_0", "num_1", "num_2"], + reference=["num_3"], + func=["add", "sub"], + ), + "num", + False, + ), + # datetime + ("DatetimeFeatures(datetime)", lambda: DatetimeFeatures(), "dt", False), + ("DatetimeFeatures(strings)", lambda: DatetimeFeatures(), "dt_str", False), + ( + "DatetimeSubtraction", + lambda: DatetimeSubtraction(variables=["dt_0"], reference=["dt_1"]), + "dt", + False, + ), + # time series + ("LagFeatures", lambda: LagFeatures(periods=[1, 2, 3]), "ts", False), + ( + "WindowFeatures", + lambda: WindowFeatures(window=[3, 7], functions=["mean", "std"]), + "ts", + False, + ), + ( + "ExpandingWindowFeatures", + lambda: ExpandingWindowFeatures(functions=["mean"]), + "ts", + False, + ), + # selection (dataframe-bound selectors; the model-based selectors are + # dominated by scikit-learn compute and are not part of this suite) + ( + "DropConstantFeatures", + lambda: DropConstantFeatures(tol=0.98), + "const", + False, + ), + ("DropDuplicateFeatures", lambda: DropDuplicateFeatures(), "dup", False), + ( + "DropCorrelatedFeatures", + lambda: DropCorrelatedFeatures(threshold=0.8), + "corr", + False, + ), + ("DropHighPSIFeatures", lambda: DropHighPSIFeatures(bins=5), "num", False), + ( + "SelectByInformationValue", + lambda: SelectByInformationValue(bins=5), + "cat", + True, + ), + # preprocessing + ("MatchCategories", lambda: MatchCategories(), "cat", False), + ("MatchVariables", lambda: MatchVariables(), "mixed", False), + # text + ( + "TextFeatures(3features)", + lambda: TextFeatures( + variables="text", + features=["word_count", "char_count", "lexical_diversity"], + ), + "text", + False, + ), +] + + +# -------------------------------------------------------------------------- +# Timing machinery +# -------------------------------------------------------------------------- + +def _time_call(func, repeats: int) -> list: + """Time func() repeats times (plus one timed warmup), in seconds. + + To keep the total runtime of the suite bounded, slow calls are repeated + fewer times: calls slower than 2s run once more, calls slower than 0.5s + run at most twice more. The warmup timing is discarded unless it is the + only measurement available. + """ + gc.collect() + start = time.perf_counter() + func() # warmup + warmup = time.perf_counter() - start + + if warmup > 2.0: + repeats = 1 + elif warmup > 0.5: + repeats = min(repeats, 2) + + timings = [] + for _ in range(repeats): + gc.collect() + start = time.perf_counter() + func() + timings.append(time.perf_counter() - start) + return timings + + +def run_benchmarks(data: dict, repeats: int, name_filter: str = "") -> dict: + results = {} + y = data["y"] + for name, factory, data_key, needs_y in CASES: + if name_filter and name_filter.lower() not in name.lower(): + continue + X = data[data_key] + target = y if needs_y else None + + fit_times = _time_call(lambda: factory().fit(X, target), repeats) + + fitted = factory().fit(X, target) + transform_times = _time_call(lambda: fitted.transform(X), repeats) + + results[name] = { + "fit": { + "min": min(fit_times), + "median": statistics.median(fit_times), + }, + "transform": { + "min": min(transform_times), + "median": statistics.median(transform_times), + }, + } + print( + f"{name:32s} fit {min(fit_times) * 1000:10.1f} ms " + f"transform {min(transform_times) * 1000:10.1f} ms" + ) + return results + + +def collect_meta(n_rows: int, repeats: int) -> dict: + return { + "date": datetime.datetime.now().isoformat(timespec="seconds"), + "platform": platform.platform(), + "processor": platform.processor(), + "python": platform.python_version(), + "feature_engine": feature_engine.__version__, + "pandas": pd.__version__, + "numpy": np.__version__, + "sklearn": sklearn.__version__, + "n_rows": n_rows, + "repeats": repeats, + } + + +def compare(results: dict, baseline_path: str, threshold: float) -> int: + with open(baseline_path) as f: + baseline = json.load(f) + + print() + print(f"Comparing against: {baseline_path}") + print(f"Baseline meta: {baseline['meta']}") + print() + header = ( + f"{'benchmark':32s} {'step':10s} {'baseline':>10s} " + f"{'current':>10s} {'ratio':>7s}" + ) + print(header) + print("-" * len(header)) + + regressions = [] + for name, res in results.items(): + base = baseline["results"].get(name) + if base is None: + print(f"{name:32s} (not in baseline)") + continue + for step in ("fit", "transform"): + old = base[step]["min"] + new = res[step]["min"] + ratio = new / old if old > 0 else float("inf") + flag = "" + if ratio > 1 + threshold: + flag = " << REGRESSION" + regressions.append((name, step, ratio)) + elif ratio < 1 - threshold: + flag = " (faster)" + print( + f"{name:32s} {step:10s} {old * 1000:8.1f}ms " + f"{new * 1000:8.1f}ms {ratio:6.2f}x{flag}" + ) + + print() + if regressions: + print(f"{len(regressions)} REGRESSION(S) above {threshold:.0%}:") + for name, step, ratio in regressions: + print(f" - {name} / {step}: {ratio:.2f}x slower") + return 1 + print(f"No regressions above {threshold:.0%}.") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, default=100_000) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--filter", type=str, default="") + parser.add_argument("--save", type=str, default=None) + parser.add_argument("--compare", type=str, default=None) + parser.add_argument("--threshold", type=float, default=0.10) + args = parser.parse_args() + + meta = collect_meta(args.rows, args.repeats) + print("Environment:", json.dumps(meta, indent=2)) + print(f"\nGenerating synthetic data with {args.rows} rows...") + data = make_data(args.rows) + print("Running benchmarks...\n") + + results = run_benchmarks(data, args.repeats, args.filter) + + if args.save: + with open(args.save, "w") as f: + json.dump({"meta": meta, "results": results}, f, indent=2) + print(f"\nSaved baseline to {args.save}") + + if args.compare: + return compare(results, args.compare, args.threshold) + + return 0 + + +if __name__ == "__main__": + sys.exit(main())