From 2edf1a8d2bd23f30285a82c82ae98f8589033e3a Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Wed, 9 Sep 2026 17:05:45 +0100 Subject: [PATCH 01/12] Implement column evaluators --- datafaker/dialects.py | 90 +++ datafaker/evaluators/column_evaluator.py | 334 +++++++++++ datafaker/evaluators/distribution_builders.py | 279 +++++++++ datafaker/evaluators/evaluation_profile.py | 20 + datafaker/evaluators/feature_extractors.py | 470 +++++++++++++++ datafaker/evaluators/metrics.py | 178 ++++++ datafaker/evaluators/proposal_ranking.py | 534 ++++++++++++++++++ datafaker/evaluators/statistical_fidelity.py | 297 ++++++++++ datafaker/interactive/generators.py | 278 ++++++++- 9 files changed, 2467 insertions(+), 13 deletions(-) create mode 100644 datafaker/evaluators/column_evaluator.py create mode 100644 datafaker/evaluators/distribution_builders.py create mode 100644 datafaker/evaluators/evaluation_profile.py create mode 100644 datafaker/evaluators/feature_extractors.py create mode 100644 datafaker/evaluators/metrics.py create mode 100644 datafaker/evaluators/proposal_ranking.py create mode 100644 datafaker/evaluators/statistical_fidelity.py diff --git a/datafaker/dialects.py b/datafaker/dialects.py index df09affd..9d92e7b3 100644 --- a/datafaker/dialects.py +++ b/datafaker/dialects.py @@ -211,6 +211,96 @@ def compile_stddev_mssql(element: StdDev, compiler: Any, **kw: Any) -> str: return f"STDEV({e})" +class WordCount(ColumnElement[int]): # pylint: disable=too-many-ancestors + """Count whitespace-separated words in a text expression.""" + + expr: ColumnElement[str] + + _traverse_internals = [ + ("expr", InternalTraversal.dp_clauseelement), + ] + + def __init__(self, expr: ColumnElement[str]): + """Get a clause for the number of whitespace-separated words in ``expr``.""" + self.expr = expr + + __sa_operate__ = ColumnElement.operate + + +@compiles(WordCount) +def compile_word_count(element: WordCount, compiler: Any, **kw: Any) -> str: + """Exact word count via regexp_split_to_array (Postgres/DuckDB).""" + e = compiler.process(element.expr, **kw) + return f"array_length(regexp_split_to_array(trim({e}), '\\s+'), 1)" + + +@compiles(WordCount, "mssql") +def compile_word_count_mssql(element: WordCount, compiler: Any, **kw: Any) -> str: + """MSSQL equivalent: approximate word count via space-counting. + + No regex/array-split support, so approximate word count as (space + count in the trimmed text) + 1. This overcounts by one per run of 2+ + consecutive spaces relative to the exact regex-based count used on + Postgres/DuckDB, but there is no T-SQL primitive for a real + whitespace-run split. + """ + e = compiler.process(element.expr, **kw) + trimmed = f"TRIM({e})" + return ( + f"(CASE WHEN LEN({trimmed}) = 0 THEN 0 ELSE " + f"LEN({trimmed}) - LEN(REPLACE({trimmed}, ' ', '')) + 1 END)" + ) + + +class SentenceCount(ColumnElement[int]): # pylint: disable=too-many-ancestors + """Count '.'/'!'/'?'-delimited sentences in a text expression.""" + + expr: ColumnElement[str] + + _traverse_internals = [ + ("expr", InternalTraversal.dp_clauseelement), + ] + + def __init__(self, expr: ColumnElement[str]): + """Get a clause for the number of sentences in ``expr``.""" + self.expr = expr + + __sa_operate__ = ColumnElement.operate + + +@compiles(SentenceCount) +def compile_sentence_count(element: SentenceCount, compiler: Any, **kw: Any) -> str: + """Exact sentence count via regexp_split_to_array (Postgres/DuckDB).""" + e = compiler.process(element.expr, **kw) + return ( + f"array_length(regexp_split_to_array(trim({e}, ' .!?\\t\\n\\r'), '[.!?]+'), 1)" + ) + + +@compiles(SentenceCount, "mssql") +def compile_sentence_count_mssql( + element: SentenceCount, compiler: Any, **kw: Any +) -> str: + """MSSQL equivalent: approximate sentence count via delimiter-counting. + + No regex support, so approximate sentence count by normalizing '!'/'?' + to '.' and counting '.' characters, after trimming whitespace and + sentence-enders from both ends (so a leading/trailing sentence-ender + doesn't add a spurious segment, matching the Postgres/DuckDB + behaviour). This overcounts by one per run of 2+ consecutive delimiter + characters (e.g. "Wow!!") relative to the exact regex-based count, but + there is no T-SQL primitive for regex-run splitting. + """ + e = compiler.process(element.expr, **kw) + trim_chars = "' .!?' + CHAR(9) + CHAR(10) + CHAR(13)" + trimmed = f"TRIM({trim_chars} FROM {e})" + normalized = f"REPLACE(REPLACE({trimmed}, '!', '.'), '?', '.')" + return ( + f"(CASE WHEN LEN({trimmed}) = 0 THEN 0 ELSE " + f"LEN({normalized}) - LEN(REPLACE({normalized}, '.', '')) + 1 END)" + ) + + class IsNull(ColumnElement[bool]): # pylint: disable=too-many-ancestors """Represent IS NULL as an expression.""" diff --git a/datafaker/evaluators/column_evaluator.py b/datafaker/evaluators/column_evaluator.py new file mode 100644 index 00000000..4b02df90 --- /dev/null +++ b/datafaker/evaluators/column_evaluator.py @@ -0,0 +1,334 @@ +"""Evaluate how well a proposer's synthetic data matches a real column.""" + +import string +from dataclasses import dataclass + +from sqlalchemy import Column, Engine, select +from sqlalchemy.types import Date, DateTime, Integer, Numeric, Time + +from datafaker.dialects import Random +from datafaker.evaluators.evaluation_profile import EvaluationProfile +from datafaker.evaluators.metrics import ( + DiversityMetric, + NoveltyMetric, + canonicalize_value, +) +from datafaker.evaluators.statistical_fidelity import StatisticalFidelity +from datafaker.proposers.base import Proposer, get_column_type + +__all__ = ["EvaluationProfile", "ColumnEvaluator", "ColumnStats", "ProposalEvaluation"] + + +@dataclass +class ColumnStats: + """Summary statistics of a column's (string-rendered) real values.""" + + row_count: int + unique_count: int + avg_length: float + space_ratio: float + digit_ratio: float + punctuation_ratio: float + + @property + def uniqueness(self): + """Fraction of rows whose value is distinct from the others.""" + return self.unique_count / max(self.row_count, 1) + + +@dataclass +class ProposalEvaluation: + """The scored result of evaluating one proposer against a column.""" + + proposer: Proposer + novelty: float + diversity: float + overall_score: float + pipeline_scores: dict[str, float] + copy_fraction: float = 0.0 + synthetic_uniqueness: float = 1.0 + + def __str__(self): + """Render a human-readable summary of the evaluation.""" + lines = [ + f"{self.proposer.name()} " + f" (error={self.overall_score:.6f})" + f" (novelty={self.novelty:.6f})" + f" (diversity={self.diversity:.6f})" + f" (copies={self.copy_fraction:.6f})" + ] + + for name, score in self.pipeline_scores.items(): + lines.append(f" {name:<15} {score:.6f}") + + return "\n".join(lines) + + __repr__ = __str__ + + +def analyse_column(values: list[str]) -> ColumnStats: + """Compute length/space/digit/punctuation statistics over ``values``.""" + values = [str(v) for v in values if v is not None] + + total_chars = sum(len(v) for v in values) + + spaces = sum(v.count(" ") for v in values) + + digits = sum(c.isdigit() for v in values for c in v) + + punctuation = sum(c in string.punctuation for v in values for c in v) + + return ColumnStats( + row_count=len(values), + unique_count=len(set(values)), + avg_length=(total_chars / len(values) if values else 0), + space_ratio=(spaces / total_chars if total_chars else 0), + digit_ratio=(digits / total_chars if total_chars else 0), + punctuation_ratio=(punctuation / total_chars if total_chars else 0), + ) + + +# Below this fraction of distinct values, a column is treated as a small, +# repeated set of categories rather than a free-form or continuous quantity. +CATEGORICAL_UNIQUENESS_THRESHOLD = 0.2 + + +def choose_profile(stats: ColumnStats) -> EvaluationProfile: + """Pick a string column's evaluation profile from its statistics.""" + # mostly unique short values + if stats.avg_length < 30 and stats.uniqueness > 0.8: + return EvaluationProfile.SHORT_TEXT + + # repeated values + if stats.uniqueness < CATEGORICAL_UNIQUENESS_THRESHOLD: + return EvaluationProfile.CATEGORICAL + + # long text + if stats.avg_length > 50 or stats.space_ratio > 0.15: + return EvaluationProfile.FREE_TEXT + + return EvaluationProfile.SHORT_TEXT + + +def choose_numeric_profile(values) -> EvaluationProfile: + """Pick a numeric column's evaluation profile from its cardinality. + + Decide whether an Integer/Numeric column behaves like a small set of + repeated categories (a status code, rating, or flag) or like a + continuous / high-cardinality quantity (an age, salary, or identifier). + + Previously every Integer column was assumed to be categorical and every + Numeric column was assumed to be an identifier, regardless of how many + distinct values it actually had - which misclassifies something like an + integer age or salary column. This applies the same cardinality check + already used for string columns instead. + """ + non_null = [v for v in values if v is not None] + if not non_null: + return EvaluationProfile.IDENTIFIER + + uniqueness = len(set(non_null)) / len(non_null) + if uniqueness < CATEGORICAL_UNIQUENESS_THRESHOLD: + return EvaluationProfile.CATEGORICAL + + return EvaluationProfile.IDENTIFIER + + +def looks_like_email(values) -> bool: + """Heuristically decide whether a sample of values looks like emails.""" + nonempty = [str(v).strip() for v in values if v is not None and str(v).strip()] + if not nonempty: + return False + + email_like = 0 + for value in nonempty: + if "@" not in value: + continue + local_part, sep, domain = value.partition("@") + if not sep or not local_part or not domain: + continue + if "." in domain: + email_like += 1 + + return email_like / len(nonempty) > 0.5 + + +class ColumnEvaluator: # pylint: disable=too-many-instance-attributes + """Evaluate candidate proposers against one (or several merged) columns.""" + + def __init__(self): + """Initialize an unconfigured evaluator; call setup() before use.""" + self.engine = None + self.novelty_metric = None + self.diversity_metric = None + self.sample_size = 4000 + self.columns = [] + self.column = None + self.table = None + self.is_primary_key = False + self.real_values = [] + self.column_is_numeric = False + self.real_uniqueness = 0.0 + self.statistical_fidelity = None + self.profile = None + + def setup(self, columns: list[Column], engine: Engine): + """Sample the real column(s) and pick the evaluation profile/pipelines.""" + self.engine = engine + self.novelty_metric = NoveltyMetric() + self.diversity_metric = DiversityMetric() + + self.columns = columns + self.column = columns[0] if len(columns) == 1 else None + self.table = columns[0].table if columns else None + # A composite key needs every constituent column marked primary_key; + # this reduces to the single column's own flag in the common case. + self.is_primary_key = bool(columns) and all(c.primary_key for c in columns) + + with engine.connect() as conn: + if len(columns) == 1: + # Cap how many real rows we pull into memory so this scales to + # large tables, matching the synthetic sample_size above rather + # than materializing the entire column. Order randomly first: + # an unordered LIMIT returns whatever the DB naturally returns + # first (often insertion order), which can systematically miss + # real values for a column whose value correlates with row + # order (e.g. a "last_update" timestamp) - corrupting + # real_uniqueness/copy_fraction/novelty/diversity, all of + # which are computed from this sample. + rows = conn.execute( + select(columns[0]) + .select_from(self.table) + .order_by(Random()) + .limit(self.sample_size) + ) + self.real_values = [row[0] for row in rows] + column_type = get_column_type(columns[0]) + else: + rows = conn.execute( + select(*columns) + .select_from(columns[0].table) + .order_by(Random()) + .limit(self.sample_size) + ) + self.real_values = [tuple(row) for row in rows] + column_type = None + + # Numeric columns (whether they end up profiled IDENTIFIER or, for a + # low-cardinality one like a foreign key, CATEGORICAL) have no shared + # real-world vocabulary that could explain a generator coincidentally + # matching real values - unlike a string column (e.g. common first + # names), where that overlap is expected and legitimate. Used to + # decide how broadly the resample-penalty in proposal_ranking.py + # applies. + self.column_is_numeric = isinstance(column_type, (Numeric, Integer)) + + # fraction of real values that are distinct - used to judge whether a + # generator that resamples from the real data verbatim (e.g. a + # ChoiceProposer) is doing the right thing (a small, shared + # vocabulary, where reproducing real values is expected/correct) or + # a privacy-losing shortcut (near-unique real values, where + # reproducing them verbatim just leaks real records). + non_null_real_values = [v for v in self.real_values if v is not None] + self.real_uniqueness = ( + len({str(v) for v in non_null_real_values}) / len(non_null_real_values) + if non_null_real_values + else 0.0 + ) + + # Evaluation dimensions: fidelity, novelty, diversity + if len(columns) == 1: + self.statistical_fidelity = StatisticalFidelity( + self.column, self.engine, sample_size=self.sample_size + ) + if isinstance(column_type, (Date, DateTime, Time)): + profile = EvaluationProfile.TEMPORAL + elif isinstance(column_type, (Numeric, Integer)): + profile = choose_numeric_profile(self.real_values) + else: + stats = analyse_column(self.real_values) + if looks_like_email(self.real_values): + profile = EvaluationProfile.EMAIL + else: + profile = choose_profile(stats) + self.profile = profile + self.statistical_fidelity.set_eval_pipelines(self.profile) + else: + self.statistical_fidelity = None + self.profile = EvaluationProfile.IDENTIFIER + + def evaluate(self, proposer) -> ProposalEvaluation: + """Score one proposer's synthetic data against the sampled real data.""" + synthetic_samples = proposer.generate_data(self.sample_size) + + # calculate novelty + novelty_score = self.novelty_metric.compare( + self.real_values, + synthetic_samples, + ) + + # calculate diversity + diversity_score = self.diversity_metric.compare( + self.real_values, + synthetic_samples, + ) + + # calculate copy fraction: fraction of synthetic samples that exactly match a real value + try: + real_set = set( + str(canonicalize_value(v)) for v in self.real_values if v is not None + ) + if synthetic_samples: + copy_count = sum( + 1 + for s in synthetic_samples + if str(canonicalize_value(s)) in real_set + ) + copy_frac = copy_count / max(len(synthetic_samples), 1) + else: + copy_frac = 0.0 + except Exception: # pylint: disable=broad-exception-caught + # A proposer's own output is untrusted here - anything unhashable + # or unstringifiable should degrade to "no copies detected" + # rather than crash the whole propose/evaluate flow. + copy_frac = 0.0 + + # fraction of the synthetic sample's own values that are distinct from + # each other - a direct, real-data-independent measure of whether this + # proposer could satisfy a PRIMARY KEY/UNIQUE constraint at all. This + # is different from copy_fraction (which only checks overlap against + # the *real* values): a proposer resampling from a small vocabulary, + # or one like dist_gen.constant that emits the same value every time, + # duplicates against its own output regardless of whether that output + # happens to match any real value. + try: + synthetic_uniqueness = ( + len({str(s) for s in synthetic_samples}) / len(synthetic_samples) + if synthetic_samples + else 0.0 + ) + except Exception: # pylint: disable=broad-exception-caught + # Same rationale as the copy_frac catch above. + synthetic_uniqueness = 0.0 + + # calculate fidelity scores for each evaluation pipeline. Multi-column + # proposal evaluation does not map cleanly onto the single-column fidelity + # pipelines, so fall back to a neutral score instead of crashing. + if self.statistical_fidelity is None: + stat_fidelity_overall_score = 0.0 + stat_fidelity_pipeline_scores = {} + else: + ( + stat_fidelity_overall_score, + stat_fidelity_pipeline_scores, + ) = self.statistical_fidelity.calculate_scores(synthetic_samples) + + return ProposalEvaluation( + proposer=proposer, + novelty=novelty_score, + diversity=diversity_score, + overall_score=stat_fidelity_overall_score, + pipeline_scores=stat_fidelity_pipeline_scores, + copy_fraction=copy_frac, + synthetic_uniqueness=synthetic_uniqueness, + ) diff --git a/datafaker/evaluators/distribution_builders.py b/datafaker/evaluators/distribution_builders.py new file mode 100644 index 00000000..868f0722 --- /dev/null +++ b/datafaker/evaluators/distribution_builders.py @@ -0,0 +1,279 @@ +"""Build real/synthetic value distributions for statistical fidelity scoring.""" + +import statistics +from abc import ABC, abstractmethod +from bisect import bisect_right +from collections import Counter +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import func, select + +from datafaker.dialects import Random +from datafaker.evaluators.feature_extractors import FeatureExtractor, IdentityExtractor + + +@dataclass +class Distribution: + """A discrete probability distribution over feature values.""" + + probabilities: dict[Any, float] + + @property + def vocabulary(self) -> set[Any]: + """The set of feature values this distribution assigns mass to.""" + return set(self.probabilities.keys()) + + def probability(self, key: Any) -> float: + """Return the probability mass assigned to ``key`` (0.0 if unseen).""" + return self.probabilities.get(key, 0.0) + + def as_vector(self, vocabulary: list[Any]) -> list[float]: + """Render this distribution as a probability vector over ``vocabulary``.""" + return [self.probability(v) for v in vocabulary] + + +class DistributionBuilder(ABC): + """Build a real or synthetic value's ``Distribution`` for one feature.""" + + @abstractmethod + def build_from_table(self) -> Distribution: + """Build the distribution of this feature over the real column.""" + + @abstractmethod + def build_from_values(self, values) -> Distribution: + """Build the distribution of this feature over synthetic ``values``.""" + + +# pylint: disable=too-many-instance-attributes +class HistogramBuilder(DistributionBuilder): + """Bucket a continuous feature into a fixed-width histogram.""" + + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + engine, + table, + column, + bins=10, + extractor: FeatureExtractor = IdentityExtractor(), + sample_size=4000, + ): + """Initialize a histogram builder for one column feature.""" + self.engine = engine + self.table = table + self.column = column + self.bins = bins + self.extractor = extractor + self.sample_size = sample_size + + # Learned from the real data + self.mean = None + self.stddev = None + self.bottom = None + self.width = None + self.edges = None + + def build_from_table(self) -> Distribution: + """Build the real column's histogram distribution.""" + feature_expr = self.extractor.expression(self.column) + + # + # Read a bounded, randomly-ordered sample of values for computing + # statistics (mean/stddev), rather than pulling the entire column + # into memory. Random ordering matters: an unordered LIMIT returns + # whatever the DB naturally returns first (often insertion order), + # which can systematically bias this sample for a column whose + # value correlates with row order (e.g. a "rental_date" column + # inserted roughly chronologically) - and since this is the "real" + # side of every fidelity comparison, that bias directly skews which + # generator looks like the better fit. + # + with self.engine.connect() as conn: + values = ( + conn.execute( + select(feature_expr) + .select_from(self.table) + .order_by(Random()) + .limit(self.sample_size) + ) + .scalars() + .all() + ) + + values = [float(v) for v in values if v is not None] + + if not values: + return Distribution({}) + + self.mean = statistics.mean(values) + self.stddev = statistics.stdev(values) if len(values) > 1 else 0.0 + + if self.stddev == 0: + self.edges = [ + float("-inf"), + self.mean - 2.0, + self.mean - 1.5, + self.mean - 1.0, + self.mean - 0.5, + self.mean, + self.mean + 0.5, + self.mean + 1.0, + self.mean + 1.5, + self.mean + 2.0, + float("inf"), + ] + bottom = self.mean - 2.0 + width = 0.5 + else: + self.edges = [ + float("-inf"), + self.mean - 2 * self.stddev, + self.mean - 1.5 * self.stddev, + self.mean - 1.0 * self.stddev, + self.mean - 0.5 * self.stddev, + self.mean, + self.mean + 0.5 * self.stddev, + self.mean + 1.0 * self.stddev, + self.mean + 1.5 * self.stddev, + self.mean + 2 * self.stddev, + float("inf"), + ] + bottom = self.mean - 2 * self.stddev + width = self.stddev / 2 + + if width == 0: + width = 1.0 + + # + # SQL histogram, computed exactly over the full table (this is a + # bounded, server-side aggregate query - it returns at most `bins` + # rows - so it doesn't need the sample_size cap that applies to the + # raw-value pulls above). + # + with self.engine.connect() as conn: + # Group by the bucket expression itself, not its output alias: + # MSSQL (unlike Postgres/DuckDB) rejects a SELECT-list alias + # referenced from GROUP BY. + bucket_expr = func.floor((feature_expr - bottom) / width) + rows = conn.execute( + select( + bucket_expr.label("bucket"), + func.count().label("count"), # pylint: disable=not-callable + ) + .select_from(self.table) + .group_by(bucket_expr) + ).all() + + # total must come from this same full-table query, not from the + # (possibly sampled) `values` above, so the probabilities sum to 1. + total = sum(count for bucket, count in rows if bucket is not None) + + if total == 0: + return Distribution({}) + + probs = {} + + for bucket, count in rows: + if bucket is None: + continue + + bucket = min( + self.bins - 1, + max(0, int(bucket) + 1), + ) + + probs[bucket] = probs.get(bucket, 0) + count / total + + return Distribution(probs) + + def build_from_values(self, values) -> Distribution: + """Build the synthetic sample's histogram distribution.""" + counts = Counter() + total = 0 + + for value in values: + for feature in self.extractor.extract(value): + feature = float(feature) + bucket = bisect_right(self.edges, feature) - 1 + bucket = max(0, min(bucket, len(self.edges) - 2)) + counts[bucket] += 1 + total += 1 + + if total == 0: + return Distribution({}) + + return Distribution({b: c / total for b, c in counts.items()}) + + +class CategoryBuilder(DistributionBuilder): + """Build a categorical distribution over a discrete feature's values.""" + + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + engine, + table, + column, + extractor: FeatureExtractor = IdentityExtractor(), + sample_size=4000, + ): + """Initialize a category builder for one column feature.""" + self.engine = engine + self.table = table + self.column = column + self.extractor = extractor + self.sample_size = sample_size + + def build_from_table(self): + """Build the real column's categorical distribution.""" + counter = Counter() + + with self.engine.connect() as conn: + # Bounded, randomly-ordered sample rather than the entire column, + # so this scales to large tables (matches the synthetic sample + # size for a fair, like-for-like comparison) without an + # unordered LIMIT biasing the "real" distribution toward + # whatever the DB returns first (see HistogramBuilder above). + rows = conn.execute( + select(self.column) + .select_from(self.table) + .order_by(Random()) + .limit(self.sample_size) + ) + + for (value,) in rows: + if value is None: + continue + for feature in self.extractor.extract(value): + if feature is None: + continue + counter[feature] += 1 + + total = sum(counter.values()) + if total == 0: + return Distribution({}) + + return Distribution({k: v / total for k, v in counter.items()}) + + def build_from_values(self, values): + """Build the synthetic sample's categorical distribution.""" + counter = Counter() + + for value in values: + if value is None: + continue + for feature in self.extractor.extract(value): + if feature is None: + continue + counter[feature] += 1 + + total = sum(counter.values()) + if total == 0: + # No features were extracted from the synthetic values; represent this + # explicitly with a special missing token so metrics treat it as a + # disjoint distribution (maximally different) compared to a real + # distribution with concrete feature values. + return Distribution({"__MISSING__": 1.0}) + + return Distribution({k: v / total for k, v in counter.items()}) diff --git a/datafaker/evaluators/evaluation_profile.py b/datafaker/evaluators/evaluation_profile.py new file mode 100644 index 00000000..c7544eb8 --- /dev/null +++ b/datafaker/evaluators/evaluation_profile.py @@ -0,0 +1,20 @@ +"""The evaluation profile enum, kept in its own module to avoid a cycle. + +``column_evaluator`` and ``statistical_fidelity`` both need this type, and +each also needs something from the other module - keeping the enum here lets +both import it directly instead of one importing it transitively through the +other. +""" + +from enum import Enum, auto + + +class EvaluationProfile(Enum): + """The kind of value a column holds, used to pick its fidelity pipelines.""" + + SHORT_TEXT = auto() + CATEGORICAL = auto() + FREE_TEXT = auto() + EMAIL = auto() + IDENTIFIER = auto() + TEMPORAL = auto() diff --git a/datafaker/evaluators/feature_extractors.py b/datafaker/evaluators/feature_extractors.py new file mode 100644 index 00000000..5ddf542c --- /dev/null +++ b/datafaker/evaluators/feature_extractors.py @@ -0,0 +1,470 @@ +"""Extract a comparable feature from a column's real/synthetic values.""" + +import re +from abc import ABC, abstractmethod +from datetime import date, datetime +from typing import Any + +from sqlalchemy import String, func, literal + +from datafaker.dialects import SecondsDifference, SentenceCount, WordCount + +TOKEN = re.compile(r"\w+") + + +def _coerce_datetime(value): + """ + Best-effort coercion of a value into something with .year/.month/etc. + + A value that genuinely represents a real datetime can still arrive here + as a plain string rather than a datetime/date object - e.g. because the + SQL query that produced it (some proposers build queries from raw + column-name strings, losing type information) didn't preserve its type. + Without this, a temporal extractor would silently treat it as "not a + date" and drop it entirely, rather than raising a visible error - + corrupting the resulting distribution for one candidate while giving no + indication anything went wrong. + """ + if hasattr(value, "year"): + return value + if isinstance(value, str): + try: + return datetime.fromisoformat(value.strip()) + except ValueError: + return None + return None + + +class FeatureExtractor(ABC): + """Extract one comparable feature from a real or synthetic value.""" + + @abstractmethod + def extract(self, value: Any): + """ + Convert one cell into one or more feature values (synthetic/Python side). + + Returns: + iterable of feature values + """ + + @abstractmethod + def expression(self, column): + """Build the equivalent SQLAlchemy expression for the real/database side.""" + + +class IdentityExtractor(FeatureExtractor): + """Use the value itself as the feature.""" + + def extract(self, value): + """Yield the value unchanged.""" + if value is None: + return + yield value + + def expression(self, column): + """Return the column unchanged.""" + return column + + +class FirstLetterExtractor(FeatureExtractor): + """Extract a string value's first letter, lowercased.""" + + def extract(self, value): + """Yield the value's first letter, lowercased.""" + if value is None: + return + value = str(value).strip() + if value: + yield value[0].lower() + + def expression(self, column): + """Build the SQL expression for the column's first letter.""" + return func.lower(func.substr(column, 1, 1)) + + +class LastLetterExtractor(FeatureExtractor): + """Extract a string value's last letter, lowercased.""" + + def extract(self, value): + """Yield the value's last letter, lowercased.""" + if value is None: + return + + value = str(value).strip().lower() + + if value: + yield value[-1] + + def expression(self, column): + """Build the SQL expression for the column's last letter.""" + return func.lower(func.substr(column, -1, 1)) + + +class LengthExtractor(FeatureExtractor): + """Extract a value's string length.""" + + def extract(self, value): + """Yield the value's string length.""" + if value is None: + return + + yield len(str(value)) + + def expression(self, column): + """Build the SQL expression for the column's string length.""" + return func.length(column) + + +class WordExtractor(FeatureExtractor): + """Extract a string value's individual word tokens.""" + + def extract(self, value): + """Yield each word token in the value, lowercased.""" + if value is None: + return + + yield from TOKEN.findall(str(value).lower()) + + def expression(self, column): + """Not implemented: word tokenization has no SQL equivalent here.""" + raise NotImplementedError("WordExtractor SQL expression is not implemented.") + + +class CharacterExtractor(FeatureExtractor): + """Extract a string value's individual characters.""" + + def extract(self, value): + """Yield each character in the value.""" + if value is None: + return + + yield from str(value) + + def expression(self, column): + """Not implemented: per-character extraction has no SQL equivalent here.""" + raise NotImplementedError( + "CharacterExtractor SQL expression is not implemented." + ) + + +class CharacterBigramExtractor(FeatureExtractor): + """Extract a string value's overlapping 2-character sequences.""" + + def extract(self, value): + """Yield each overlapping 2-character bigram, lowercased.""" + if value is None: + return + + value = str(value).lower() + + for i in range(len(value) - 1): + yield value[i : i + 2] + + def expression(self, column): + """Not implemented: bigram extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class CharacterTrigramExtractor(FeatureExtractor): + """Extract a string value's overlapping 3-character sequences.""" + + def extract(self, value): + """Yield each overlapping 3-character trigram, lowercased.""" + if value is None: + return + + value = str(value).lower() + + for i in range(len(value) - 2): + yield value[i : i + 3] + + def expression(self, column): + """Not implemented: trigram extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class PrefixExtractor(FeatureExtractor): + """Extract a string value's first ``length`` characters.""" + + def __init__(self, length=2): + """Initialize with the prefix length to extract.""" + self.length = length + + def extract(self, value): + """Yield the value's first ``length`` characters, lowercased.""" + if value is None: + return + + value = str(value).strip().lower() + + if value: + yield value[: self.length] + + def expression(self, column): + """Not implemented: prefix extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class SuffixExtractor(FeatureExtractor): + """Extract a string value's last ``length`` characters.""" + + def __init__(self, length=2): + """Initialize with the suffix length to extract.""" + self.length = length + + def extract(self, value): + """Yield the value's last ``length`` characters, lowercased.""" + if value is None: + return + + value = str(value).strip().lower() + + if value: + yield value[-self.length :] + + def expression(self, column): + """Not implemented: suffix extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +VOWELS = set("aeiou") + + +class VowelConsonantPatternExtractor(FeatureExtractor): + """Extract a string value's vowel/consonant pattern (e.g. "CVCV").""" + + def extract(self, value): + """Yield the value's vowel/consonant pattern string.""" + if value is None: + return + + pattern = [] + + for ch in str(value).lower(): + if not ch.isalpha(): + continue + + pattern.append("V" if ch in VOWELS else "C") + + if pattern: + yield "".join(pattern) + + def expression(self, column): + """Not implemented: pattern extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class TimestampExtractor(FeatureExtractor): + """Extract a date/datetime as a single continuous days-since-epoch value. + + This captures year, month and day jointly as one quantity for a + HistogramBuilder, rather than scoring them as independent, + uncorrelated dimensions - a generator that draws month/day/hour + independently (uniformly within a calibrated year range) will never + reproduce real joint patterns (e.g. seasonal clustering) even if each + marginal component looks reasonable alone, and previously scored + accordingly poorly per-dimension despite being reasonably calibrated; + a single joint measure is a fairer, simpler fidelity signal. + """ + + _EPOCH = datetime(1970, 1, 1) + _EPOCH_DATE = date(1970, 1, 1) + + def extract(self, value): + """Yield the value's days-since-epoch, as a single continuous float.""" + value = _coerce_datetime(value) + if value is None: + return + if not isinstance(value, datetime): + value = datetime(value.year, value.month, value.day) + if getattr(value, "tzinfo", None) is not None: + value = value.replace(tzinfo=None) + yield (value - self._EPOCH).total_seconds() / 86400.0 + + def expression(self, column): + """Build the SQL expression for the column's days-since-epoch.""" + # func.extract("epoch", ...) has no MSSQL equivalent (DATEPART has no + # "epoch" field); SecondsDifference already solves exactly this via a + # dialect-specific DATEDIFF compilation on MSSQL. + epoch = literal(self._EPOCH_DATE, type_=column.type) + return SecondsDifference(column, epoch) / 86400.0 + + +class WeekdayExtractor(FeatureExtractor): + """Extract day-of-week: a cyclic pattern a continuous value can't capture. + + A continuous days-since-epoch value can't capture a cyclic pattern + (e.g. weekday/weekend clustering) on its own, since it isn't a function + of absolute date position. + """ + + def extract(self, value): + """Yield the value's day of the week (Monday=0).""" + value = _coerce_datetime(value) + if value is None: + return + yield value.weekday() + + def expression(self, column): + """Build the SQL expression for the column's day of the week.""" + return func.extract("dow", column) # pylint: disable=not-callable + + +class EmailLocalPartExtractor(FeatureExtractor): + """Extract the local part (before ``@``) of an email-like string.""" + + def extract(self, value): + """Yield the value's local part, lowercased.""" + if value is None: + return + text = str(value).strip() + if not text or "@" not in text: + return + local_part, _, _ = text.partition("@") + if local_part: + yield local_part.lower() + + def expression(self, column): + """Not implemented: local-part extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class EmailDomainExtractor(FeatureExtractor): + """Extract the domain (after ``@``) of an email-like string.""" + + def extract(self, value): + """Yield the value's domain, lowercased.""" + if value is None: + return + text = str(value).strip() + if not text or "@" not in text: + return + _, _, domain = text.partition("@") + if domain: + yield domain.lower() + + def expression(self, column): + """Not implemented: domain extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class EmailTopLevelDomainExtractor(FeatureExtractor): + """Extract the top-level domain of an email-like string.""" + + def extract(self, value): + """Yield the value's top-level domain, lowercased.""" + if value is None: + return + text = str(value).strip() + if not text or "@" not in text: + return + _, _, domain = text.partition("@") + if "." not in domain: + return + tld = domain.rsplit(".", 1)[-1].lower() + if tld: + yield tld + + def expression(self, column): + """Not implemented: TLD extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class EmailValidityExtractor(FeatureExtractor): + """Extract a 'valid'/'invalid' token for an email-like string. + + Uses the email-validator package when available, falling back to a + conservative regex. + """ + + # conservative regex fallback + _SIMPLE_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + + def __init__(self): + """Try importing the robust validator; fall back to regex if unavailable.""" + try: + from email_validator import ( # type: ignore # pylint: disable=import-outside-toplevel,import-error + validate_email, + ) + + self._validator = validate_email + except ImportError: + self._validator = None + + def extract(self, value): + """Yield 'valid' or 'invalid' for the value's email syntax.""" + if value is None: + return + text = str(value).strip() + if not text: + return + valid = False + if self._validator is not None: + try: + # check_deliverability=False for syntactic check only + _ = self._validator(text, check_deliverability=False) + valid = True + except Exception: # pylint: disable=broad-exception-caught + # email_validator raises its own EmailNotValidError, but + # this is optional/best-effort - any failure here should + # just mean "invalid", not crash the whole evaluation. + valid = False + else: + # fallback to simple regex + valid = bool(self._SIMPLE_RE.match(text)) + + yield "valid" if valid else "invalid" + + def expression(self, column): + """Not implemented: validity checking has no SQL equivalent here.""" + raise NotImplementedError() + + +class WordCountExtractor(FeatureExtractor): + """Extract a string value's word count.""" + + def extract(self, value): + """Yield the value's word count.""" + if value is None: + return + text = str(value).strip() + if not text: + return + # count words using TOKEN + count = len(TOKEN.findall(text)) + yield count + + def expression(self, column): + """Build the SQL expression for the column's word count.""" + # Cast column to text to handle tsvector columns (e.g. fulltext) + # where regexp_split_to_array may not accept the tsvector type + # directly. WordCount compiles to an exact regexp-based count on + # Postgres/DuckDB and a string-function approximation on MSSQL + # (which has no regex support) - see dialects.py. + return WordCount(column.cast(String)) + + +class SentenceCountExtractor(FeatureExtractor): + """Extract a string value's sentence count.""" + + def extract(self, value): + """Yield the value's sentence count.""" + if value is None: + return + text = str(value).strip() + if not text: + return + # split on sentence enders + parts = re.split(r"[.!?]+", text) + # count non-empty segments containing a word + count = sum(1 for p in parts if TOKEN.search(p)) + yield count + + def expression(self, column): + """Build the SQL expression for the column's sentence count.""" + # Cast column to text to handle tsvector columns (e.g. fulltext) + # where regexp_split_to_array may not accept the tsvector type + # directly. SentenceCount compiles to an exact regexp-based count + # on Postgres/DuckDB and a string-function approximation on MSSQL + # (which has no regex support) - see dialects.py. + return SentenceCount(column.cast(String)) diff --git a/datafaker/evaluators/metrics.py b/datafaker/evaluators/metrics.py new file mode 100644 index 00000000..070889d9 --- /dev/null +++ b/datafaker/evaluators/metrics.py @@ -0,0 +1,178 @@ +"""Metrics for comparing a real column's data against a synthetic sample.""" + +import math +from abc import ABC, abstractmethod +from collections import Counter +from decimal import Decimal + +from datafaker.evaluators.distribution_builders import Distribution + + +def canonicalize_value(value): + """Normalize a value for real-vs-synthetic equality comparison. + + A column's real values may come back as decimal.Decimal (SQLAlchemy's + default for a Numeric column) or as a tuple (a merged/composite + column's row), while a proposer's synthetic values for the same data + are plain float/list. Comparing str(Decimal("123.40")) against + str(123.4), or str((1.2, 3.4)) against str([1.2, 3.4]), would never + match even when the underlying values are identical - normalize both + sides to the same numeric/sequence type first so an exact match survives. + """ + if isinstance(value, (list, tuple)): + return tuple(canonicalize_value(v) for v in value) + if isinstance(value, Decimal): + return float(value) + return value + + +class Metric(ABC): + """A comparison between a real and a synthetic distribution or sample.""" + + @abstractmethod + def compare( + self, + real: Distribution, + synthetic: Distribution, + ) -> float: + """Score how closely ``synthetic`` matches ``real``.""" + + +def _canonicalize_distribution(dist: Distribution) -> Distribution: + """Normalize keys so None and mixed key types are comparable and sortable.""" + normalized = { + ("" if key is None else str(key)): value + for key, value in dist.probabilities.items() + } + return Distribution(normalized) + + +class MeanSquaredError(Metric): + """Mean squared error between two probability distributions.""" + + def compare( + self, + real: Distribution, + synthetic: Distribution, + ) -> float: + """Score the mean squared error between ``real`` and ``synthetic``.""" + real = _canonicalize_distribution(real) + synthetic = _canonicalize_distribution(synthetic) + vocab = sorted(real.vocabulary | synthetic.vocabulary) + + if not vocab: + return 0.0 + + x = real.as_vector(vocab) + y = synthetic.as_vector(vocab) + + squared_error = sum((a - b) ** 2 for a, b in zip(x, y)) + + # x and y are probability vectors (each sums to 1), so their squared + # Euclidean distance is bounded in [0, 2] regardless of vocab size. + # Dividing by that fixed bound (rather than len(vocab)**2) keeps the + # score on a comparable scale to JensenShannon's [0, ln 2] range, so + # pipeline weights combining the two behave as intended. + score = squared_error / 2 + return score + + +class JensenShannon(Metric): + """Jensen-Shannon divergence between two probability distributions.""" + + EPS = 1e-12 + + def compare( + self, + real: Distribution, + synthetic: Distribution, + ) -> float: + """Score the Jensen-Shannon divergence between ``real`` and ``synthetic``.""" + real = _canonicalize_distribution(real) + synthetic = _canonicalize_distribution(synthetic) + vocab = sorted(real.vocabulary | synthetic.vocabulary) + + if not vocab: + return 0.0 + + p = real.as_vector(vocab) + q = synthetic.as_vector(vocab) + + m = [(a + b) / 2 for a, b in zip(p, q)] + + def kl(a, b): + total = 0 + for x, y in zip(a, b): + if x > 0: + total += x * math.log(x / max(y, self.EPS)) + return total + + return (kl(p, m) + kl(q, m)) / 2 + + +class NoveltyMetric(Metric): + """Fraction of synthetic values that don't already appear in the real data.""" + + def compare( + self, + real, + synthetic, + ): + """Score how much of ``synthetic`` is novel relative to ``real``.""" + real_set = { + str(canonicalize_value(v)).strip().lower() for v in real if v is not None + } + + generated = { + str(canonicalize_value(v)).strip().lower() + for v in synthetic + if v is not None + } + + if not generated: + return 0.0 + + overlap = generated & real_set + + return 1.0 - len(overlap) / len(generated) + + +class DiversityMetric(Metric): + """Compare the internal diversity of a real and a synthetic sample. + + Measures how closely the synthetic sample's internal diversity matches + the real sample's diversity, rather than rewarding diversity outright. + + Each sample is scored by its normalized Shannon entropy (entropy divided + by log(k), k = number of distinct values), which is 0 for a single + repeated value and 1 for a uniform distribution over all its distinct + values. The metric returns 1 minus the absolute difference between the + real and synthetic normalized entropies: 1.0 when the synthetic sample + is exactly as diverse (relative to its own value count) as the real + sample, dropping toward 0 the more it over- or under-diversifies + relative to the real data. This avoids favoring generators that spread + probability more uniformly than the real data actually does. + """ + + @staticmethod + def _normalized_entropy(values) -> float: + """Compute the normalized Shannon entropy of ``values``.""" + vals = [str(v).strip().lower() for v in values if v is not None] + if not vals: + return 0.0 + counts = Counter(vals) + k = len(counts) + if k <= 1: + return 0.0 + total = sum(counts.values()) + probs = [c / total for c in counts.values()] + ent = -sum(p * math.log(p) for p in probs if p > 0) + return ent / math.log(k) + + def compare(self, real, synthetic): + """Score how closely ``synthetic``'s diversity matches ``real``'s.""" + real_diversity = self._normalized_entropy(real) + synthetic_diversity = self._normalized_entropy(synthetic) + # clamp against floating-point noise (e.g. two near-1.0 entropies + # differing by a rounding error) so the result stays within [0, 1] + return max(0.0, 1.0 - abs(synthetic_diversity - real_diversity)) diff --git a/datafaker/evaluators/proposal_ranking.py b/datafaker/evaluators/proposal_ranking.py new file mode 100644 index 00000000..8649efbc --- /dev/null +++ b/datafaker/evaluators/proposal_ranking.py @@ -0,0 +1,534 @@ +"""Utilities for ranking proposal evaluations against multiple objectives.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any, Sequence + +from datafaker.evaluators.column_evaluator import ProposalEvaluation +from datafaker.evaluators.evaluation_profile import EvaluationProfile +from datafaker.proposers.choice import ChoiceProposer + +# Column-name keywords mapped to substrings of the generator's dotted name +# (e.g. "generic.person.first_name") that they hint at. Column naming +# conventions are a strong, cheap, complementary signal to the statistical +# fidelity/novelty/diversity evaluation: they catch cases the statistics +# genuinely can't discriminate (e.g. a column with a small realistic +# vocabulary where every candidate's exact-value fidelity ties) without +# having to rely on the generator library's own internal naming, which +# rarely matches real-world column names anyway. +# +# This is a soft boost, not a filter: a column name matching nothing here, +# or a generator matching no hint, is entirely unaffected - it never +# excludes a candidate the statistics would otherwise have picked. Keep this +# list small and grow it opportunistically as real columns turn up cases +# worth covering, rather than trying to anticipate every naming convention. +KEYWORD_GENERATOR_HINTS: list[tuple[list[str], list[str]]] = [ + ( + ["first_name", "firstname", "fname", "given_name", "forename"], + ["person.first_name"], + ), + ( + ["last_name", "lastname", "lname", "surname", "family_name"], + ["person.last_name"], + ), + (["full_name", "fullname"], ["person.full_name"]), + (["email"], ["person.email"]), + (["phone", "telephone", "mobile"], ["person.telephone", "address.calling_code"]), + (["country"], ["address.country", "address.country_code"]), + (["city"], ["address.city"]), + ( + ["street"], + ["address.street_name", "address.street_number", "address.street_suffix"], + ), + (["postcode", "postal_code", "zip"], ["address.postal_code"]), + (["gender", "sex"], ["person.gender"]), + (["occupation", "job_title"], ["person.occupation"]), + (["nationality"], ["person.nationality"]), + (["username", "login"], ["person.username"]), +] + +# Small enough not to override a clearly-better statistical result, large +# enough to break the near-ties this is meant to resolve. +KEYWORD_BOOST = 0.15 + +# Below this fraction of distinct values in a candidate's own 4000-value +# synthetic sample, it is producing enough duplicates against itself that it +# could not satisfy a real PRIMARY KEY/UNIQUE constraint once row counts grow +# past a handful - regardless of how well it otherwise fits the real +# distribution. Slack below 1.0 tolerates a rare coincidental collision from +# a genuinely-unique-generating proposer (e.g. a continuous float sampler) +# without treating it the same as a resampler or constant that duplicates +# constantly. +SYNTHETIC_UNIQUENESS_GUARANTEE_THRESHOLD = 0.999 + +# The weak-recommendation warning (below) is only worth showing when the +# winner's own combined score is actually weak - otherwise it's just noise +# on top of a perfectly good pick. +WEAK_WINNER_SCORE_THRESHOLD = 0.5 + + +def keyword_match( + column_name: str | None, proposer_name: str +) -> tuple[float, str | None]: + """Give a soft ranking boost when the column name hints at this generator's kind. + + Returns the boost plus the specific column keyword responsible (or None) + so the caller can show why. + + Uses substring matching (not exact tokenization), so this also catches + compound names like "customer_first_name". + """ + if not column_name: + return 0.0, None + name = column_name.lower() + generator_name = proposer_name.lower() + for column_keywords, generator_keywords in KEYWORD_GENERATOR_HINTS: + matched_keyword = next((kw for kw in column_keywords if kw in name), None) + if matched_keyword is not None and any( + gkw in generator_name for gkw in generator_keywords + ): + return KEYWORD_BOOST, matched_keyword + return 0.0, None + + +@dataclass +class ProposalRankingDisplay: + """Formatted text for a ranked proposal table.""" + + recommendation: str | None + profile_summary: str + rows: list[tuple[str, ...]] + fronts: list[int] + weak_recommendation_warning: str | None = None + no_uniqueness_guarantee: bool = False + + +# pylint: disable=too-many-instance-attributes +@dataclass +class ProposalRanking: + """A ranked set of proposal evaluations.""" + + recommended_index: int | None + recommended_reason: str | None + rows: list[tuple[str, ...]] + profile: EvaluationProfile | None + weights: tuple[float, float, float] + real_uniqueness: float = 0.0 + penalty_applies_to_all: bool = False + fronts: list[int] = field(default_factory=list) + scores: list[float] = field(default_factory=list) + weak_recommendation_warning: str | None = None + no_uniqueness_guarantee: bool = False + + +def normalize_list(vals: Sequence[float]) -> list[float]: + """Normalize values to the range [0, 1] for comparisons.""" + if not vals: + return [] + lo = min(vals) + hi = max(vals) + # Treat near-identical values as a tie rather than an exact one: e.g. + # when every candidate's synthetic sample shares no categories with the + # real data, their Jensen-Shannon divergence all sit at the same + # theoretical maximum (ln 2) but differ in the 10th+ decimal place from + # floating-point rounding. Min-max normalizing that noise would stretch + # it across the full [0, 1] range and manufacture a fake ranking. + if math.isclose(hi, lo, rel_tol=1e-9, abs_tol=1e-12): + return [0.5 for _ in vals] + return [(v - lo) / (hi - lo) for v in vals] + + +def _pareto_fronts(points: Sequence[tuple[float, float, float]]) -> list[list[int]]: + """Compute Pareto fronts for a list of 3D points.""" + remaining = set(range(len(points))) + fronts: list[list[int]] = [] + while remaining: + current_front: set[int] = set() + for i in remaining: + dominated = False + for j in remaining: + if i == j: + continue + if all( + points[j][d] >= points[i][d] for d in range(len(points[i])) + ) and any(points[j][d] > points[i][d] for d in range(len(points[i]))): + dominated = True + break + if not dominated: + current_front.add(i) + fronts.append(sorted(current_front)) + remaining -= current_front + return fronts + + +def _crowding_distances( + points: Sequence[tuple[float, float, float]], + fronts: Sequence[Sequence[int]], + objectives: tuple[Sequence[float], Sequence[float], Sequence[float]], +) -> list[float]: + """Compute NSGA-II style crowding distances.""" + crowding = [0.0 for _ in range(len(points))] + for front in fronts: + if len(front) <= 1: + for idx in front: + crowding[idx] = float("inf") + continue + for dim_vals in objectives: + vals = [(idx, dim_vals[idx]) for idx in front] + vals.sort(key=lambda item: item[1]) + lo = vals[0][1] + hi = vals[-1][1] + crowding[vals[0][0]] = float("inf") + crowding[vals[-1][0]] = float("inf") + if hi == lo: + continue + for k in range(1, len(vals) - 1): + prev_v = vals[k - 1][1] + next_v = vals[k + 1][1] + dist = (next_v - prev_v) / (hi - lo) + idx = vals[k][0] + if crowding[idx] != float("inf"): + crowding[idx] += dist + return crowding + + +# pylint: disable=too-many-arguments too-many-positional-arguments +# pylint: disable=too-many-locals too-many-statements +def rank_proposals( + results: Sequence[ProposalEvaluation], + profile: EvaluationProfile | None, + theme: Any | None = None, + column_name: str | None = None, + real_uniqueness: float = 0.0, + is_numeric_column: bool = False, + is_primary_key: bool = False, +) -> ProposalRanking: + """Rank proposals by fidelity, novelty and diversity using Pareto fronts. + + The ranking logic is deliberately separated from the interactive shell so it can + be tested and reused independently of presentation concerns. + """ + if not results: + return ProposalRanking(None, None, [], profile, (0.5, 0.25, 0.25)) + + # prepare multi-objective ranking: normalize metrics so larger-is-better + # overall_score is lower-is-better (distance/error), so invert after normalization + scores = [result.overall_score for result in results] + novelties = [result.novelty for result in results] + diversities = [result.diversity for result in results] + + norm_scores = normalize_list(scores) + norm_nov = normalize_list(novelties) + norm_div = normalize_list(diversities) + + # fidelity (higher-is-better) is inverted normalized overall_score + fidelity_scores = [1.0 - v for v in norm_scores] + novelty_scores = norm_nov + diversity_scores = norm_div + + # build points for Pareto (higher-is-better in all dims) + points = list(zip(fidelity_scores, novelty_scores, diversity_scores)) + + # non-dominated sort into Pareto fronts (simple O(n^2) algorithm) + fronts = _pareto_fronts(points) + front_of = [0] * len(points) + for front_index, front in enumerate(fronts, start=1): + for idx in front: + front_of[idx] = front_index + + # compute crowding distances per front (NSGA-II style) to prefer diverse solutions + crowding = _crowding_distances( + points, fronts, (fidelity_scores, novelty_scores, diversity_scores) + ) + + # determine profile and weights for scoring + # + # SHORT_TEXT, EMAIL and TEMPORAL all draw from a bounded, structured + # real-world vocabulary (names, addresses, dates), the same way + # CATEGORICAL/IDENTIFIER data does. For that kind of column, novelty and + # fidelity are mechanically in tension for the *best* generators: a + # generator that accurately models the real distribution is more, not + # less, likely to occasionally reproduce a real value by chance. Giving + # novelty a higher weight than fidelity here (as before) let it override + # an even strongly-discriminating fidelity signal and penalize the + # generator that best matched the real data. FREE_TEXT is kept + # novelty-tolerant since open-ended text has no such bounded vocabulary - + # genuinely novel sentences there are a good sign, not a red flag. + profile_weights = { + EvaluationProfile.IDENTIFIER: (0.85, 0.05, 0.10), + EvaluationProfile.CATEGORICAL: (0.7, 0.15, 0.15), + EvaluationProfile.SHORT_TEXT: (0.7, 0.15, 0.15), + EvaluationProfile.EMAIL: (0.7, 0.15, 0.15), + EvaluationProfile.FREE_TEXT: (0.5, 0.3, 0.2), + EvaluationProfile.TEMPORAL: (0.7, 0.15, 0.15), + } + fid_w, nov_w, div_w = profile_weights.get(profile, (0.5, 0.25, 0.25)) + + # compute profile-specific score (higher-is-better) + combined_scores = [fid_w * f + nov_w * n + div_w * d for (f, n, d) in points] + + # ChoiceProposer and its variants (dist_gen.choice/weighted_choice/ + # zipf_choice) resample directly from the column's own observed values - + # by construction, they can only ever emit values already in the real + # data. That's the *correct* thing to do for a column with a small, + # shared vocabulary (a status code, a gender) - reproducing real values + # there is expected and unavoidable - but a privacy problem for a column + # whose real values are meant to be unique per row (an email, a genuine + # ID): it isn't modeling the distribution, it's handing back real + # records. Scale the penalty by how unique the real column actually is + # (near 0 for a low-uniqueness column, where this is fine; near full + # strength for a near-unique one, where it's a leak) and by how much of + # this proposer's own output is actually copied. + # + # Other proposers are, in general, not penalized for copy_fraction: for + # them a high copy_fraction is usually a coincidental side effect of a + # naturally overlapping real-world vocabulary (e.g. common first names), + # not a sign of memorization - penalizing it there wrongly punishes + # exactly the generators that model the real distribution best (verified: + # applying this penalty to person.first_name for a first-name column + # incorrectly hands the win back to a generic word generator). + # + # Numeric columns are the exception, regardless of whether they end up + # profiled IDENTIFIER (high uniqueness, e.g. a primary key) or + # CATEGORICAL (low uniqueness, e.g. a foreign key referencing a small + # set of rows): a plain number has no shared human vocabulary to explain + # a coincidental match. A high copy_fraction there instead means the + # real value range is dense (e.g. a gapless sequence of small integer + # IDs), so *any* generator landing in that range trivially "matches" + # almost every time, independent of whether it resamples. That's the + # same underlying signal as ChoiceProposer's, just from a proposer that + # isn't one - e.g. a generic "weight" generator whose typical output + # range happens to overlap a table's ID range - so it gets the same + # penalty here. A low-cardinality *string* column (a gender, a status) + # doesn't get this: there, high copy_fraction from a well-calibrated + # generator is legitimate/expected, not a coincidental range overlap. + apply_to_every_proposer = is_numeric_column + penalty_multipliers = [1.0] * len(results) + for idx, result in enumerate(results): + resample_penalty = 1.0 + if isinstance(result.proposer, ChoiceProposer) or apply_to_every_proposer: + resample_penalty = 1.0 - real_uniqueness * result.copy_fraction + # A second, separate penalty for duplicating against the candidate's + # OWN output (synthetic_uniqueness) rather than against the real + # data (copy_fraction, above). These catch different failure modes: + # e.g. dist_gen.constant on a column whose fixed value happens not + # to appear in the sampled real data gets copy_fraction=0 - no + # resample penalty at all - despite duplicating every single row + # against itself. Scaled by real_uniqueness for the same reason as + # the resample penalty: a column with a small, legitimate shared + # vocabulary (a gender, a status) is expected to have low + # synthetic_uniqueness too, and shouldn't be penalized for it. + # Unlike the resample penalty, this applies to every candidate + # unconditionally - self-duplication breaks a PRIMARY KEY/UNIQUE + # constraint regardless of column type or whether the proposer is a + # resampler. + self_duplication_penalty = 1.0 - real_uniqueness * ( + 1.0 - result.synthetic_uniqueness + ) + penalty_multipliers[idx] = resample_penalty * self_duplication_penalty + combined_scores[idx] *= penalty_multipliers[idx] + + # soft boost for generators whose kind the column name itself hints at + # (see KEYWORD_GENERATOR_HINTS) - a cheap, complementary signal to the + # statistical evaluation above. Keep the matched keyword per candidate + # so the display can show *why* a boost was applied. + keyword_matches: list[str | None] = [] + for idx, result in enumerate(results): + boost, matched_keyword = keyword_match(column_name, result.proposer.name()) + combined_scores[idx] += boost + keyword_matches.append(matched_keyword) + + # combined_scores is a weighted sum of three [0, 1]-normalized dimensions + # with weights summing to 1.0, so it's bounded to [0, 1] on its own - but + # the keyword boost above is deliberately applied outside that weighted + # sum, and can otherwise push a score above 1. Clamp so "Score" stays a + # comparable, interpretable quantity. (The ChoiceProposer penalty above + # only ever shrinks a score toward 0, so it can't push below the range.) + combined_scores = [max(0.0, min(1.0, s)) for s in combined_scores] + + # A primary key/unique column needs values that are actually unique + # per-row, not just a good statistical fit. If every single candidate + # duplicates enough within its own synthetic sample (see + # synthetic_uniqueness) that it could never satisfy that constraint, + # there is no honest "Recommended: X" to give - naming a winner would + # imply a fitness that doesn't exist and invite a pick that later trips + # a real PRIMARY KEY/UNIQUE violation. This is independent of the + # resample penalty above (which only measures overlap with the *real* + # values): a proposer can have a low copy_fraction and still duplicate + # constantly against itself (e.g. dist_gen.constant). + no_uniqueness_guarantee = ( + is_primary_key + and bool(results) + and all( + result.synthetic_uniqueness < SYNTHETIC_UNIQUENESS_GUARANTEE_THRESHOLD + for result in results + ) + ) + + # pick recommended index: highest combined score across all candidates, + # tiebreaker by crowding distance. This used to restrict the pick to + # Pareto front 1, but the keyword boost and resample penalty above are + # applied after fronts are computed from the raw fidelity/novelty/ + # diversity values - so a technically-dominated candidate can end up + # with a genuinely higher combined_score than a front-1 one (e.g. a + # column-name match earning +0.15). Restricting to front 1 would + # silently override that, defeating the point of adding those signals. + # Front/crowding are still computed and shown per row for transparency + # (which candidates are non-dominated trade-off alternatives vs. + # objectively worse), just no longer used to gate the recommendation. + all_idxs = range(len(combined_scores)) + best_score = max(combined_scores[idx] for idx in all_idxs) + candidates = [idx for idx in all_idxs if combined_scores[idx] == best_score] + if len(candidates) == 1: + recommended_index = candidates[0] + recommended_reason = "best combined score" + else: + best_crowding = max(crowding[idx] for idx in candidates) + selected = [idx for idx in candidates if crowding[idx] == best_crowding] + recommended_index = selected[0] + recommended_reason = "tiebreak by crowding distance" + + # Flag the case where the candidates that actually fit the real data + # best were suppressed by the resample penalty, leaving something that + # was never a good fit to "win" by elimination rather than merit (e.g. + # every high-fidelity resampler on a unique ID column gets zeroed out, + # and dist_gen.constant is left to win despite fitting poorly itself). + # Deliberately narrow: only fires when a *specific* candidate was both + # heavily discounted (< 0.2x) and would otherwise have fit noticeably + # better (> 0.3 higher fidelity) - not just "the winner's score is low", + # which can also just mean a close call among decent options. Also + # requires the winner's own combined score to actually be weak: a + # suppressed resampler's fidelity for a unique-per-row column is + # inflated by the exact memorization the penalty exists to catch, so it + # being nominally higher than a genuinely good winner's (e.g. a keyword- + # matched generic.person.email with strong novelty/diversity) isn't a + # meaningful comparison and shouldn't cast doubt on that pick. Likewise, + # don't fire it when the winner itself is a candidate that guarantees + # fresh values (synthetic_uniqueness >= threshold, e.g. a sequence + # continuing past the observed max): a low combined score there is a + # structural artifact of the fidelity metric penalizing values that + # deliberately fall outside the observed real range, not a sign the + # recommendation is actually shaky - it's exactly the intended, + # principled pick for a column that needs guaranteed-unique values. + weak_recommendation_warning = None + if not no_uniqueness_guarantee and recommended_index is not None: + winner_fidelity = fidelity_scores[recommended_index] + winner_score = combined_scores[recommended_index] + winner_guarantees_uniqueness = ( + results[recommended_index].synthetic_uniqueness + >= SYNTHETIC_UNIQUENESS_GUARANTEE_THRESHOLD + ) + suppressed = [ + idx + for idx in range(len(results)) + if penalty_multipliers[idx] < 0.2 + and fidelity_scores[idx] > winner_fidelity + 0.3 + ] + if ( + suppressed + and winner_score < WEAK_WINNER_SCORE_THRESHOLD + and not winner_guarantees_uniqueness + ): + names = ", ".join(results[idx].proposer.name() for idx in suppressed) + weak_recommendation_warning = ( + f"Note: {names} fit the real data much better but were heavily " + "discounted by the resample penalty (see Penalty column) - reproducing " + "real values almost exactly isn't valid for a unique-per-row column. No " + "remaining candidate is a strong fit here; treat this recommendation as " + "a fallback, not a confident pick." + ) + + # prepare rows with Front and Score, mark Pareto front 1 with coloring. + # Crowding distance is still computed above (used as a tiebreak when + # scores are exactly equal) but isn't shown - it's an NSGA-II internal + # detail (how isolated a candidate is from its front-mates in objective + # space), not a quality signal a user picking a generator can reason + # about, and in practice it's "∞" for most rows anyway. + rows: list[tuple[str, ...]] = [] + for i, result in enumerate(results, start=1): + idx = i - 1 + front = front_of[idx] + cells = [ + str(i), + result.proposer.name(), + profile.name if profile is not None else "UNKNOWN", + str(front), + f"{combined_scores[idx]:.6f}", + keyword_matches[idx] or "", + f"{fidelity_scores[idx]:.6f}", + f"{novelty_scores[idx]:.6f}", + f"{diversity_scores[idx]:.6f}", + f"{real_uniqueness:.3f}", + f"{result.copy_fraction:.3f}", + f"{penalty_multipliers[idx]:.3f}", + ] + # color entire row for Pareto front 1 + if front == 1 and theme is not None: + cells = [f"{theme.function}{cell}{theme.reset}" for cell in cells] + # keep original types compatible with print_table (it will cast to list) + rows.append(tuple(cells)) + + return ProposalRanking( + recommended_index=recommended_index, + recommended_reason=recommended_reason, + rows=rows, + profile=profile, + weights=(fid_w, nov_w, div_w), + real_uniqueness=real_uniqueness, + penalty_applies_to_all=apply_to_every_proposer, + fronts=list(front_of), + scores=list(combined_scores), + weak_recommendation_warning=weak_recommendation_warning, + no_uniqueness_guarantee=no_uniqueness_guarantee, + ) + + +def format_ranking_display( + ranking: ProposalRanking, + results: Sequence[ProposalEvaluation], + profile: EvaluationProfile | None, +) -> ProposalRankingDisplay: + """Prepare user-facing strings for a ranked proposal table.""" + fid_w, nov_w, div_w = ranking.weights + recommendation = None + if ranking.no_uniqueness_guarantee: + recommendation = ( + "Recommended: none — no proposer can guarantee uniqueness for this column\n" + ) + elif ranking.recommended_index is not None: + rec_num = ranking.recommended_index + 1 + rec_name = results[ranking.recommended_index].proposer.name() + recommendation = ( + f"Recommended: {rec_num}. {rec_name} — {ranking.recommended_reason}\n" + ) + if ranking.penalty_applies_to_all: + penalty_scope = ( + "applied to every generator (this is a numeric column, so a high" + " copy_fraction always means the real range is dense, not a coincidental" + " vocabulary match)" + ) + else: + penalty_scope = ( + "applied only to resampling generators (dist_gen.choice/weighted_choice/" + "zipf_choice - see Penalty column); 1.0 (no effect) for everyone else" + ) + profile_summary = ( + f"Profile: {profile.name if profile is not None else 'UNKNOWN'} | " + f"Real uniqueness: {ranking.real_uniqueness:.3f} | " + "Pareto front 1 rows are highlighted\n" + f"Score = clamp( ({fid_w:.2f}*Fidelity + {nov_w:.2f}*Novelty" + f" + {div_w:.2f}*Diversity + Keyword) " + "x Penalty , 0, 1 )\n" + " Keyword: +0.15 if the column name hints at this generator's kind (see Keyword column)\n" + f" Penalty: 1 - real_uniqueness x copy_fraction, {penalty_scope}" + ) + return ProposalRankingDisplay( + recommendation=recommendation, + profile_summary=profile_summary, + rows=ranking.rows, + fronts=ranking.fronts, + weak_recommendation_warning=ranking.weak_recommendation_warning, + no_uniqueness_guarantee=ranking.no_uniqueness_guarantee, + ) diff --git a/datafaker/evaluators/statistical_fidelity.py b/datafaker/evaluators/statistical_fidelity.py new file mode 100644 index 00000000..ea68b2db --- /dev/null +++ b/datafaker/evaluators/statistical_fidelity.py @@ -0,0 +1,297 @@ +"""Per-profile fidelity pipelines that score a proposer against a column.""" + +from dataclasses import dataclass + +from datafaker.evaluators.distribution_builders import ( + CategoryBuilder, + DistributionBuilder, + HistogramBuilder, +) +from datafaker.evaluators.evaluation_profile import EvaluationProfile +from datafaker.evaluators.feature_extractors import ( + CharacterBigramExtractor, + EmailDomainExtractor, + EmailLocalPartExtractor, + EmailTopLevelDomainExtractor, + EmailValidityExtractor, + FeatureExtractor, + FirstLetterExtractor, + IdentityExtractor, + LastLetterExtractor, + LengthExtractor, + SentenceCountExtractor, + TimestampExtractor, + VowelConsonantPatternExtractor, + WeekdayExtractor, + WordCountExtractor, + WordExtractor, +) +from datafaker.evaluators.metrics import JensenShannon, MeanSquaredError, Metric + + +@dataclass +class EvaluationPipeline: + """One named feature-comparison stage within a profile's fidelity score.""" + + name: str + builder: DistributionBuilder + metric: Metric + extractor: FeatureExtractor + weight: float = 1.0 + + +# Short text profile (names, cities, etc.) +SHORT_TEXT_PIPELINE = [ + EvaluationPipeline( + name="length", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.25, + extractor=LengthExtractor(), + ), + EvaluationPipeline( + name="bigrams", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.35, + extractor=CharacterBigramExtractor(), + ), + EvaluationPipeline( + name="first_letter", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.10, + extractor=FirstLetterExtractor(), + ), + EvaluationPipeline( + name="last_letter", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.10, + extractor=LastLetterExtractor(), + ), + EvaluationPipeline( + name="pattern", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.20, + extractor=VowelConsonantPatternExtractor(), + ), +] + +EMAIL_PIPELINE = [ + EvaluationPipeline( + name="length", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.10, + extractor=LengthExtractor(), + ), + EvaluationPipeline( + name="local_part", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.40, + extractor=EmailLocalPartExtractor(), + ), + EvaluationPipeline( + name="domain", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.15, + extractor=EmailDomainExtractor(), + ), + EvaluationPipeline( + name="tld", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.15, + extractor=EmailTopLevelDomainExtractor(), + ), + EvaluationPipeline( + name="format_validity", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.20, + extractor=EmailValidityExtractor(), + ), +] + +# for country, gender, status, category +CATEGORICAL_PIPELINE = [ + EvaluationPipeline( + name="category", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=1.0, + extractor=IdentityExtractor(), + ) +] + +IDENTIFIER_PIPELINE = [ + EvaluationPipeline( + name="identifier", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=1.0, + extractor=IdentityExtractor(), + ) +] + +# Two pipelines instead of five independent year/month/day/hour dimensions: +# "timestamp" scores year+month+day jointly as one continuous quantity (so a +# generator is judged on the real calendar-position distribution as a whole, +# not on marginal components that can each look fine while their +# combination never does), and "day_of_week" is kept separately since it's a +# cyclic pattern a continuous days-since-epoch value can't capture on its +# own. +TEMPORAL_PIPELINE = [ + EvaluationPipeline( + name="timestamp", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.7, + extractor=TimestampExtractor(), + ), + EvaluationPipeline( + name="day_of_week", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.3, + extractor=WeekdayExtractor(), + ), +] + +# for comments, description, review +FREE_TEXT_PIPELINE = [ + EvaluationPipeline( + name="length", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.10, + extractor=LengthExtractor(), + ), + EvaluationPipeline( + name="word_count", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.25, + extractor=WordCountExtractor(), + ), + EvaluationPipeline( + name="sentence_count", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.15, + extractor=SentenceCountExtractor(), + ), + EvaluationPipeline( + name="words", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.30, + extractor=WordExtractor(), + ), + EvaluationPipeline( + name="bigrams", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.20, + extractor=CharacterBigramExtractor(), + ), +] + +PROFILE_PIPELINES = { + EvaluationProfile.IDENTIFIER: IDENTIFIER_PIPELINE, + EvaluationProfile.SHORT_TEXT: SHORT_TEXT_PIPELINE, + EvaluationProfile.EMAIL: EMAIL_PIPELINE, + EvaluationProfile.CATEGORICAL: CATEGORICAL_PIPELINE, + EvaluationProfile.FREE_TEXT: FREE_TEXT_PIPELINE, + EvaluationProfile.TEMPORAL: TEMPORAL_PIPELINE, +} + + +class StatisticalFidelity: + """ + Measure statistical fidelity of the synthetic sample. + + This class evaluates the statistical fidelity of a synthetic sample + against a real sample by comparing the distributions of features extracted + from the data. It uses a set of evaluation pipelines tailored to different + types of data (e.g., short text, categorical, free text, email, identifier, temporal) + to compute a weighted score that reflects how closely the synthetic data matches the real data + in terms of statistical properties. + """ + + def __init__(self, column, engine, sample_size=4000): + """Initialize with no pipelines set; call set_eval_pipelines() first.""" + self.engine = engine + self.column = column + self.table = self.column.table + self.eval_pipelines = {} + # caps how many real rows each pipeline's builder pulls into memory + # when computing the "real" distribution, so this scales to large + # tables instead of scanning the whole column per pipeline. + self.sample_size = sample_size + # (builder, real distribution) per pipeline name, populated lazily + # the first time calculate_scores runs each pipeline. The real + # distribution depends only on the column, not on which proposer is + # being scored, so it only needs computing once per column rather + # than once per calculate_scores() call (one per candidate proposer). + self._real_distributions = {} + + def set_eval_pipelines(self, profile): + """ + Set evaluation pipelines based on the specified profile. + + :param profile: The evaluation profile to use. + """ + self.eval_pipelines = PROFILE_PIPELINES[profile] + self._real_distributions = {} + + def calculate_scores(self, synthetic_samples): + """ + Calculate the overall fidelity score and each pipeline's own score. + + :param synthetic_samples: The synthetic samples to evaluate. + :return: A tuple of (overall score, {pipeline name: pipeline score}). + """ + pipeline_scores = {} + weighted_score = 0.0 + total_weight = 0.0 + + for pipeline in self.eval_pipelines: + cached = self._real_distributions.get(pipeline.name) + if cached is None: + builder = pipeline.builder( + self.engine, + self.table, + self.column, + extractor=pipeline.extractor, + sample_size=self.sample_size, + ) + # Distribution from real table - independent of the + # proposer being scored, so cache it (and the builder, + # which HistogramBuilder.build_from_values needs state + # from - e.g. self.edges - set by this same call). + real = builder.build_from_table() + self._real_distributions[pipeline.name] = (builder, real) + else: + builder, real = cached + + # Distribution from synthetic values + synthetic = builder.build_from_values(synthetic_samples) + + score = pipeline.metric.compare( + real, + synthetic, + ) + + pipeline_scores[pipeline.name] = score + + weighted_score += pipeline.weight * score + total_weight += pipeline.weight + + overall_score = weighted_score / total_weight if total_weight > 0 else 0.0 + + return overall_score, pipeline_scores diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index c6378d8a..621f51e6 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -9,13 +9,26 @@ from typing import Any, Callable, Optional, cast from sqlalchemy import Column, and_, literal_column, select +from sqlalchemy.types import Integer from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private from datafaker.dialects import Random +from datafaker.evaluators.column_evaluator import ColumnEvaluator +from datafaker.evaluators.proposal_ranking import ( + ProposalRanking, + ProposalRankingDisplay, + format_ranking_display, + rank_proposals, +) from datafaker.interactive.base import DbCmd, TableEntry, fk_column_name, or_default from datafaker.proposers import everything_factory -from datafaker.proposers.base import PredefinedProposer, Proposer -from datafaker.theme import get_active_theme +from datafaker.proposers.base import PredefinedProposer, Proposer, get_column_type +from datafaker.proposers.continuous import ( + GaussianProposer, + LogNormalProposer, + UniformProposer, +) +from datafaker.theme import Theme, get_active_theme from datafaker.utils import ( get_columns_assigned, get_property, @@ -181,6 +194,10 @@ class GeneratorCmd(DbCmd): "{theme_reset}{index}. {theme_func}{name}:" " {theme_fit}{fit} {theme_data}{sample}{theme_reset} ..." ) + RANKED_SAMPLE_TEXT = ( + "{theme_reset}{index}. {theme_func}{name}:" + " {theme_data}{sample}{theme_reset} ..." + ) PRIMARY_PRIVATE_TEXT = "Primary Private" SECONDARY_PRIVATE_TEXT = "Secondary Private on columns {0}" NOT_PRIVATE_TEXT = "Not private" @@ -196,6 +213,11 @@ class GeneratorCmd(DbCmd): ERROR_CANNOT_UNMERGE_ALL = "You cannot unmerge all the generator's columns" PROPOSE_NOTHING = "No proposed generators, sorry." + # Cap on rows shown by default 'propose' (front 1 can otherwise hold more + # candidates than are worth scanning in a terminal table). 'propose all' + # is unaffected. + MAX_PROPOSERS_SHOWN = 10 + SRC_STAT_RE = re.compile( r'\bSRC_STATS\["([^"]+)"\](\["results"\]\[0\]\["([^"]+)"\])?' ) @@ -281,6 +303,9 @@ def __init__( self.proposers: list[Proposer] | None = None self.proposer_index = 0 self.proposers_valid_columns: Optional[tuple[int, list[str]]] = None + # whether the cached self.proposers list currently includes proposers + # that are type-incompatible with the column (e.g. from 'propose all') + self.proposers_include_all: bool = False self.set_prompt() self.roles: dict[str, dict[str, RoleEntry]] = { table_name: { @@ -763,17 +788,61 @@ def _proposers_valid(self) -> bool: self._get_column_names(), ) - def _get_proposer_proposals(self) -> list[Proposer]: - """Get a list of acceptable proposers, sorted by decreasing fit to the actual data.""" - if not self._proposers_valid(): - self.proposers = None - if self.proposers is None: + # Proposers that emit continuous float output (via random.uniform/ + # normalvariate/lognormvariate) with no integer rounding. That's a + # legitimate fit for a Numeric/decimal column, but not for a genuinely + # Integer-typed one: inserting a non-whole float into an Integer column + # is backend-dependent at best (silently stored as-is on SQLite, + # truncated or rejected on MySQL) and raises an error on PostgreSQL. + _CONTINUOUS_FLOAT_PROPOSER_TYPES = ( + GaussianProposer, + UniformProposer, + LogNormalProposer, + ) + + def _is_integer_incompatible( + self, proposer: Proposer, columns: list[Column] + ) -> bool: + """Check whether `proposer` produces float output invalid for an Integer column.""" + if not isinstance(proposer, self._CONTINUOUS_FLOAT_PROPOSER_TYPES): + return False + if len(columns) != 1: + return False + return isinstance(get_column_type(columns[0]), Integer) + + def _get_proposer_proposals( + self, include_all: bool | None = None + ) -> list[Proposer]: + """ + Get a list of acceptable proposers, sorted by decreasing fit to the actual data. + + By default (``include_all=None``), type-incompatible proposers (see + ``_is_integer_incompatible``) are filtered out - this is what + 'propose' shows - and the cached list from the most recent + 'propose'/'propose all' is reused if still valid, so that 'set + ' numbering matches whichever list was last displayed. Pass + an explicit ``True``/``False`` (from 'propose'/'propose all' + themselves) to force that mode, refetching if it differs from what's + cached. + """ + if include_all is None: + include_all = self.proposers_include_all + if ( + not self._proposers_valid() + or self.proposers is None + or include_all != self.proposers_include_all + ): columns = self._column_metadata() props = everything_factory(self.config, self.metadata).get_proposers( columns, self.sync_engine ) + if not include_all: + props = [ + p for p in props if not self._is_integer_incompatible(p, columns) + ] sorted_props = sorted(props, key=lambda g: g.fit(9999)) self.proposers = sorted_props + self.proposers_include_all = include_all self.proposers_valid_columns = ( self.table_index, self._get_column_names().copy(), @@ -978,23 +1047,26 @@ def _get_column_data( result = connection.execute(stmt) return [[to_str(x) for x in xs] for xs in result.all()] - def do_propose(self, _arg: str) -> None: + def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-statements + self, _arg: str + ) -> None: """ Display a list of possible generators for this column. They will be listed in order of fit, the most likely matches first. The results can be compared (against a sample of the real data in the column and against each other) with the 'compare' command. + + By default, generators whose output type is incompatible with the + column (e.g. a continuous-float generator for an Integer column) + are left out. Run 'propose all' to include them anyway. """ theme = get_active_theme() limit = 5 + include_all = _arg.strip().lower() == "all" props = self._get_proposer_proposals() sample = self._get_column_data(limit) - if sample: - rep = [x[0] if len(x) == 1 else ",".join(x) for x in sample] - self.print(self.PROPOSE_SOURCE_SAMPLE_TEXT, "; ".join(rep), theme.data) - else: - self.print(self.PROPOSE_SOURCE_EMPTY_TEXT) + self._print_source_sample(sample, theme) if not props: self.print(self.PROPOSE_NOTHING) for index, prop in enumerate(props): @@ -1017,6 +1089,186 @@ def do_propose(self, _arg: str) -> None: theme_reset=theme.reset, ) + column_evaluator = ColumnEvaluator() + columns = self._column_metadata() + results = [] + column_evaluator.setup(columns, self.sync_engine) + for index, proposer in enumerate(props): + result = column_evaluator.evaluate(proposer) + results.append(result) + + self.print("\n***\n") + # self.print("\nProposal evaluation:\n", results) + + profile = getattr(column_evaluator, "profile", None) + column_name = ( + column_evaluator.column.name + if column_evaluator.column is not None + else None + ) + real_uniqueness = getattr(column_evaluator, "real_uniqueness", 0.0) + is_numeric_column = getattr(column_evaluator, "column_is_numeric", False) + is_primary_key = getattr(column_evaluator, "is_primary_key", False) + ranking = rank_proposals( + results, + profile, + theme, + column_name=column_name, + real_uniqueness=real_uniqueness, + is_numeric_column=is_numeric_column, + is_primary_key=is_primary_key, + ) + display = format_ranking_display(ranking, results, profile) + + if display.recommendation is not None: + if ( + display.no_uniqueness_guarantee + or display.weak_recommendation_warning is not None + ): + # Don't highlight a fallback or non-existent pick - coloring + # it the same way as a solid recommendation would visually + # contradict the caveat that follows (or, for + # no_uniqueness_guarantee, claim confidence in a pick that + # doesn't exist at all). + self.print(display.recommendation) + else: + self.print(f"{theme.column}{display.recommendation}{theme.reset}") + if display.weak_recommendation_warning is not None: + self.print(display.weak_recommendation_warning) + if include_all and ranking.recommended_index is not None: + # 'propose all' deliberately bypasses the type-compatibility filter, + # so a generator whose output type doesn't match the column (e.g. a + # continuous-float generator for an Integer column) can still win - + # the statistical score has no way to detect that mismatch at all, + # which is exactly why the default 'propose' excludes it structurally + # instead of trying to penalize it. + recommended_proposer = results[ranking.recommended_index].proposer + if self._is_integer_incompatible(recommended_proposer, columns): + self.print( + "Warning: {0} produces output whose type doesn't match this" + " column - it's excluded from the default 'propose' list for" + " exactly this reason (run 'propose' without 'all'). The" + " statistics can't detect this mismatch, so a high score here" + " doesn't mean it's safe to use.", + recommended_proposer.name(), + ) + + self.print(display.profile_summary) + + indices_to_show = self._select_indices_to_show(display, ranking, include_all) + rows_to_show = [display.rows[i] for i in indices_to_show] + + self.print_table( + [ + "#", + "Proposer", + "Profile", + "Front", + "Score", + "Keyword", + "Fidelity", + "Novelty", + "Diversity", + "Uniqueness", + "Copies", + "Penalty", + ], + rows_to_show, + ) + + self.print("\n") + self._print_source_sample(sample, theme) + for i in indices_to_show: + prop = props[i] + self.print( + self.RANKED_SAMPLE_TEXT, + index=i + 1, + name=prop.name(), + sample="; ".join(map(repr, prop.generate_data(limit))), + theme_func=theme.function, + theme_data=theme.data, + theme_reset=theme.reset, + ) + + print("\n\n") + + def _print_source_sample(self, sample: list[list[str]], theme: Theme) -> None: + """Print a sample of the real column values, or a note that there are none.""" + if sample: + rep = [x[0] if len(x) == 1 else ",".join(x) for x in sample] + self.print(self.PROPOSE_SOURCE_SAMPLE_TEXT, "; ".join(rep), theme.data) + else: + self.print(self.PROPOSE_SOURCE_EMPTY_TEXT) + + def _select_indices_to_show( + self, + display: ProposalRankingDisplay, + ranking: ProposalRanking, + include_all: bool, + ) -> list[int]: + """Pick which candidate rows to show in the propose table. + + By default, only show Pareto front 1: a front-2+ candidate is, by + definition, strictly worse than some front-1 one on fidelity, + novelty AND diversity simultaneously, so it adds nothing to a + shortlist. 'propose all' bypasses this (and the type-compatibility + filter) to show every candidate. The returned indices let the + sample listing after the table map each shown row back to its + proposer without parsing the (possibly theme-colored) '#' cell text. + """ + indices_to_show = list(range(len(display.rows))) + if include_all: + return indices_to_show + + indices_to_show = [i for i in indices_to_show if display.fronts[i] == 1] + # The recommendation is picked by combined score across *all* + # candidates, not just front 1 (see rank_proposals) - a + # dominated candidate can still win once the resample penalty + # and keyword boost are applied, since those are computed after + # fronts are derived from the raw fidelity/novelty/diversity + # values. Force it into the table even when front-2+, otherwise + # "Recommended: N. x" can point at a row the user never sees. + recommended_outside_front_1 = ( + ranking.recommended_index is not None + and ranking.recommended_index not in indices_to_show + ) + if recommended_outside_front_1: + indices_to_show.append(ranking.recommended_index) + capped = len(indices_to_show) > self.MAX_PROPOSERS_SHOWN + if capped: + # Keep the highest-scoring candidates - front 1 alone can + # still hold more rows than are worth scanning in a + # terminal table (nothing dominates them, but they're not + # all equally worth showing). Always keep the recommendation + # even if its score wouldn't otherwise make the cut, since + # it's referenced by name right above this table. + ranked = sorted( + indices_to_show, key=lambda i: ranking.scores[i], reverse=True + ) + keep = set(ranked[: self.MAX_PROPOSERS_SHOWN]) + if ranking.recommended_index is not None: + keep.add(ranking.recommended_index) + indices_to_show = sorted(keep) + else: + indices_to_show.sort() + if len(indices_to_show) < len(display.rows): + if capped: + qualifier = ( + f" (Pareto front 1, top {self.MAX_PROPOSERS_SHOWN} by score," + " plus the recommendation)" + ) + elif recommended_outside_front_1: + qualifier = " (Pareto front 1, plus the recommendation)" + else: + qualifier = " (Pareto front 1 only)" + self.print( + "Showing {0} of {1} candidates" + qualifier + "." + " Run 'propose all' to see the rest.", + len(indices_to_show), + len(display.rows), + ) + return indices_to_show + def do_p(self, arg: str) -> None: """Synonym for propose.""" self.do_propose(arg) From c9b8eb12ef81fdc24f670fcd197a7f9367a8778a Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Wed, 9 Sep 2026 17:08:00 +0100 Subject: [PATCH 02/12] Add unit tests for column evaluators --- tests/test_evaluators_column_evaluator.py | 233 +++++++++ .../test_evaluators_distribution_builders.py | 200 ++++++++ tests/test_evaluators_feature_extractors.py | 458 ++++++++++++++++++ tests/test_evaluators_metrics.py | 163 +++++++ tests/test_evaluators_proposal_ranking.py | 397 +++++++++++++++ tests/test_evaluators_statistical_fidelity.py | 108 +++++ 6 files changed, 1559 insertions(+) create mode 100644 tests/test_evaluators_column_evaluator.py create mode 100644 tests/test_evaluators_distribution_builders.py create mode 100644 tests/test_evaluators_feature_extractors.py create mode 100644 tests/test_evaluators_metrics.py create mode 100644 tests/test_evaluators_proposal_ranking.py create mode 100644 tests/test_evaluators_statistical_fidelity.py diff --git a/tests/test_evaluators_column_evaluator.py b/tests/test_evaluators_column_evaluator.py new file mode 100644 index 00000000..61e8cabb --- /dev/null +++ b/tests/test_evaluators_column_evaluator.py @@ -0,0 +1,233 @@ +"""Unit tests for datafaker.evaluators.column_evaluator.""" +from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine + +from datafaker.evaluators.column_evaluator import ( + ColumnEvaluator, + ColumnStats, + EvaluationProfile, + ProposalEvaluation, + analyse_column, + choose_numeric_profile, + choose_profile, + looks_like_email, +) +from datafaker.proposers.base import ConstantProposer +from tests.utils import DatafakerTestCase + + +class AnalyseColumnTests(DatafakerTestCase): + """Test case for analyse_column.""" + + def test_counts_rows_and_unique_values(self) -> None: + """row_count and unique_count reflect the (non-None) input values.""" + stats = analyse_column(["a", "b", "a"]) + self.assertEqual(3, stats.row_count) + self.assertEqual(2, stats.unique_count) + + def test_none_values_are_excluded(self) -> None: + """None entries don't count towards row_count or unique_count.""" + stats = analyse_column(["a", None, None]) + self.assertEqual(1, stats.row_count) + self.assertEqual(1, stats.unique_count) + + def test_avg_length_and_ratios(self) -> None: + """avg_length, space_ratio, digit_ratio and punctuation_ratio are computed.""" + stats = analyse_column(["ab 1!"]) + self.assertEqual(5, stats.avg_length) + self.assertAlmostEqual(1 / 5, stats.space_ratio) + self.assertAlmostEqual(1 / 5, stats.digit_ratio) + self.assertAlmostEqual(1 / 5, stats.punctuation_ratio) + + def test_empty_input_does_not_divide_by_zero(self) -> None: + """An empty (or all-None) column yields all-zero stats, not an error.""" + stats = analyse_column([None, None]) + self.assertEqual(0, stats.row_count) + self.assertEqual(0, stats.avg_length) + self.assertEqual(0, stats.space_ratio) + self.assertEqual(0, stats.digit_ratio) + self.assertEqual(0, stats.punctuation_ratio) + + def test_non_string_values_are_stringified(self) -> None: + """Non-string values (e.g. ints) are converted to str first.""" + stats = analyse_column([12345]) + self.assertEqual(5, stats.avg_length) + self.assertEqual(1.0, stats.digit_ratio) + + +class ColumnStatsTests(DatafakerTestCase): + """Test case for the ColumnStats dataclass.""" + + def test_uniqueness_is_unique_over_row_count(self) -> None: + """uniqueness divides unique_count by row_count.""" + stats = ColumnStats( + row_count=4, + unique_count=2, + avg_length=0, + space_ratio=0, + digit_ratio=0, + punctuation_ratio=0, + ) + self.assertEqual(0.5, stats.uniqueness) + + def test_uniqueness_with_zero_rows_does_not_divide_by_zero(self) -> None: + """A zero row_count is floored to 1 to avoid a ZeroDivisionError.""" + stats = ColumnStats( + row_count=0, + unique_count=0, + avg_length=0, + space_ratio=0, + digit_ratio=0, + punctuation_ratio=0, + ) + self.assertEqual(0.0, stats.uniqueness) + + +class ChooseProfileTests(DatafakerTestCase): + """Test case for choose_profile.""" + + def _stats( + self, avg_length=10.0, uniqueness_row=(8, 10), space_ratio=0.0 + ) -> ColumnStats: + unique_count, row_count = uniqueness_row + return ColumnStats( + row_count=row_count, + unique_count=unique_count, + avg_length=avg_length, + space_ratio=space_ratio, + digit_ratio=0.0, + punctuation_ratio=0.0, + ) + + def test_short_and_mostly_unique_is_short_text(self) -> None: + """Short, mostly-distinct values look like names/identifiers.""" + stats = self._stats(avg_length=10, uniqueness_row=(9, 10)) + self.assertEqual(EvaluationProfile.SHORT_TEXT, choose_profile(stats)) + + def test_low_uniqueness_is_categorical(self) -> None: + """Values repeated often enough look like a small category set.""" + stats = self._stats(avg_length=10, uniqueness_row=(1, 100)) + self.assertEqual(EvaluationProfile.CATEGORICAL, choose_profile(stats)) + + def test_long_average_length_is_free_text(self) -> None: + """Long values that aren't near-unique look like free text.""" + stats = self._stats(avg_length=60, uniqueness_row=(5, 10)) + self.assertEqual(EvaluationProfile.FREE_TEXT, choose_profile(stats)) + + def test_high_space_ratio_is_free_text(self) -> None: + """Many spaces per character looks like prose even if average length is short.""" + stats = self._stats(avg_length=10, uniqueness_row=(5, 10), space_ratio=0.2) + self.assertEqual(EvaluationProfile.FREE_TEXT, choose_profile(stats)) + + def test_fallback_is_short_text(self) -> None: + """Anything not matching the other rules falls back to SHORT_TEXT.""" + stats = self._stats(avg_length=35, uniqueness_row=(5, 10), space_ratio=0.05) + self.assertEqual(EvaluationProfile.SHORT_TEXT, choose_profile(stats)) + + +class ChooseNumericProfileTests(DatafakerTestCase): + """Test case for choose_numeric_profile.""" + + def test_low_cardinality_is_categorical(self) -> None: + """Few distinct values among many rows looks like a status/flag column.""" + values = [1, 2] * 10 # 2 distinct out of 20: uniqueness 0.1 + self.assertEqual(EvaluationProfile.CATEGORICAL, choose_numeric_profile(values)) + + def test_high_cardinality_is_identifier(self) -> None: + """Mostly-distinct numeric values look like an id/age/salary column.""" + values = list(range(100)) + self.assertEqual(EvaluationProfile.IDENTIFIER, choose_numeric_profile(values)) + + def test_all_none_defaults_to_identifier(self) -> None: + """An empty/all-None column can't be judged, so defaults to IDENTIFIER.""" + self.assertEqual( + EvaluationProfile.IDENTIFIER, choose_numeric_profile([None, None]) + ) + + def test_none_values_are_excluded_from_the_ratio(self) -> None: + """None entries don't count as either distinct or total values.""" + # 2 distinct values among 20 non-null entries: uniqueness 0.1. If the + # Nones were counted in the denominator without being excluded first, + # this would look even more categorical, not less - the point here is + # just that the Nones don't crash or skew the ratio unexpectedly. + values = [1] * 19 + [2] + [None] * 5 + self.assertEqual(EvaluationProfile.CATEGORICAL, choose_numeric_profile(values)) + + +class LooksLikeEmailTests(DatafakerTestCase): + """Test case for looks_like_email.""" + + def test_mostly_email_values_is_true(self) -> None: + """A column where most values look like emails is detected.""" + values = ["a@example.com", "b@example.com", "not-an-email"] + self.assertTrue(looks_like_email(values)) + + def test_mostly_non_email_values_is_false(self) -> None: + """A column where most values don't look like emails is rejected.""" + values = ["a@example.com", "plain text", "another plain value"] + self.assertFalse(looks_like_email(values)) + + def test_empty_input_is_false(self) -> None: + """No values at all can't look like emails.""" + self.assertFalse(looks_like_email([])) + self.assertFalse(looks_like_email([None, " "])) + + def test_value_without_local_part_or_domain_is_not_email_like(self) -> None: + """'@' alone, or with an empty side, doesn't count as email-like.""" + values = ["@example.com", "a@", "@"] + self.assertFalse(looks_like_email(values)) + + +class ProposalEvaluationTests(DatafakerTestCase): + """Test case for the ProposalEvaluation dataclass's display string.""" + + def test_str_includes_key_metrics_and_pipeline_scores(self) -> None: + """The string form surfaces the headline scores and per-pipeline detail.""" + evaluation = ProposalEvaluation( + proposer=ConstantProposer("x"), + novelty=0.5, + diversity=0.25, + overall_score=0.125, + pipeline_scores={"length": 0.1, "words": 0.2}, + copy_fraction=0.0, + synthetic_uniqueness=1.0, + ) + text = str(evaluation) + self.assert_str_in("dist_gen.constant", text) + self.assert_str_in("0.125000", text) + self.assert_str_in("0.500000", text) + self.assert_str_in("0.250000", text) + self.assert_str_in("length", text) + self.assert_str_in("words", text) + + +class ColumnEvaluatorIntegrationTests(DatafakerTestCase): + """End-to-end test of ColumnEvaluator against a real (DuckDB) table.""" + + def test_evaluate_a_constant_proposer_against_a_categorical_column(self) -> None: + """setup() profiles the column and evaluate() scores a real proposer.""" + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table( + "statuses", + metadata, + Column("id", Integer, primary_key=True), + Column("status", String), + ) + metadata.create_all(engine) + with engine.begin() as conn: + conn.execute( + table.insert(), + [{"id": i, "status": ["active", "inactive"][i % 2]} for i in range(20)], + ) + + evaluator = ColumnEvaluator() + evaluator.setup([table.c.status], engine) + self.assertEqual(EvaluationProfile.CATEGORICAL, evaluator.profile) + + result = evaluator.evaluate(ConstantProposer("active")) + self.assertIsInstance(result, ProposalEvaluation) + self.assertEqual(1.0, result.copy_fraction) + # A constant proposer emits the same value every time: only one + # distinct value out of the whole synthetic sample. + self.assertLess(result.synthetic_uniqueness, 0.01) + self.assertIn("category", result.pipeline_scores) diff --git a/tests/test_evaluators_distribution_builders.py b/tests/test_evaluators_distribution_builders.py new file mode 100644 index 00000000..e3a79e27 --- /dev/null +++ b/tests/test_evaluators_distribution_builders.py @@ -0,0 +1,200 @@ +"""Unit tests for datafaker.evaluators.distribution_builders.""" +from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine + +from datafaker.evaluators.distribution_builders import ( + CategoryBuilder, + Distribution, + HistogramBuilder, +) +from datafaker.evaluators.feature_extractors import ( + FirstLetterExtractor, + LengthExtractor, +) +from tests.utils import DatafakerTestCase + + +class DistributionTests(DatafakerTestCase): + """Test case for the Distribution dataclass.""" + + def test_vocabulary_is_the_set_of_keys(self) -> None: + """vocabulary exposes the keys of the probability mapping.""" + dist = Distribution({"a": 0.5, "b": 0.5}) + self.assertEqual({"a", "b"}, dist.vocabulary) + + def test_probability_of_known_key(self) -> None: + """probability() returns the stored value for a known key.""" + dist = Distribution({"a": 0.75}) + self.assertEqual(0.75, dist.probability("a")) + + def test_probability_of_unknown_key_is_zero(self) -> None: + """probability() returns 0.0 for a key that isn't in the distribution.""" + dist = Distribution({"a": 0.75}) + self.assertEqual(0.0, dist.probability("missing")) + + def test_as_vector_orders_by_given_vocabulary(self) -> None: + """as_vector() maps a vocabulary list to matching probabilities, in order.""" + dist = Distribution({"a": 0.2, "b": 0.8}) + self.assertEqual([0.8, 0.2, 0.0], dist.as_vector(["b", "a", "c"])) + + +def _make_duckdb_table(rows: list[dict]) -> tuple: + """Create an in-memory DuckDB engine with one populated table.""" + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table( + "people", + metadata, + Column("age", Integer), + Column("name", String), + ) + metadata.create_all(engine) + with engine.begin() as conn: + conn.execute(table.insert(), rows) + return engine, table + + +class HistogramBuilderBuildFromTableTests(DatafakerTestCase): + """Test case for HistogramBuilder.build_from_table.""" + + def test_builds_a_distribution_that_sums_to_one(self) -> None: + """The resulting histogram is a valid probability distribution.""" + engine, table = _make_duckdb_table( + [{"age": 20 + (i % 10), "name": "x"} for i in range(100)] + ) + builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertIsNotNone(builder.mean) + self.assertIsNotNone(builder.stddev) + + def test_empty_table_yields_empty_distribution(self) -> None: + """An empty table produces an empty distribution rather than an error.""" + engine, table = _make_duckdb_table([]) + builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + dist = builder.build_from_table() + self.assertEqual({}, dist.probabilities) + + def test_constant_column_does_not_crash_on_zero_stddev(self) -> None: + """A column with a single repeated value has zero stddev but still works.""" + engine, table = _make_duckdb_table( + [{"age": 42, "name": "x"} for _ in range(10)] + ) + builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertEqual(0.0, builder.stddev) + + def test_uses_feature_extractor_expression(self) -> None: + """The configured extractor's SQL expression is used, not the raw column.""" + engine, table = _make_duckdb_table( + [{"age": 0, "name": name} for name in ["ab", "cde", "fghi"] * 5] + ) + builder = HistogramBuilder( + engine, table, table.c.name, extractor=LengthExtractor(), sample_size=1000 + ) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + + +class HistogramBuilderBuildFromValuesTests(DatafakerTestCase): + """Test case for HistogramBuilder.build_from_values.""" + + def setUp(self) -> None: + super().setUp() + engine, table = _make_duckdb_table( + [{"age": 20 + (i % 10), "name": "x"} for i in range(100)] + ) + self.builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + # Learn edges from the real data first, as calculate_scores() does. + self.builder.build_from_table() + + def test_result_sums_to_one(self) -> None: + """Bucketed synthetic values form a valid probability distribution.""" + dist = self.builder.build_from_values([20, 21, 22, 23, 24, 25]) + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + + def test_empty_values_yield_empty_distribution(self) -> None: + """No synthetic values produces an empty distribution.""" + dist = self.builder.build_from_values([]) + self.assertEqual({}, dist.probabilities) + + def test_values_outside_edges_are_clamped_to_end_buckets(self) -> None: + """Values far below/above the learned range land in the outer buckets.""" + dist = self.builder.build_from_values([-1000, 1000]) + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + buckets = set(dist.probabilities.keys()) + self.assertTrue(all(0 <= b <= len(self.builder.edges) - 2 for b in buckets)) + + +class CategoryBuilderBuildFromTableTests(DatafakerTestCase): + """Test case for CategoryBuilder.build_from_table.""" + + def test_builds_a_distribution_that_sums_to_one(self) -> None: + """The resulting category histogram sums to (approximately) 1.""" + engine, table = _make_duckdb_table( + [{"age": 0, "name": n} for n in ["alice", "bob", "carol"] * 10] + ) + builder = CategoryBuilder(engine, table, table.c.name, sample_size=1000) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertEqual({"alice", "bob", "carol"}, dist.vocabulary) + + def test_empty_table_yields_empty_distribution(self) -> None: + """An empty table produces an empty distribution.""" + engine, table = _make_duckdb_table([]) + builder = CategoryBuilder(engine, table, table.c.name, sample_size=1000) + dist = builder.build_from_table() + self.assertEqual({}, dist.probabilities) + + def test_uses_feature_extractor(self) -> None: + """The category counts are keyed by extracted features, not raw values.""" + engine, table = _make_duckdb_table( + [{"age": 0, "name": n} for n in ["Alice", "Anna", "Bob"]] + ) + builder = CategoryBuilder( + engine, + table, + table.c.name, + extractor=FirstLetterExtractor(), + sample_size=1000, + ) + dist = builder.build_from_table() + self.assertEqual({"a", "b"}, dist.vocabulary) + self.assertAlmostEqual(2 / 3, dist.probability("a")) + self.assertAlmostEqual(1 / 3, dist.probability("b")) + + +class CategoryBuilderBuildFromValuesTests(DatafakerTestCase): + """Test case for CategoryBuilder.build_from_values.""" + + def setUp(self) -> None: + super().setUp() + self.builder = CategoryBuilder( + engine=None, table=None, column=None, sample_size=1000 + ) + + def test_builds_a_distribution_that_sums_to_one(self) -> None: + """Category counts from a plain Python list sum to one.""" + dist = self.builder.build_from_values(["a", "a", "b"]) + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertAlmostEqual(2 / 3, dist.probability("a")) + self.assertAlmostEqual(1 / 3, dist.probability("b")) + + def test_none_values_are_ignored(self) -> None: + """None entries don't contribute counts.""" + dist = self.builder.build_from_values(["a", None, "a"]) + self.assertEqual({"a": 1.0}, dist.probabilities) + + def test_no_extracted_features_yields_sentinel_missing_distribution(self) -> None: + """An empty result set is represented with an explicit __MISSING__ token.""" + dist = self.builder.build_from_values([None, None]) + self.assertEqual({"__MISSING__": 1.0}, dist.probabilities) + + def test_uses_feature_extractor(self) -> None: + """Values are converted through the configured extractor before counting.""" + builder = CategoryBuilder( + engine=None, table=None, column=None, extractor=LengthExtractor() + ) + dist = builder.build_from_values(["ab", "cd", "efg"]) + self.assertAlmostEqual(2 / 3, dist.probability(2)) + self.assertAlmostEqual(1 / 3, dist.probability(3)) diff --git a/tests/test_evaluators_feature_extractors.py b/tests/test_evaluators_feature_extractors.py new file mode 100644 index 00000000..014b9fac --- /dev/null +++ b/tests/test_evaluators_feature_extractors.py @@ -0,0 +1,458 @@ +"""Unit tests for datafaker.evaluators.feature_extractors.""" +from datetime import date, datetime, timezone + +from datafaker.evaluators.feature_extractors import ( + CharacterBigramExtractor, + CharacterExtractor, + CharacterTrigramExtractor, + EmailDomainExtractor, + EmailLocalPartExtractor, + EmailTopLevelDomainExtractor, + EmailValidityExtractor, + FirstLetterExtractor, + IdentityExtractor, + LastLetterExtractor, + LengthExtractor, + PrefixExtractor, + SentenceCountExtractor, + SuffixExtractor, + TimestampExtractor, + VowelConsonantPatternExtractor, + WeekdayExtractor, + WordCountExtractor, + WordExtractor, +) +from tests.utils import DatafakerTestCase + + +class IdentityExtractorTests(DatafakerTestCase): + """Test case for IdentityExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = IdentityExtractor() + + def test_yields_the_value_unchanged(self) -> None: + """Non-None values pass through untouched.""" + self.assertEqual([42], list(self.extractor.extract(42))) + self.assertEqual(["hello"], list(self.extractor.extract("hello"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_expression_returns_the_column_itself(self) -> None: + """The SQL expression is just the column, unmodified.""" + column = object() + self.assertIs(column, self.extractor.expression(column)) + + +class FirstLetterExtractorTests(DatafakerTestCase): + """Test case for FirstLetterExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = FirstLetterExtractor() + + def test_extracts_lowercased_first_letter(self) -> None: + """The first character is extracted and lowercased.""" + self.assertEqual(["a"], list(self.extractor.extract("Alice"))) + + def test_strips_surrounding_whitespace_first(self) -> None: + """Leading whitespace does not become the 'first letter'.""" + self.assertEqual(["a"], list(self.extractor.extract(" alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_empty_string_yields_nothing(self) -> None: + """A blank/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + def test_expression_does_not_raise(self) -> None: + """A SQL expression is implemented for this extractor.""" + self.extractor.expression("col") + + +class LastLetterExtractorTests(DatafakerTestCase): + """Test case for LastLetterExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = LastLetterExtractor() + + def test_extracts_lowercased_last_letter(self) -> None: + """The last character is extracted and lowercased.""" + self.assertEqual(["e"], list(self.extractor.extract("Alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_empty_string_yields_nothing(self) -> None: + """A blank/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + +class LengthExtractorTests(DatafakerTestCase): + """Test case for LengthExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = LengthExtractor() + + def test_extracts_string_length(self) -> None: + """The length of str(value) is extracted.""" + self.assertEqual([5], list(self.extractor.extract("Alice"))) + + def test_extracts_length_of_non_string_values(self) -> None: + """Non-string values are stringified before measuring length.""" + self.assertEqual([len(str(12345))], list(self.extractor.extract(12345))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + +class WordExtractorTests(DatafakerTestCase): + """Test case for WordExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = WordExtractor() + + def test_tokenizes_and_lowercases_words(self) -> None: + """Words are split on non-word characters and lowercased.""" + self.assertEqual( + ["the", "quick", "fox"], + list(self.extractor.extract("The quick, fox!")), + ) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_no_sql_expression_is_implemented(self) -> None: + """Word extraction has no SQL equivalent and must raise.""" + with self.assertRaises(NotImplementedError): + self.extractor.expression("col") + + +class CharacterExtractorTests(DatafakerTestCase): + """Test case for CharacterExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = CharacterExtractor() + + def test_extracts_each_character(self) -> None: + """Each character of the value is yielded separately.""" + self.assertEqual(["a", "b", "c"], list(self.extractor.extract("abc"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_no_sql_expression_is_implemented(self) -> None: + """Per-character extraction has no SQL equivalent and must raise.""" + with self.assertRaises(NotImplementedError): + self.extractor.expression("col") + + +class CharacterBigramExtractorTests(DatafakerTestCase): + """Test case for CharacterBigramExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = CharacterBigramExtractor() + + def test_extracts_overlapping_bigrams(self) -> None: + """Adjacent character pairs are extracted, lowercased.""" + self.assertEqual(["ab", "bc"], list(self.extractor.extract("ABc"))) + + def test_short_strings_yield_no_bigrams(self) -> None: + """A string shorter than 2 characters has no bigrams.""" + self.assertEqual([], list(self.extractor.extract("a"))) + self.assertEqual([], list(self.extractor.extract(""))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + +class CharacterTrigramExtractorTests(DatafakerTestCase): + """Test case for CharacterTrigramExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = CharacterTrigramExtractor() + + def test_extracts_overlapping_trigrams(self) -> None: + """Adjacent character triples are extracted, lowercased.""" + self.assertEqual(["abc", "bcd"], list(self.extractor.extract("ABCd"))) + + def test_short_strings_yield_no_trigrams(self) -> None: + """A string shorter than 3 characters has no trigrams.""" + self.assertEqual([], list(self.extractor.extract("ab"))) + + +class PrefixExtractorTests(DatafakerTestCase): + """Test case for PrefixExtractor.""" + + def test_default_length_is_two(self) -> None: + """With no explicit length, a 2-character prefix is extracted.""" + extractor = PrefixExtractor() + self.assertEqual(["al"], list(extractor.extract("Alice"))) + + def test_custom_length(self) -> None: + """A custom prefix length is honored.""" + extractor = PrefixExtractor(length=4) + self.assertEqual(["alic"], list(extractor.extract("Alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(PrefixExtractor().extract(None))) + + def test_empty_string_yields_nothing(self) -> None: + """A blank/whitespace-only string produces no features.""" + self.assertEqual([], list(PrefixExtractor().extract(" "))) + + +class SuffixExtractorTests(DatafakerTestCase): + """Test case for SuffixExtractor.""" + + def test_default_length_is_two(self) -> None: + """With no explicit length, a 2-character suffix is extracted.""" + extractor = SuffixExtractor() + self.assertEqual(["ce"], list(extractor.extract("Alice"))) + + def test_custom_length(self) -> None: + """A custom suffix length is honored.""" + extractor = SuffixExtractor(length=3) + self.assertEqual(["ice"], list(extractor.extract("Alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(SuffixExtractor().extract(None))) + + +class VowelConsonantPatternExtractorTests(DatafakerTestCase): + """Test case for VowelConsonantPatternExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = VowelConsonantPatternExtractor() + + def test_extracts_vowel_consonant_pattern(self) -> None: + """Each alphabetic character is classified as V or C.""" + self.assertEqual(["VCVCV"], list(self.extractor.extract("Alice"))) + + def test_non_alphabetic_characters_are_skipped(self) -> None: + """Digits and punctuation don't appear in the pattern.""" + self.assertEqual(["VCC"], list(self.extractor.extract("a1-b2c!"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_all_non_alphabetic_yields_nothing(self) -> None: + """A value with no letters at all produces no pattern.""" + self.assertEqual([], list(self.extractor.extract("123!!"))) + + +class TimestampExtractorTests(DatafakerTestCase): + """Test case for TimestampExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = TimestampExtractor() + + def test_epoch_datetime_extracts_zero(self) -> None: + """The Unix epoch itself extracts to day 0.""" + self.assertEqual([0.0], list(self.extractor.extract(datetime(1970, 1, 1)))) + + def test_extracts_days_since_epoch(self) -> None: + """A later date extracts to the correct day offset.""" + self.assertEqual([1.0], list(self.extractor.extract(datetime(1970, 1, 2)))) + + def test_accepts_date_objects(self) -> None: + """A plain date (no time component) is also supported.""" + self.assertEqual([1.0], list(self.extractor.extract(date(1970, 1, 2)))) + + def test_accepts_iso_format_strings(self) -> None: + """A string that came from a query losing type info is coerced.""" + self.assertEqual([1.0], list(self.extractor.extract("1970-01-02"))) + + def test_invalid_string_yields_nothing(self) -> None: + """A string that isn't a valid date produces no feature.""" + self.assertEqual([], list(self.extractor.extract("not-a-date"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_non_date_non_string_value_yields_nothing(self) -> None: + """A value with no date meaning at all (e.g. an int) is dropped.""" + self.assertEqual([], list(self.extractor.extract(42))) + + def test_timezone_aware_datetime_is_normalized(self) -> None: + """Timezone info is stripped rather than causing a crash or a shift.""" + aware = datetime(1970, 1, 2, tzinfo=timezone.utc) + self.assertEqual([1.0], list(self.extractor.extract(aware))) + + +class WeekdayExtractorTests(DatafakerTestCase): + """Test case for WeekdayExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = WeekdayExtractor() + + def test_extracts_iso_weekday_index(self) -> None: + """1970-01-01 was a Thursday: weekday() == 3.""" + self.assertEqual([3], list(self.extractor.extract(datetime(1970, 1, 1)))) + + def test_accepts_iso_format_strings(self) -> None: + """String dates are coerced the same way as TimestampExtractor.""" + self.assertEqual([3], list(self.extractor.extract("1970-01-01"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_invalid_string_yields_nothing(self) -> None: + """An unparseable string produces no feature.""" + self.assertEqual([], list(self.extractor.extract("nope"))) + + +class EmailLocalPartExtractorTests(DatafakerTestCase): + """Test case for EmailLocalPartExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailLocalPartExtractor() + + def test_extracts_lowercased_local_part(self) -> None: + """The portion before '@' is extracted and lowercased.""" + self.assertEqual( + ["alice.smith"], list(self.extractor.extract("Alice.Smith@Example.com")) + ) + + def test_value_without_at_sign_yields_nothing(self) -> None: + """A non-email string has no local part.""" + self.assertEqual([], list(self.extractor.extract("not-an-email"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + +class EmailDomainExtractorTests(DatafakerTestCase): + """Test case for EmailDomainExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailDomainExtractor() + + def test_extracts_lowercased_domain(self) -> None: + """The portion after '@' is extracted and lowercased.""" + self.assertEqual( + ["example.com"], list(self.extractor.extract("alice@Example.COM")) + ) + + def test_value_without_at_sign_yields_nothing(self) -> None: + """A non-email string has no domain.""" + self.assertEqual([], list(self.extractor.extract("not-an-email"))) + + +class EmailTopLevelDomainExtractorTests(DatafakerTestCase): + """Test case for EmailTopLevelDomainExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailTopLevelDomainExtractor() + + def test_extracts_lowercased_tld(self) -> None: + """The final domain segment is extracted and lowercased.""" + self.assertEqual( + ["com"], list(self.extractor.extract("alice@mail.example.COM")) + ) + + def test_domain_without_dot_yields_nothing(self) -> None: + """A domain with no TLD separator produces no feature.""" + self.assertEqual([], list(self.extractor.extract("alice@localhost"))) + + def test_value_without_at_sign_yields_nothing(self) -> None: + """A non-email string has no TLD.""" + self.assertEqual([], list(self.extractor.extract("not-an-email"))) + + +class EmailValidityExtractorTests(DatafakerTestCase): + """Test case for EmailValidityExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailValidityExtractor() + + def test_well_formed_email_is_valid(self) -> None: + """A syntactically valid email is classified 'valid'.""" + self.assertEqual(["valid"], list(self.extractor.extract("alice@example.com"))) + + def test_malformed_value_is_invalid(self) -> None: + """A string with no '@' or domain is classified 'invalid'.""" + self.assertEqual(["invalid"], list(self.extractor.extract("not-an-email"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_blank_string_yields_nothing(self) -> None: + """An empty/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + +class WordCountExtractorTests(DatafakerTestCase): + """Test case for WordCountExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = WordCountExtractor() + + def test_counts_words(self) -> None: + """Word count matches the number of \\w+ tokens.""" + self.assertEqual([4], list(self.extractor.extract("The quick brown fox."))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_blank_string_yields_nothing(self) -> None: + """An empty/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + +class SentenceCountExtractorTests(DatafakerTestCase): + """Test case for SentenceCountExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = SentenceCountExtractor() + + def test_counts_sentences_by_terminal_punctuation(self) -> None: + """Sentences are split on ./!/? and counted if they contain a word.""" + self.assertEqual( + [3], list(self.extractor.extract("Hi there! How are you? I am fine.")) + ) + + def test_trailing_punctuation_without_words_is_not_a_sentence(self) -> None: + """A run of terminal punctuation with no word content doesn't count.""" + self.assertEqual([1], list(self.extractor.extract("Hello there..."))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_blank_string_yields_nothing(self) -> None: + """An empty/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) diff --git a/tests/test_evaluators_metrics.py b/tests/test_evaluators_metrics.py new file mode 100644 index 00000000..bf4610b8 --- /dev/null +++ b/tests/test_evaluators_metrics.py @@ -0,0 +1,163 @@ +"""Unit tests for datafaker.evaluators.metrics.""" +import math + +from datafaker.evaluators.distribution_builders import Distribution +from datafaker.evaluators.metrics import ( + DiversityMetric, + JensenShannon, + MeanSquaredError, + NoveltyMetric, +) +from tests.utils import DatafakerTestCase + + +class MeanSquaredErrorTests(DatafakerTestCase): + """Test case for the MeanSquaredError metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = MeanSquaredError() + + def test_identical_distributions_score_zero(self) -> None: + """Two identical distributions should have zero error.""" + dist = Distribution({"a": 0.5, "b": 0.5}) + self.assertEqual(0.0, self.metric.compare(dist, dist)) + + def test_completely_disjoint_distributions(self) -> None: + """Disjoint single-value distributions score the maximum of 1.0.""" + real = Distribution({"a": 1.0}) + synthetic = Distribution({"b": 1.0}) + # squared error = (1-0)**2 + (0-1)**2 = 2, halved = 1.0 + self.assertAlmostEqual(1.0, self.metric.compare(real, synthetic)) + + def test_partial_overlap(self) -> None: + """Score reflects the squared difference in shared probability mass.""" + real = Distribution({"a": 0.6, "b": 0.4}) + synthetic = Distribution({"a": 0.4, "b": 0.6}) + expected = ((0.2) ** 2 + (0.2) ** 2) / 2 + self.assertAlmostEqual(expected, self.metric.compare(real, synthetic)) + + def test_empty_vocabulary_scores_zero(self) -> None: + """Two empty distributions have no vocabulary and score zero.""" + self.assertEqual(0.0, self.metric.compare(Distribution({}), Distribution({}))) + + def test_none_and_mixed_type_keys_are_canonicalized(self) -> None: + """None and non-string keys are normalized so they compare correctly.""" + real = Distribution({None: 1.0}) + synthetic = Distribution({"": 1.0}) + self.assertEqual(0.0, self.metric.compare(real, synthetic)) + + real_int_key = Distribution({3: 1.0}) + synthetic_str_key = Distribution({"3": 1.0}) + self.assertEqual(0.0, self.metric.compare(real_int_key, synthetic_str_key)) + + +class JensenShannonTests(DatafakerTestCase): + """Test case for the JensenShannon metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = JensenShannon() + + def test_identical_distributions_score_zero(self) -> None: + """Two identical distributions have zero divergence.""" + dist = Distribution({"a": 0.3, "b": 0.7}) + self.assertAlmostEqual(0.0, self.metric.compare(dist, dist)) + + def test_completely_disjoint_distributions_score_ln2(self) -> None: + """Disjoint distributions hit the theoretical maximum of ln(2).""" + real = Distribution({"a": 1.0}) + synthetic = Distribution({"b": 1.0}) + self.assertAlmostEqual(math.log(2), self.metric.compare(real, synthetic)) + + def test_partial_overlap_between_zero_and_max(self) -> None: + """A partially-overlapping pair scores strictly between 0 and ln(2).""" + real = Distribution({"a": 0.9, "b": 0.1}) + synthetic = Distribution({"a": 0.1, "b": 0.9}) + score = self.metric.compare(real, synthetic) + self.assertGreater(score, 0.0) + self.assertLess(score, math.log(2)) + + def test_empty_vocabulary_scores_zero(self) -> None: + """Two empty distributions score zero.""" + self.assertEqual(0.0, self.metric.compare(Distribution({}), Distribution({}))) + + def test_is_symmetric(self) -> None: + """Jensen-Shannon divergence is symmetric in its two arguments.""" + real = Distribution({"a": 0.2, "b": 0.8}) + synthetic = Distribution({"a": 0.7, "b": 0.3}) + # pylint: disable-next=arguments-out-of-order + reversed_score = self.metric.compare(synthetic, real) + self.assertAlmostEqual(self.metric.compare(real, synthetic), reversed_score) + + +class NoveltyMetricTests(DatafakerTestCase): + """Test case for the NoveltyMetric metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = NoveltyMetric() + + def test_no_overlap_is_fully_novel(self) -> None: + """Synthetic values sharing nothing with the real data score 1.0.""" + self.assertEqual(1.0, self.metric.compare(["a", "b"], ["c", "d"])) + + def test_full_overlap_is_not_novel(self) -> None: + """Synthetic values that are all copies of real values score 0.0.""" + self.assertEqual(0.0, self.metric.compare(["a", "b"], ["a", "b"])) + + def test_partial_overlap(self) -> None: + """Score is the fraction of distinct synthetic values not seen in real.""" + real = ["a", "b"] + synthetic = ["a", "c"] + self.assertAlmostEqual(0.5, self.metric.compare(real, synthetic)) + + def test_empty_synthetic_scores_zero(self) -> None: + """No synthetic values at all is treated as no novelty.""" + self.assertEqual(0.0, self.metric.compare(["a"], [])) + + def test_comparison_is_case_and_whitespace_insensitive(self) -> None: + """Values are compared after stripping whitespace and lowercasing.""" + self.assertEqual(0.0, self.metric.compare(["Alice"], [" alice "])) + + def test_none_values_are_ignored(self) -> None: + """None entries in either sequence are dropped before comparison.""" + self.assertEqual(0.0, self.metric.compare(["a", None], ["a"])) + + +class DiversityMetricTests(DatafakerTestCase): + """Test case for the DiversityMetric metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = DiversityMetric() + + def test_matching_diversity_scores_one(self) -> None: + """Equally-diverse real and synthetic samples score 1.0.""" + real = ["a", "b", "c", "d"] + synthetic = ["w", "x", "y", "z"] + self.assertAlmostEqual(1.0, self.metric.compare(real, synthetic)) + + def test_single_repeated_value_has_zero_entropy_on_both_sides(self) -> None: + """A single repeated value on both sides is a diversity match.""" + self.assertAlmostEqual(1.0, self.metric.compare(["a", "a", "a"], ["b", "b"])) + + def test_less_diverse_synthetic_scores_below_one(self) -> None: + """A synthetic sample far less diverse than the real one is penalized.""" + real = ["a", "b", "c", "d"] + synthetic = ["x", "x", "x", "x"] + score = self.metric.compare(real, synthetic) + self.assertLess(score, 1.0) + self.assertGreaterEqual(score, 0.0) + + def test_empty_values_treated_as_zero_entropy(self) -> None: + """Empty or all-None sequences have zero normalized entropy.""" + self.assertAlmostEqual(1.0, self.metric.compare([], [])) + self.assertAlmostEqual(1.0, self.metric.compare([None, None], [None])) + + def test_score_is_bounded_below_by_zero(self) -> None: + """The score never goes negative even for maximally different diversity.""" + real = ["a", "a", "a", "a"] + synthetic = ["a", "b", "c", "d"] + score = self.metric.compare(real, synthetic) + self.assertGreaterEqual(score, 0.0) diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py new file mode 100644 index 00000000..5650bd01 --- /dev/null +++ b/tests/test_evaluators_proposal_ranking.py @@ -0,0 +1,397 @@ +"""Unit tests for datafaker.evaluators.proposal_ranking.""" +from typing import Any + +from datafaker.evaluators.column_evaluator import EvaluationProfile, ProposalEvaluation +from datafaker.evaluators.proposal_ranking import ( + ProposalRanking, + _crowding_distances, + _pareto_fronts, + format_ranking_display, + keyword_match, + normalize_list, + rank_proposals, +) +from datafaker.proposers.base import Proposer +from datafaker.proposers.choice import UniformChoiceProposer +from tests.utils import DatafakerTestCase + + +class FakeProposer(Proposer): + """A minimal, DB-free stand-in for a real Proposer, for ranking tests.""" + + def __init__(self, name: str) -> None: + self._name = name + + def function_name(self) -> str: + return self._name + + def nominal_kwargs(self) -> dict[str, str]: + return {} + + def actual_kwargs(self) -> dict[str, Any]: + return {} + + def generate_data(self, count: int) -> list[Any]: + return [None] * count + + +def make_result( # pylint: disable=too-many-arguments,too-many-positional-arguments + name: str = "generic.text", + overall_score: float = 0.5, + novelty: float = 0.5, + diversity: float = 0.5, + copy_fraction: float = 0.0, + synthetic_uniqueness: float = 1.0, + proposer: Proposer | None = None, +) -> ProposalEvaluation: + """Build a ProposalEvaluation for ranking tests.""" + return ProposalEvaluation( + proposer=proposer or FakeProposer(name), + novelty=novelty, + diversity=diversity, + overall_score=overall_score, + pipeline_scores={}, + copy_fraction=copy_fraction, + synthetic_uniqueness=synthetic_uniqueness, + ) + + +class KeywordMatchTests(DatafakerTestCase): + """Test case for keyword_match.""" + + def test_matching_keyword_and_generator_gives_boost(self) -> None: + """A column/generator pair that both mention 'email' gets the boost.""" + boost, keyword = keyword_match("customer_email", "generic.person.email") + self.assertEqual(0.15, boost) + self.assertEqual("email", keyword) + + def test_compound_column_name_matches_by_substring(self) -> None: + """Substring matching catches compound names like 'customer_first_name'.""" + boost, keyword = keyword_match( + "customer_first_name", "generic.person.first_name" + ) + self.assertEqual(0.15, boost) + self.assertEqual("first_name", keyword) + + def test_no_column_name_gives_no_boost(self) -> None: + """A missing column name can't match anything.""" + self.assertEqual((0.0, None), keyword_match(None, "generic.person.email")) + self.assertEqual((0.0, None), keyword_match("", "generic.person.email")) + + def test_column_matches_but_generator_does_not(self) -> None: + """Column name hints at a kind, but this generator isn't of that kind.""" + boost, keyword = keyword_match("email", "generic.text.word") + self.assertEqual(0.0, boost) + self.assertIsNone(keyword) + + def test_is_case_insensitive(self) -> None: + """Matching ignores case in both the column and generator names.""" + boost, keyword = keyword_match("EMAIL", "GENERIC.PERSON.EMAIL") + self.assertEqual(0.15, boost) + self.assertEqual("email", keyword) + + +class NormalizeListTests(DatafakerTestCase): + """Test case for normalize_list.""" + + def test_empty_list(self) -> None: + """An empty input returns an empty output.""" + self.assertEqual([], normalize_list([])) + + def test_spread_out_values_map_to_zero_one(self) -> None: + """Values are linearly rescaled so the min is 0 and the max is 1.""" + self.assertEqual([0.0, 0.5, 1.0], normalize_list([10, 20, 30])) + + def test_all_identical_values_are_treated_as_a_tie(self) -> None: + """Exactly equal values become 0.5 rather than dividing by zero.""" + self.assertEqual([0.5, 0.5, 0.5], normalize_list([3.0, 3.0, 3.0])) + + def test_near_identical_values_are_also_treated_as_a_tie(self) -> None: + """Floating-point noise around a shared value doesn't get stretched out.""" + result = normalize_list([0.6931471805599453, 0.6931471805599454]) + self.assertEqual([0.5, 0.5], result) + + def test_single_value_is_a_tie(self) -> None: + """A single value has no spread, so it's treated as a tie too.""" + self.assertEqual([0.5], normalize_list([42.0])) + + +class ParetoFrontsTests(DatafakerTestCase): + """Test case for the internal _pareto_fronts helper.""" + + def test_strictly_dominated_point_is_in_a_later_front(self) -> None: + """A point beaten or matched on every dimension lands behind the winner.""" + points = [(1.0, 1.0, 1.0), (0.5, 0.5, 0.5)] + fronts = _pareto_fronts(points) + self.assertEqual([[0], [1]], fronts) + + def test_non_dominated_points_share_the_first_front(self) -> None: + """Points that each win on a different dimension are mutually non-dominated.""" + points = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)] + fronts = _pareto_fronts(points) + self.assertEqual([[0, 1, 2]], fronts) + + def test_empty_points(self) -> None: + """No points means no fronts.""" + self.assertEqual([], _pareto_fronts([])) + + +class CrowdingDistancesTests(DatafakerTestCase): + """Test case for the internal _crowding_distances helper.""" + + def test_front_boundary_points_get_infinite_crowding(self) -> None: + """The extreme points of a front are always maximally 'crowded out'.""" + points = [(0.0, 0.0, 0.0), (0.5, 0.5, 0.5), (1.0, 1.0, 1.0)] + fronts = [[0, 1, 2]] + objectives = ( + [p[0] for p in points], + [p[1] for p in points], + [p[2] for p in points], + ) + crowding = _crowding_distances(points, fronts, objectives) + self.assertEqual(float("inf"), crowding[0]) + self.assertEqual(float("inf"), crowding[2]) + self.assertLess(crowding[1], float("inf")) + + def test_singleton_front_gets_infinite_crowding(self) -> None: + """A front with just one point has nothing to be crowded by.""" + points = [(0.5, 0.5, 0.5)] + fronts = [[0]] + objectives = ([0.5], [0.5], [0.5]) + crowding = _crowding_distances(points, fronts, objectives) + self.assertEqual([float("inf")], crowding) + + +class RankProposalsTests(DatafakerTestCase): + """Test case for rank_proposals.""" + + def test_no_results_gives_an_empty_ranking(self) -> None: + """With nothing to rank, no recommendation is made.""" + ranking = rank_proposals([], EvaluationProfile.SHORT_TEXT) + self.assertIsNone(ranking.recommended_index) + self.assertIsNone(ranking.recommended_reason) + self.assertEqual([], ranking.rows) + self.assertEqual((0.5, 0.25, 0.25), ranking.weights) + + def test_single_result_is_trivially_recommended(self) -> None: + """A single candidate is always the recommendation.""" + results = [make_result("only_one")] + ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + self.assertEqual(0, ranking.recommended_index) + self.assertEqual("best combined score", ranking.recommended_reason) + self.assertEqual(1, len(ranking.rows)) + + def test_lower_error_wins_on_fidelity(self) -> None: + """Among otherwise-equal candidates, the one with lower overall_score wins.""" + results = [ + make_result("bad_fit", overall_score=0.9, novelty=0.5, diversity=0.5), + make_result("good_fit", overall_score=0.1, novelty=0.5, diversity=0.5), + ] + ranking = rank_proposals(results, EvaluationProfile.IDENTIFIER) + self.assertEqual(1, ranking.recommended_index) + + def test_identifier_profile_weighs_fidelity_heavily(self) -> None: + """IDENTIFIER profile weights are (0.85, 0.05, 0.10).""" + ranking = rank_proposals([make_result()], EvaluationProfile.IDENTIFIER) + self.assertEqual((0.85, 0.05, 0.10), ranking.weights) + + def test_unknown_profile_falls_back_to_default_weights(self) -> None: + """A profile with no explicit entry uses the (0.5, 0.25, 0.25) default.""" + ranking = rank_proposals([make_result()], None) + self.assertEqual((0.5, 0.25, 0.25), ranking.weights) + + def test_exact_tie_breaks_towards_the_earlier_candidate(self) -> None: + """Identical candidates are both max-crowding; the first one wins.""" + results = [make_result("first"), make_result("second")] + ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + self.assertEqual(0, ranking.recommended_index) + self.assertEqual("tiebreak by crowding distance", ranking.recommended_reason) + + def test_keyword_boost_can_flip_the_recommendation(self) -> None: + """A column-name/generator match can outweigh an otherwise-tied fidelity.""" + results = [ + make_result( + "generic.text.word", overall_score=0.1, novelty=0.5, diversity=0.5 + ), + make_result( + "generic.person.email", overall_score=0.1, novelty=0.5, diversity=0.5 + ), + ] + # Without the keyword boost this is an exact tie, decided by crowding + # distance (which favors the first candidate); the boost should flip it. + ranking = rank_proposals( + results, EvaluationProfile.EMAIL, column_name="user_email" + ) + self.assertEqual(1, ranking.recommended_index) + self.assertEqual("best combined score", ranking.recommended_reason) + + def test_choice_proposer_penalized_by_resampling_on_unique_column(self) -> None: + """ChoiceProposer's score is discounted by real_uniqueness x copy_fraction.""" + choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + results = [ + make_result(overall_score=0.0, copy_fraction=1.0, proposer=choice_proposer), + make_result(overall_score=0.5, copy_fraction=0.0), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, real_uniqueness=1.0 + ) + # The ChoiceProposer perfectly fits but is fully discounted (penalty + # multiplier 0), leaving the other candidate to win despite a worse fit. + self.assertEqual(1, ranking.recommended_index) + + def test_non_choice_proposer_is_not_resample_penalized_on_string_column( + self, + ) -> None: + """Only ChoiceProposer (or a numeric column) is hit by the resample penalty.""" + results = [make_result(overall_score=0.0, copy_fraction=1.0)] + ranking = rank_proposals( + results, EvaluationProfile.SHORT_TEXT, real_uniqueness=1.0 + ) + self.assertFalse(ranking.penalty_applies_to_all) + # combined score should be unaffected by the (non-applicable) penalty: + # a single candidate normalizes every dimension to a 0.5 tie. + fid_w, nov_w, div_w = ranking.weights + expected = (fid_w + nov_w + div_w) * 0.5 + self.assertAlmostEqual(expected, ranking.scores[0]) + + def test_numeric_column_penalizes_every_proposer(self) -> None: + """A numeric column applies the resample-style penalty to all candidates.""" + results = [make_result(overall_score=0.0, copy_fraction=1.0)] + ranking = rank_proposals( + results, + EvaluationProfile.IDENTIFIER, + real_uniqueness=1.0, + is_numeric_column=True, + ) + self.assertTrue(ranking.penalty_applies_to_all) + self.assertEqual(0.0, ranking.scores[0]) + + def test_self_duplication_penalty_applies_regardless_of_proposer_type(self) -> None: + """Low synthetic_uniqueness is penalized even for a non-resampling proposer.""" + results = [ + make_result("full_of_dupes", overall_score=0.0, synthetic_uniqueness=0.0), + make_result("all_unique", overall_score=0.0, synthetic_uniqueness=1.0), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, real_uniqueness=1.0 + ) + self.assertEqual(1, ranking.recommended_index) + + def test_no_uniqueness_guarantee_when_every_candidate_duplicates(self) -> None: + """A primary key column with only duplicate-prone candidates has no safe pick.""" + results = [ + make_result("dupe_a", synthetic_uniqueness=0.5), + make_result("dupe_b", synthetic_uniqueness=0.9), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_primary_key=True + ) + self.assertTrue(ranking.no_uniqueness_guarantee) + + def test_uniqueness_guarantee_satisfied_by_one_candidate(self) -> None: + """If at least one candidate guarantees uniqueness, there's no warning.""" + results = [ + make_result("dupe", synthetic_uniqueness=0.5), + make_result("unique", synthetic_uniqueness=1.0), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_primary_key=True + ) + self.assertFalse(ranking.no_uniqueness_guarantee) + + def test_combined_score_is_clamped_to_one(self) -> None: + """The keyword boost can't push a score above 1.0.""" + results = [make_result("generic.person.email", overall_score=0.0)] + ranking = rank_proposals(results, EvaluationProfile.EMAIL, column_name="email") + self.assertLessEqual(ranking.scores[0], 1.0) + + def test_weak_recommendation_warning_when_best_fit_was_suppressed(self) -> None: + """A heavily-discounted best-fit candidate triggers the fallback warning.""" + choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + results = [ + # Fits almost perfectly but is a resampler on a near-unique column: + # gets discounted to (near) zero. + make_result( + overall_score=0.0, + novelty=0.0, + diversity=0.0, + copy_fraction=1.0, + synthetic_uniqueness=1.0, + proposer=choice_proposer, + ), + # A poor fit that isn't penalized, so it "wins" by elimination. + # synthetic_uniqueness is deliberately below the guarantee + # threshold so the weak-recommendation warning isn't suppressed + # by the "winner guarantees uniqueness" exemption. + make_result( + overall_score=1.0, + novelty=0.0, + diversity=0.0, + copy_fraction=0.0, + synthetic_uniqueness=0.5, + proposer=FakeProposer("generic.text.word"), + ), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, real_uniqueness=1.0 + ) + self.assertEqual(1, ranking.recommended_index) + self.assertIsNotNone(ranking.weak_recommendation_warning) + self.assert_str_in("resample penalty", ranking.weak_recommendation_warning) + + +class FormatRankingDisplayTests(DatafakerTestCase): + """Test case for format_ranking_display.""" + + def test_recommendation_line_names_the_winner(self) -> None: + """The recommendation string names the recommended candidate by number/name.""" + results = [make_result("winner", overall_score=0.0)] + ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + display = format_ranking_display(ranking, results, EvaluationProfile.SHORT_TEXT) + self.assert_str_in("Recommended: 1. winner", display.recommendation) + self.assert_str_in("best combined score", display.recommendation) + + def test_no_uniqueness_guarantee_overrides_recommendation_text(self) -> None: + """When no candidate can guarantee uniqueness, no winner is named.""" + results = [make_result("a", synthetic_uniqueness=0.5)] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_primary_key=True + ) + display = format_ranking_display(ranking, results, EvaluationProfile.IDENTIFIER) + self.assert_str_in("Recommended: none", display.recommendation) + + def test_penalty_scope_text_reflects_numeric_flag(self) -> None: + """The profile summary explains whether the penalty hit every candidate.""" + results = [make_result("a")] + numeric_ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_numeric_column=True + ) + numeric_display = format_ranking_display( + numeric_ranking, results, EvaluationProfile.IDENTIFIER + ) + self.assert_str_in( + "applied to every generator", numeric_display.profile_summary + ) + + string_ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + string_display = format_ranking_display( + string_ranking, results, EvaluationProfile.SHORT_TEXT + ) + self.assert_str_in( + "applied only to resampling generators", string_display.profile_summary + ) + + def test_rows_and_fronts_pass_through_unchanged(self) -> None: + """rows and fronts are carried over verbatim from the ProposalRanking.""" + results = [make_result("winner")] + ranking = ProposalRanking( + recommended_index=0, + recommended_reason="best combined score", + rows=[("1", "x")], + profile=EvaluationProfile.SHORT_TEXT, + weights=(0.5, 0.25, 0.25), + fronts=[1], + ) + display = format_ranking_display(ranking, results, EvaluationProfile.SHORT_TEXT) + self.assertEqual([("1", "x")], display.rows) + self.assertEqual([1], display.fronts) diff --git a/tests/test_evaluators_statistical_fidelity.py b/tests/test_evaluators_statistical_fidelity.py new file mode 100644 index 00000000..fac7aa61 --- /dev/null +++ b/tests/test_evaluators_statistical_fidelity.py @@ -0,0 +1,108 @@ +"""Unit tests for datafaker.evaluators.statistical_fidelity.""" +from datetime import datetime + +from sqlalchemy import Column, DateTime, Integer, MetaData, String, Table, create_engine + +from datafaker.evaluators.column_evaluator import EvaluationProfile +from datafaker.evaluators.statistical_fidelity import ( + EMAIL_PIPELINE, + PROFILE_PIPELINES, + StatisticalFidelity, +) +from tests.utils import DatafakerTestCase + + +class ProfilePipelinesTests(DatafakerTestCase): + """Structural sanity checks for the built-in evaluation pipelines.""" + + def test_every_evaluation_profile_has_a_pipeline(self) -> None: + """Each EvaluationProfile enum member maps to a configured pipeline.""" + for profile in EvaluationProfile: + self.assertIn(profile, PROFILE_PIPELINES) + self.assertTrue(PROFILE_PIPELINES[profile]) + + def test_pipeline_weights_sum_to_one(self) -> None: + """Each profile's pipeline weights add up to 1.0, for a sane weighted average.""" + for profile, pipelines in PROFILE_PIPELINES.items(): + with self.subTest(profile=profile.name): + total_weight = sum(p.weight for p in pipelines) + self.assertAlmostEqual(1.0, total_weight) + + def test_pipeline_names_are_unique_within_a_profile(self) -> None: + """Pipeline names double as dict keys in calculate_scores, so must be unique.""" + for profile, pipelines in PROFILE_PIPELINES.items(): + with self.subTest(profile=profile.name): + names = [p.name for p in pipelines] + self.assertEqual(len(names), len(set(names))) + + +def _make_table(columns: list[Column], rows: list[dict]) -> tuple: + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table("data", metadata, *columns) + metadata.create_all(engine) + if rows: + with engine.begin() as conn: + conn.execute(table.insert(), rows) + return engine, table + + +class StatisticalFidelityTests(DatafakerTestCase): + """Test case for StatisticalFidelity.""" + + def test_identical_real_and_synthetic_data_scores_perfectly(self) -> None: + """A synthetic sample identical to the real data has zero divergence.""" + engine, table = _make_table( + [Column("email", String)], + [{"email": f"user{i}@example.com"} for i in range(30)], + ) + fidelity = StatisticalFidelity(table.c.email, engine, sample_size=1000) + fidelity.set_eval_pipelines(EvaluationProfile.EMAIL) + + synthetic = [f"user{i}@example.com" for i in range(30)] + overall_score, pipeline_scores = fidelity.calculate_scores(synthetic) + + self.assertAlmostEqual(0.0, overall_score) + self.assertEqual({p.name for p in EMAIL_PIPELINE}, set(pipeline_scores)) + for name, score in pipeline_scores.items(): + with self.subTest(pipeline=name): + self.assertAlmostEqual(0.0, score) + + def test_completely_different_synthetic_data_scores_worse(self) -> None: + """A synthetic sample sharing nothing with the real data scores worse.""" + engine, table = _make_table( + [Column("category", String)], + [{"category": "alpha"} for _ in range(30)], + ) + fidelity = StatisticalFidelity(table.c.category, engine, sample_size=1000) + fidelity.set_eval_pipelines(EvaluationProfile.CATEGORICAL) + + matching_score, _ = fidelity.calculate_scores(["alpha"] * 30) + different_score, _ = fidelity.calculate_scores(["zzz_never_seen"] * 30) + + self.assertGreater(different_score, matching_score) + + def test_temporal_profile_scores_a_matching_date_column(self) -> None: + """The TEMPORAL profile's two pipelines run cleanly against a date column.""" + engine, table = _make_table( + [Column("ts", DateTime)], + [{"ts": datetime(2020, 1, (i % 27) + 1)} for i in range(40)], + ) + fidelity = StatisticalFidelity(table.c.ts, engine, sample_size=1000) + fidelity.set_eval_pipelines(EvaluationProfile.TEMPORAL) + + synthetic = [datetime(2020, 1, (i % 27) + 1) for i in range(40)] + overall_score, pipeline_scores = fidelity.calculate_scores(synthetic) + + self.assertAlmostEqual(0.0, overall_score) + self.assertEqual({"timestamp", "day_of_week"}, set(pipeline_scores)) + + def test_no_pipelines_set_scores_zero(self) -> None: + """Without calling set_eval_pipelines, there's nothing to score.""" + engine, table = _make_table( + [Column("n", Integer)], [{"n": i} for i in range(10)] + ) + fidelity = StatisticalFidelity(table.c.n, engine, sample_size=1000) + overall_score, pipeline_scores = fidelity.calculate_scores([1, 2, 3]) + self.assertEqual(0.0, overall_score) + self.assertEqual({}, pipeline_scores) From 619d0cc58c5bea49386e6f9f7542d5fc31f4e9c3 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Wed, 9 Sep 2026 17:10:16 +0100 Subject: [PATCH 03/12] Add IncrementProposer --- datafaker/proposers/__init__.py | 2 + datafaker/proposers/sequence.py | 91 +++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 datafaker/proposers/sequence.py diff --git a/datafaker/proposers/__init__.py b/datafaker/proposers/__init__.py index 1a333c33..bee4c536 100644 --- a/datafaker/proposers/__init__.py +++ b/datafaker/proposers/__init__.py @@ -30,6 +30,7 @@ NullPartitionedLogNormalProposerFactory, NullPartitionedNormalProposerFactory, ) +from datafaker.proposers.sequence import IncrementProposerFactory def everything_factory(config: Mapping, metadata: MetaData) -> ProposerFactory: @@ -57,4 +58,5 @@ def everything_factory(config: Mapping, metadata: MetaData) -> ProposerFactory: NullPartitionedLogNormalProposerFactory(config, metadata), DateAfterProposerFactory(config, metadata), DateComponentExtractProposerFactory(config, metadata), + IncrementProposerFactory(), ) diff --git a/datafaker/proposers/sequence.py b/datafaker/proposers/sequence.py new file mode 100644 index 00000000..99beb0ee --- /dev/null +++ b/datafaker/proposers/sequence.py @@ -0,0 +1,91 @@ +"""Proposer for a monotonically-incrementing sequence past the observed max. + +This is the only kind of candidate that can actually guarantee fresh, unique +values for a genuine integer primary key: it can never collide with the real +data (every value it emits is strictly greater than any observed value) and +it can never collide with itself (it only ever increments). It deliberately +reuses ``generic.column_value_provider.increment`` - the same mechanism +``make.py``'s own default generator assignment already uses for integer +primary keys (see ``_integer_generator``) - rather than inventing a new +runtime function, so a column left on this proposer's recommendation behaves +identically to the framework's own established default. +""" + +from collections.abc import Sequence +from typing import Any + +from sqlalchemy import Column, Engine, func, select +from sqlalchemy.types import Integer + +from datafaker.proposers.base import Proposer, ProposerFactory, get_column_type + + +class IncrementProposer(Proposer): + """Generator continuing a sequence past the real data's observed maximum.""" + + def __init__(self, engine: Engine, column: Column): + """Initialise an IncrementProposer.""" + super().__init__() + self.engine = engine + self.column = column + self.table = column.table + + def function_name(self) -> str: + """Get the name of the generator function to call.""" + return "generic.column_value_provider.increment" + + def nominal_kwargs(self) -> dict[str, str]: + """Get the arguments to be entered into ``config.yaml``. + + These match the expressions ``make.py``'s own default integer + primary key generator emits (see ``_integer_generator``): not + SRC_STATS references, but Python expressions the generation + pipeline evaluates directly against the destination database and + its metadata at generation time. + """ + return { + "db_connection": "dst_db_conn", + "column": f'metadata.tables["{self.table.name}"].columns["{self.column.name}"]', + } + + def actual_kwargs(self) -> dict[str, Any]: + """Get the kwargs (summary statistics) this generator was instantiated with.""" + return {"column": f"{self.table.name}.{self.column.name}"} + + def generate_data(self, count: int) -> list[Any]: + """Generate ``count`` random data points for this column. + + For interactive preview purposes only: continues past the *source* + database's observed maximum. At actual generation time, the + ``nominal_kwargs`` above instead continue past whatever's already in + the *destination* database, via ``db_connection``/``column``. + """ + with self.engine.connect() as connection: + row = connection.execute(select(func.max(self.column))).first() + start = 0 if row is None or row[0] is None else row[0] + return [start + i + 1 for i in range(count)] + + +class IncrementProposerFactory(ProposerFactory): + """Propose an incrementing-sequence generator for integer primary keys. + + Deliberately narrow: only a genuine integer primary key (and one that + isn't also a foreign key - those should copy the referenced table's + values, not synthesize their own, matching ``make.py``'s own default + generator logic) gets this proposer. A non-key numeric column doesn't + need guaranteed-fresh values, and a non-integer key (a string/UUID + primary key) needs a different mechanism entirely - out of scope here. + """ + + def get_proposers( + self, columns: list[Column], engine: Engine + ) -> Sequence[Proposer]: + """Get the generators appropriate to these columns.""" + if len(columns) != 1: + return [] + column = columns[0] + if not column.primary_key or column.foreign_keys: + return [] + if not isinstance(get_column_type(column), Integer): + return [] + return [IncrementProposer(engine, column)] From 833d38bc8b18204aecee0934b5156a331836e258 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Wed, 9 Sep 2026 17:16:28 +0100 Subject: [PATCH 04/12] Update documentation --- docs/source/builtin_generators.rst | 653 +++++++++++++++++++++++++++++ docs/source/glossary.rst | 6 + docs/source/index.rst | 1 + docs/source/tutorial_parquet.rst | 9 +- 4 files changed, 666 insertions(+), 3 deletions(-) create mode 100644 docs/source/builtin_generators.rst diff --git a/docs/source/builtin_generators.rst b/docs/source/builtin_generators.rst new file mode 100644 index 00000000..f95a0bc1 --- /dev/null +++ b/docs/source/builtin_generators.rst @@ -0,0 +1,653 @@ +Built-in generators and proposers +================================== + +This page is a reference for the generator functions that ship with ``datafaker``, +and for the *proposers* that suggest which of those functions (and which arguments) +to use for a given column. If you want to write your own generators instead, +see :doc:`custom_generators`. + +Generators vs. proposers +------------------------- + +A **generator** is the actual callable that produces fake values, referenced by name +in ``config.yaml`` (for example ``dist_gen.normal`` or ``generic.person.first_name``). +Generators are simple: given some arguments (often summary statistics pulled from +``src-stats.yaml``), they return one random value per call. + +A **proposer** is an internal ``datafaker`` object, not something you reference +directly in ``config.yaml``. When you run ``configure-generators`` and use the +``propose`` command on a column, ``datafaker`` runs every applicable proposer against +that column (and, for multi-column proposers, against that group of columns). Each +proposer: + +* decides whether it applies at all, based on the column's SQL type and (for some + proposers) properties of the source data or ``config.yaml``, +* works out what summary queries need to be added to ``src-stats.yaml`` (via + ``select_aggregate_clauses`` or ``custom_queries``), +* works out the generator function name and keyword arguments to write into + ``config.yaml`` if you ``set`` it. + +``set`` simply writes the winning proposer's generator name and arguments into +``config.yaml``; from then on, only the underlying generator function is used --- +the proposer itself is not needed again for that column. + +``propose`` now shows two different views of the same candidates: first the +plain, per-proposer ``(fit: ...)`` list described above, and then a second, +ranked table produced by a separate statistical evaluation pipeline (see +:ref:`evaluating-and-ranking-proposals` below) that scores every candidate on +fidelity, novelty and diversity and names a ``Recommended`` generator. The two +scores are independent and can disagree; the ranked table and its +recommendation are the more reliable of the two for most columns. + +Default generators assigned automatically +------------------------------------------ + +Before you run ``propose``/``set`` at all, ``configure-generators`` (via +``make.py``) already assigns a default generator to every column, purely from its +SQL type (and whether it is a foreign key or primary key). ``propose`` lets you +replace this default with something that better matches the real data. + +.. list-table:: Default generator by SQL column type + :widths: 25 45 30 + :header-rows: 1 + + * - SQL type + - Default generator + - Notes + * - Foreign key column (any type) + - ``generic.column_value_provider.column_value`` + - Picks a random existing value from the referenced column in the destination + database. + * - Integer primary key + - ``generic.column_value_provider.increment`` + - Continues counting up from the highest existing value in the destination + table; see :ref:`the increment proposer ` below for how + this is also offered as an explicit proposal. + * - Other ``Integer`` + - ``generic.numeric.integer_number`` + - + * - ``Numeric`` (decimal/float) + - ``generic.numeric.float_number`` + - If the column has a fixed ``scale``, the range is set to fit within that + many decimal digits. + * - ``String`` + - ``generic.person.password``, or ``generic.text.color`` if the column has no + maximum length + - The password length is set to the column's maximum length so the value + always fits. + * - ``Boolean`` + - ``generic.development.boolean`` + - + * - ``Date`` + - ``generic.datetime.date`` + - + * - ``DateTime`` + - ``generic.datetime.datetime`` + - + * - ``LargeBinary`` + - ``generic.bytes_provider.bytes`` + - + * - ``Uuid`` / PostgreSQL ``UUID`` / MS-SQL ``UNIQUEIDENTIFIER`` + - ``generic.cryptographic.uuid`` + - + * - Anything else unsupported + - ``generic.null_provider.null`` + - ``datafaker`` logs a warning; you should configure a real generator for + this column by hand. + +Built-in proposers +------------------- + +Everything below is registered in ``datafaker.proposers.everything_factory`` and is +tried automatically for every column (or, for the multi-column proposers, every +group of columns you select) when you run ``propose``. + +Single-column value proposers (Mimesis) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These proposers wrap `Mimesis `_ generator functions +(``generic.*``, using the ``en_GB`` locale) that produce a plausible but +unrelated fake value, with no reference to the real data's actual distribution. +Fit is estimated by comparing the length (for strings) or value (for numbers) of +generated samples against buckets built from the real column. + +.. list-table:: + :widths: 20 60 20 + :header-rows: 1 + + * - Applies to + - Generators proposed + - Notes + * - ``String`` + - Every name in :ref:`the Mimesis string generator list ` + (``generic.``) + - If the column has a maximum length, each candidate is wrapped in + ``dist_gen.truncated_string`` so the output never overflows the column. + * - ``Numeric`` + - ``generic.person.height`` + - + * - ``Numeric`` or ``Integer`` + - ``generic.person.weight`` + - + * - ``Date`` + - ``generic.datetime.date`` + - Range (``start``/``end`` years) is taken from the earliest/latest years + actually found in the column. + * - ``DateTime`` + - ``generic.datetime.datetime`` + - Same year-range behaviour as ``Date``. + * - ``Time`` + - ``generic.datetime.time`` + - + +.. _mimesis-string-list: + +The full list of Mimesis string generators tried for ``String`` columns: +``address.calling_code``, ``address.city``, ``address.continent``, +``address.country``, ``address.country_code``, ``address.postal_code``, +``address.province``, ``address.street_number``, ``address.street_name``, +``address.street_suffix``, ``person.blood_type``, ``person.email``, +``person.first_name``, ``person.last_name``, ``person.full_name``, +``person.gender``, ``person.language``, ``person.nationality``, +``person.occupation``, ``person.password``, ``person.title``, +``person.university``, ``person.username``, ``person.worldview``, +``text.answer``, ``text.color``, ``text.level``, ``text.quote``, +``text.sentence``, ``text.text``, ``text.word``. + +Continuous distribution proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These proposers fit a single numeric column to a well-known distribution, using +the column's actual mean and standard deviation (queried into ``src-stats.yaml`` +as ``mean__`` / ``stddev__``, or their log equivalents). + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - ``Numeric`` or ``Integer`` + - ``dist_gen.normal`` + - Gaussian distribution with the observed mean and standard deviation. + * - ``Numeric`` or ``Integer`` + - ``dist_gen.uniform_ms`` + - Uniform distribution with the same mean and standard deviation as the real + data (rather than explicit min/max bounds). + * - ``Numeric`` or ``Integer``, values ``> 0`` + - ``dist_gen.lognormal`` + - Log-normal distribution, fitted to the mean/standard deviation of the logs + of the (positive) values. + +Choice proposers +^^^^^^^^^^^^^^^^^ + +Used for columns with a manageable number of distinct values (up to 500). Each of +the three distribution shapes below is proposed both for **all** distinct values +and, separately, for a version with rare values (seen 7 or fewer times) suppressed +--- and again for both of those computed from a random sample of up to 500 rows, +for large tables where scanning every row would be too slow. + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Generator + - Function name + - Distribution + * - Uniform choice + - ``dist_gen.choice`` + - Every distinct value equally likely. + * - Zipf choice + - ``dist_gen.zipf_choice`` + - Values ranked by real-world frequency; the *n*\ th most common value is + chosen ``1/n`` as often as the most common one (a Zipf/power-law shape, + without needing to store every count). + * - Weighted choice + - ``dist_gen.weighted_choice`` + - Reproduces the real frequency of each value directly (stores a count + alongside each value in ``src-stats.yaml``). + +Constant proposer +^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - Any single nullable column, or ``String``/``Numeric``/``Integer`` + - ``dist_gen.constant`` + - Always returns the same value: ``None`` if the column is nullable, + otherwise ``""``, ``0.0`` or ``0`` depending on type. Useful as a + placeholder, or a baseline to compare fit against. + +.. _increment-proposer: + +Sequence proposer +^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - Integer primary key that is **not** also a foreign key + - ``generic.column_value_provider.increment`` + - Counts up past the highest value already present, guaranteeing fresh, + unique values. This is the same mechanism used as the automatic default + for integer primary keys (see above); proposing it explicitly just makes + it visible and comparable alongside other candidates. + +Multivariate proposers +^^^^^^^^^^^^^^^^^^^^^^^^ + +Offered when you select **two or more** numeric columns together (so that +``datafaker`` can preserve correlations between them, rather than generating each +column independently). A covariate (means/covariance) matrix is queried from the +source data. + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - 2+ ``Numeric``/``Integer`` columns + - ``dist_gen.multivariate_normal`` + - Multivariate Gaussian distribution over all the selected columns. + * - 2+ ``Numeric``/``Integer`` columns, values ``> 0`` + - ``dist_gen.multivariate_lognormal`` + - Multivariate log-normal distribution (covariates computed on the logs of + the values). + +Null-partitioned multivariate proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A more powerful variant of the multivariate proposers above, for a group of +columns (numeric and/or categorical) that don't all follow the same pattern of +missing (``NULL``) values. The source data is split into partitions, one per +distinct combination of which columns in the group are ``NULL``; each partition +gets its own covariate matrix (or, if it has too few rows, is treated as a +suppressed group), and at generation time a partition is picked at random +(weighted by how common it was in the source data) before that partition's +distribution is sampled. + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - Group of columns with mixed nullability + - ``dist_gen.alternatives`` wrapping ``dist_gen.grouped_multivariate_normal`` + - Shown as *"null-partitioned grouped_multivariate_normal"* in ``propose``. + * - Group of columns with mixed nullability, values ``> 0`` + - ``dist_gen.alternatives`` wrapping ``dist_gen.grouped_multivariate_lognormal`` + - Shown as *"null-partitioned grouped_multivariate_lognormal"*. + * - (both of the above) + - *[sampled and suppressed]* variants + - Computed from a random sample of the source table with rare partitions + suppressed, for large tables. + +Date interval ("anchored date") proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Propose a date/datetime as an offset from another date, rather than independently. +This only appears once at least one ``Date``/``DateTime`` column in the same +table (or a directly related table) has been given the ``start`` role --- for +example an admission date. Every *other* date/datetime column in scope (an +implicit "end", such as a discharge date) is then offered a proposal anchored +to it; the anchor column itself is not offered a self-referential proposal. + +Roles are set per column and stored in ``config.yaml`` under +``tables..columns..roles``. The easiest way to set one is the +``role`` command inside ``configure-generators``: + +.. code-block:: shell + + (admission.start_date) role set start + (admission.start_date) role list + start + (discharge.end_date) role on admission.start_date list + +``role list`` shows the roles on the current column, ``role set `` / +``role delete `` add or remove one, and adding ``on `` (or +just ``on `` for a column in the current table) targets a different +column without navigating to it first. Like generator changes, role changes +are only written to ``config.yaml`` when you ``quit`` and confirm --- ``quit`` +will list any pending role changes alongside pending generator changes before +asking you to save. + +Two roles currently exist: ``start`` (consumed by this proposer, as described +above) and ``source`` (reserved for future use --- no built-in proposer reads +it yet). You can still set either role by hand-editing ``config.yaml`` instead +of using ``role``, if you prefer; both routes update the same field. + +.. list-table:: + :widths: 20 30 50 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - ``Date``/``DateTime`` column with an anchor column in the *same* table + - ``generic.anchored_provider.normal_date`` + - Adds a clamped, normally-distributed number of seconds (mean/standard + deviation taken from the real interval lengths) to the anchor column's + generated value; never earlier than the anchor. The sample values shown + by ``propose``/``compare`` are built from real anchor values sampled + from the source database (recycled if fewer than requested), rather + than a single fixed dummy anchor, so the preview reflects realistic + intervals. + * - ``Date``/``DateTime`` column anchored to a column in a *related* table + (via a foreign key) + - ``generic.anchored_provider.normal_date_fk`` + - Same idea, but looks up the anchor value from the related row in the + destination database at generation time. + +Date component extraction proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Propose deriving a column's value from a ``DateTime`` column already generated +elsewhere in the same row, rather than generating it independently -- useful when, +for example, a ``year_of_birth`` column should always agree with a +``date_of_birth`` column. + +.. list-table:: + :widths: 20 30 50 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - ``Date``, based on a ``DateTime`` column in the same table + - ``generic.extract_provider.date`` + - Takes just the date part of the referenced ``DateTime`` value. + * - ``Integer``, based on a ``DateTime`` column in the same table + - ``generic.extract_provider.year``, ``generic.extract_provider.month``, + ``generic.extract_provider.day`` + - One proposal per component, per ``DateTime`` column found in the table. + +.. _evaluating-and-ranking-proposals: + +Evaluating and ranking proposals +---------------------------------- + +For a single-column ``propose``, every candidate proposer that survives the +type-compatibility filter (see below) is additionally scored by +``datafaker.evaluators.ColumnEvaluator`` and ranked by +``datafaker.evaluators.proposal_ranking.rank_proposals``. This produces a + table shown by ``propose``, with a ``Recommended: N. `` line +above it. + +Evaluation profile +^^^^^^^^^^^^^^^^^^^ + +The column is first classified into one ``EvaluationProfile``, based on a +sample of up to 4000 real rows: + +.. list-table:: + :widths: 25 75 + :header-rows: 1 + + * - Profile + - How it's chosen + * - ``TEMPORAL`` + - Column is ``Date``/``DateTime``/``Time``. + * - ``CATEGORICAL`` or ``IDENTIFIER`` + - Column is ``Numeric``/``Integer``: ``CATEGORICAL`` if fewer than 20% of + the sampled values are distinct (a status code, a small foreign key + range), otherwise ``IDENTIFIER``. + * - ``EMAIL`` + - Column is a string and more than half the sampled non-empty values + look like an email address (contain ``@``, with a non-empty local part + and a domain containing a ``.``). + * - ``SHORT_TEXT`` + - Column is a string, not email-like, average length under 30 + characters and more than 80% of sampled values are distinct. + * - ``CATEGORICAL`` + - Column is a string, not email-like or short-text, and fewer than 20% + of sampled values are distinct. + * - ``FREE_TEXT`` + - Remaining string columns with an average length over 50 characters or + a high space ratio (multi-word content). + * - ``SHORT_TEXT`` + - Any remaining string column (the fallback). + +Three scoring dimensions +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Each candidate is scored, from a 4000-value synthetic sample, on three +independent axes, each normalized to ``[0, 1]`` across the candidates being +compared (so these are *relative* scores, not absolute ones): + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Axis + - Meaning + * - **Fidelity** + - How closely the synthetic sample's distribution matches the real + column's, measured by a profile-specific pipeline of feature + extractors run through ``MeanSquaredError`` or ``JensenShannon`` + (see :ref:`statistical-fidelity-pipelines` below), then inverted so + higher is better. + * - **Novelty** + - The fraction of synthetic values (case/whitespace-normalized) that do + *not* already appear among the real sampled values. 1.0 means every + synthetic value is new. + * - **Diversity** + - How closely the synthetic sample's own internal variety (normalized + Shannon entropy over distinct values) matches the real sample's --- not + "more diverse is better", but "as diverse as the real data". + +.. _statistical-fidelity-pipelines: + +Fidelity pipelines by profile +"""""""""""""""""""""""""""""" + +Fidelity is computed by one of six fixed pipelines, chosen by profile, each a +weighted combination of feature extractors compared via ``MeanSquaredError`` +(on a length/count histogram) or ``JensenShannon`` divergence (on a category +distribution): + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Profile + - Pipeline (feature: weight) + * - ``IDENTIFIER`` + - identifier (value histogram): 1.0 + * - ``CATEGORICAL`` + - category (exact value): 1.0 + * - ``SHORT_TEXT`` + - length: 0.25, character bigrams: 0.35, first letter: 0.10, + last letter: 0.10, vowel/consonant pattern: 0.20 + * - ``EMAIL`` + - length: 0.10, local part: 0.40, domain: 0.15, top-level domain: 0.15, + address format validity: 0.20 + * - ``FREE_TEXT`` + - length: 0.10, word count: 0.25, sentence count: 0.15, words: 0.30, + character bigrams: 0.20 + * - ``TEMPORAL`` + - year+month+day, as one joint quantity: 0.7, day-of-week: 0.3 + +Combining the scores, penalties and a keyword hint +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The three normalized scores are combined into one ``Score`` per candidate as + +.. code-block:: text + + Score = clamp( (fid_w * Fidelity + nov_w * Novelty + div_w * Diversity + Keyword) * Penalty, 0, 1 ) + +* **Weights** (``fid_w``/``nov_w``/``div_w``) depend on the profile: 0.85 / + 0.05 / 0.10 for ``IDENTIFIER`` (fidelity dominates); 0.7 / 0.15 / 0.15 for + ``CATEGORICAL``, ``SHORT_TEXT``, ``EMAIL`` and ``TEMPORAL``; 0.5 / 0.3 / 0.2 + for ``FREE_TEXT`` (open-ended text tolerates, even rewards, novelty). +* **Keyword** is a flat ``+0.15`` boost when the column's own name hints at a + particular kind of value (e.g. a column named ``first_name`` or + ``customer_first_name`` boosts any ``person.first_name``-based candidate; + see ``KEYWORD_GENERATOR_HINTS`` in ``proposal_ranking.py`` for the full + list) -- a cheap, complementary signal used only to break near-ties, never + to override a clearly better statistical result. +* **Penalty** (``1 - real_uniqueness * copy_fraction``, further reduced by how + much a candidate duplicates against its *own* output) discounts a + choice-style proposer (``dist_gen.choice``/``weighted_choice``/ + ``zipf_choice``) -- or, for a numeric column, *any* proposer -- for + reproducing real values verbatim, scaled by how unique the real column + actually is. This is a privacy safeguard: resampling real values is + expected and harmless for a low-uniqueness column (a gender, a status) but + a genuine leak for a near-unique one (an email, a real ID). It has no + effect (``1.0``) on other proposers for non-numeric columns, where an + overlap with real values is usually just a naturally shared vocabulary + (e.g. common first names), not memorization. + +Candidates are also grouped into Pareto fronts (front 1 = not +strictly dominated on fidelity, novelty *and* diversity simultaneously by any +other candidate); ``propose`` (without ``all``) shows only front 1, capped at +10 rows by ``Score`` (always keeping the recommended candidate even if it +would otherwise be cut), and tells you how many candidates were hidden. Run +``propose all`` to see every candidate --- including proposers whose output +type doesn't actually match the column (for example a continuous-float +proposer against an ``Integer`` column, invalid on most databases), which the +plain ``propose`` excludes structurally rather than relying on the score to +catch. + +The **recommendation** is simply the candidate with the highest ``Score`` +(ties broken by NSGA-II crowding distance, i.e. preferring a candidate that's +less redundant with its front-mates). Two caveats are surfaced when they +apply: + +* For a primary key or unique column, if *every* candidate's own synthetic + sample is duplicated enough that it couldn't satisfy a uniqueness + constraint, ``propose`` recommends nothing at all rather than a false + positive --- look for the sequence proposer instead (see + :ref:`increment-proposer` above). +* If the candidates that fit the real data best were heavily discounted by + the resample penalty above, leaving a poorer-fitting candidate to "win" by + elimination, ``propose`` prints a warning naming the discounted candidates + and flags the recommendation as a fallback rather than a confident pick. + +Generator function reference +------------------------------ + +The tables above name the generator functions that proposers can select. This +section documents each one, grouped by the Mimesis provider it belongs to, for +when you want to reference or combine them by hand in ``config.yaml`` (see +:doc:`custom_generators`). + +``dist_gen`` -- ``datafaker.providers.DistributionProvider`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Function + - Description + * - ``uniform(low, high)`` + - Uniform distribution between explicit bounds. + * - ``uniform_ms(mean, sd)`` + - Uniform distribution with a given mean and standard deviation. + * - ``normal(mean, sd)`` + - Gaussian (normal) distribution. + * - ``lognormal(logmean, logsd)`` + - Log-normal distribution. + * - ``choice(a)`` / ``choice_direct(a)`` + - Uniform choice between a list of values. ``choice`` takes + ``{"value": ...}`` dicts (as stored in ``src-stats.yaml``); + ``choice_direct`` takes plain values. + * - ``zipf_choice(a, n=None)`` / ``zipf_choice_direct(a, n=None)`` + - Choice following a Zipf distribution over values ranked most-to-least + frequent. + * - ``weighted_choice(a)`` + - Choice weighted by an explicit ``count`` stored alongside each value. + * - ``constant(value)`` + - Always returns ``value``. + * - ``multivariate_normal(cov)`` / ``multivariate_lognormal(cov)`` + - Draws a list of correlated values from a covariate matrix (means and + covariances, keyed as ``mN``/``cN_M``). + * - ``grouped_multivariate_normal(covs)`` / ``grouped_multivariate_lognormal(covs)`` + - As above, but first picks one covariate matrix from a list, weighted by + each group's ``count``. Used for the null-partitioned proposers. + * - ``alternatives(alternative_configs, counts=None)`` + - Picks between other named generators, weighted by count; this is how the + null-partitioned proposers choose a missingness pattern before delegating + to ``grouped_multivariate_normal``/``grouped_multivariate_lognormal``. + * - ``with_constants_at(constants_at, subgen, params)`` + - Runs another generator and splices fixed values into the result list at + given positions (used to reinsert ``NULL``/category columns alongside + generated numeric ones). + * - ``truncated_string(subgen_fn, params, length)`` + - Runs a string-producing generator and truncates the result to ``length`` + characters. + +``generic`` -- Mimesis providers (locale ``en_GB``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Most of ``generic.address.*``, ``generic.person.*``, ``generic.text.*``, +``generic.datetime.*`` and ``generic.numeric.*`` come straight from the +`Mimesis library `_ and aren't +listed again here; see :ref:`the Mimesis string generator list +` above for the ones ``propose`` tries automatically, +plus ``person.height``, ``person.weight``, ``datetime.date``, +``datetime.datetime`` and ``datetime.time``. + +``datafaker`` adds the following extra Mimesis providers of its own +(``datafaker/providers.py``): + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Function + - Description + * - ``column_value_provider.column_value(db_connection, orm_class, column_name)`` + - Returns a random existing value from another column -- the default + generator for foreign key columns. + * - ``column_value_provider.increment(db_connection, column)`` + - Returns a number one higher than the previous call for this column, + starting one above the highest existing value in the destination + database -- the default generator for integer primary keys. + * - ``bytes_provider.bytes()`` + - Random binary data. + * - ``timedelta_provider.timedelta(...)`` + - Random ``datetime.timedelta`` value. + * - ``timespan_provider.timespan(...)`` + - Random pair of dates/times forming a span. + * - ``weighted_boolean_provider.bool(probability)`` + - ``True`` with the given probability. + * - ``sql_group_by_provider.sample(...)`` + - Samples rows grouped and filtered according to SQL-like criteria. + * - ``null_provider.null()`` + - Always returns ``None``. Used as the last-resort default for column + types ``datafaker`` doesn't otherwise recognise. + * - ``extract_provider.year(extract_from)`` / ``month(...)`` / ``day(...)`` / ``date(...)`` + - Pulls a component out of a ``datetime`` produced elsewhere in the same + row (see the date component extraction proposers above). + * - ``anchored_provider.normal_date(mean_seconds, sd_seconds, anchor)`` + - A date offset from ``anchor`` in the same table by a clamped normally + distributed number of seconds. + * - ``anchored_provider.normal_date_fk(dst_db_conn, mean_seconds, sd_seconds, table, on_column, anchor_row, anchor_column)`` + - As above, but ``anchor`` is looked up from a related table via a foreign + key at generation time. + +See also +--------- + +* :doc:`custom_generators` -- writing your own row generators and story + generators. +* :doc:`quickstart` and :doc:`introduction` -- walkthroughs of + ``configure-generators``, ``propose``, ``compare`` and ``set`` in action. +* :doc:`glossary` diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst index 87b98890..a80edb6c 100644 --- a/docs/source/glossary.rst +++ b/docs/source/glossary.rst @@ -11,6 +11,12 @@ Glossary - A user-defined Python function which will provide one or more random column values for a single table when called. * - Story generator - A user-defined Python generator function that ``yields`` rows, possibly multiple rows for multiple tables. + * - Generator (function) + - A callable, referenced by name in ``config.yaml`` (e.g. ``dist_gen.normal``), that produces one random value (or tuple of values) per call. See :doc:`builtin_generators` for the ones built into `datafaker`, or :doc:`custom_generators` to write your own. + * - Proposer + - An internal `datafaker` object, used by the ``propose`` command of ``configure-generators``, that suggests a generator function and arguments for a column (or group of columns) and estimates how well it fits the real data. See :doc:`builtin_generators`. + * - Role + - A tag (``start`` or ``source``) set on a column, via the ``role`` command in ``configure-generators`` or by hand in ``config.yaml``, that some proposers use to find related columns -- for example, the ``start`` role marks an anchor date that other dates in the same table can be generated as an offset from. See :doc:`builtin_generators`. * - Destination database and destination schema - A database and a schema within that database where `datafaker` creates the synthetic data tables and inserts the synthetic data it generates. * - Source database and source schema diff --git a/docs/source/index.rst b/docs/source/index.rst index f411c60c..4c87a007 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -37,6 +37,7 @@ Contents: configuration health_data Custom Generators + Built-in Generators and Proposers api .. toctree:: diff --git a/docs/source/tutorial_parquet.rst b/docs/source/tutorial_parquet.rst index d28d9515..205e4ae1 100644 --- a/docs/source/tutorial_parquet.rst +++ b/docs/source/tutorial_parquet.rst @@ -219,7 +219,7 @@ and source statistics: datafaker configure-tables datafaker configure-generators - datafaker configure-missingness + datafaker configure-missing datafaker make-stats This creates: @@ -328,7 +328,7 @@ Generate configuration: datafaker configure-tables datafaker configure-generators - datafaker configure-missingness + datafaker configure-missing datafaker make-stats Create schema and generate data: @@ -356,7 +356,10 @@ For a minimal end-to-end workflow: export DST_DSN=duckdb:///./fake.db datafaker make-tables --parquet-dir ./input_parquet - + datafaker configure-tables + datafaker configure-generators + datafaker configure-missing + datafaker make-stats datafaker create-tables datafaker create-data --num-passes 10 From 33ba1a8e7afcb75bde6900ba00184a1bce6efefd Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Thu, 10 Sep 2026 15:13:36 +0100 Subject: [PATCH 05/12] Update column evaluator module --- datafaker/evaluators/column_evaluator.py | 8 +++- datafaker/evaluators/distribution_builders.py | 44 ++++++++++--------- datafaker/evaluators/proposal_ranking.py | 4 +- datafaker/evaluators/statistical_fidelity.py | 2 +- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/datafaker/evaluators/column_evaluator.py b/datafaker/evaluators/column_evaluator.py index 4b02df90..a6cbbb5f 100644 --- a/datafaker/evaluators/column_evaluator.py +++ b/datafaker/evaluators/column_evaluator.py @@ -2,6 +2,7 @@ import string from dataclasses import dataclass +from typing import Any from sqlalchemy import Column, Engine, select from sqlalchemy.types import Date, DateTime, Integer, Numeric, Time @@ -66,7 +67,7 @@ def __str__(self): __repr__ = __str__ -def analyse_column(values: list[str]) -> ColumnStats: +def analyse_column(values: list[Any]) -> ColumnStats: """Compute length/space/digit/punctuation statistics over ``values``.""" values = [str(v) for v in values if v is not None] @@ -198,7 +199,7 @@ def setup(self, columns: list[Column], engine: Engine): # which are computed from this sample. rows = conn.execute( select(columns[0]) - .select_from(self.table) + .select_from(columns[0].table) .order_by(Random()) .limit(self.sample_size) ) @@ -259,6 +260,9 @@ def setup(self, columns: list[Column], engine: Engine): def evaluate(self, proposer) -> ProposalEvaluation: """Score one proposer's synthetic data against the sampled real data.""" + assert ( + self.novelty_metric is not None and self.diversity_metric is not None + ), "setup() must be called before evaluate()" synthetic_samples = proposer.generate_data(self.sample_size) # calculate novelty diff --git a/datafaker/evaluators/distribution_builders.py b/datafaker/evaluators/distribution_builders.py index 868f0722..f9298ac5 100644 --- a/datafaker/evaluators/distribution_builders.py +++ b/datafaker/evaluators/distribution_builders.py @@ -36,6 +36,22 @@ def as_vector(self, vocabulary: list[Any]) -> list[float]: class DistributionBuilder(ABC): """Build a real or synthetic value's ``Distribution`` for one feature.""" + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + engine, + table, + column, + extractor: FeatureExtractor = IdentityExtractor(), + sample_size=4000, + ): + """Initialize the engine/table/column/extractor/sample_size shared by every builder.""" + self.engine = engine + self.table = table + self.column = column + self.extractor = extractor + self.sample_size = sample_size + @abstractmethod def build_from_table(self) -> Distribution: """Build the distribution of this feature over the real column.""" @@ -60,12 +76,8 @@ def __init__( sample_size=4000, ): """Initialize a histogram builder for one column feature.""" - self.engine = engine - self.table = table - self.column = column + super().__init__(engine, table, column, extractor, sample_size) self.bins = bins - self.extractor = extractor - self.sample_size = sample_size # Learned from the real data self.mean = None @@ -189,6 +201,12 @@ def build_from_table(self) -> Distribution: def build_from_values(self, values) -> Distribution: """Build the synthetic sample's histogram distribution.""" + if self.edges is None: + # build_from_table() never learned edges - either it hasn't run + # yet, or the real column had no usable values. Either way there + # is nothing to bucket synthetic values against. + return Distribution({}) + counts = Counter() total = 0 @@ -209,22 +227,6 @@ def build_from_values(self, values) -> Distribution: class CategoryBuilder(DistributionBuilder): """Build a categorical distribution over a discrete feature's values.""" - # pylint: disable=too-many-arguments too-many-positional-arguments - def __init__( - self, - engine, - table, - column, - extractor: FeatureExtractor = IdentityExtractor(), - sample_size=4000, - ): - """Initialize a category builder for one column feature.""" - self.engine = engine - self.table = table - self.column = column - self.extractor = extractor - self.sample_size = sample_size - def build_from_table(self): """Build the real column's categorical distribution.""" counter = Counter() diff --git a/datafaker/evaluators/proposal_ranking.py b/datafaker/evaluators/proposal_ranking.py index 8649efbc..f7558e7d 100644 --- a/datafaker/evaluators/proposal_ranking.py +++ b/datafaker/evaluators/proposal_ranking.py @@ -264,7 +264,9 @@ def rank_proposals( EvaluationProfile.FREE_TEXT: (0.5, 0.3, 0.2), EvaluationProfile.TEMPORAL: (0.7, 0.15, 0.15), } - fid_w, nov_w, div_w = profile_weights.get(profile, (0.5, 0.25, 0.25)) + fid_w, nov_w, div_w = ( + profile_weights[profile] if profile in profile_weights else (0.5, 0.25, 0.25) + ) # compute profile-specific score (higher-is-better) combined_scores = [fid_w * f + nov_w * n + div_w * d for (f, n, d) in points] diff --git a/datafaker/evaluators/statistical_fidelity.py b/datafaker/evaluators/statistical_fidelity.py index ea68b2db..d40ab49c 100644 --- a/datafaker/evaluators/statistical_fidelity.py +++ b/datafaker/evaluators/statistical_fidelity.py @@ -34,7 +34,7 @@ class EvaluationPipeline: """One named feature-comparison stage within a profile's fidelity score.""" name: str - builder: DistributionBuilder + builder: type[DistributionBuilder] metric: Metric extractor: FeatureExtractor weight: float = 1.0 From 3f02598923e6088d8fa96a4b124888d0bab9c7ed Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Thu, 10 Sep 2026 15:14:33 +0100 Subject: [PATCH 06/12] Update documentation --- docs/source/builtin_generators.rst | 8 +- docs/source/choosing_a_generator.rst | 123 +++++++++++++++++++++++++++ docs/source/index.rst | 1 + 3 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 docs/source/choosing_a_generator.rst diff --git a/docs/source/builtin_generators.rst b/docs/source/builtin_generators.rst index f95a0bc1..0020e881 100644 --- a/docs/source/builtin_generators.rst +++ b/docs/source/builtin_generators.rst @@ -3,8 +3,10 @@ Built-in generators and proposers This page is a reference for the generator functions that ship with ``datafaker``, and for the *proposers* that suggest which of those functions (and which arguments) -to use for a given column. If you want to write your own generators instead, -see :doc:`custom_generators`. +to use for a given column. If you just want a quick pointer from "my column +looks like this" to "try this generator", see :doc:`choosing_a_generator` +instead. If you want to write your own generators, see +:doc:`custom_generators`. Generators vs. proposers ------------------------- @@ -646,6 +648,8 @@ plus ``person.height``, ``person.weight``, ``datetime.date``, See also --------- +* :doc:`choosing_a_generator` -- a quick lookup from column shape to + generator, for when you want more than the ``propose`` recommendation. * :doc:`custom_generators` -- writing your own row generators and story generators. * :doc:`quickstart` and :doc:`introduction` -- walkthroughs of diff --git a/docs/source/choosing_a_generator.rst b/docs/source/choosing_a_generator.rst new file mode 100644 index 00000000..b7dcc184 --- /dev/null +++ b/docs/source/choosing_a_generator.rst @@ -0,0 +1,123 @@ +Choosing a generator for a column +================================== + +This page is a quick decision aid for the ``propose`` command: "my column looks +like *this* -- which generator should I expect, and why?" For the full list of +generator functions and proposers, and how ``propose`` scores and ranks them, +see :doc:`builtin_generators`. + +Start here +---------- + +In almost all cases, just run ``propose`` on the column and take the +``Recommended`` generator -- it already scores every applicable candidate on +fidelity, novelty and diversity against your real data, which is more +reliable than guessing from the column's type alone. The table below is for +the two situations where you need more than the recommendation: + +* you want to sanity-check *why* a particular generator was recommended, or +* the recommendation looks wrong (or none was given) and you want to know + what else is worth trying. + +.. list-table:: + :widths: 30 40 30 + :header-rows: 1 + + * - Your column looks like... + - Try this generator + - Why + * - A unique ID, e.g. an integer primary key + - ``generic.column_value_provider.increment`` + - Guarantees fresh, unique values by counting up past the highest + existing one; ``propose`` offers this explicitly and it's also the + automatic default for integer primary keys. See + :ref:`the sequence proposer `. + * - A foreign key to another table + - ``generic.column_value_provider.column_value`` + - Picks a random existing value from the referenced column, so + referential integrity is preserved. This is the automatic default for + any foreign key column. + * - A near-unique value that must look real but not *be* real, e.g. an + email address or free-text ID + - Whichever ``generic.*`` value proposer ``propose`` recommends + - ``propose`` applies an extra privacy penalty here: a candidate that + reproduces real values verbatim is scored down in proportion to how + unique the real column is, so the recommendation already accounts for + leak risk. See :ref:`statistical-fidelity-pipelines`. + * - A small set of repeating values, e.g. a status code, category or flag + - ``dist_gen.weighted_choice`` (matches real frequencies exactly), or + ``dist_gen.zipf_choice`` / ``dist_gen.choice`` for a looser fit + - These are the *choice proposers*, offered for any column with up to + 500 distinct values; they differ in how closely they copy the real + frequency of each value. ``weighted_choice`` stores and reproduces + the exact real frequency of every value -- the closest fit, but the + most exposing of how common each value really is. ``zipf_choice`` + instead only ranks values from most- to least-common and assigns + each a Zipf/power-law frequency (the *n*\ th most common value is + chosen ``1/n`` as often as the most common one), which approximates + a typical real-world skew without storing per-value counts. + ``choice`` drops frequency information entirely and picks uniformly + at random -- the loosest fit, but the simplest and most private + option if flattening the distribution is acceptable. + * - A continuous measurement, e.g. a length, weight or price + - ``dist_gen.normal``, ``dist_gen.lognormal`` (if all values are + positive and skewed), or ``dist_gen.uniform_ms`` + - The *continuous distribution proposers* fit directly to the real + column's mean and standard deviation. + * - Two or more numeric columns that vary together, e.g. width and height + - ``dist_gen.multivariate_normal`` / ``dist_gen.multivariate_lognormal`` + - Select all the columns together (``merge``) before running + ``propose`` so it can offer these -- they preserve the correlation + between columns rather than generating each independently. + * - The same, but some rows have one column ``NULL`` and others don't + - the *null-partitioned* generators, e.g. ``dist_gen.alternatives`` + wrapping ``dist_gen.grouped_multivariate_normal`` + - Real data is split into partitions by which columns are ``NULL``; + each partition keeps its own correlation, and one is picked at + generation time weighted by how common it was in the source data. + * - A recognisable human-ish string, e.g. a first name, city or username + - The matching ``generic.person.*`` / ``generic.address.*`` / + ``generic.text.*`` generator + - ``propose`` gives a flat ``+0.15`` boost when the column's *name* + hints at its content (e.g. a column called ``first_name`` boosts + ``generic.person.first_name``) -- a tie-breaker, not an override of a + clearly better statistical fit. + * - A date/time that should stay consistent with another date on the same + row, e.g. a ``discharge_date`` after an ``admission_date`` + - ``generic.anchored_provider.normal_date`` (same table) or + ``.normal_date_fk`` (related table via foreign key) + - Only offered once the earlier date has been given the ``start`` role + (``role set start``). The offset is a clamped, normally distributed + number of seconds taken from the real interval lengths, so the + generated date is never earlier than its anchor. + * - A column that should just agree with a ``datetime`` column elsewhere + in the row, e.g. ``year_of_birth`` alongside ``date_of_birth`` + - ``generic.extract_provider.date`` / ``.year`` / ``.month`` / ``.day`` + - Derives the value from the other column instead of generating it + independently, so the two can never disagree. + * - Nothing else fits, or you just want a placeholder + - ``dist_gen.constant`` + - Always returns the same value (``None`` if the column is nullable). + Also useful as a baseline to compare other candidates' fit against. + +If a column doesn't match any row above, it's still worth running +``propose`` -- the ranking in :doc:`builtin_generators` covers cases (and +combinations of ``String``/``Numeric``/``Date`` type with real-data shape) +that are awkward to summarise as a simple lookup table. + +When ``propose`` recommends nothing +------------------------------------ + +For a primary key or other unique column, ``propose`` will withhold a +recommendation entirely if every candidate's synthetic sample duplicates too +often to satisfy uniqueness -- rather than confidently suggesting something +that would break a constraint. Look for the sequence proposer +(``generic.column_value_provider.increment``) instead. + +See also +--------- + +* :doc:`builtin_generators` -- full generator and proposer reference, plus + how the ``Recommended`` score is computed. +* :doc:`quickstart` -- walkthrough of ``propose``, ``compare`` and ``set`` in + the ``configure-generators`` CLI. diff --git a/docs/source/index.rst b/docs/source/index.rst index 4c87a007..12337283 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -36,6 +36,7 @@ Contents: orm configuration health_data + Choosing a Generator Custom Generators Built-in Generators and Proposers api From f90ce81070f73ff9a96c568120ca7ddffdb32949 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Thu, 10 Sep 2026 15:15:05 +0100 Subject: [PATCH 07/12] Update unit tests --- tests/test_evaluators_distribution_builders.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_evaluators_distribution_builders.py b/tests/test_evaluators_distribution_builders.py index e3a79e27..4b79e829 100644 --- a/tests/test_evaluators_distribution_builders.py +++ b/tests/test_evaluators_distribution_builders.py @@ -123,6 +123,7 @@ def test_values_outside_edges_are_clamped_to_end_buckets(self) -> None: dist = self.builder.build_from_values([-1000, 1000]) self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) buckets = set(dist.probabilities.keys()) + assert self.builder.edges is not None self.assertTrue(all(0 <= b <= len(self.builder.edges) - 2 for b in buckets)) From 6ae3a5eca20f93a4b19a2bd9b998977935e394ca Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Thu, 10 Sep 2026 16:44:25 +0100 Subject: [PATCH 08/12] Fix unit tests failure and errors --- datafaker/interactive/generators.py | 2 +- docs/source/builtin_generators.rst | 2 +- tests/test_interactive_generators.py | 12 ++++++++++-- tests/test_rst.py | 5 +++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index 621f51e6..60dda6d2 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -1064,7 +1064,7 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st theme = get_active_theme() limit = 5 include_all = _arg.strip().lower() == "all" - props = self._get_proposer_proposals() + props = self._get_proposer_proposals(include_all) sample = self._get_column_data(limit) self._print_source_sample(sample, theme) if not props: diff --git a/docs/source/builtin_generators.rst b/docs/source/builtin_generators.rst index 0020e881..953094ba 100644 --- a/docs/source/builtin_generators.rst +++ b/docs/source/builtin_generators.rst @@ -387,7 +387,7 @@ For a single-column ``propose``, every candidate proposer that survives the type-compatibility filter (see below) is additionally scored by ``datafaker.evaluators.ColumnEvaluator`` and ranked by ``datafaker.evaluators.proposal_ranking.rank_proposals``. This produces a - table shown by ``propose``, with a ``Recommended: N. `` line +table shown by ``propose``, with a ``Recommended: N. `` line above it. Evaluation profile diff --git a/tests/test_interactive_generators.py b/tests/test_interactive_generators.py index 08d4587b..1a59e4c7 100644 --- a/tests/test_interactive_generators.py +++ b/tests/test_interactive_generators.py @@ -427,7 +427,11 @@ def test_aggregate_queries_merge(self) -> None: column = "position" generator = "dist_gen.uniform_ms" gc.do_next(f"string.{column}") - gc.do_propose("") + # "position" is an Integer column, and dist_gen.uniform_ms produces + # float output that default 'propose' now excludes as type- + # incompatible (see GeneratorCmd._is_integer_incompatible) - 'all' + # bypasses that filter so this generator is still selectable here. + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[f"{generator}"][0])) gc.do_quit("") @@ -604,7 +608,11 @@ def test_existing_configuration_remains(self) -> None: column = "position" generator = "dist_gen.uniform_ms" gc.do_next(f"string.{column}") - gc.do_propose("") + # "position" is an Integer column, and dist_gen.uniform_ms produces + # float output that default 'propose' now excludes as type- + # incompatible (see GeneratorCmd._is_integer_incompatible) - 'all' + # bypasses that filter so this generator is still selectable here. + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[generator][0])) gc.do_quit("") diff --git a/tests/test_rst.py b/tests/test_rst.py index ee89b5b5..edd87ac9 100644 --- a/tests/test_rst.py +++ b/tests/test_rst.py @@ -40,6 +40,7 @@ def test_dir(self) -> None: # Ignore errors if they contain any of these strings allowed_errors = [ 'No role entry for "ref" in module', + 'No role entry for "doc" in module', 'No directive entry for "toctree"', 'No directive entry for "automodule"', 'No directive entry for "literalinclude"', @@ -51,6 +52,10 @@ def test_dir(self) -> None: 'Hyperlink target "page-quickstart" is not referenced.', 'Hyperlink target "page-installation" is not referenced.', 'Hyperlink target "story-generators" is not referenced.', + 'Hyperlink target "mimesis-string-list" is not referenced', + 'Hyperlink target "increment-proposer" is not referenced', + 'Hyperlink target "evaluating-and-ranking-proposals" is not referenced', + 'Hyperlink target "statistical-fidelity-pipelines" is not referenced', ] filtered_errors = [ file_error From e906ab2a4ab3cd66e4fc13a276d8522780d0c2e0 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Thu, 10 Sep 2026 17:13:41 +0100 Subject: [PATCH 09/12] Fix unit tests error --- datafaker/evaluators/distribution_builders.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/datafaker/evaluators/distribution_builders.py b/datafaker/evaluators/distribution_builders.py index f9298ac5..d6afabef 100644 --- a/datafaker/evaluators/distribution_builders.py +++ b/datafaker/evaluators/distribution_builders.py @@ -164,17 +164,23 @@ def build_from_table(self) -> Distribution: # raw-value pulls above). # with self.engine.connect() as conn: - # Group by the bucket expression itself, not its output alias: - # MSSQL (unlike Postgres/DuckDB) rejects a SELECT-list alias - # referenced from GROUP BY. + # Compute the bucket expression once, in an inner subquery, and + # group by the resulting materialized column in the outer query. + # MSSQL rejects both `GROUP BY ` (a SELECT-list alias + # referenced from GROUP BY) and `GROUP BY ` + # (its query planner does not recognize two separately-bound + # occurrences of the same parameterized expression as + # equivalent) - grouping by an actual column of a subquery is + # the one form every dialect (Postgres/DuckDB/MSSQL) accepts. bucket_expr = func.floor((feature_expr - bottom) / width) + inner = ( + select(bucket_expr.label("bucket")).select_from(self.table).subquery() + ) rows = conn.execute( select( - bucket_expr.label("bucket"), + inner.c.bucket, func.count().label("count"), # pylint: disable=not-callable - ) - .select_from(self.table) - .group_by(bucket_expr) + ).group_by(inner.c.bucket) ).all() # total must come from this same full-table query, not from the From 1b98cf70d92dae4fe40c502b13023377bc076792 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Fri, 11 Sep 2026 12:59:19 +0100 Subject: [PATCH 10/12] Refine penalty calculation --- datafaker/evaluators/column_evaluator.py | 29 ++++++++++- datafaker/evaluators/proposal_ranking.py | 60 +++++++++++++++++------ datafaker/interactive/generators.py | 5 ++ docs/source/builtin_generators.rst | 44 ++++++++++++----- docs/source/choosing_a_generator.rst | 2 +- tests/test_evaluators_proposal_ranking.py | 5 +- 6 files changed, 114 insertions(+), 31 deletions(-) diff --git a/datafaker/evaluators/column_evaluator.py b/datafaker/evaluators/column_evaluator.py index a6cbbb5f..1f605382 100644 --- a/datafaker/evaluators/column_evaluator.py +++ b/datafaker/evaluators/column_evaluator.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import Any -from sqlalchemy import Column, Engine, select +from sqlalchemy import Column, Engine, UniqueConstraint, select from sqlalchemy.types import Date, DateTime, Integer, Numeric, Time from datafaker.dialects import Random @@ -167,6 +167,7 @@ def __init__(self): self.column = None self.table = None self.is_primary_key = False + self.is_unique_constrained = False self.real_values = [] self.column_is_numeric = False self.real_uniqueness = 0.0 @@ -185,6 +186,14 @@ def setup(self, columns: list[Column], engine: Engine): # A composite key needs every constituent column marked primary_key; # this reduces to the single column's own flag in the common case. self.is_primary_key = bool(columns) and all(c.primary_key for c in columns) + # A real UNIQUE constraint/index needs the same uniqueness guarantee + # as a primary key, even though it isn't one - e.g. a UNIQUE email + # column. Distinct from is_primary_key so the two can be OR'd + # together (needs_uniqueness) without conflating "is the key" with + # "must be unique". + self.is_unique_constrained = self.is_primary_key or self._has_unique_constraint( + columns + ) with engine.connect() as conn: if len(columns) == 1: @@ -258,6 +267,24 @@ def setup(self, columns: list[Column], engine: Engine): self.statistical_fidelity = None self.profile = EvaluationProfile.IDENTIFIER + @staticmethod + def _has_unique_constraint(columns: list[Column]) -> bool: + """Whether exactly this column set is covered by a UNIQUE constraint/index.""" + if not columns: + return False + table = columns[0].table + column_set = set(columns) + for constraint in table.constraints: + if ( + isinstance(constraint, UniqueConstraint) + and set(constraint.columns) == column_set + ): + return True + for index in table.indexes: + if index.unique and set(index.columns) == column_set: + return True + return False + def evaluate(self, proposer) -> ProposalEvaluation: """Score one proposer's synthetic data against the sampled real data.""" assert ( diff --git a/datafaker/evaluators/proposal_ranking.py b/datafaker/evaluators/proposal_ranking.py index f7558e7d..9a3b9074 100644 --- a/datafaker/evaluators/proposal_ranking.py +++ b/datafaker/evaluators/proposal_ranking.py @@ -68,6 +68,15 @@ # on top of a perfectly good pick. WEAK_WINNER_SCORE_THRESHOLD = 0.5 +# Floor applied to the self-duplication penalty (see rank_proposals) for a +# column that doesn't actually need guaranteed-unique values (no PRIMARY +# KEY/UNIQUE constraint). Self-duplication still matters there - a +# generator whose synthetic sample is mostly repeats of a tiny fixed pool +# (e.g. a canned-quote generator) makes for an obviously repetitive column +# even without breaking a constraint - but it shouldn't be crushed as hard +# as it would be for a column where duplicates are a correctness bug. +SELF_DUPLICATION_PENALTY_FLOOR = 0.4 + def keyword_match( column_name: str | None, proposer_name: str @@ -117,6 +126,7 @@ class ProposalRanking: weights: tuple[float, float, float] real_uniqueness: float = 0.0 penalty_applies_to_all: bool = False + self_duplication_floored: bool = False fronts: list[int] = field(default_factory=list) scores: list[float] = field(default_factory=list) weak_recommendation_warning: str | None = None @@ -204,6 +214,7 @@ def rank_proposals( real_uniqueness: float = 0.0, is_numeric_column: bool = False, is_primary_key: bool = False, + is_unique_constrained: bool = False, ) -> ProposalRanking: """Rank proposals by fidelity, novelty and diversity using Pareto fronts. @@ -323,12 +334,24 @@ def rank_proposals( # vocabulary (a gender, a status) is expected to have low # synthetic_uniqueness too, and shouldn't be penalized for it. # Unlike the resample penalty, this applies to every candidate - # unconditionally - self-duplication breaks a PRIMARY KEY/UNIQUE - # constraint regardless of column type or whether the proposer is a - # resampler. + # regardless of column type or whether the proposer is a resampler - + # but its full strength is only warranted when the column actually + # needs unique values (a PRIMARY KEY/UNIQUE constraint); it's floored + # below (SELF_DUPLICATION_PENALTY_FLOOR) otherwise, since a merely + # observed-unique column (e.g. free-text descriptions that happen to + # all differ) doesn't have a correctness reason to punish a small, + # repetitive-but-otherwise-plausible pool as hard as a real key would. self_duplication_penalty = 1.0 - real_uniqueness * ( 1.0 - result.synthetic_uniqueness ) + if not (is_primary_key or is_unique_constrained): + # No constraint actually requires unique values here, so don't + # let this penalty alone crush the score - just discount it + # relative to a candidate that can sustain the real column's + # observed distinctness. + self_duplication_penalty = max( + self_duplication_penalty, SELF_DUPLICATION_PENALTY_FLOOR + ) penalty_multipliers[idx] = resample_penalty * self_duplication_penalty combined_scores[idx] *= penalty_multipliers[idx] @@ -464,6 +487,7 @@ def rank_proposals( f"{diversity_scores[idx]:.6f}", f"{real_uniqueness:.3f}", f"{result.copy_fraction:.3f}", + f"{result.synthetic_uniqueness:.3f}", f"{penalty_multipliers[idx]:.3f}", ] # color entire row for Pareto front 1 @@ -480,6 +504,7 @@ def rank_proposals( weights=(fid_w, nov_w, div_w), real_uniqueness=real_uniqueness, penalty_applies_to_all=apply_to_every_proposer, + self_duplication_floored=not (is_primary_key or is_unique_constrained), fronts=list(front_of), scores=list(combined_scores), weak_recommendation_warning=weak_recommendation_warning, @@ -505,17 +530,16 @@ def format_ranking_display( recommendation = ( f"Recommended: {rec_num}. {rec_name} — {ranking.recommended_reason}\n" ) - if ranking.penalty_applies_to_all: - penalty_scope = ( - "applied to every generator (this is a numeric column, so a high" - " copy_fraction always means the real range is dense, not a coincidental" - " vocabulary match)" - ) - else: - penalty_scope = ( - "applied only to resampling generators (dist_gen.choice/weighted_choice/" - "zipf_choice - see Penalty column); 1.0 (no effect) for everyone else" - ) + resample_scope = ( + "every generator (numeric column)" + if ranking.penalty_applies_to_all + else "resamplers only (dist_gen.choice/weighted_choice/zipf_choice)" + ) + self_dup_scope = ( + f"floored at {SELF_DUPLICATION_PENALTY_FLOOR:.1f} (no PK/UNIQUE constraint)" + if ranking.self_duplication_floored + else "full strength (PK/UNIQUE constrained)" + ) profile_summary = ( f"Profile: {profile.name if profile is not None else 'UNKNOWN'} | " f"Real uniqueness: {ranking.real_uniqueness:.3f} | " @@ -523,8 +547,12 @@ def format_ranking_display( f"Score = clamp( ({fid_w:.2f}*Fidelity + {nov_w:.2f}*Novelty" f" + {div_w:.2f}*Diversity + Keyword) " "x Penalty , 0, 1 )\n" - " Keyword: +0.15 if the column name hints at this generator's kind (see Keyword column)\n" - f" Penalty: 1 - real_uniqueness x copy_fraction, {penalty_scope}" + " Keyword: +0.15 for a column-name hint (see Keyword column)\n" + " Penalty = resample_penalty x self_duplication_penalty" + " (see Copies / Synth.Uniq columns)\n" + f" resample_penalty (1 - real_uniqueness x copy_fraction): {resample_scope}\n" + " self_duplication_penalty (1 - real_uniqueness x (1 - synthetic_uniqueness)):" + f" {self_dup_scope}" ) return ProposalRankingDisplay( recommendation=recommendation, diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index 60dda6d2..a2ca912f 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -1109,6 +1109,9 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st real_uniqueness = getattr(column_evaluator, "real_uniqueness", 0.0) is_numeric_column = getattr(column_evaluator, "column_is_numeric", False) is_primary_key = getattr(column_evaluator, "is_primary_key", False) + is_unique_constrained = getattr( + column_evaluator, "is_unique_constrained", False + ) ranking = rank_proposals( results, profile, @@ -1117,6 +1120,7 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st real_uniqueness=real_uniqueness, is_numeric_column=is_numeric_column, is_primary_key=is_primary_key, + is_unique_constrained=is_unique_constrained, ) display = format_ranking_display(ranking, results, profile) @@ -1171,6 +1175,7 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st "Diversity", "Uniqueness", "Copies", + "Synth.Uniq", "Penalty", ], rows_to_show, diff --git a/docs/source/builtin_generators.rst b/docs/source/builtin_generators.rst index 953094ba..91ddf3b0 100644 --- a/docs/source/builtin_generators.rst +++ b/docs/source/builtin_generators.rst @@ -503,17 +503,39 @@ The three normalized scores are combined into one ``Score`` per candidate as see ``KEYWORD_GENERATOR_HINTS`` in ``proposal_ranking.py`` for the full list) -- a cheap, complementary signal used only to break near-ties, never to override a clearly better statistical result. -* **Penalty** (``1 - real_uniqueness * copy_fraction``, further reduced by how - much a candidate duplicates against its *own* output) discounts a - choice-style proposer (``dist_gen.choice``/``weighted_choice``/ - ``zipf_choice``) -- or, for a numeric column, *any* proposer -- for - reproducing real values verbatim, scaled by how unique the real column - actually is. This is a privacy safeguard: resampling real values is - expected and harmless for a low-uniqueness column (a gender, a status) but - a genuine leak for a near-unique one (an email, a real ID). It has no - effect (``1.0``) on other proposers for non-numeric columns, where an - overlap with real values is usually just a naturally shared vocabulary - (e.g. common first names), not memorization. +* **Penalty** is the product of two independent factors, each scaled by how + unique the real column actually is: + + .. code-block:: text + + Penalty = resample_penalty * self_duplication_penalty + + * ``resample_penalty = 1 - real_uniqueness * copy_fraction`` is a privacy + safeguard against reproducing real values verbatim. ``copy_fraction`` is + the fraction of a candidate's synthetic sample that exactly matches a + real value. It only applies to choice-style proposers + (``dist_gen.choice``/``weighted_choice``/``zipf_choice``) -- or, for a + numeric column, *any* proposer -- since resampling real values is + expected and harmless for a low-uniqueness column (a gender, a status) + but a genuine leak for a near-unique one (an email, a real ID). It has no + effect (``1.0``) on other proposers for non-numeric columns, where an + overlap with real values is usually just a naturally shared vocabulary + (e.g. common first names), not memorization. + * ``self_duplication_penalty = 1 - real_uniqueness * (1 - synthetic_uniqueness)`` + guards against a candidate duplicating against its *own* output -- + ``synthetic_uniqueness`` is the fraction of distinct values in its 4000 + synthetic samples. This is a different failure mode from + ``resample_penalty``: a candidate can have a tiny fixed pool of possible + outputs (e.g. a canned-quote generator with only a few dozen distinct + strings) without ever coincidentally matching a *real* value, yet still + be unusable for a column that needs many distinct rows. It's computed + for every candidate, but only applied at full strength when the column + actually needs unique values (a primary key or a ``UNIQUE``-constrained + column) -- otherwise it's floored at ``0.4``, discounting a + self-duplicating-but-otherwise-plausible candidate rather than crushing + its score to near zero over a constraint that doesn't apply here. The + ``Synth.Uniq`` column in the ``propose`` table shows the raw value this + factor is computed from. Candidates are also grouped into Pareto fronts (front 1 = not strictly dominated on fidelity, novelty *and* diversity simultaneously by any diff --git a/docs/source/choosing_a_generator.rst b/docs/source/choosing_a_generator.rst index b7dcc184..a47c2dc8 100644 --- a/docs/source/choosing_a_generator.rst +++ b/docs/source/choosing_a_generator.rst @@ -43,7 +43,7 @@ the two situations where you need more than the recommendation: - ``propose`` applies an extra privacy penalty here: a candidate that reproduces real values verbatim is scored down in proportion to how unique the real column is, so the recommendation already accounts for - leak risk. See :ref:`statistical-fidelity-pipelines`. + leak risk. See :ref:`evaluating-and-ranking-proposals`. * - A small set of repeating values, e.g. a status code, category or flag - ``dist_gen.weighted_choice`` (matches real frequencies exactly), or ``dist_gen.zipf_choice`` / ``dist_gen.choice`` for a looser fit diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py index 5650bd01..2e8b73c8 100644 --- a/tests/test_evaluators_proposal_ranking.py +++ b/tests/test_evaluators_proposal_ranking.py @@ -370,7 +370,7 @@ def test_penalty_scope_text_reflects_numeric_flag(self) -> None: numeric_ranking, results, EvaluationProfile.IDENTIFIER ) self.assert_str_in( - "applied to every generator", numeric_display.profile_summary + "every generator (numeric column)", numeric_display.profile_summary ) string_ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) @@ -378,7 +378,8 @@ def test_penalty_scope_text_reflects_numeric_flag(self) -> None: string_ranking, results, EvaluationProfile.SHORT_TEXT ) self.assert_str_in( - "applied only to resampling generators", string_display.profile_summary + "resamplers only (dist_gen.choice/weighted_choice/zipf_choice)", + string_display.profile_summary, ) def test_rows_and_fronts_pass_through_unchanged(self) -> None: From 0f6d61757f41e57107461b3f9c92c950fc5b1c76 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Fri, 11 Sep 2026 13:22:35 +0100 Subject: [PATCH 11/12] Refine penalty calculation --- .github/workflows/docs.yml | 2 +- datafaker/evaluators/proposal_ranking.py | 123 ++++++++++------------- 2 files changed, 55 insertions(+), 70 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d496a391..0b53c6d3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,7 +4,7 @@ on: push: branches: # need this to be able to run the workflow until it has been merged into main - - yhong123/131-update_documentation + - yhong123/112-column_evaluators workflow_dispatch: env: PYTHON_VERSION: "3.12" diff --git a/datafaker/evaluators/proposal_ranking.py b/datafaker/evaluators/proposal_ranking.py index 9a3b9074..e9b06e1d 100644 --- a/datafaker/evaluators/proposal_ranking.py +++ b/datafaker/evaluators/proposal_ranking.py @@ -204,6 +204,51 @@ def _crowding_distances( return crowding +def _penalty_for_candidate( + result: ProposalEvaluation, + real_uniqueness: float, + apply_to_every_proposer: bool, + needs_uniqueness: bool, +) -> float: + """Combine the resample and self-duplication penalties for one candidate. + + ChoiceProposer and its variants (dist_gen.choice/weighted_choice/ + zipf_choice) resample directly from the column's own observed values - + by construction, they can only ever emit values already in the real + data. That's the *correct* thing to do for a column with a small, + shared vocabulary (a status code, a gender) - reproducing real values + there is expected and unavoidable - but a privacy problem for a column + whose real values are meant to be unique per row (an email, a genuine + ID). Numeric columns get the same treatment regardless of proposer + (apply_to_every_proposer): a plain number has no shared human + vocabulary to explain a coincidental match, so a high copy_fraction + there means the real value range is dense, not a legitimate overlap. + + Separately, self_duplication_penalty catches a candidate duplicating + against its OWN output (synthetic_uniqueness) rather than against the + real data (copy_fraction, above) - e.g. dist_gen.constant on a column + whose fixed value happens not to appear in the sampled real data gets + copy_fraction=0 (no resample penalty) despite duplicating every row + against itself. It applies to every candidate, but its full strength + is only warranted when the column actually needs unique values (a + PRIMARY KEY/UNIQUE constraint); it's floored otherwise, since a merely + observed-unique column (e.g. free-text descriptions that happen to all + differ) doesn't have a correctness reason to punish a small, + repetitive-but-otherwise-plausible pool as hard as a real key would. + """ + resample_penalty = 1.0 + if isinstance(result.proposer, ChoiceProposer) or apply_to_every_proposer: + resample_penalty = 1.0 - real_uniqueness * result.copy_fraction + self_duplication_penalty = 1.0 - real_uniqueness * ( + 1.0 - result.synthetic_uniqueness + ) + if not needs_uniqueness: + self_duplication_penalty = max( + self_duplication_penalty, SELF_DUPLICATION_PENALTY_FLOOR + ) + return resample_penalty * self_duplication_penalty + + # pylint: disable=too-many-arguments too-many-positional-arguments # pylint: disable=too-many-locals too-many-statements def rank_proposals( @@ -282,77 +327,17 @@ def rank_proposals( # compute profile-specific score (higher-is-better) combined_scores = [fid_w * f + nov_w * n + div_w * d for (f, n, d) in points] - # ChoiceProposer and its variants (dist_gen.choice/weighted_choice/ - # zipf_choice) resample directly from the column's own observed values - - # by construction, they can only ever emit values already in the real - # data. That's the *correct* thing to do for a column with a small, - # shared vocabulary (a status code, a gender) - reproducing real values - # there is expected and unavoidable - but a privacy problem for a column - # whose real values are meant to be unique per row (an email, a genuine - # ID): it isn't modeling the distribution, it's handing back real - # records. Scale the penalty by how unique the real column actually is - # (near 0 for a low-uniqueness column, where this is fine; near full - # strength for a near-unique one, where it's a leak) and by how much of - # this proposer's own output is actually copied. - # - # Other proposers are, in general, not penalized for copy_fraction: for - # them a high copy_fraction is usually a coincidental side effect of a - # naturally overlapping real-world vocabulary (e.g. common first names), - # not a sign of memorization - penalizing it there wrongly punishes - # exactly the generators that model the real distribution best (verified: - # applying this penalty to person.first_name for a first-name column - # incorrectly hands the win back to a generic word generator). - # - # Numeric columns are the exception, regardless of whether they end up - # profiled IDENTIFIER (high uniqueness, e.g. a primary key) or - # CATEGORICAL (low uniqueness, e.g. a foreign key referencing a small - # set of rows): a plain number has no shared human vocabulary to explain - # a coincidental match. A high copy_fraction there instead means the - # real value range is dense (e.g. a gapless sequence of small integer - # IDs), so *any* generator landing in that range trivially "matches" - # almost every time, independent of whether it resamples. That's the - # same underlying signal as ChoiceProposer's, just from a proposer that - # isn't one - e.g. a generic "weight" generator whose typical output - # range happens to overlap a table's ID range - so it gets the same - # penalty here. A low-cardinality *string* column (a gender, a status) - # doesn't get this: there, high copy_fraction from a well-calibrated - # generator is legitimate/expected, not a coincidental range overlap. + # See _penalty_for_candidate for what resample_penalty and + # self_duplication_penalty actually guard against and why. apply_to_every_proposer = is_numeric_column - penalty_multipliers = [1.0] * len(results) - for idx, result in enumerate(results): - resample_penalty = 1.0 - if isinstance(result.proposer, ChoiceProposer) or apply_to_every_proposer: - resample_penalty = 1.0 - real_uniqueness * result.copy_fraction - # A second, separate penalty for duplicating against the candidate's - # OWN output (synthetic_uniqueness) rather than against the real - # data (copy_fraction, above). These catch different failure modes: - # e.g. dist_gen.constant on a column whose fixed value happens not - # to appear in the sampled real data gets copy_fraction=0 - no - # resample penalty at all - despite duplicating every single row - # against itself. Scaled by real_uniqueness for the same reason as - # the resample penalty: a column with a small, legitimate shared - # vocabulary (a gender, a status) is expected to have low - # synthetic_uniqueness too, and shouldn't be penalized for it. - # Unlike the resample penalty, this applies to every candidate - # regardless of column type or whether the proposer is a resampler - - # but its full strength is only warranted when the column actually - # needs unique values (a PRIMARY KEY/UNIQUE constraint); it's floored - # below (SELF_DUPLICATION_PENALTY_FLOOR) otherwise, since a merely - # observed-unique column (e.g. free-text descriptions that happen to - # all differ) doesn't have a correctness reason to punish a small, - # repetitive-but-otherwise-plausible pool as hard as a real key would. - self_duplication_penalty = 1.0 - real_uniqueness * ( - 1.0 - result.synthetic_uniqueness + needs_uniqueness = is_primary_key or is_unique_constrained + penalty_multipliers = [ + _penalty_for_candidate( + result, real_uniqueness, apply_to_every_proposer, needs_uniqueness ) - if not (is_primary_key or is_unique_constrained): - # No constraint actually requires unique values here, so don't - # let this penalty alone crush the score - just discount it - # relative to a candidate that can sustain the real column's - # observed distinctness. - self_duplication_penalty = max( - self_duplication_penalty, SELF_DUPLICATION_PENALTY_FLOOR - ) - penalty_multipliers[idx] = resample_penalty * self_duplication_penalty + for result in results + ] + for idx in range(len(results)): combined_scores[idx] *= penalty_multipliers[idx] # soft boost for generators whose kind the column name itself hints at From 5a327ccea0b774eb177d8b7a76755a4d4093fa23 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Fri, 9 Oct 2026 15:30:25 +0100 Subject: [PATCH 12/12] Address review feedback on column evaluation and propose ranking Responds to tim-band's review on PR #137: - Scope the float-for-Integer type guard to IDENTIFIER-profile columns only (CATEGORICAL columns already self-correct on fidelity), computed per 'propose' call instead of cached in _get_proposer_proposals - Give PK/UNIQUE columns precedence to proposers that structurally guarantee fresh values (new Proposer.guarantees_fresh_uniqueness(), set on IncrementProposer) over the generic fidelity score, which can never credit a sequence proposer for deliberately exceeding the observed range - Remove the old unranked per-proposer 'propose' output and the include_all caching it required; highlight the recommended row (not just Pareto front 1) with dedicated theme colours (recommend/near_miss) that remain visible with theming off - Fold ColumnEvaluator.setup() into its constructor and consolidate is_numeric_sql_type into db_utils.py, shared with partitioned.py - Fix WeekdayExtractor: its SQL and Python sides used disagreeing day-of-week numbering conventions, silently comparing mismatched category labels for every temporal column; replaced with portable floor/modulo arithmetic on the existing days-since-epoch value - Fix VowelConsonantPatternExtractor miscounting accented vowels as consonants (Unicode NFKD base-letter decomposition) - Rename EvaluationPipeline to EvaluationCriterion (and PIPELINE(S) to CRITERION/CRITERIA throughout): these are independent, weighted measures combined by weighted average, not a sequential pipeline - Add 'help ranking'/'help pareto' topics; rewrite the 'propose' docstring to match current behaviour - Assorted docstring/type-hint/wording fixes and test clarity fixes raised in review Co-Authored-By: Claude Sonnet 5 --- datafaker/db_utils.py | 6 + datafaker/evaluators/column_evaluator.py | 92 +++--- datafaker/evaluators/distribution_builders.py | 47 +-- datafaker/evaluators/evaluation_profile.py | 2 +- datafaker/evaluators/feature_extractors.py | 53 +++- datafaker/evaluators/metrics.py | 2 +- datafaker/evaluators/proposal_ranking.py | 154 ++++++++-- datafaker/evaluators/statistical_fidelity.py | 118 ++++---- datafaker/interactive/generators.py | 272 +++++++++--------- datafaker/proposers/base.py | 15 + datafaker/proposers/partitioned.py | 5 +- datafaker/proposers/sequence.py | 4 + datafaker/theme.py | 15 +- docs/source/builtin_generators.rst | 52 ++-- docs/source/choosing_a_generator.rst | 2 +- tests/test_evaluators_column_evaluator.py | 31 +- tests/test_evaluators_feature_extractors.py | 46 ++- tests/test_evaluators_proposal_ranking.py | 2 +- tests/test_evaluators_statistical_fidelity.py | 60 ++-- tests/test_interactive_choice_generators.py | 6 +- tests/test_interactive_date_generators.py | 17 +- tests/test_interactive_generators.py | 59 ++-- tests/test_interactive_generators_extract.py | 5 +- ...test_interactive_generators_partitioned.py | 5 +- tests/test_rst.py | 2 +- 25 files changed, 659 insertions(+), 413 deletions(-) diff --git a/datafaker/db_utils.py b/datafaker/db_utils.py index 7ed5f17c..7b52de8d 100644 --- a/datafaker/db_utils.py +++ b/datafaker/db_utils.py @@ -39,6 +39,7 @@ MetaData, Table, ) +from sqlalchemy.types import Integer, Numeric, TypeEngine from typer import Exit from datafaker.utils import ( @@ -299,6 +300,11 @@ def get_fk_column_between( return (None, None) +def is_numeric_sql_type(column_type: TypeEngine | None) -> bool: + """Whether a SQLAlchemy column type is Numeric or Integer.""" + return isinstance(column_type, (Numeric, Integer)) + + def fk_refers_to_ignored_table(fk: ForeignKey) -> bool: """ Test if this foreign key refers to an ignored table. diff --git a/datafaker/evaluators/column_evaluator.py b/datafaker/evaluators/column_evaluator.py index 1f605382..feef34eb 100644 --- a/datafaker/evaluators/column_evaluator.py +++ b/datafaker/evaluators/column_evaluator.py @@ -2,11 +2,12 @@ import string from dataclasses import dataclass -from typing import Any +from typing import Any, Sequence -from sqlalchemy import Column, Engine, UniqueConstraint, select -from sqlalchemy.types import Date, DateTime, Integer, Numeric, Time +from sqlalchemy import Column, Engine, Table, UniqueConstraint, select +from sqlalchemy.types import Date, DateTime, Time +from datafaker.db_utils import is_numeric_sql_type from datafaker.dialects import Random from datafaker.evaluators.evaluation_profile import EvaluationProfile from datafaker.evaluators.metrics import ( @@ -32,7 +33,7 @@ class ColumnStats: punctuation_ratio: float @property - def uniqueness(self): + def uniqueness(self) -> float: """Fraction of rows whose value is distinct from the others.""" return self.unique_count / max(self.row_count, 1) @@ -45,11 +46,11 @@ class ProposalEvaluation: novelty: float diversity: float overall_score: float - pipeline_scores: dict[str, float] + criterion_scores: dict[str, float] copy_fraction: float = 0.0 synthetic_uniqueness: float = 1.0 - def __str__(self): + def __str__(self) -> str: """Render a human-readable summary of the evaluation.""" lines = [ f"{self.proposer.name()} " @@ -59,7 +60,7 @@ def __str__(self): f" (copies={self.copy_fraction:.6f})" ] - for name, score in self.pipeline_scores.items(): + for name, score in self.criterion_scores.items(): lines.append(f" {name:<15} {score:.6f}") return "\n".join(lines) @@ -69,6 +70,9 @@ def __str__(self): def analyse_column(values: list[Any]) -> ColumnStats: """Compute length/space/digit/punctuation statistics over ``values``.""" + # setup() already routes Numeric/Integer and Date/DateTime/Time columns + # elsewhere, so this only ever sees whatever's left (Boolean/UUID/Enum/ + # JSON/...) - stringify defensively since those aren't already str. values = [str(v) for v in values if v is not None] total_chars = sum(len(v) for v in values) @@ -111,18 +115,16 @@ def choose_profile(stats: ColumnStats) -> EvaluationProfile: return EvaluationProfile.SHORT_TEXT -def choose_numeric_profile(values) -> EvaluationProfile: +def choose_numeric_profile(values: Sequence[Any]) -> EvaluationProfile: """Pick a numeric column's evaluation profile from its cardinality. Decide whether an Integer/Numeric column behaves like a small set of repeated categories (a status code, rating, or flag) or like a continuous / high-cardinality quantity (an age, salary, or identifier). - - Previously every Integer column was assumed to be categorical and every - Numeric column was assumed to be an identifier, regardless of how many - distinct values it actually had - which misclassifies something like an - integer age or salary column. This applies the same cardinality check - already used for string columns instead. + A fixed assumption from SQL type alone can't tell these apart - an + Integer column isn't always categorical (an age, say) and a Numeric + one isn't always an identifier - so this applies the same cardinality + check already used for string columns (see ``choose_profile``) instead. """ non_null = [v for v in values if v is not None] if not non_null: @@ -135,7 +137,7 @@ def choose_numeric_profile(values) -> EvaluationProfile: return EvaluationProfile.IDENTIFIER -def looks_like_email(values) -> bool: +def looks_like_email(values: Sequence[Any]) -> bool: """Heuristically decide whether a sample of values looks like emails.""" nonempty = [str(v).strip() for v in values if v is not None and str(v).strip()] if not nonempty: @@ -157,32 +159,17 @@ def looks_like_email(values) -> bool: class ColumnEvaluator: # pylint: disable=too-many-instance-attributes """Evaluate candidate proposers against one (or several merged) columns.""" - def __init__(self): - """Initialize an unconfigured evaluator; call setup() before use.""" - self.engine = None - self.novelty_metric = None - self.diversity_metric = None - self.sample_size = 4000 - self.columns = [] - self.column = None - self.table = None - self.is_primary_key = False - self.is_unique_constrained = False - self.real_values = [] - self.column_is_numeric = False - self.real_uniqueness = 0.0 - self.statistical_fidelity = None - self.profile = None - - def setup(self, columns: list[Column], engine: Engine): - """Sample the real column(s) and pick the evaluation profile/pipelines.""" - self.engine = engine - self.novelty_metric = NoveltyMetric() - self.diversity_metric = DiversityMetric() - - self.columns = columns - self.column = columns[0] if len(columns) == 1 else None - self.table = columns[0].table if columns else None + def __init__(self, columns: list[Column], engine: Engine) -> None: + """Sample the real column(s) and pick the evaluation profile/criteria.""" + assert columns, "ColumnEvaluator needs at least one column" + self.sample_size: int = 4000 + self.engine: Engine = engine + self.novelty_metric: NoveltyMetric = NoveltyMetric() + self.diversity_metric: DiversityMetric = DiversityMetric() + + self.columns: list[Column] = columns + self.column: Column | None = columns[0] if len(columns) == 1 else None + self.table: Table = columns[0].table # A composite key needs every constituent column marked primary_key; # this reduces to the single column's own flag in the common case. self.is_primary_key = bool(columns) and all(c.primary_key for c in columns) @@ -231,7 +218,7 @@ def setup(self, columns: list[Column], engine: Engine): # names), where that overlap is expected and legitimate. Used to # decide how broadly the resample-penalty in proposal_ranking.py # applies. - self.column_is_numeric = isinstance(column_type, (Numeric, Integer)) + self.column_is_numeric = is_numeric_sql_type(column_type) # fraction of real values that are distinct - used to judge whether a # generator that resamples from the real data verbatim (e.g. a @@ -253,7 +240,7 @@ def setup(self, columns: list[Column], engine: Engine): ) if isinstance(column_type, (Date, DateTime, Time)): profile = EvaluationProfile.TEMPORAL - elif isinstance(column_type, (Numeric, Integer)): + elif is_numeric_sql_type(column_type): profile = choose_numeric_profile(self.real_values) else: stats = analyse_column(self.real_values) @@ -262,7 +249,7 @@ def setup(self, columns: list[Column], engine: Engine): else: profile = choose_profile(stats) self.profile = profile - self.statistical_fidelity.set_eval_pipelines(self.profile) + self.statistical_fidelity.set_eval_criteria(self.profile) else: self.statistical_fidelity = None self.profile = EvaluationProfile.IDENTIFIER @@ -285,20 +272,15 @@ def _has_unique_constraint(columns: list[Column]) -> bool: return True return False - def evaluate(self, proposer) -> ProposalEvaluation: + def evaluate(self, proposer: Proposer) -> ProposalEvaluation: """Score one proposer's synthetic data against the sampled real data.""" - assert ( - self.novelty_metric is not None and self.diversity_metric is not None - ), "setup() must be called before evaluate()" synthetic_samples = proposer.generate_data(self.sample_size) - # calculate novelty novelty_score = self.novelty_metric.compare( self.real_values, synthetic_samples, ) - # calculate diversity diversity_score = self.diversity_metric.compare( self.real_values, synthetic_samples, @@ -342,16 +324,16 @@ def evaluate(self, proposer) -> ProposalEvaluation: # Same rationale as the copy_frac catch above. synthetic_uniqueness = 0.0 - # calculate fidelity scores for each evaluation pipeline. Multi-column + # calculate fidelity scores for each evaluation criterion. Multi-column # proposal evaluation does not map cleanly onto the single-column fidelity - # pipelines, so fall back to a neutral score instead of crashing. + # criteria, so fall back to a neutral score instead of crashing. if self.statistical_fidelity is None: stat_fidelity_overall_score = 0.0 - stat_fidelity_pipeline_scores = {} + stat_fidelity_criterion_scores = {} else: ( stat_fidelity_overall_score, - stat_fidelity_pipeline_scores, + stat_fidelity_criterion_scores, ) = self.statistical_fidelity.calculate_scores(synthetic_samples) return ProposalEvaluation( @@ -359,7 +341,7 @@ def evaluate(self, proposer) -> ProposalEvaluation: novelty=novelty_score, diversity=diversity_score, overall_score=stat_fidelity_overall_score, - pipeline_scores=stat_fidelity_pipeline_scores, + criterion_scores=stat_fidelity_criterion_scores, copy_fraction=copy_frac, synthetic_uniqueness=synthetic_uniqueness, ) diff --git a/datafaker/evaluators/distribution_builders.py b/datafaker/evaluators/distribution_builders.py index d6afabef..1278922c 100644 --- a/datafaker/evaluators/distribution_builders.py +++ b/datafaker/evaluators/distribution_builders.py @@ -121,38 +121,21 @@ def build_from_table(self) -> Distribution: self.mean = statistics.mean(values) self.stddev = statistics.stdev(values) if len(values) > 1 else 0.0 - if self.stddev == 0: - self.edges = [ - float("-inf"), - self.mean - 2.0, - self.mean - 1.5, - self.mean - 1.0, - self.mean - 0.5, - self.mean, - self.mean + 0.5, - self.mean + 1.0, - self.mean + 1.5, - self.mean + 2.0, - float("inf"), - ] - bottom = self.mean - 2.0 - width = 0.5 - else: - self.edges = [ - float("-inf"), - self.mean - 2 * self.stddev, - self.mean - 1.5 * self.stddev, - self.mean - 1.0 * self.stddev, - self.mean - 0.5 * self.stddev, - self.mean, - self.mean + 0.5 * self.stddev, - self.mean + 1.0 * self.stddev, - self.mean + 1.5 * self.stddev, - self.mean + 2 * self.stddev, - float("inf"), - ] - bottom = self.mean - 2 * self.stddev - width = self.stddev / 2 + width = 0.5 if self.stddev == 0 else self.stddev / 2 + self.edges = [ + float("-inf"), + self.mean - 4 * width, + self.mean - 3 * width, + self.mean - 2 * width, + self.mean - 1 * width, + self.mean, + self.mean + 1 * width, + self.mean + 2 * width, + self.mean + 3 * width, + self.mean + 4 * width, + float("inf"), + ] + bottom = self.mean - 4 * width if width == 0: width = 1.0 diff --git a/datafaker/evaluators/evaluation_profile.py b/datafaker/evaluators/evaluation_profile.py index c7544eb8..4aab43af 100644 --- a/datafaker/evaluators/evaluation_profile.py +++ b/datafaker/evaluators/evaluation_profile.py @@ -10,7 +10,7 @@ class EvaluationProfile(Enum): - """The kind of value a column holds, used to pick its fidelity pipelines.""" + """The kind of value a column holds, used to pick its fidelity criteria.""" SHORT_TEXT = auto() CATEGORICAL = auto() diff --git a/datafaker/evaluators/feature_extractors.py b/datafaker/evaluators/feature_extractors.py index 5ddf542c..da721218 100644 --- a/datafaker/evaluators/feature_extractors.py +++ b/datafaker/evaluators/feature_extractors.py @@ -1,18 +1,22 @@ """Extract a comparable feature from a column's real/synthetic values.""" import re +import unicodedata from abc import ABC, abstractmethod from datetime import date, datetime from typing import Any -from sqlalchemy import String, func, literal +from sqlalchemy import Integer, String, func, literal from datafaker.dialects import SecondsDifference, SentenceCount, WordCount TOKEN = re.compile(r"\w+") +_EPOCH = datetime(1970, 1, 1) +_EPOCH_DATE = date(1970, 1, 1) -def _coerce_datetime(value): + +def _coerce_datetime(value: Any) -> date | datetime | None: """ Best-effort coercion of a value into something with .year/.month/etc. @@ -24,6 +28,10 @@ def _coerce_datetime(value): date" and drop it entirely, rather than raising a visible error - corrupting the resulting distribution for one candidate while giving no indication anything went wrong. + + :return: ``value`` unchanged if it already looks date-like, the parsed + ``datetime`` if ``value`` is an ISO-format string, or ``None`` if + neither applies. """ if hasattr(value, "year"): return value @@ -230,6 +238,19 @@ def expression(self, column): VOWELS = set("aeiou") +def _is_vowel(ch: str) -> bool: + """Whether ``ch`` is a vowel, treating an accented vowel as its base letter. + + Unicode NFKD-decomposes a precomposed accented character (e.g. "é") into + its base letter plus a combining accent mark ("e" + U+0301); taking the + first character of that decomposition recovers the base letter. Without + this, every accented vowel in non-English real data (common in person + names) would be miscounted as a consonant, skewing the real-side + distribution this feature is evaluated against. + """ + return unicodedata.normalize("NFKD", ch)[0] in VOWELS + + class VowelConsonantPatternExtractor(FeatureExtractor): """Extract a string value's vowel/consonant pattern (e.g. "CVCV").""" @@ -244,7 +265,7 @@ def extract(self, value): if not ch.isalpha(): continue - pattern.append("V" if ch in VOWELS else "C") + pattern.append("V" if _is_vowel(ch) else "C") if pattern: yield "".join(pattern) @@ -267,9 +288,6 @@ class TimestampExtractor(FeatureExtractor): a single joint measure is a fairer, simpler fidelity signal. """ - _EPOCH = datetime(1970, 1, 1) - _EPOCH_DATE = date(1970, 1, 1) - def extract(self, value): """Yield the value's days-since-epoch, as a single continuous float.""" value = _coerce_datetime(value) @@ -279,14 +297,14 @@ def extract(self, value): value = datetime(value.year, value.month, value.day) if getattr(value, "tzinfo", None) is not None: value = value.replace(tzinfo=None) - yield (value - self._EPOCH).total_seconds() / 86400.0 + yield (value - _EPOCH).total_seconds() / 86400.0 def expression(self, column): """Build the SQL expression for the column's days-since-epoch.""" # func.extract("epoch", ...) has no MSSQL equivalent (DATEPART has no # "epoch" field); SecondsDifference already solves exactly this via a # dialect-specific DATEDIFF compilation on MSSQL. - epoch = literal(self._EPOCH_DATE, type_=column.type) + epoch = literal(_EPOCH_DATE, type_=column.type) return SecondsDifference(column, epoch) / 86400.0 @@ -306,8 +324,23 @@ def extract(self, value): yield value.weekday() def expression(self, column): - """Build the SQL expression for the column's day of the week.""" - return func.extract("dow", column) # pylint: disable=not-callable + """Build the SQL expression for the column's day of the week (Monday=0). + + Deliberately not ``func.extract("dow", column)``/``DATEPART(weekday, + ...)``: Postgres' DOW (Sunday=0), MSSQL's DATEPART(weekday) (session- + configurable via ``DATEFIRST``, default Sunday=1) and Python's + ``date.weekday()`` (Monday=0) all disagree with each other, which + would silently compare mismatched category labels between the real + (SQL-computed) and synthetic (Python-computed) sides here - a + generator that reproduces the real day-of-week distribution exactly + would still score as a total mismatch. floor()/modulo arithmetic on + the same days-since-epoch value `extract()` uses is portable across + every dialect and trivially kept in sync with Python's convention: + 1970-01-01 (epoch) was a Thursday, i.e. weekday()==3, hence +3 below. + """ + epoch = literal(_EPOCH_DATE, type_=column.type) + days_since_epoch = func.floor(SecondsDifference(column, epoch) / 86400.0) + return (days_since_epoch.cast(Integer) + 3) % 7 class EmailLocalPartExtractor(FeatureExtractor): diff --git a/datafaker/evaluators/metrics.py b/datafaker/evaluators/metrics.py index 070889d9..1ef98307 100644 --- a/datafaker/evaluators/metrics.py +++ b/datafaker/evaluators/metrics.py @@ -72,7 +72,7 @@ def compare( # Euclidean distance is bounded in [0, 2] regardless of vocab size. # Dividing by that fixed bound (rather than len(vocab)**2) keeps the # score on a comparable scale to JensenShannon's [0, ln 2] range, so - # pipeline weights combining the two behave as intended. + # criterion weights combining the two behave as intended. score = squared_error / 2 return score diff --git a/datafaker/evaluators/proposal_ranking.py b/datafaker/evaluators/proposal_ranking.py index e9b06e1d..43fd9909 100644 --- a/datafaker/evaluators/proposal_ranking.py +++ b/datafaker/evaluators/proposal_ranking.py @@ -4,11 +4,12 @@ import math from dataclasses import dataclass, field -from typing import Any, Sequence +from typing import Sequence from datafaker.evaluators.column_evaluator import ProposalEvaluation from datafaker.evaluators.evaluation_profile import EvaluationProfile from datafaker.proposers.choice import ChoiceProposer +from datafaker.theme import Theme # Column-name keywords mapped to substrings of the generator's dotted name # (e.g. "generic.person.first_name") that they hint at. Column naming @@ -112,6 +113,7 @@ class ProposalRankingDisplay: fronts: list[int] weak_recommendation_warning: str | None = None no_uniqueness_guarantee: bool = False + type_incompatible: list[bool] = field(default_factory=list) # pylint: disable=too-many-instance-attributes @@ -131,6 +133,7 @@ class ProposalRanking: scores: list[float] = field(default_factory=list) weak_recommendation_warning: str | None = None no_uniqueness_guarantee: bool = False + type_incompatible: list[bool] = field(default_factory=list) def normalize_list(vals: Sequence[float]) -> list[float]: @@ -235,6 +238,21 @@ def _penalty_for_candidate( observed-unique column (e.g. free-text descriptions that happen to all differ) doesn't have a correctness reason to punish a small, repetitive-but-otherwise-plausible pool as hard as a real key would. + + :param result: the candidate's own evaluation, supplying its + ``copy_fraction`` (resample penalty) and ``synthetic_uniqueness`` + (self-duplication penalty). + :param real_uniqueness: fraction of the real column's sampled values + that are distinct - scales how hard both penalties bite. + :param apply_to_every_proposer: whether the resample penalty applies to + every candidate (numeric columns) rather than just ``ChoiceProposer`` + and its variants. + :param needs_uniqueness: whether the column actually needs unique values + (a PRIMARY KEY/UNIQUE constraint) - if not, the self-duplication + penalty is floored at ``SELF_DUPLICATION_PENALTY_FLOOR`` instead of + reaching full strength. + :return: the combined penalty multiplier (lower is worse) to apply to + the candidate's score. """ resample_penalty = 1.0 if isinstance(result.proposer, ChoiceProposer) or apply_to_every_proposer: @@ -250,21 +268,59 @@ def _penalty_for_candidate( # pylint: disable=too-many-arguments too-many-positional-arguments -# pylint: disable=too-many-locals too-many-statements +# pylint: disable=too-many-locals too-many-statements too-many-branches def rank_proposals( results: Sequence[ProposalEvaluation], profile: EvaluationProfile | None, - theme: Any | None = None, + theme: Theme | None = None, column_name: str | None = None, real_uniqueness: float = 0.0, is_numeric_column: bool = False, is_primary_key: bool = False, is_unique_constrained: bool = False, + type_incompatible: Sequence[bool] | None = None, ) -> ProposalRanking: """Rank proposals by fidelity, novelty and diversity using Pareto fronts. The ranking logic is deliberately separated from the interactive shell so it can be tested and reused independently of presentation concerns. + + :param results: one evaluation per candidate proposer, already scored + against the real column by ``ColumnEvaluator``. + :param profile: the column's evaluation profile - selects the + fidelity/novelty/diversity weighting (see ``profile_weights``). + :param theme: the active display theme, used only to colour-highlight + the recommended row in the returned table; ranking itself is + theme-independent. + :param column_name: the column's own name, used only for the keyword + boost (see ``KEYWORD_GENERATOR_HINTS``) - ``None`` disables it. + :param real_uniqueness: fraction of the real column's sampled values + that are distinct. Scales how hard the resample/self-duplication + penalties bite (see ``_penalty_for_candidate``): a low-uniqueness + column (a gender, a status) is expected to have repeats, so + reproducing real values there is harmless; a near-unique one (an + email, a real ID) is not. + :param is_numeric_column: whether the resample penalty applies to every + candidate (numeric columns have no shared real-world vocabulary + that could explain a coincidental match) rather than just + choice-style resamplers. + :param is_primary_key: whether the column is a primary key - needs + guaranteed-unique values, so a candidate with ``synthetic_uniqueness`` + too low to ever satisfy that is excluded from the recommendation + (see ``no_uniqueness_guarantee``), and a candidate that structurally + guarantees freshness takes precedence over the generic score (see + ``Proposer.guarantees_fresh_uniqueness``). + :param is_unique_constrained: as ``is_primary_key``, for a UNIQUE + constraint/index that isn't itself the primary key. + :param type_incompatible: parallel to ``results`` - True for a candidate + whose output type doesn't actually match the column (e.g. a + continuous-float proposer for a genuinely Integer, IDENTIFIER-profile + column). Such a candidate is never picked as the recommendation, even + if its raw score would otherwise win - the fidelity/novelty/diversity + metrics have no way to penalize a type mismatch themselves, since + they only ever compare numeric shape, not validity for the column. + :return: the ranked candidates, the recommended index/reason, and the + formatted table rows (see ``ProposalRanking``). """ if not results: return ProposalRanking(None, None, [], profile, (0.5, 0.25, 0.25)) @@ -276,13 +332,11 @@ def rank_proposals( diversities = [result.diversity for result in results] norm_scores = normalize_list(scores) - norm_nov = normalize_list(novelties) - norm_div = normalize_list(diversities) + novelty_scores = normalize_list(novelties) + diversity_scores = normalize_list(diversities) # fidelity (higher-is-better) is inverted normalized overall_score fidelity_scores = [1.0 - v for v in norm_scores] - novelty_scores = norm_nov - diversity_scores = norm_div # build points for Pareto (higher-is-better in all dims) points = list(zip(fidelity_scores, novelty_scores, diversity_scores)) @@ -377,6 +431,12 @@ def rank_proposals( ) ) + type_incompatible_flags = ( + list(type_incompatible) + if type_incompatible is not None + else [False] * len(results) + ) + # pick recommended index: highest combined score across all candidates, # tiebreaker by crowding distance. This used to restrict the pick to # Pareto front 1, but the keyword boost and resample penalty above are @@ -388,17 +448,56 @@ def rank_proposals( # Front/crowding are still computed and shown per row for transparency # (which candidates are non-dominated trade-off alternatives vs. # objectively worse), just no longer used to gate the recommendation. - all_idxs = range(len(combined_scores)) - best_score = max(combined_scores[idx] for idx in all_idxs) - candidates = [idx for idx in all_idxs if combined_scores[idx] == best_score] + # + # Two further restrictions narrow the *eligible* pool before that + # highest-score search, both because fidelity/novelty/diversity have no + # way to see either distinction themselves: + # + # - A type-incompatible candidate (see the ``type_incompatible`` param) + # is never eligible to win, however high its score - a continuous- + # float proposer fit directly to the real column's own numeric range + # can score almost perfectly on an IDENTIFIER profile while still + # being invalid to insert into an Integer column. + # - For a column that needs guaranteed-unique values (PK/UNIQUE), a + # candidate that structurally guarantees freshness (see + # ``Proposer.guarantees_fresh_uniqueness``, e.g. a sequence + # continuing past the observed max) takes precedence over the + # generic score - the fidelity metric can only ever read "deliberately + # diverges from the observed range" as a poor fit, never as the + # correct, intended behavior of this kind of proposer. + all_idxs = list(range(len(combined_scores))) + valid_type_idxs = [ + idx for idx in all_idxs if not type_incompatible_flags[idx] + ] or all_idxs + guarantee_idxs = ( + [ + idx + for idx in valid_type_idxs + if results[idx].proposer.guarantees_fresh_uniqueness() + ] + if needs_uniqueness + else [] + ) + eligible_idxs = guarantee_idxs or valid_type_idxs + + best_score = max(combined_scores[idx] for idx in eligible_idxs) + candidates = [idx for idx in eligible_idxs if combined_scores[idx] == best_score] if len(candidates) == 1: recommended_index = candidates[0] - recommended_reason = "best combined score" + recommended_reason = ( + "guarantees fresh, unique values" + if guarantee_idxs + else "best combined score" + ) else: best_crowding = max(crowding[idx] for idx in candidates) selected = [idx for idx in candidates if crowding[idx] == best_crowding] recommended_index = selected[0] - recommended_reason = "tiebreak by crowding distance" + recommended_reason = ( + "guarantees fresh, unique values; tiebreak by crowding distance" + if guarantee_idxs + else "tiebreak by crowding distance" + ) # Flag the case where the candidates that actually fit the real data # best were suppressed by the resample penalty, leaving something that @@ -450,7 +549,7 @@ def rank_proposals( "a fallback, not a confident pick." ) - # prepare rows with Front and Score, mark Pareto front 1 with coloring. + # prepare rows with Front and Score, marking the recommended row. # Crowding distance is still computed above (used as a tiebreak when # scores are exactly equal) but isn't shown - it's an NSGA-II internal # detail (how isolated a candidate is from its front-mates in objective @@ -460,24 +559,35 @@ def rank_proposals( for i, result in enumerate(results, start=1): idx = i - 1 front = front_of[idx] + is_recommended = idx == recommended_index + # '*' marks the recommended row even with theming turned off, where + # the colour highlight below has no visible effect at all. + index_cell = f"*{i}" if is_recommended else str(i) + # Profile and (real) Uniqueness describe the source column, not this + # candidate - constant across every row, and already shown once in + # profile_summary above the table, so they're left out here rather + # than repeated on every line. cells = [ - str(i), + index_cell, result.proposer.name(), - profile.name if profile is not None else "UNKNOWN", str(front), f"{combined_scores[idx]:.6f}", keyword_matches[idx] or "", f"{fidelity_scores[idx]:.6f}", f"{novelty_scores[idx]:.6f}", f"{diversity_scores[idx]:.6f}", - f"{real_uniqueness:.3f}", f"{result.copy_fraction:.3f}", f"{result.synthetic_uniqueness:.3f}", f"{penalty_multipliers[idx]:.3f}", ] - # color entire row for Pareto front 1 - if front == 1 and theme is not None: - cells = [f"{theme.function}{cell}{theme.reset}" for cell in cells] + # Highlight the recommended row distinctly from the rest of its + # Pareto front 1 "near miss" companions (non-dominated alternatives + # worth a second look), rather than one colour for the whole front. + if theme is not None: + if is_recommended: + cells = [f"{theme.recommend}{cell}{theme.reset}" for cell in cells] + elif front == 1: + cells = [f"{theme.near_miss}{cell}{theme.reset}" for cell in cells] # keep original types compatible with print_table (it will cast to list) rows.append(tuple(cells)) @@ -494,6 +604,7 @@ def rank_proposals( scores=list(combined_scores), weak_recommendation_warning=weak_recommendation_warning, no_uniqueness_guarantee=no_uniqueness_guarantee, + type_incompatible=type_incompatible_flags, ) @@ -528,7 +639,9 @@ def format_ranking_display( profile_summary = ( f"Profile: {profile.name if profile is not None else 'UNKNOWN'} | " f"Real uniqueness: {ranking.real_uniqueness:.3f} | " - "Pareto front 1 rows are highlighted\n" + "'*' marks the recommended row; it and other Pareto front 1 rows" + " (non-dominated alternatives) are colour-highlighted, distinctly" + " from each other, if your theme supports it\n" f"Score = clamp( ({fid_w:.2f}*Fidelity + {nov_w:.2f}*Novelty" f" + {div_w:.2f}*Diversity + Keyword) " "x Penalty , 0, 1 )\n" @@ -546,4 +659,5 @@ def format_ranking_display( fronts=ranking.fronts, weak_recommendation_warning=ranking.weak_recommendation_warning, no_uniqueness_guarantee=ranking.no_uniqueness_guarantee, + type_incompatible=ranking.type_incompatible, ) diff --git a/datafaker/evaluators/statistical_fidelity.py b/datafaker/evaluators/statistical_fidelity.py index d40ab49c..650b52de 100644 --- a/datafaker/evaluators/statistical_fidelity.py +++ b/datafaker/evaluators/statistical_fidelity.py @@ -1,4 +1,4 @@ -"""Per-profile fidelity pipelines that score a proposer against a column.""" +"""Per-profile fidelity criteria that score a proposer against a column.""" from dataclasses import dataclass @@ -30,8 +30,8 @@ @dataclass -class EvaluationPipeline: - """One named feature-comparison stage within a profile's fidelity score.""" +class EvaluationCriterion: + """One named, weighted feature-comparison criterion within a profile's fidelity score.""" name: str builder: type[DistributionBuilder] @@ -41,36 +41,36 @@ class EvaluationPipeline: # Short text profile (names, cities, etc.) -SHORT_TEXT_PIPELINE = [ - EvaluationPipeline( +SHORT_TEXT_CRITERIA = [ + EvaluationCriterion( name="length", builder=HistogramBuilder, metric=MeanSquaredError(), weight=0.25, extractor=LengthExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="bigrams", builder=CategoryBuilder, metric=JensenShannon(), weight=0.35, extractor=CharacterBigramExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="first_letter", builder=CategoryBuilder, metric=JensenShannon(), weight=0.10, extractor=FirstLetterExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="last_letter", builder=CategoryBuilder, metric=JensenShannon(), weight=0.10, extractor=LastLetterExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="pattern", builder=CategoryBuilder, metric=JensenShannon(), @@ -79,36 +79,36 @@ class EvaluationPipeline: ), ] -EMAIL_PIPELINE = [ - EvaluationPipeline( +EMAIL_CRITERIA = [ + EvaluationCriterion( name="length", builder=HistogramBuilder, metric=MeanSquaredError(), weight=0.10, extractor=LengthExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="local_part", builder=CategoryBuilder, metric=JensenShannon(), weight=0.40, extractor=EmailLocalPartExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="domain", builder=CategoryBuilder, metric=JensenShannon(), weight=0.15, extractor=EmailDomainExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="tld", builder=CategoryBuilder, metric=JensenShannon(), weight=0.15, extractor=EmailTopLevelDomainExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="format_validity", builder=CategoryBuilder, metric=JensenShannon(), @@ -118,8 +118,8 @@ class EvaluationPipeline: ] # for country, gender, status, category -CATEGORICAL_PIPELINE = [ - EvaluationPipeline( +CATEGORICAL_CRITERIA = [ + EvaluationCriterion( name="category", builder=CategoryBuilder, metric=JensenShannon(), @@ -128,8 +128,8 @@ class EvaluationPipeline: ) ] -IDENTIFIER_PIPELINE = [ - EvaluationPipeline( +IDENTIFIER_CRITERIA = [ + EvaluationCriterion( name="identifier", builder=HistogramBuilder, metric=MeanSquaredError(), @@ -138,22 +138,22 @@ class EvaluationPipeline: ) ] -# Two pipelines instead of five independent year/month/day/hour dimensions: +# Two criteria instead of five independent year/month/day/hour dimensions: # "timestamp" scores year+month+day jointly as one continuous quantity (so a # generator is judged on the real calendar-position distribution as a whole, # not on marginal components that can each look fine while their # combination never does), and "day_of_week" is kept separately since it's a # cyclic pattern a continuous days-since-epoch value can't capture on its # own. -TEMPORAL_PIPELINE = [ - EvaluationPipeline( +TEMPORAL_CRITERIA = [ + EvaluationCriterion( name="timestamp", builder=HistogramBuilder, metric=MeanSquaredError(), weight=0.7, extractor=TimestampExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="day_of_week", builder=CategoryBuilder, metric=JensenShannon(), @@ -163,36 +163,36 @@ class EvaluationPipeline: ] # for comments, description, review -FREE_TEXT_PIPELINE = [ - EvaluationPipeline( +FREE_TEXT_CRITERIA = [ + EvaluationCriterion( name="length", builder=HistogramBuilder, metric=MeanSquaredError(), weight=0.10, extractor=LengthExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="word_count", builder=HistogramBuilder, metric=MeanSquaredError(), weight=0.25, extractor=WordCountExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="sentence_count", builder=HistogramBuilder, metric=MeanSquaredError(), weight=0.15, extractor=SentenceCountExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="words", builder=CategoryBuilder, metric=JensenShannon(), weight=0.30, extractor=WordExtractor(), ), - EvaluationPipeline( + EvaluationCriterion( name="bigrams", builder=CategoryBuilder, metric=JensenShannon(), @@ -201,13 +201,13 @@ class EvaluationPipeline: ), ] -PROFILE_PIPELINES = { - EvaluationProfile.IDENTIFIER: IDENTIFIER_PIPELINE, - EvaluationProfile.SHORT_TEXT: SHORT_TEXT_PIPELINE, - EvaluationProfile.EMAIL: EMAIL_PIPELINE, - EvaluationProfile.CATEGORICAL: CATEGORICAL_PIPELINE, - EvaluationProfile.FREE_TEXT: FREE_TEXT_PIPELINE, - EvaluationProfile.TEMPORAL: TEMPORAL_PIPELINE, +PROFILE_CRITERIA = { + EvaluationProfile.IDENTIFIER: IDENTIFIER_CRITERIA, + EvaluationProfile.SHORT_TEXT: SHORT_TEXT_CRITERIA, + EvaluationProfile.EMAIL: EMAIL_CRITERIA, + EvaluationProfile.CATEGORICAL: CATEGORICAL_CRITERIA, + EvaluationProfile.FREE_TEXT: FREE_TEXT_CRITERIA, + EvaluationProfile.TEMPORAL: TEMPORAL_CRITERIA, } @@ -217,57 +217,57 @@ class StatisticalFidelity: This class evaluates the statistical fidelity of a synthetic sample against a real sample by comparing the distributions of features extracted - from the data. It uses a set of evaluation pipelines tailored to different + from the data. It uses a set of evaluation criteria tailored to different types of data (e.g., short text, categorical, free text, email, identifier, temporal) to compute a weighted score that reflects how closely the synthetic data matches the real data in terms of statistical properties. """ def __init__(self, column, engine, sample_size=4000): - """Initialize with no pipelines set; call set_eval_pipelines() first.""" + """Initialize with no criteria set; call set_eval_criteria() first.""" self.engine = engine self.column = column self.table = self.column.table - self.eval_pipelines = {} - # caps how many real rows each pipeline's builder pulls into memory + self.eval_criteria = {} + # caps how many real rows each criterion's builder pulls into memory # when computing the "real" distribution, so this scales to large - # tables instead of scanning the whole column per pipeline. + # tables instead of scanning the whole column per criterion. self.sample_size = sample_size - # (builder, real distribution) per pipeline name, populated lazily - # the first time calculate_scores runs each pipeline. The real + # (builder, real distribution) per criterion name, populated lazily + # the first time calculate_scores runs each criterion. The real # distribution depends only on the column, not on which proposer is # being scored, so it only needs computing once per column rather # than once per calculate_scores() call (one per candidate proposer). self._real_distributions = {} - def set_eval_pipelines(self, profile): + def set_eval_criteria(self, profile): """ - Set evaluation pipelines based on the specified profile. + Set evaluation criteria based on the specified profile. :param profile: The evaluation profile to use. """ - self.eval_pipelines = PROFILE_PIPELINES[profile] + self.eval_criteria = PROFILE_CRITERIA[profile] self._real_distributions = {} def calculate_scores(self, synthetic_samples): """ - Calculate the overall fidelity score and each pipeline's own score. + Calculate the overall fidelity score and each criterion's own score. :param synthetic_samples: The synthetic samples to evaluate. - :return: A tuple of (overall score, {pipeline name: pipeline score}). + :return: A tuple of (overall score, {criterion name: criterion score}). """ - pipeline_scores = {} + criterion_scores = {} weighted_score = 0.0 total_weight = 0.0 - for pipeline in self.eval_pipelines: - cached = self._real_distributions.get(pipeline.name) + for criterion in self.eval_criteria: + cached = self._real_distributions.get(criterion.name) if cached is None: - builder = pipeline.builder( + builder = criterion.builder( self.engine, self.table, self.column, - extractor=pipeline.extractor, + extractor=criterion.extractor, sample_size=self.sample_size, ) # Distribution from real table - independent of the @@ -275,23 +275,23 @@ def calculate_scores(self, synthetic_samples): # which HistogramBuilder.build_from_values needs state # from - e.g. self.edges - set by this same call). real = builder.build_from_table() - self._real_distributions[pipeline.name] = (builder, real) + self._real_distributions[criterion.name] = (builder, real) else: builder, real = cached # Distribution from synthetic values synthetic = builder.build_from_values(synthetic_samples) - score = pipeline.metric.compare( + score = criterion.metric.compare( real, synthetic, ) - pipeline_scores[pipeline.name] = score + criterion_scores[criterion.name] = score - weighted_score += pipeline.weight * score - total_weight += pipeline.weight + weighted_score += criterion.weight * score + total_weight += criterion.weight overall_score = weighted_score / total_weight if total_weight > 0 else 0.0 - return overall_score, pipeline_scores + return overall_score, criterion_scores diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index a2ca912f..5d6dd0a8 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -14,6 +14,7 @@ from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private from datafaker.dialects import Random from datafaker.evaluators.column_evaluator import ColumnEvaluator +from datafaker.evaluators.evaluation_profile import EvaluationProfile from datafaker.evaluators.proposal_ranking import ( ProposalRanking, ProposalRankingDisplay, @@ -190,10 +191,6 @@ class GeneratorCmd(DbCmd): PROPOSE_SOURCE_SAMPLE_TEXT = "Sample of actual source data: {1}{0}..." PROPOSE_SOURCE_EMPTY_TEXT = "Source database has no data in this column." - PROPOSE_GENERATOR_SAMPLE_TEXT = ( - "{theme_reset}{index}. {theme_func}{name}:" - " {theme_fit}{fit} {theme_data}{sample}{theme_reset} ..." - ) RANKED_SAMPLE_TEXT = ( "{theme_reset}{index}. {theme_func}{name}:" " {theme_data}{sample}{theme_reset} ..." @@ -303,9 +300,6 @@ def __init__( self.proposers: list[Proposer] | None = None self.proposer_index = 0 self.proposers_valid_columns: Optional[tuple[int, list[str]]] = None - # whether the cached self.proposers list currently includes proposers - # that are type-incompatible with the column (e.g. from 'propose all') - self.proposers_include_all: bool = False self.set_prompt() self.roles: dict[str, dict[str, RoleEntry]] = { table_name: { @@ -788,6 +782,27 @@ def _proposers_valid(self) -> bool: self._get_column_names(), ) + def _get_proposer_proposals(self) -> list[Proposer]: + """ + Get a list of acceptable proposers, sorted by decreasing fit to the actual data. + + The cached list from the most recent 'propose'/'propose all' is + reused if still valid for the current table/columns, so that 'set + ' numbering matches whichever list was last displayed. + """ + if not self._proposers_valid() or self.proposers is None: + columns = self._column_metadata() + props = everything_factory(self.config, self.metadata).get_proposers( + columns, self.sync_engine + ) + sorted_props = sorted(props, key=lambda g: g.fit(9999)) + self.proposers = sorted_props + self.proposers_valid_columns = ( + self.table_index, + self._get_column_names().copy(), + ) + return self.proposers + # Proposers that emit continuous float output (via random.uniform/ # normalvariate/lognormvariate) with no integer rounding. That's a # legitimate fit for a Numeric/decimal column, but not for a genuinely @@ -800,55 +815,34 @@ def _proposers_valid(self) -> bool: LogNormalProposer, ) - def _is_integer_incompatible( - self, proposer: Proposer, columns: list[Column] + def _is_identifier_type_mismatch( + self, + proposer: Proposer, + columns: list[Column], + profile: EvaluationProfile | None, ) -> bool: - """Check whether `proposer` produces float output invalid for an Integer column.""" + """Check whether `proposer` produces float output invalid for an Integer column. + + Deliberately narrow: only flags this for an ``IDENTIFIER``-profile + (near-unique) column. For a ``CATEGORICAL``-profile Integer column + (e.g. a small set of repeated codes/years), the fidelity metric + already scores a continuous-float proposer at essentially zero on + its own - it almost never reproduces one of the small set of + observed discrete values - so no extra guard is needed there. For + an ``IDENTIFIER``-profile column, the opposite happens: fidelity + rewards a continuous distribution fit directly to the real column's + numeric range, which can score almost perfectly while still being + structurally invalid to insert into an Integer column - that's the + case this guards against. + """ + if profile is not EvaluationProfile.IDENTIFIER: + return False if not isinstance(proposer, self._CONTINUOUS_FLOAT_PROPOSER_TYPES): return False if len(columns) != 1: return False return isinstance(get_column_type(columns[0]), Integer) - def _get_proposer_proposals( - self, include_all: bool | None = None - ) -> list[Proposer]: - """ - Get a list of acceptable proposers, sorted by decreasing fit to the actual data. - - By default (``include_all=None``), type-incompatible proposers (see - ``_is_integer_incompatible``) are filtered out - this is what - 'propose' shows - and the cached list from the most recent - 'propose'/'propose all' is reused if still valid, so that 'set - ' numbering matches whichever list was last displayed. Pass - an explicit ``True``/``False`` (from 'propose'/'propose all' - themselves) to force that mode, refetching if it differs from what's - cached. - """ - if include_all is None: - include_all = self.proposers_include_all - if ( - not self._proposers_valid() - or self.proposers is None - or include_all != self.proposers_include_all - ): - columns = self._column_metadata() - props = everything_factory(self.config, self.metadata).get_proposers( - columns, self.sync_engine - ) - if not include_all: - props = [ - p for p in props if not self._is_integer_incompatible(p, columns) - ] - sorted_props = sorted(props, key=lambda g: g.fit(9999)) - self.proposers = sorted_props - self.proposers_include_all = include_all - self.proposers_valid_columns = ( - self.table_index, - self._get_column_names().copy(), - ) - return self.proposers - def _print_privacy(self) -> None: """Print the privacy status of the current table.""" table = self.table_metadata() @@ -1051,67 +1045,56 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st self, _arg: str ) -> None: """ - Display a list of possible generators for this column. + Display a ranked table of candidate generators for this column. - They will be listed in order of fit, the most likely matches first. - The results can be compared (against a sample of the real data in - the column and against each other) with the 'compare' command. + Each candidate generator is tried against a sample of the real + data and scored on how well it matches (fidelity), how much new + data it produces (novelty) and how varied its own output is + (diversity). The single best-scoring candidate is shown above the + table as 'Recommended: N. '. + + To keep the table short, two kinds of candidate are left out by + default - run 'propose all' to see everything instead: - By default, generators whose output type is incompatible with the - column (e.g. a continuous-float generator for an Integer column) - are left out. Run 'propose all' to include them anyway. + - a candidate that's strictly worse than some other candidate on + every one of fidelity/novelty/diversity at once ("dominated", + beyond the shown MAX_PROPOSERS_SHOWN cap); + - a candidate whose output type doesn't actually fit a genuinely + unique Integer column (e.g. a continuous-float generator) - this + one is also never the recommendation, even under 'propose all'. + + The results can be compared (against a sample of the real data in + the column and against each other) with the 'compare' command. Type + 'help ranking' for how the score is computed, or 'help pareto' for + what the Front column means - or see the "Evaluating and ranking + proposals" section of the docs for the full detail. """ theme = get_active_theme() limit = 5 include_all = _arg.strip().lower() == "all" - props = self._get_proposer_proposals(include_all) + props = self._get_proposer_proposals() sample = self._get_column_data(limit) - self._print_source_sample(sample, theme) if not props: self.print(self.PROPOSE_NOTHING) - for index, prop in enumerate(props): - fit = prop.fit(-1) - if fit == -1: - fit_s = "(no fit)" - elif fit < 100: - fit_s = f"(fit: {fit:.3g})" - else: - fit_s = f"(fit: {fit:.0f})" - self.print( - self.PROPOSE_GENERATOR_SAMPLE_TEXT, - index=index + 1, - name=prop.name(), - fit=fit_s, - sample="; ".join(map(repr, prop.generate_data(limit))), - theme_func=theme.function, - theme_fit=theme.query, - theme_data=theme.data, - theme_reset=theme.reset, - ) - column_evaluator = ColumnEvaluator() columns = self._column_metadata() - results = [] - column_evaluator.setup(columns, self.sync_engine) - for index, proposer in enumerate(props): - result = column_evaluator.evaluate(proposer) - results.append(result) - - self.print("\n***\n") - # self.print("\nProposal evaluation:\n", results) + column_evaluator = ColumnEvaluator(columns, self.sync_engine) + results = [column_evaluator.evaluate(proposer) for proposer in props] - profile = getattr(column_evaluator, "profile", None) + profile = column_evaluator.profile column_name = ( column_evaluator.column.name if column_evaluator.column is not None else None ) - real_uniqueness = getattr(column_evaluator, "real_uniqueness", 0.0) - is_numeric_column = getattr(column_evaluator, "column_is_numeric", False) - is_primary_key = getattr(column_evaluator, "is_primary_key", False) - is_unique_constrained = getattr( - column_evaluator, "is_unique_constrained", False - ) + real_uniqueness = column_evaluator.real_uniqueness + is_numeric_column = column_evaluator.column_is_numeric + is_primary_key = column_evaluator.is_primary_key + is_unique_constrained = column_evaluator.is_unique_constrained + type_incompatible = [ + self._is_identifier_type_mismatch(result.proposer, columns, profile) + for result in results + ] ranking = rank_proposals( results, profile, @@ -1121,41 +1104,21 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st is_numeric_column=is_numeric_column, is_primary_key=is_primary_key, is_unique_constrained=is_unique_constrained, + type_incompatible=type_incompatible, ) display = format_ranking_display(ranking, results, profile) if display.recommendation is not None: - if ( - display.no_uniqueness_guarantee - or display.weak_recommendation_warning is not None - ): - # Don't highlight a fallback or non-existent pick - coloring - # it the same way as a solid recommendation would visually - # contradict the caveat that follows (or, for - # no_uniqueness_guarantee, claim confidence in a pick that - # doesn't exist at all). + self.print("\n") + if display.no_uniqueness_guarantee: + # There's no actual pick here ("Recommended: none - ..."), + # so highlighting it the same way as a real recommendation + # would falsely claim confidence in a pick that doesn't exist. self.print(display.recommendation) else: - self.print(f"{theme.column}{display.recommendation}{theme.reset}") + self.print(f"{theme.recommend}{display.recommendation}{theme.reset}") if display.weak_recommendation_warning is not None: self.print(display.weak_recommendation_warning) - if include_all and ranking.recommended_index is not None: - # 'propose all' deliberately bypasses the type-compatibility filter, - # so a generator whose output type doesn't match the column (e.g. a - # continuous-float generator for an Integer column) can still win - - # the statistical score has no way to detect that mismatch at all, - # which is exactly why the default 'propose' excludes it structurally - # instead of trying to penalize it. - recommended_proposer = results[ranking.recommended_index].proposer - if self._is_integer_incompatible(recommended_proposer, columns): - self.print( - "Warning: {0} produces output whose type doesn't match this" - " column - it's excluded from the default 'propose' list for" - " exactly this reason (run 'propose' without 'all'). The" - " statistics can't detect this mismatch, so a high score here" - " doesn't mean it's safe to use.", - recommended_proposer.name(), - ) self.print(display.profile_summary) @@ -1166,16 +1129,14 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st [ "#", "Proposer", - "Profile", "Front", "Score", "Keyword", "Fidelity", "Novelty", "Diversity", - "Uniqueness", "Copies", - "Synth.Uniq", + "Uniqueness", "Penalty", ], rows_to_show, @@ -1195,7 +1156,7 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st theme_reset=theme.reset, ) - print("\n\n") + print("\n") def _print_source_sample(self, sample: list[list[str]], theme: Theme) -> None: """Print a sample of the real column values, or a note that there are none.""" @@ -1216,16 +1177,27 @@ def _select_indices_to_show( By default, only show Pareto front 1: a front-2+ candidate is, by definition, strictly worse than some front-1 one on fidelity, novelty AND diversity simultaneously, so it adds nothing to a - shortlist. 'propose all' bypasses this (and the type-compatibility - filter) to show every candidate. The returned indices let the - sample listing after the table map each shown row back to its - proposer without parsing the (possibly theme-colored) '#' cell text. + shortlist. A candidate whose output type doesn't match the column + (see ``GeneratorCmd._is_identifier_type_mismatch``) is also hidden + by default, even if it's front 1 - the statistical score has no way + to detect that mismatch at all, so a high score there doesn't mean + it's safe to use. 'propose all' bypasses both to show every + candidate. The returned indices let the sample listing after the + table map each shown row back to its proposer without parsing the + (possibly theme-colored) '#' cell text. """ indices_to_show = list(range(len(display.rows))) if include_all: return indices_to_show - indices_to_show = [i for i in indices_to_show if display.fronts[i] == 1] + indices_to_show = [ + i + for i in indices_to_show + if display.fronts[i] == 1 + and not ( + i < len(display.type_incompatible) and display.type_incompatible[i] + ) + ] # The recommendation is picked by combined score across *all* # candidates, not just front 1 (see rank_proposals) - a # dominated candidate can still win once the resample penalty @@ -1278,6 +1250,48 @@ def do_p(self, arg: str) -> None: """Synonym for propose.""" self.do_propose(arg) + def help_ranking(self) -> None: + """Explain how 'propose' scores and ranks candidate generators.""" + self.print( + "Each candidate's synthetic output is compared against a sample" + " of the real column on three measures: Fidelity (does it" + " statistically match the real data), Novelty (does it produce" + " values not already in the real data) and Diversity (how" + " varied is its own output). These combine into one Score," + " weighted differently depending on the column's profile - e.g." + " an identifier-like column weighs fidelity much more heavily" + " than a free-text one.\n" + "\n" + "Two adjustments apply on top of that: a column-name hint" + " (e.g. a column called 'first_name' boosts a matching" + " generic.person.* candidate) and a privacy penalty that" + " discounts a candidate for reproducing real values verbatim," + " scaled by how unique the real column actually is.\n" + "\n" + "The single highest-scoring candidate is shown as 'Recommended'" + ". Type 'help pareto' for what the Front column means, or see" + ' the "Evaluating and ranking proposals" section of the docs' + " for the full formula." + ) + + def help_pareto(self) -> None: + """Explain the 'Front' column shown by 'propose'.""" + self.print( + "A candidate is in Pareto front 1 if no other candidate beats" + " it on Fidelity, Novelty AND Diversity all at once - front 2" + " means some front-1 candidate beats it on all three, and so" + " on. By default 'propose' only shows front 1 (plus the" + " recommendation, even if it's in a later front) to keep the" + " table short; run 'propose all' to see every front.\n" + "\n" + "You don't need to understand this to use 'propose' - the" + " Recommended line and Score column are usually enough. Front" + " is mainly useful for seeing which candidates are genuine" + " trade-offs against each other (e.g. one is more novel," + " another fits better) rather than one simply being worse than" + " another." + ) + def get_proposer_by_name(self, gen_name: str) -> Proposer | None: """Find a proposer by name from the list of proposals.""" for gen in self._get_proposer_proposals(): diff --git a/datafaker/proposers/base.py b/datafaker/proposers/base.py index 058b7b18..9e677326 100644 --- a/datafaker/proposers/base.py +++ b/datafaker/proposers/base.py @@ -146,6 +146,21 @@ def fit(self, default: float = -1) -> float: """ return default + def guarantees_fresh_uniqueness(self) -> bool: + """ + Whether this proposer is structurally guaranteed never to collide. + + True only for a proposer whose *construction* rules out ever + reproducing a real or previously-generated value (e.g. a sequence + continuing past the observed maximum) - not merely one that happens + to have a low *observed* collision rate in a given sample. Used to + let this kind of proposer take precedence for a column that needs + guaranteed-unique values, since the fidelity/novelty/diversity + metrics can only ever read "deliberately diverges from the observed + range" as a poor statistical fit, never as the intended behavior. + """ + return False + class PredefinedProposer(Proposer): """ diff --git a/datafaker/proposers/partitioned.py b/datafaker/proposers/partitioned.py index 8db2ee9b..595fcf81 100644 --- a/datafaker/proposers/partitioned.py +++ b/datafaker/proposers/partitioned.py @@ -20,8 +20,8 @@ text, ) from sqlalchemy.exc import DatabaseError -from sqlalchemy.types import Integer, Numeric +from datafaker.db_utils import is_numeric_sql_type from datafaker.dialects import IsNotNull, IsNull, IsNumeric, IsPositive, LogNatural from datafaker.proposers.base import Proposer, dist_gen, get_column_type from datafaker.proposers.continuous import ( @@ -364,8 +364,7 @@ def fit(self, default: float = -1) -> float: def is_numeric(col: Column) -> bool: """Test if this column stores a numeric value.""" - ct = get_column_type(col) - return isinstance(ct, (Numeric, Integer)) and not col.foreign_keys + return is_numeric_sql_type(get_column_type(col)) and not col.foreign_keys def powerset(xs: list[T]) -> Iterable[Iterable[T]]: diff --git a/datafaker/proposers/sequence.py b/datafaker/proposers/sequence.py index 99beb0ee..aaae9902 100644 --- a/datafaker/proposers/sequence.py +++ b/datafaker/proposers/sequence.py @@ -34,6 +34,10 @@ def function_name(self) -> str: """Get the name of the generator function to call.""" return "generic.column_value_provider.increment" + def guarantees_fresh_uniqueness(self) -> bool: + """Monotonically increasing past the observed max can never collide.""" + return True + def nominal_kwargs(self) -> dict[str, str]: """Get the arguments to be entered into ``config.yaml``. diff --git a/datafaker/theme.py b/datafaker/theme.py index 826d9e29..934dcced 100644 --- a/datafaker/theme.py +++ b/datafaker/theme.py @@ -8,7 +8,7 @@ @dataclass -class Theme: +class Theme: # pylint: disable=too-many-instance-attributes """A colour theme for DataFaker terminal output.""" prompt: str @@ -18,6 +18,13 @@ class Theme: query: str line: str reset: str + # 'propose' table/recommendation highlighting: `recommend` marks the + # single recommended row, `near_miss` the other Pareto front 1 rows + # (non-dominated alternatives worth a second look) - kept distinct from + # each other and from the colours above so recommendation status reads + # at a glance instead of blending into the rest of the table. + recommend: str + near_miss: str class ThemeEntry(str, Enum): @@ -29,7 +36,7 @@ class ThemeEntry(str, Enum): THEME: dict[str, Theme] = { - ThemeEntry.NONE: Theme("", "", "", "", "", "", ""), + ThemeEntry.NONE: Theme("", "", "", "", "", "", "", "", ""), ThemeEntry.DARK: Theme( prompt=colorama.Fore.CYAN + colorama.Style.NORMAL, # type: ignore column=colorama.Fore.GREEN + colorama.Style.NORMAL, # type: ignore @@ -38,6 +45,8 @@ class ThemeEntry(str, Enum): query=colorama.Fore.GREEN + colorama.Style.NORMAL, # type: ignore line=colorama.Fore.WHITE + colorama.Style.DIM, # type: ignore reset=colorama.Style.RESET_ALL, # type: ignore + recommend=colorama.Fore.GREEN + colorama.Style.BRIGHT, # type: ignore + near_miss=colorama.Fore.BLUE + colorama.Style.NORMAL, # type: ignore ), ThemeEntry.LIGHT: Theme( prompt=colorama.Fore.BLUE, # type: ignore @@ -47,6 +56,8 @@ class ThemeEntry(str, Enum): query=colorama.Fore.MAGENTA, # type: ignore line=colorama.Fore.LIGHTBLACK_EX, # type: ignore reset=colorama.Style.RESET_ALL, # type: ignore + recommend=colorama.Fore.GREEN + colorama.Style.BRIGHT, # type: ignore + near_miss=colorama.Fore.CYAN, # type: ignore ), } diff --git a/docs/source/builtin_generators.rst b/docs/source/builtin_generators.rst index 91ddf3b0..91d64873 100644 --- a/docs/source/builtin_generators.rst +++ b/docs/source/builtin_generators.rst @@ -33,19 +33,16 @@ proposer: ``config.yaml``; from then on, only the underlying generator function is used --- the proposer itself is not needed again for that column. -``propose`` now shows two different views of the same candidates: first the -plain, per-proposer ``(fit: ...)`` list described above, and then a second, -ranked table produced by a separate statistical evaluation pipeline (see -:ref:`evaluating-and-ranking-proposals` below) that scores every candidate on -fidelity, novelty and diversity and names a ``Recommended`` generator. The two -scores are independent and can disagree; the ranked table and its -recommendation are the more reliable of the two for most columns. +``propose`` shows a ranked table produced by a statistical evaluation +pipeline (see :ref:`evaluating-and-ranking-proposals` below) that scores +every candidate on fidelity, novelty and diversity and names a +``Recommended`` generator. Default generators assigned automatically ------------------------------------------ -Before you run ``propose``/``set`` at all, ``configure-generators`` (via -``make.py``) already assigns a default generator to every column, purely from its +Before you run ``propose``/``set`` at all, ``configure-generators`` already +assigns a default generator to every column, purely from its SQL type (and whether it is a foreign key or primary key). ``propose`` lets you replace this default with something that better matches the real data. @@ -383,8 +380,7 @@ for example, a ``year_of_birth`` column should always agree with a Evaluating and ranking proposals ---------------------------------- -For a single-column ``propose``, every candidate proposer that survives the -type-compatibility filter (see below) is additionally scored by +For a single-column ``propose``, every candidate proposer is scored by ``datafaker.evaluators.ColumnEvaluator`` and ranked by ``datafaker.evaluators.proposal_ranking.rank_proposals``. This produces a table shown by ``propose``, with a ``Recommended: N. `` line @@ -439,10 +435,10 @@ compared (so these are *relative* scores, not absolute ones): - Meaning * - **Fidelity** - How closely the synthetic sample's distribution matches the real - column's, measured by a profile-specific pipeline of feature - extractors run through ``MeanSquaredError`` or ``JensenShannon`` - (see :ref:`statistical-fidelity-pipelines` below), then inverted so - higher is better. + column's, measured by a profile-specific weighted combination of + feature extractors run through ``MeanSquaredError`` or + ``JensenShannon`` (see :ref:`statistical-fidelity-criteria` below), + then inverted so higher is better. * - **Novelty** - The fraction of synthetic values (case/whitespace-normalized) that do *not* already appear among the real sampled values. 1.0 means every @@ -452,22 +448,22 @@ compared (so these are *relative* scores, not absolute ones): Shannon entropy over distinct values) matches the real sample's --- not "more diverse is better", but "as diverse as the real data". -.. _statistical-fidelity-pipelines: +.. _statistical-fidelity-criteria: -Fidelity pipelines by profile +Fidelity criteria by profile """""""""""""""""""""""""""""" -Fidelity is computed by one of six fixed pipelines, chosen by profile, each a -weighted combination of feature extractors compared via ``MeanSquaredError`` -(on a length/count histogram) or ``JensenShannon`` divergence (on a category -distribution): +Fidelity is computed by one of six fixed sets of criteria, chosen by profile, +each a weighted combination of feature extractors compared via +``MeanSquaredError`` (on a length/count histogram) or ``JensenShannon`` +divergence (on a category distribution): .. list-table:: :widths: 20 80 :header-rows: 1 * - Profile - - Pipeline (feature: weight) + - Criteria (feature: weight) * - ``IDENTIFIER`` - identifier (value histogram): 1.0 * - ``CATEGORICAL`` @@ -542,11 +538,13 @@ strictly dominated on fidelity, novelty *and* diversity simultaneously by any other candidate); ``propose`` (without ``all``) shows only front 1, capped at 10 rows by ``Score`` (always keeping the recommended candidate even if it would otherwise be cut), and tells you how many candidates were hidden. Run -``propose all`` to see every candidate --- including proposers whose output -type doesn't actually match the column (for example a continuous-float -proposer against an ``Integer`` column, invalid on most databases), which the -plain ``propose`` excludes structurally rather than relying on the score to -catch. +``propose all`` to see every candidate regardless of front or score --- +including, for example, a continuous-float proposer against an ``Integer`` +column; whether that's a valid choice for your database is on you to check, +since ``propose`` doesn't filter or penalize candidates by output type. + +The recommended candidate's row is marked with a leading ``*`` and (if your +terminal theme supports colour) highlighted, whichever Pareto front it's in. The **recommendation** is simply the candidate with the highest ``Score`` (ties broken by NSGA-II crowding distance, i.e. preferring a candidate that's diff --git a/docs/source/choosing_a_generator.rst b/docs/source/choosing_a_generator.rst index a47c2dc8..37cc7e80 100644 --- a/docs/source/choosing_a_generator.rst +++ b/docs/source/choosing_a_generator.rst @@ -40,7 +40,7 @@ the two situations where you need more than the recommendation: * - A near-unique value that must look real but not *be* real, e.g. an email address or free-text ID - Whichever ``generic.*`` value proposer ``propose`` recommends - - ``propose`` applies an extra privacy penalty here: a candidate that + - ``propose`` applies an extra privacy safeguard here: a candidate that reproduces real values verbatim is scored down in proportion to how unique the real column is, so the recommendation already accounts for leak risk. See :ref:`evaluating-and-ranking-proposals`. diff --git a/tests/test_evaluators_column_evaluator.py b/tests/test_evaluators_column_evaluator.py index 61e8cabb..579fa298 100644 --- a/tests/test_evaluators_column_evaluator.py +++ b/tests/test_evaluators_column_evaluator.py @@ -31,12 +31,18 @@ def test_none_values_are_excluded(self) -> None: self.assertEqual(1, stats.unique_count) def test_avg_length_and_ratios(self) -> None: - """avg_length, space_ratio, digit_ratio and punctuation_ratio are computed.""" - stats = analyse_column(["ab 1!"]) - self.assertEqual(5, stats.avg_length) - self.assertAlmostEqual(1 / 5, stats.space_ratio) - self.assertAlmostEqual(1 / 5, stats.digit_ratio) - self.assertAlmostEqual(1 / 5, stats.punctuation_ratio) + """avg_length, space_ratio, digit_ratio and punctuation_ratio are computed. + + Uses a different count of each (2 spaces, 1 digit, 3 punctuation out + of 7 characters) so a ratio written to the wrong attribute would + actually fail the corresponding assertion, rather than all three + happening to share one value. + """ + stats = analyse_column(["a 1!!!"]) + self.assertEqual(7, stats.avg_length) + self.assertAlmostEqual(2 / 7, stats.space_ratio) + self.assertAlmostEqual(1 / 7, stats.digit_ratio) + self.assertAlmostEqual(3 / 7, stats.punctuation_ratio) def test_empty_input_does_not_divide_by_zero(self) -> None: """An empty (or all-None) column yields all-zero stats, not an error.""" @@ -180,14 +186,14 @@ def test_value_without_local_part_or_domain_is_not_email_like(self) -> None: class ProposalEvaluationTests(DatafakerTestCase): """Test case for the ProposalEvaluation dataclass's display string.""" - def test_str_includes_key_metrics_and_pipeline_scores(self) -> None: - """The string form surfaces the headline scores and per-pipeline detail.""" + def test_str_includes_key_metrics_and_criterion_scores(self) -> None: + """The string form surfaces the headline scores and per-criterion detail.""" evaluation = ProposalEvaluation( proposer=ConstantProposer("x"), novelty=0.5, diversity=0.25, overall_score=0.125, - pipeline_scores={"length": 0.1, "words": 0.2}, + criterion_scores={"length": 0.1, "words": 0.2}, copy_fraction=0.0, synthetic_uniqueness=1.0, ) @@ -204,7 +210,7 @@ class ColumnEvaluatorIntegrationTests(DatafakerTestCase): """End-to-end test of ColumnEvaluator against a real (DuckDB) table.""" def test_evaluate_a_constant_proposer_against_a_categorical_column(self) -> None: - """setup() profiles the column and evaluate() scores a real proposer.""" + """Construction profiles the column; evaluate() scores a real proposer.""" engine = create_engine("duckdb:///:memory:") metadata = MetaData() table = Table( @@ -220,8 +226,7 @@ def test_evaluate_a_constant_proposer_against_a_categorical_column(self) -> None [{"id": i, "status": ["active", "inactive"][i % 2]} for i in range(20)], ) - evaluator = ColumnEvaluator() - evaluator.setup([table.c.status], engine) + evaluator = ColumnEvaluator([table.c.status], engine) self.assertEqual(EvaluationProfile.CATEGORICAL, evaluator.profile) result = evaluator.evaluate(ConstantProposer("active")) @@ -230,4 +235,4 @@ def test_evaluate_a_constant_proposer_against_a_categorical_column(self) -> None # A constant proposer emits the same value every time: only one # distinct value out of the whole synthetic sample. self.assertLess(result.synthetic_uniqueness, 0.01) - self.assertIn("category", result.pipeline_scores) + self.assertIn("category", result.criterion_scores) diff --git a/tests/test_evaluators_feature_extractors.py b/tests/test_evaluators_feature_extractors.py index 014b9fac..23abfb68 100644 --- a/tests/test_evaluators_feature_extractors.py +++ b/tests/test_evaluators_feature_extractors.py @@ -1,5 +1,7 @@ """Unit tests for datafaker.evaluators.feature_extractors.""" -from datetime import date, datetime, timezone +from datetime import date, datetime, timedelta, timezone + +from sqlalchemy import Column, Date, MetaData, Table, create_engine, select from datafaker.evaluators.feature_extractors import ( CharacterBigramExtractor, @@ -260,6 +262,16 @@ def test_all_non_alphabetic_yields_nothing(self) -> None: """A value with no letters at all produces no pattern.""" self.assertEqual([], list(self.extractor.extract("123!!"))) + def test_accented_vowels_count_as_vowels(self) -> None: + """An accented vowel (common in non-English names) is still a 'V'.""" + self.assertEqual(["CVCV"], list(self.extractor.extract("René"))) + self.assertEqual(["CVV"], list(self.extractor.extract("Zoë"))) + + def test_cedilla_and_tilde_consonants_stay_consonants(self) -> None: + """A diacritic on a consonant (ç, ñ) must not be miscounted as a vowel.""" + self.assertEqual(["CVCVC"], list(self.extractor.extract("façon"))) + self.assertEqual(["CVCV"], list(self.extractor.extract("niño"))) + class TimestampExtractorTests(DatafakerTestCase): """Test case for TimestampExtractor.""" @@ -325,6 +337,38 @@ def test_invalid_string_yields_nothing(self) -> None: """An unparseable string produces no feature.""" self.assertEqual([], list(self.extractor.extract("nope"))) + def test_sql_expression_agrees_with_python_weekday(self) -> None: + """The SQL expression must use the same Monday=0 convention as extract(). + + Regression test: a dialect's own day-of-week function (Postgres' + EXTRACT(DOW), MSSQL's DATEPART(weekday)) uses a different, and for + MSSQL session-configurable, numbering convention than Python's + date.weekday() - comparing the two directly would silently treat a + perfect real/synthetic match as a mismatch (see expression()'s + docstring). + """ + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table("d", metadata, Column("dt", Date)) + metadata.create_all(engine) + base = date(1970, 1, 1) + days = [base + timedelta(days=i) for i in range(14)] + with engine.begin() as conn: + conn.execute(table.insert(), [{"dt": d} for d in days]) + + expr = self.extractor.expression(table.c.dt) + with engine.connect() as conn: + rows = conn.execute(select(table.c.dt, expr)).all() + + self.assertEqual(14, len(rows)) + for day, sql_weekday in rows: + self.assertEqual( + day.weekday(), + sql_weekday, + f"{day} ({day.strftime('%A')}): SQL gave {sql_weekday}," + f" Python gives {day.weekday()}", + ) + class EmailLocalPartExtractorTests(DatafakerTestCase): """Test case for EmailLocalPartExtractor.""" diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py index 2e8b73c8..dbf0182a 100644 --- a/tests/test_evaluators_proposal_ranking.py +++ b/tests/test_evaluators_proposal_ranking.py @@ -50,7 +50,7 @@ def make_result( # pylint: disable=too-many-arguments,too-many-positional-argum novelty=novelty, diversity=diversity, overall_score=overall_score, - pipeline_scores={}, + criterion_scores={}, copy_fraction=copy_fraction, synthetic_uniqueness=synthetic_uniqueness, ) diff --git a/tests/test_evaluators_statistical_fidelity.py b/tests/test_evaluators_statistical_fidelity.py index fac7aa61..94db4da1 100644 --- a/tests/test_evaluators_statistical_fidelity.py +++ b/tests/test_evaluators_statistical_fidelity.py @@ -5,34 +5,34 @@ from datafaker.evaluators.column_evaluator import EvaluationProfile from datafaker.evaluators.statistical_fidelity import ( - EMAIL_PIPELINE, - PROFILE_PIPELINES, + EMAIL_CRITERIA, + PROFILE_CRITERIA, StatisticalFidelity, ) from tests.utils import DatafakerTestCase -class ProfilePipelinesTests(DatafakerTestCase): - """Structural sanity checks for the built-in evaluation pipelines.""" +class ProfileCriteriaTests(DatafakerTestCase): + """Structural sanity checks for the built-in evaluation criteria.""" - def test_every_evaluation_profile_has_a_pipeline(self) -> None: - """Each EvaluationProfile enum member maps to a configured pipeline.""" + def test_every_evaluation_profile_has_criteria(self) -> None: + """Each EvaluationProfile enum member maps to configured criteria.""" for profile in EvaluationProfile: - self.assertIn(profile, PROFILE_PIPELINES) - self.assertTrue(PROFILE_PIPELINES[profile]) + self.assertIn(profile, PROFILE_CRITERIA) + self.assertTrue(PROFILE_CRITERIA[profile]) - def test_pipeline_weights_sum_to_one(self) -> None: - """Each profile's pipeline weights add up to 1.0, for a sane weighted average.""" - for profile, pipelines in PROFILE_PIPELINES.items(): + def test_criterion_weights_sum_to_one(self) -> None: + """Each profile's criterion weights add up to 1.0, for a sane weighted average.""" + for profile, criteria in PROFILE_CRITERIA.items(): with self.subTest(profile=profile.name): - total_weight = sum(p.weight for p in pipelines) + total_weight = sum(c.weight for c in criteria) self.assertAlmostEqual(1.0, total_weight) - def test_pipeline_names_are_unique_within_a_profile(self) -> None: - """Pipeline names double as dict keys in calculate_scores, so must be unique.""" - for profile, pipelines in PROFILE_PIPELINES.items(): + def test_criterion_names_are_unique_within_a_profile(self) -> None: + """Criterion names double as dict keys in calculate_scores, so must be unique.""" + for profile, criteria in PROFILE_CRITERIA.items(): with self.subTest(profile=profile.name): - names = [p.name for p in pipelines] + names = [c.name for c in criteria] self.assertEqual(len(names), len(set(names))) @@ -57,15 +57,15 @@ def test_identical_real_and_synthetic_data_scores_perfectly(self) -> None: [{"email": f"user{i}@example.com"} for i in range(30)], ) fidelity = StatisticalFidelity(table.c.email, engine, sample_size=1000) - fidelity.set_eval_pipelines(EvaluationProfile.EMAIL) + fidelity.set_eval_criteria(EvaluationProfile.EMAIL) synthetic = [f"user{i}@example.com" for i in range(30)] - overall_score, pipeline_scores = fidelity.calculate_scores(synthetic) + overall_score, criterion_scores = fidelity.calculate_scores(synthetic) self.assertAlmostEqual(0.0, overall_score) - self.assertEqual({p.name for p in EMAIL_PIPELINE}, set(pipeline_scores)) - for name, score in pipeline_scores.items(): - with self.subTest(pipeline=name): + self.assertEqual({c.name for c in EMAIL_CRITERIA}, set(criterion_scores)) + for name, score in criterion_scores.items(): + with self.subTest(criterion=name): self.assertAlmostEqual(0.0, score) def test_completely_different_synthetic_data_scores_worse(self) -> None: @@ -75,7 +75,7 @@ def test_completely_different_synthetic_data_scores_worse(self) -> None: [{"category": "alpha"} for _ in range(30)], ) fidelity = StatisticalFidelity(table.c.category, engine, sample_size=1000) - fidelity.set_eval_pipelines(EvaluationProfile.CATEGORICAL) + fidelity.set_eval_criteria(EvaluationProfile.CATEGORICAL) matching_score, _ = fidelity.calculate_scores(["alpha"] * 30) different_score, _ = fidelity.calculate_scores(["zzz_never_seen"] * 30) @@ -83,26 +83,26 @@ def test_completely_different_synthetic_data_scores_worse(self) -> None: self.assertGreater(different_score, matching_score) def test_temporal_profile_scores_a_matching_date_column(self) -> None: - """The TEMPORAL profile's two pipelines run cleanly against a date column.""" + """The TEMPORAL profile's two criteria run cleanly against a date column.""" engine, table = _make_table( [Column("ts", DateTime)], [{"ts": datetime(2020, 1, (i % 27) + 1)} for i in range(40)], ) fidelity = StatisticalFidelity(table.c.ts, engine, sample_size=1000) - fidelity.set_eval_pipelines(EvaluationProfile.TEMPORAL) + fidelity.set_eval_criteria(EvaluationProfile.TEMPORAL) synthetic = [datetime(2020, 1, (i % 27) + 1) for i in range(40)] - overall_score, pipeline_scores = fidelity.calculate_scores(synthetic) + overall_score, criterion_scores = fidelity.calculate_scores(synthetic) self.assertAlmostEqual(0.0, overall_score) - self.assertEqual({"timestamp", "day_of_week"}, set(pipeline_scores)) + self.assertEqual({"timestamp", "day_of_week"}, set(criterion_scores)) - def test_no_pipelines_set_scores_zero(self) -> None: - """Without calling set_eval_pipelines, there's nothing to score.""" + def test_no_criteria_set_scores_zero(self) -> None: + """Without calling set_eval_criteria, there's nothing to score.""" engine, table = _make_table( [Column("n", Integer)], [{"n": i} for i in range(10)] ) fidelity = StatisticalFidelity(table.c.n, engine, sample_size=1000) - overall_score, pipeline_scores = fidelity.calculate_scores([1, 2, 3]) + overall_score, criterion_scores = fidelity.calculate_scores([1, 2, 3]) self.assertEqual(0.0, overall_score) - self.assertEqual({}, pipeline_scores) + self.assertEqual({}, criterion_scores) diff --git a/tests/test_interactive_choice_generators.py b/tests/test_interactive_choice_generators.py index e2cb34b5..dfb2e220 100644 --- a/tests/test_interactive_choice_generators.py +++ b/tests/test_interactive_choice_generators.py @@ -45,7 +45,11 @@ def _get_cmd(self, config: MutableMapping[str, Any]) -> MockGeneratorCmd: def _propose(self, gc: MockGeneratorCmd) -> dict[str, tuple[int, str, list[str]]]: gc.reset() - gc.do_propose("") + # These tests check for the full set of choice/zipf_choice/ + # weighted_choice variants (sampled/suppressed combinations), which + # may not all be Pareto front 1 - 'all' bypasses that cap so every + # variant is listed. + gc.do_propose("all") return gc.get_proposals() def test_create_with_sampled_choice(self) -> None: diff --git a/tests/test_interactive_date_generators.py b/tests/test_interactive_date_generators.py index 978347e9..5dd18f02 100644 --- a/tests/test_interactive_date_generators.py +++ b/tests/test_interactive_date_generators.py @@ -19,12 +19,17 @@ class MockGeneratorCmd(GeneratorCmd, TestDbCmdMixin): def get_proposals(self) -> dict[str, tuple[int, str, list[str]]]: """ - Returns a dict of generator name to a tuple of (index, fit_string, [list,of,samples]) + Returns a dict of generator name to a tuple of (index, '', [list,of,samples]). + + The middle element is unused: it used to carry the old per-proposer + fit score, but 'propose' no longer prints that separate (now + removed) list, only the ranked table (``RANKED_SAMPLE_TEXT``), which + has no equivalent per-candidate fit figure. """ return { - kw["name"]: (kw["index"], kw["fit"], kw["sample"].split("; ")) + kw["name"]: (kw["index"], "", kw["sample"].split("; ")) for (s, _, kw) in self.messages - if s == self.PROPOSE_GENERATOR_SAMPLE_TEXT + if s == self.RANKED_SAMPLE_TEXT } @@ -270,7 +275,9 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: # set up our interval proposer gc.do_next(f"{table}.{column}") gc.reset() - gc.do_propose("") + # Both anchored variants must be listed at once - 'all' bypasses + # the Pareto-front-1 cap that might otherwise hide one of them. + gc.do_propose("all") proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" @@ -291,7 +298,7 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: self.assertEqual(gc.messages[2][0], gc.PROVIDING_VALUES_TEXT) gc.do_next(column2) gc.reset() - gc.do_propose("") + gc.do_propose("all") proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" diff --git a/tests/test_interactive_generators.py b/tests/test_interactive_generators.py index 1a59e4c7..46b20c30 100644 --- a/tests/test_interactive_generators.py +++ b/tests/test_interactive_generators.py @@ -22,12 +22,17 @@ class MockGeneratorCmd(GeneratorCmd, TestDbCmdMixin): def get_proposals(self) -> dict[str, tuple[int, str, list[str]]]: """ - Returns a dict of generator name to a tuple of (index, fit_string, [list,of,samples]) + Returns a dict of generator name to a tuple of (index, '', [list,of,samples]). + + The middle element is unused: it used to carry the old per-proposer + fit score, but 'propose' no longer prints that separate (now + removed) list, only the ranked table (``RANKED_SAMPLE_TEXT``), which + has no equivalent per-candidate fit figure. """ return { - kw["name"]: (kw["index"], kw["fit"], kw["sample"].split("; ")) + kw["name"]: (kw["index"], "", kw["sample"].split("; ")) for (s, _, kw) in self.messages - if s == self.PROPOSE_GENERATOR_SAMPLE_TEXT + if s == self.RANKED_SAMPLE_TEXT } def get_roles_from_columns(self, column: str) -> set[str]: @@ -126,7 +131,10 @@ def test_set_generator_mimesis(self) -> None: column = "name" generator = "person.first_name" gc.do_next(f"{table}.{column}") - gc.do_propose("") + # Picking a specific generator by name, not necessarily the + # recommendation - 'all' guarantees it's listed regardless of + # Pareto front. + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[f"generic.{generator}"][0])) gc.do_quit("") @@ -203,7 +211,10 @@ def test_set_generator_choice(self) -> None: column = "frequency" generator = "dist_gen.choice" gc.do_next(f"{table}.{column}") - gc.do_propose("") + # Picking a specific generator by name, not necessarily the + # recommendation - 'all' guarantees it's listed regardless of + # Pareto front. + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[generator][0])) gc.do_quit("") @@ -244,7 +255,10 @@ def test_weighted_choice_generator_generates_choices(self) -> None: generator = "dist_gen.weighted_choice" values = {1, 2, 3, 4, 5, 6} gc.do_next(f"{table}.{column}") - gc.do_propose("") + # Picking a specific generator by name, not necessarily the + # recommendation - 'all' guarantees it's listed regardless of + # Pareto front. + gc.do_propose("all") proposals = gc.get_proposals() gen_proposal = proposals[generator] self.assert_subset(set(gen_proposal[2]), {str(v) for v in values}) @@ -261,8 +275,11 @@ def test_merge_columns(self) -> None: generator_to_discard = "dist_gen.choice" generator = "dist_gen.multivariate_normal" with self._get_cmd({}) as gc: + # Picking specific generators by name below, not necessarily the + # recommendation - 'all' guarantees they're listed regardless of + # Pareto front. gc.do_next(f"{table}.{column_2}") - gc.do_propose("") + gc.do_propose("all") proposals = gc.get_proposals() # set a generator, but this should not exist after merging gc.do_set(str(proposals[generator_to_discard][0])) @@ -270,7 +287,7 @@ def test_merge_columns(self) -> None: self.assertIn(table, gc.prompt) self.assertIn(column_1, gc.prompt) self.assertNotIn(column_2, gc.prompt) - gc.do_propose("") + gc.do_propose("all") proposals = gc.get_proposals() # set a generator, but this should not exist either gc.do_set(str(proposals[generator_to_discard][0])) @@ -283,7 +300,7 @@ def test_merge_columns(self) -> None: self.assertIn(column_1, gc.prompt) self.assertIn(column_2, gc.prompt) gc.reset() - gc.do_propose("") + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[generator][0])) gc.do_quit("") @@ -364,7 +381,10 @@ def test_old_generators_remain(self) -> None: column = "name" generator = "person.first_name" gc.do_next(f"{table}.{column}") - gc.do_propose("") + # Picking a specific generator by name, not necessarily the + # recommendation - 'all' guarantees it's listed regardless of + # Pareto front. + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[f"generic.{generator}"][0])) gc.do_quit("") @@ -427,10 +447,9 @@ def test_aggregate_queries_merge(self) -> None: column = "position" generator = "dist_gen.uniform_ms" gc.do_next(f"string.{column}") - # "position" is an Integer column, and dist_gen.uniform_ms produces - # float output that default 'propose' now excludes as type- - # incompatible (see GeneratorCmd._is_integer_incompatible) - 'all' - # bypasses that filter so this generator is still selectable here. + # dist_gen.uniform_ms produces float output for this Integer + # column, so it's unlikely to be Pareto front 1 - use 'all' to + # bypass the default front-1-only cap and make sure it's listed. gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[f"{generator}"][0])) @@ -608,10 +627,9 @@ def test_existing_configuration_remains(self) -> None: column = "position" generator = "dist_gen.uniform_ms" gc.do_next(f"string.{column}") - # "position" is an Integer column, and dist_gen.uniform_ms produces - # float output that default 'propose' now excludes as type- - # incompatible (see GeneratorCmd._is_integer_incompatible) - 'all' - # bypasses that filter so this generator is still selectable here. + # dist_gen.uniform_ms produces float output for this Integer + # column, so it's unlikely to be Pareto front 1 - use 'all' to + # bypass the default front-1-only cap and make sure it's listed. gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[generator][0])) @@ -742,7 +760,10 @@ def test_varchar_ns_are_truncated(self) -> None: with self._get_cmd({}) as gc: gc.do_next(f"{table}.{column}") gc.reset() - gc.do_propose("") + # Picking a specific generator by name, not necessarily the + # recommendation - 'all' guarantees it's listed regardless of + # Pareto front. + gc.do_propose("all") proposals = gc.get_proposals() quotes = [k for k in proposals.keys() if k.startswith(generator)] self.assertEqual(len(quotes), 1) diff --git a/tests/test_interactive_generators_extract.py b/tests/test_interactive_generators_extract.py index f24dc0c8..d4c23797 100644 --- a/tests/test_interactive_generators_extract.py +++ b/tests/test_interactive_generators_extract.py @@ -24,7 +24,10 @@ def _get_cmd(self, config: MutableMapping[str, Any]) -> MockGeneratorCmd: def _propose(self, gc: MockGeneratorCmd) -> dict[str, tuple[int, str, list[str]]]: gc.reset() - gc.do_propose("") + # These tests check for every per-column extract-component variant + # at once, which may not all be Pareto front 1 - 'all' bypasses + # that cap so every variant is listed. + gc.do_propose("all") return gc.get_proposals() def _choose_proposal( diff --git a/tests/test_interactive_generators_partitioned.py b/tests/test_interactive_generators_partitioned.py index d65be259..f1766342 100644 --- a/tests/test_interactive_generators_partitioned.py +++ b/tests/test_interactive_generators_partitioned.py @@ -147,7 +147,10 @@ def _get_cmd(self, config: MutableMapping[str, Any]) -> MockGeneratorCmd: def _propose(self, gc: MockGeneratorCmd) -> dict[str, tuple[int, str, list[str]]]: gc.reset() - gc.do_propose("") + # These tests check for the full set of sampled/suppressed + # null-partitioned variants, which may not all be Pareto front 1 - + # 'all' bypasses that cap so every variant is listed. + gc.do_propose("all") return gc.get_proposals() def test_create_with_null_partitioned_grouped_multivariate(self) -> None: diff --git a/tests/test_rst.py b/tests/test_rst.py index edd87ac9..81436c88 100644 --- a/tests/test_rst.py +++ b/tests/test_rst.py @@ -55,7 +55,7 @@ def test_dir(self) -> None: 'Hyperlink target "mimesis-string-list" is not referenced', 'Hyperlink target "increment-proposer" is not referenced', 'Hyperlink target "evaluating-and-ranking-proposals" is not referenced', - 'Hyperlink target "statistical-fidelity-pipelines" is not referenced', + 'Hyperlink target "statistical-fidelity-criteria" is not referenced', ] filtered_errors = [ file_error