From 2edf1a8d2bd23f30285a82c82ae98f8589033e3a Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Wed, 9 Sep 2026 17:05:45 +0100 Subject: [PATCH 01/23] Implement column evaluators --- datafaker/dialects.py | 90 +++ datafaker/evaluators/column_evaluator.py | 334 +++++++++++ datafaker/evaluators/distribution_builders.py | 279 +++++++++ datafaker/evaluators/evaluation_profile.py | 20 + datafaker/evaluators/feature_extractors.py | 470 +++++++++++++++ datafaker/evaluators/metrics.py | 178 ++++++ datafaker/evaluators/proposal_ranking.py | 534 ++++++++++++++++++ datafaker/evaluators/statistical_fidelity.py | 297 ++++++++++ datafaker/interactive/generators.py | 278 ++++++++- 9 files changed, 2467 insertions(+), 13 deletions(-) create mode 100644 datafaker/evaluators/column_evaluator.py create mode 100644 datafaker/evaluators/distribution_builders.py create mode 100644 datafaker/evaluators/evaluation_profile.py create mode 100644 datafaker/evaluators/feature_extractors.py create mode 100644 datafaker/evaluators/metrics.py create mode 100644 datafaker/evaluators/proposal_ranking.py create mode 100644 datafaker/evaluators/statistical_fidelity.py diff --git a/datafaker/dialects.py b/datafaker/dialects.py index df09aff..9d92e7b 100644 --- a/datafaker/dialects.py +++ b/datafaker/dialects.py @@ -211,6 +211,96 @@ def compile_stddev_mssql(element: StdDev, compiler: Any, **kw: Any) -> str: return f"STDEV({e})" +class WordCount(ColumnElement[int]): # pylint: disable=too-many-ancestors + """Count whitespace-separated words in a text expression.""" + + expr: ColumnElement[str] + + _traverse_internals = [ + ("expr", InternalTraversal.dp_clauseelement), + ] + + def __init__(self, expr: ColumnElement[str]): + """Get a clause for the number of whitespace-separated words in ``expr``.""" + self.expr = expr + + __sa_operate__ = ColumnElement.operate + + +@compiles(WordCount) +def compile_word_count(element: WordCount, compiler: Any, **kw: Any) -> str: + """Exact word count via regexp_split_to_array (Postgres/DuckDB).""" + e = compiler.process(element.expr, **kw) + return f"array_length(regexp_split_to_array(trim({e}), '\\s+'), 1)" + + +@compiles(WordCount, "mssql") +def compile_word_count_mssql(element: WordCount, compiler: Any, **kw: Any) -> str: + """MSSQL equivalent: approximate word count via space-counting. + + No regex/array-split support, so approximate word count as (space + count in the trimmed text) + 1. This overcounts by one per run of 2+ + consecutive spaces relative to the exact regex-based count used on + Postgres/DuckDB, but there is no T-SQL primitive for a real + whitespace-run split. + """ + e = compiler.process(element.expr, **kw) + trimmed = f"TRIM({e})" + return ( + f"(CASE WHEN LEN({trimmed}) = 0 THEN 0 ELSE " + f"LEN({trimmed}) - LEN(REPLACE({trimmed}, ' ', '')) + 1 END)" + ) + + +class SentenceCount(ColumnElement[int]): # pylint: disable=too-many-ancestors + """Count '.'/'!'/'?'-delimited sentences in a text expression.""" + + expr: ColumnElement[str] + + _traverse_internals = [ + ("expr", InternalTraversal.dp_clauseelement), + ] + + def __init__(self, expr: ColumnElement[str]): + """Get a clause for the number of sentences in ``expr``.""" + self.expr = expr + + __sa_operate__ = ColumnElement.operate + + +@compiles(SentenceCount) +def compile_sentence_count(element: SentenceCount, compiler: Any, **kw: Any) -> str: + """Exact sentence count via regexp_split_to_array (Postgres/DuckDB).""" + e = compiler.process(element.expr, **kw) + return ( + f"array_length(regexp_split_to_array(trim({e}, ' .!?\\t\\n\\r'), '[.!?]+'), 1)" + ) + + +@compiles(SentenceCount, "mssql") +def compile_sentence_count_mssql( + element: SentenceCount, compiler: Any, **kw: Any +) -> str: + """MSSQL equivalent: approximate sentence count via delimiter-counting. + + No regex support, so approximate sentence count by normalizing '!'/'?' + to '.' and counting '.' characters, after trimming whitespace and + sentence-enders from both ends (so a leading/trailing sentence-ender + doesn't add a spurious segment, matching the Postgres/DuckDB + behaviour). This overcounts by one per run of 2+ consecutive delimiter + characters (e.g. "Wow!!") relative to the exact regex-based count, but + there is no T-SQL primitive for regex-run splitting. + """ + e = compiler.process(element.expr, **kw) + trim_chars = "' .!?' + CHAR(9) + CHAR(10) + CHAR(13)" + trimmed = f"TRIM({trim_chars} FROM {e})" + normalized = f"REPLACE(REPLACE({trimmed}, '!', '.'), '?', '.')" + return ( + f"(CASE WHEN LEN({trimmed}) = 0 THEN 0 ELSE " + f"LEN({normalized}) - LEN(REPLACE({normalized}, '.', '')) + 1 END)" + ) + + class IsNull(ColumnElement[bool]): # pylint: disable=too-many-ancestors """Represent IS NULL as an expression.""" diff --git a/datafaker/evaluators/column_evaluator.py b/datafaker/evaluators/column_evaluator.py new file mode 100644 index 0000000..4b02df9 --- /dev/null +++ b/datafaker/evaluators/column_evaluator.py @@ -0,0 +1,334 @@ +"""Evaluate how well a proposer's synthetic data matches a real column.""" + +import string +from dataclasses import dataclass + +from sqlalchemy import Column, Engine, select +from sqlalchemy.types import Date, DateTime, Integer, Numeric, Time + +from datafaker.dialects import Random +from datafaker.evaluators.evaluation_profile import EvaluationProfile +from datafaker.evaluators.metrics import ( + DiversityMetric, + NoveltyMetric, + canonicalize_value, +) +from datafaker.evaluators.statistical_fidelity import StatisticalFidelity +from datafaker.proposers.base import Proposer, get_column_type + +__all__ = ["EvaluationProfile", "ColumnEvaluator", "ColumnStats", "ProposalEvaluation"] + + +@dataclass +class ColumnStats: + """Summary statistics of a column's (string-rendered) real values.""" + + row_count: int + unique_count: int + avg_length: float + space_ratio: float + digit_ratio: float + punctuation_ratio: float + + @property + def uniqueness(self): + """Fraction of rows whose value is distinct from the others.""" + return self.unique_count / max(self.row_count, 1) + + +@dataclass +class ProposalEvaluation: + """The scored result of evaluating one proposer against a column.""" + + proposer: Proposer + novelty: float + diversity: float + overall_score: float + pipeline_scores: dict[str, float] + copy_fraction: float = 0.0 + synthetic_uniqueness: float = 1.0 + + def __str__(self): + """Render a human-readable summary of the evaluation.""" + lines = [ + f"{self.proposer.name()} " + f" (error={self.overall_score:.6f})" + f" (novelty={self.novelty:.6f})" + f" (diversity={self.diversity:.6f})" + f" (copies={self.copy_fraction:.6f})" + ] + + for name, score in self.pipeline_scores.items(): + lines.append(f" {name:<15} {score:.6f}") + + return "\n".join(lines) + + __repr__ = __str__ + + +def analyse_column(values: list[str]) -> ColumnStats: + """Compute length/space/digit/punctuation statistics over ``values``.""" + values = [str(v) for v in values if v is not None] + + total_chars = sum(len(v) for v in values) + + spaces = sum(v.count(" ") for v in values) + + digits = sum(c.isdigit() for v in values for c in v) + + punctuation = sum(c in string.punctuation for v in values for c in v) + + return ColumnStats( + row_count=len(values), + unique_count=len(set(values)), + avg_length=(total_chars / len(values) if values else 0), + space_ratio=(spaces / total_chars if total_chars else 0), + digit_ratio=(digits / total_chars if total_chars else 0), + punctuation_ratio=(punctuation / total_chars if total_chars else 0), + ) + + +# Below this fraction of distinct values, a column is treated as a small, +# repeated set of categories rather than a free-form or continuous quantity. +CATEGORICAL_UNIQUENESS_THRESHOLD = 0.2 + + +def choose_profile(stats: ColumnStats) -> EvaluationProfile: + """Pick a string column's evaluation profile from its statistics.""" + # mostly unique short values + if stats.avg_length < 30 and stats.uniqueness > 0.8: + return EvaluationProfile.SHORT_TEXT + + # repeated values + if stats.uniqueness < CATEGORICAL_UNIQUENESS_THRESHOLD: + return EvaluationProfile.CATEGORICAL + + # long text + if stats.avg_length > 50 or stats.space_ratio > 0.15: + return EvaluationProfile.FREE_TEXT + + return EvaluationProfile.SHORT_TEXT + + +def choose_numeric_profile(values) -> EvaluationProfile: + """Pick a numeric column's evaluation profile from its cardinality. + + Decide whether an Integer/Numeric column behaves like a small set of + repeated categories (a status code, rating, or flag) or like a + continuous / high-cardinality quantity (an age, salary, or identifier). + + Previously every Integer column was assumed to be categorical and every + Numeric column was assumed to be an identifier, regardless of how many + distinct values it actually had - which misclassifies something like an + integer age or salary column. This applies the same cardinality check + already used for string columns instead. + """ + non_null = [v for v in values if v is not None] + if not non_null: + return EvaluationProfile.IDENTIFIER + + uniqueness = len(set(non_null)) / len(non_null) + if uniqueness < CATEGORICAL_UNIQUENESS_THRESHOLD: + return EvaluationProfile.CATEGORICAL + + return EvaluationProfile.IDENTIFIER + + +def looks_like_email(values) -> bool: + """Heuristically decide whether a sample of values looks like emails.""" + nonempty = [str(v).strip() for v in values if v is not None and str(v).strip()] + if not nonempty: + return False + + email_like = 0 + for value in nonempty: + if "@" not in value: + continue + local_part, sep, domain = value.partition("@") + if not sep or not local_part or not domain: + continue + if "." in domain: + email_like += 1 + + return email_like / len(nonempty) > 0.5 + + +class ColumnEvaluator: # pylint: disable=too-many-instance-attributes + """Evaluate candidate proposers against one (or several merged) columns.""" + + def __init__(self): + """Initialize an unconfigured evaluator; call setup() before use.""" + self.engine = None + self.novelty_metric = None + self.diversity_metric = None + self.sample_size = 4000 + self.columns = [] + self.column = None + self.table = None + self.is_primary_key = False + self.real_values = [] + self.column_is_numeric = False + self.real_uniqueness = 0.0 + self.statistical_fidelity = None + self.profile = None + + def setup(self, columns: list[Column], engine: Engine): + """Sample the real column(s) and pick the evaluation profile/pipelines.""" + self.engine = engine + self.novelty_metric = NoveltyMetric() + self.diversity_metric = DiversityMetric() + + self.columns = columns + self.column = columns[0] if len(columns) == 1 else None + self.table = columns[0].table if columns else None + # A composite key needs every constituent column marked primary_key; + # this reduces to the single column's own flag in the common case. + self.is_primary_key = bool(columns) and all(c.primary_key for c in columns) + + with engine.connect() as conn: + if len(columns) == 1: + # Cap how many real rows we pull into memory so this scales to + # large tables, matching the synthetic sample_size above rather + # than materializing the entire column. Order randomly first: + # an unordered LIMIT returns whatever the DB naturally returns + # first (often insertion order), which can systematically miss + # real values for a column whose value correlates with row + # order (e.g. a "last_update" timestamp) - corrupting + # real_uniqueness/copy_fraction/novelty/diversity, all of + # which are computed from this sample. + rows = conn.execute( + select(columns[0]) + .select_from(self.table) + .order_by(Random()) + .limit(self.sample_size) + ) + self.real_values = [row[0] for row in rows] + column_type = get_column_type(columns[0]) + else: + rows = conn.execute( + select(*columns) + .select_from(columns[0].table) + .order_by(Random()) + .limit(self.sample_size) + ) + self.real_values = [tuple(row) for row in rows] + column_type = None + + # Numeric columns (whether they end up profiled IDENTIFIER or, for a + # low-cardinality one like a foreign key, CATEGORICAL) have no shared + # real-world vocabulary that could explain a generator coincidentally + # matching real values - unlike a string column (e.g. common first + # names), where that overlap is expected and legitimate. Used to + # decide how broadly the resample-penalty in proposal_ranking.py + # applies. + self.column_is_numeric = isinstance(column_type, (Numeric, Integer)) + + # fraction of real values that are distinct - used to judge whether a + # generator that resamples from the real data verbatim (e.g. a + # ChoiceProposer) is doing the right thing (a small, shared + # vocabulary, where reproducing real values is expected/correct) or + # a privacy-losing shortcut (near-unique real values, where + # reproducing them verbatim just leaks real records). + non_null_real_values = [v for v in self.real_values if v is not None] + self.real_uniqueness = ( + len({str(v) for v in non_null_real_values}) / len(non_null_real_values) + if non_null_real_values + else 0.0 + ) + + # Evaluation dimensions: fidelity, novelty, diversity + if len(columns) == 1: + self.statistical_fidelity = StatisticalFidelity( + self.column, self.engine, sample_size=self.sample_size + ) + if isinstance(column_type, (Date, DateTime, Time)): + profile = EvaluationProfile.TEMPORAL + elif isinstance(column_type, (Numeric, Integer)): + profile = choose_numeric_profile(self.real_values) + else: + stats = analyse_column(self.real_values) + if looks_like_email(self.real_values): + profile = EvaluationProfile.EMAIL + else: + profile = choose_profile(stats) + self.profile = profile + self.statistical_fidelity.set_eval_pipelines(self.profile) + else: + self.statistical_fidelity = None + self.profile = EvaluationProfile.IDENTIFIER + + def evaluate(self, proposer) -> ProposalEvaluation: + """Score one proposer's synthetic data against the sampled real data.""" + synthetic_samples = proposer.generate_data(self.sample_size) + + # calculate novelty + novelty_score = self.novelty_metric.compare( + self.real_values, + synthetic_samples, + ) + + # calculate diversity + diversity_score = self.diversity_metric.compare( + self.real_values, + synthetic_samples, + ) + + # calculate copy fraction: fraction of synthetic samples that exactly match a real value + try: + real_set = set( + str(canonicalize_value(v)) for v in self.real_values if v is not None + ) + if synthetic_samples: + copy_count = sum( + 1 + for s in synthetic_samples + if str(canonicalize_value(s)) in real_set + ) + copy_frac = copy_count / max(len(synthetic_samples), 1) + else: + copy_frac = 0.0 + except Exception: # pylint: disable=broad-exception-caught + # A proposer's own output is untrusted here - anything unhashable + # or unstringifiable should degrade to "no copies detected" + # rather than crash the whole propose/evaluate flow. + copy_frac = 0.0 + + # fraction of the synthetic sample's own values that are distinct from + # each other - a direct, real-data-independent measure of whether this + # proposer could satisfy a PRIMARY KEY/UNIQUE constraint at all. This + # is different from copy_fraction (which only checks overlap against + # the *real* values): a proposer resampling from a small vocabulary, + # or one like dist_gen.constant that emits the same value every time, + # duplicates against its own output regardless of whether that output + # happens to match any real value. + try: + synthetic_uniqueness = ( + len({str(s) for s in synthetic_samples}) / len(synthetic_samples) + if synthetic_samples + else 0.0 + ) + except Exception: # pylint: disable=broad-exception-caught + # Same rationale as the copy_frac catch above. + synthetic_uniqueness = 0.0 + + # calculate fidelity scores for each evaluation pipeline. Multi-column + # proposal evaluation does not map cleanly onto the single-column fidelity + # pipelines, so fall back to a neutral score instead of crashing. + if self.statistical_fidelity is None: + stat_fidelity_overall_score = 0.0 + stat_fidelity_pipeline_scores = {} + else: + ( + stat_fidelity_overall_score, + stat_fidelity_pipeline_scores, + ) = self.statistical_fidelity.calculate_scores(synthetic_samples) + + return ProposalEvaluation( + proposer=proposer, + novelty=novelty_score, + diversity=diversity_score, + overall_score=stat_fidelity_overall_score, + pipeline_scores=stat_fidelity_pipeline_scores, + copy_fraction=copy_frac, + synthetic_uniqueness=synthetic_uniqueness, + ) diff --git a/datafaker/evaluators/distribution_builders.py b/datafaker/evaluators/distribution_builders.py new file mode 100644 index 0000000..868f072 --- /dev/null +++ b/datafaker/evaluators/distribution_builders.py @@ -0,0 +1,279 @@ +"""Build real/synthetic value distributions for statistical fidelity scoring.""" + +import statistics +from abc import ABC, abstractmethod +from bisect import bisect_right +from collections import Counter +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import func, select + +from datafaker.dialects import Random +from datafaker.evaluators.feature_extractors import FeatureExtractor, IdentityExtractor + + +@dataclass +class Distribution: + """A discrete probability distribution over feature values.""" + + probabilities: dict[Any, float] + + @property + def vocabulary(self) -> set[Any]: + """The set of feature values this distribution assigns mass to.""" + return set(self.probabilities.keys()) + + def probability(self, key: Any) -> float: + """Return the probability mass assigned to ``key`` (0.0 if unseen).""" + return self.probabilities.get(key, 0.0) + + def as_vector(self, vocabulary: list[Any]) -> list[float]: + """Render this distribution as a probability vector over ``vocabulary``.""" + return [self.probability(v) for v in vocabulary] + + +class DistributionBuilder(ABC): + """Build a real or synthetic value's ``Distribution`` for one feature.""" + + @abstractmethod + def build_from_table(self) -> Distribution: + """Build the distribution of this feature over the real column.""" + + @abstractmethod + def build_from_values(self, values) -> Distribution: + """Build the distribution of this feature over synthetic ``values``.""" + + +# pylint: disable=too-many-instance-attributes +class HistogramBuilder(DistributionBuilder): + """Bucket a continuous feature into a fixed-width histogram.""" + + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + engine, + table, + column, + bins=10, + extractor: FeatureExtractor = IdentityExtractor(), + sample_size=4000, + ): + """Initialize a histogram builder for one column feature.""" + self.engine = engine + self.table = table + self.column = column + self.bins = bins + self.extractor = extractor + self.sample_size = sample_size + + # Learned from the real data + self.mean = None + self.stddev = None + self.bottom = None + self.width = None + self.edges = None + + def build_from_table(self) -> Distribution: + """Build the real column's histogram distribution.""" + feature_expr = self.extractor.expression(self.column) + + # + # Read a bounded, randomly-ordered sample of values for computing + # statistics (mean/stddev), rather than pulling the entire column + # into memory. Random ordering matters: an unordered LIMIT returns + # whatever the DB naturally returns first (often insertion order), + # which can systematically bias this sample for a column whose + # value correlates with row order (e.g. a "rental_date" column + # inserted roughly chronologically) - and since this is the "real" + # side of every fidelity comparison, that bias directly skews which + # generator looks like the better fit. + # + with self.engine.connect() as conn: + values = ( + conn.execute( + select(feature_expr) + .select_from(self.table) + .order_by(Random()) + .limit(self.sample_size) + ) + .scalars() + .all() + ) + + values = [float(v) for v in values if v is not None] + + if not values: + return Distribution({}) + + self.mean = statistics.mean(values) + self.stddev = statistics.stdev(values) if len(values) > 1 else 0.0 + + if self.stddev == 0: + self.edges = [ + float("-inf"), + self.mean - 2.0, + self.mean - 1.5, + self.mean - 1.0, + self.mean - 0.5, + self.mean, + self.mean + 0.5, + self.mean + 1.0, + self.mean + 1.5, + self.mean + 2.0, + float("inf"), + ] + bottom = self.mean - 2.0 + width = 0.5 + else: + self.edges = [ + float("-inf"), + self.mean - 2 * self.stddev, + self.mean - 1.5 * self.stddev, + self.mean - 1.0 * self.stddev, + self.mean - 0.5 * self.stddev, + self.mean, + self.mean + 0.5 * self.stddev, + self.mean + 1.0 * self.stddev, + self.mean + 1.5 * self.stddev, + self.mean + 2 * self.stddev, + float("inf"), + ] + bottom = self.mean - 2 * self.stddev + width = self.stddev / 2 + + if width == 0: + width = 1.0 + + # + # SQL histogram, computed exactly over the full table (this is a + # bounded, server-side aggregate query - it returns at most `bins` + # rows - so it doesn't need the sample_size cap that applies to the + # raw-value pulls above). + # + with self.engine.connect() as conn: + # Group by the bucket expression itself, not its output alias: + # MSSQL (unlike Postgres/DuckDB) rejects a SELECT-list alias + # referenced from GROUP BY. + bucket_expr = func.floor((feature_expr - bottom) / width) + rows = conn.execute( + select( + bucket_expr.label("bucket"), + func.count().label("count"), # pylint: disable=not-callable + ) + .select_from(self.table) + .group_by(bucket_expr) + ).all() + + # total must come from this same full-table query, not from the + # (possibly sampled) `values` above, so the probabilities sum to 1. + total = sum(count for bucket, count in rows if bucket is not None) + + if total == 0: + return Distribution({}) + + probs = {} + + for bucket, count in rows: + if bucket is None: + continue + + bucket = min( + self.bins - 1, + max(0, int(bucket) + 1), + ) + + probs[bucket] = probs.get(bucket, 0) + count / total + + return Distribution(probs) + + def build_from_values(self, values) -> Distribution: + """Build the synthetic sample's histogram distribution.""" + counts = Counter() + total = 0 + + for value in values: + for feature in self.extractor.extract(value): + feature = float(feature) + bucket = bisect_right(self.edges, feature) - 1 + bucket = max(0, min(bucket, len(self.edges) - 2)) + counts[bucket] += 1 + total += 1 + + if total == 0: + return Distribution({}) + + return Distribution({b: c / total for b, c in counts.items()}) + + +class CategoryBuilder(DistributionBuilder): + """Build a categorical distribution over a discrete feature's values.""" + + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + engine, + table, + column, + extractor: FeatureExtractor = IdentityExtractor(), + sample_size=4000, + ): + """Initialize a category builder for one column feature.""" + self.engine = engine + self.table = table + self.column = column + self.extractor = extractor + self.sample_size = sample_size + + def build_from_table(self): + """Build the real column's categorical distribution.""" + counter = Counter() + + with self.engine.connect() as conn: + # Bounded, randomly-ordered sample rather than the entire column, + # so this scales to large tables (matches the synthetic sample + # size for a fair, like-for-like comparison) without an + # unordered LIMIT biasing the "real" distribution toward + # whatever the DB returns first (see HistogramBuilder above). + rows = conn.execute( + select(self.column) + .select_from(self.table) + .order_by(Random()) + .limit(self.sample_size) + ) + + for (value,) in rows: + if value is None: + continue + for feature in self.extractor.extract(value): + if feature is None: + continue + counter[feature] += 1 + + total = sum(counter.values()) + if total == 0: + return Distribution({}) + + return Distribution({k: v / total for k, v in counter.items()}) + + def build_from_values(self, values): + """Build the synthetic sample's categorical distribution.""" + counter = Counter() + + for value in values: + if value is None: + continue + for feature in self.extractor.extract(value): + if feature is None: + continue + counter[feature] += 1 + + total = sum(counter.values()) + if total == 0: + # No features were extracted from the synthetic values; represent this + # explicitly with a special missing token so metrics treat it as a + # disjoint distribution (maximally different) compared to a real + # distribution with concrete feature values. + return Distribution({"__MISSING__": 1.0}) + + return Distribution({k: v / total for k, v in counter.items()}) diff --git a/datafaker/evaluators/evaluation_profile.py b/datafaker/evaluators/evaluation_profile.py new file mode 100644 index 0000000..c7544eb --- /dev/null +++ b/datafaker/evaluators/evaluation_profile.py @@ -0,0 +1,20 @@ +"""The evaluation profile enum, kept in its own module to avoid a cycle. + +``column_evaluator`` and ``statistical_fidelity`` both need this type, and +each also needs something from the other module - keeping the enum here lets +both import it directly instead of one importing it transitively through the +other. +""" + +from enum import Enum, auto + + +class EvaluationProfile(Enum): + """The kind of value a column holds, used to pick its fidelity pipelines.""" + + SHORT_TEXT = auto() + CATEGORICAL = auto() + FREE_TEXT = auto() + EMAIL = auto() + IDENTIFIER = auto() + TEMPORAL = auto() diff --git a/datafaker/evaluators/feature_extractors.py b/datafaker/evaluators/feature_extractors.py new file mode 100644 index 0000000..5ddf542 --- /dev/null +++ b/datafaker/evaluators/feature_extractors.py @@ -0,0 +1,470 @@ +"""Extract a comparable feature from a column's real/synthetic values.""" + +import re +from abc import ABC, abstractmethod +from datetime import date, datetime +from typing import Any + +from sqlalchemy import String, func, literal + +from datafaker.dialects import SecondsDifference, SentenceCount, WordCount + +TOKEN = re.compile(r"\w+") + + +def _coerce_datetime(value): + """ + Best-effort coercion of a value into something with .year/.month/etc. + + A value that genuinely represents a real datetime can still arrive here + as a plain string rather than a datetime/date object - e.g. because the + SQL query that produced it (some proposers build queries from raw + column-name strings, losing type information) didn't preserve its type. + Without this, a temporal extractor would silently treat it as "not a + date" and drop it entirely, rather than raising a visible error - + corrupting the resulting distribution for one candidate while giving no + indication anything went wrong. + """ + if hasattr(value, "year"): + return value + if isinstance(value, str): + try: + return datetime.fromisoformat(value.strip()) + except ValueError: + return None + return None + + +class FeatureExtractor(ABC): + """Extract one comparable feature from a real or synthetic value.""" + + @abstractmethod + def extract(self, value: Any): + """ + Convert one cell into one or more feature values (synthetic/Python side). + + Returns: + iterable of feature values + """ + + @abstractmethod + def expression(self, column): + """Build the equivalent SQLAlchemy expression for the real/database side.""" + + +class IdentityExtractor(FeatureExtractor): + """Use the value itself as the feature.""" + + def extract(self, value): + """Yield the value unchanged.""" + if value is None: + return + yield value + + def expression(self, column): + """Return the column unchanged.""" + return column + + +class FirstLetterExtractor(FeatureExtractor): + """Extract a string value's first letter, lowercased.""" + + def extract(self, value): + """Yield the value's first letter, lowercased.""" + if value is None: + return + value = str(value).strip() + if value: + yield value[0].lower() + + def expression(self, column): + """Build the SQL expression for the column's first letter.""" + return func.lower(func.substr(column, 1, 1)) + + +class LastLetterExtractor(FeatureExtractor): + """Extract a string value's last letter, lowercased.""" + + def extract(self, value): + """Yield the value's last letter, lowercased.""" + if value is None: + return + + value = str(value).strip().lower() + + if value: + yield value[-1] + + def expression(self, column): + """Build the SQL expression for the column's last letter.""" + return func.lower(func.substr(column, -1, 1)) + + +class LengthExtractor(FeatureExtractor): + """Extract a value's string length.""" + + def extract(self, value): + """Yield the value's string length.""" + if value is None: + return + + yield len(str(value)) + + def expression(self, column): + """Build the SQL expression for the column's string length.""" + return func.length(column) + + +class WordExtractor(FeatureExtractor): + """Extract a string value's individual word tokens.""" + + def extract(self, value): + """Yield each word token in the value, lowercased.""" + if value is None: + return + + yield from TOKEN.findall(str(value).lower()) + + def expression(self, column): + """Not implemented: word tokenization has no SQL equivalent here.""" + raise NotImplementedError("WordExtractor SQL expression is not implemented.") + + +class CharacterExtractor(FeatureExtractor): + """Extract a string value's individual characters.""" + + def extract(self, value): + """Yield each character in the value.""" + if value is None: + return + + yield from str(value) + + def expression(self, column): + """Not implemented: per-character extraction has no SQL equivalent here.""" + raise NotImplementedError( + "CharacterExtractor SQL expression is not implemented." + ) + + +class CharacterBigramExtractor(FeatureExtractor): + """Extract a string value's overlapping 2-character sequences.""" + + def extract(self, value): + """Yield each overlapping 2-character bigram, lowercased.""" + if value is None: + return + + value = str(value).lower() + + for i in range(len(value) - 1): + yield value[i : i + 2] + + def expression(self, column): + """Not implemented: bigram extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class CharacterTrigramExtractor(FeatureExtractor): + """Extract a string value's overlapping 3-character sequences.""" + + def extract(self, value): + """Yield each overlapping 3-character trigram, lowercased.""" + if value is None: + return + + value = str(value).lower() + + for i in range(len(value) - 2): + yield value[i : i + 3] + + def expression(self, column): + """Not implemented: trigram extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class PrefixExtractor(FeatureExtractor): + """Extract a string value's first ``length`` characters.""" + + def __init__(self, length=2): + """Initialize with the prefix length to extract.""" + self.length = length + + def extract(self, value): + """Yield the value's first ``length`` characters, lowercased.""" + if value is None: + return + + value = str(value).strip().lower() + + if value: + yield value[: self.length] + + def expression(self, column): + """Not implemented: prefix extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class SuffixExtractor(FeatureExtractor): + """Extract a string value's last ``length`` characters.""" + + def __init__(self, length=2): + """Initialize with the suffix length to extract.""" + self.length = length + + def extract(self, value): + """Yield the value's last ``length`` characters, lowercased.""" + if value is None: + return + + value = str(value).strip().lower() + + if value: + yield value[-self.length :] + + def expression(self, column): + """Not implemented: suffix extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +VOWELS = set("aeiou") + + +class VowelConsonantPatternExtractor(FeatureExtractor): + """Extract a string value's vowel/consonant pattern (e.g. "CVCV").""" + + def extract(self, value): + """Yield the value's vowel/consonant pattern string.""" + if value is None: + return + + pattern = [] + + for ch in str(value).lower(): + if not ch.isalpha(): + continue + + pattern.append("V" if ch in VOWELS else "C") + + if pattern: + yield "".join(pattern) + + def expression(self, column): + """Not implemented: pattern extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class TimestampExtractor(FeatureExtractor): + """Extract a date/datetime as a single continuous days-since-epoch value. + + This captures year, month and day jointly as one quantity for a + HistogramBuilder, rather than scoring them as independent, + uncorrelated dimensions - a generator that draws month/day/hour + independently (uniformly within a calibrated year range) will never + reproduce real joint patterns (e.g. seasonal clustering) even if each + marginal component looks reasonable alone, and previously scored + accordingly poorly per-dimension despite being reasonably calibrated; + a single joint measure is a fairer, simpler fidelity signal. + """ + + _EPOCH = datetime(1970, 1, 1) + _EPOCH_DATE = date(1970, 1, 1) + + def extract(self, value): + """Yield the value's days-since-epoch, as a single continuous float.""" + value = _coerce_datetime(value) + if value is None: + return + if not isinstance(value, datetime): + value = datetime(value.year, value.month, value.day) + if getattr(value, "tzinfo", None) is not None: + value = value.replace(tzinfo=None) + yield (value - self._EPOCH).total_seconds() / 86400.0 + + def expression(self, column): + """Build the SQL expression for the column's days-since-epoch.""" + # func.extract("epoch", ...) has no MSSQL equivalent (DATEPART has no + # "epoch" field); SecondsDifference already solves exactly this via a + # dialect-specific DATEDIFF compilation on MSSQL. + epoch = literal(self._EPOCH_DATE, type_=column.type) + return SecondsDifference(column, epoch) / 86400.0 + + +class WeekdayExtractor(FeatureExtractor): + """Extract day-of-week: a cyclic pattern a continuous value can't capture. + + A continuous days-since-epoch value can't capture a cyclic pattern + (e.g. weekday/weekend clustering) on its own, since it isn't a function + of absolute date position. + """ + + def extract(self, value): + """Yield the value's day of the week (Monday=0).""" + value = _coerce_datetime(value) + if value is None: + return + yield value.weekday() + + def expression(self, column): + """Build the SQL expression for the column's day of the week.""" + return func.extract("dow", column) # pylint: disable=not-callable + + +class EmailLocalPartExtractor(FeatureExtractor): + """Extract the local part (before ``@``) of an email-like string.""" + + def extract(self, value): + """Yield the value's local part, lowercased.""" + if value is None: + return + text = str(value).strip() + if not text or "@" not in text: + return + local_part, _, _ = text.partition("@") + if local_part: + yield local_part.lower() + + def expression(self, column): + """Not implemented: local-part extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class EmailDomainExtractor(FeatureExtractor): + """Extract the domain (after ``@``) of an email-like string.""" + + def extract(self, value): + """Yield the value's domain, lowercased.""" + if value is None: + return + text = str(value).strip() + if not text or "@" not in text: + return + _, _, domain = text.partition("@") + if domain: + yield domain.lower() + + def expression(self, column): + """Not implemented: domain extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class EmailTopLevelDomainExtractor(FeatureExtractor): + """Extract the top-level domain of an email-like string.""" + + def extract(self, value): + """Yield the value's top-level domain, lowercased.""" + if value is None: + return + text = str(value).strip() + if not text or "@" not in text: + return + _, _, domain = text.partition("@") + if "." not in domain: + return + tld = domain.rsplit(".", 1)[-1].lower() + if tld: + yield tld + + def expression(self, column): + """Not implemented: TLD extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class EmailValidityExtractor(FeatureExtractor): + """Extract a 'valid'/'invalid' token for an email-like string. + + Uses the email-validator package when available, falling back to a + conservative regex. + """ + + # conservative regex fallback + _SIMPLE_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + + def __init__(self): + """Try importing the robust validator; fall back to regex if unavailable.""" + try: + from email_validator import ( # type: ignore # pylint: disable=import-outside-toplevel,import-error + validate_email, + ) + + self._validator = validate_email + except ImportError: + self._validator = None + + def extract(self, value): + """Yield 'valid' or 'invalid' for the value's email syntax.""" + if value is None: + return + text = str(value).strip() + if not text: + return + valid = False + if self._validator is not None: + try: + # check_deliverability=False for syntactic check only + _ = self._validator(text, check_deliverability=False) + valid = True + except Exception: # pylint: disable=broad-exception-caught + # email_validator raises its own EmailNotValidError, but + # this is optional/best-effort - any failure here should + # just mean "invalid", not crash the whole evaluation. + valid = False + else: + # fallback to simple regex + valid = bool(self._SIMPLE_RE.match(text)) + + yield "valid" if valid else "invalid" + + def expression(self, column): + """Not implemented: validity checking has no SQL equivalent here.""" + raise NotImplementedError() + + +class WordCountExtractor(FeatureExtractor): + """Extract a string value's word count.""" + + def extract(self, value): + """Yield the value's word count.""" + if value is None: + return + text = str(value).strip() + if not text: + return + # count words using TOKEN + count = len(TOKEN.findall(text)) + yield count + + def expression(self, column): + """Build the SQL expression for the column's word count.""" + # Cast column to text to handle tsvector columns (e.g. fulltext) + # where regexp_split_to_array may not accept the tsvector type + # directly. WordCount compiles to an exact regexp-based count on + # Postgres/DuckDB and a string-function approximation on MSSQL + # (which has no regex support) - see dialects.py. + return WordCount(column.cast(String)) + + +class SentenceCountExtractor(FeatureExtractor): + """Extract a string value's sentence count.""" + + def extract(self, value): + """Yield the value's sentence count.""" + if value is None: + return + text = str(value).strip() + if not text: + return + # split on sentence enders + parts = re.split(r"[.!?]+", text) + # count non-empty segments containing a word + count = sum(1 for p in parts if TOKEN.search(p)) + yield count + + def expression(self, column): + """Build the SQL expression for the column's sentence count.""" + # Cast column to text to handle tsvector columns (e.g. fulltext) + # where regexp_split_to_array may not accept the tsvector type + # directly. SentenceCount compiles to an exact regexp-based count + # on Postgres/DuckDB and a string-function approximation on MSSQL + # (which has no regex support) - see dialects.py. + return SentenceCount(column.cast(String)) diff --git a/datafaker/evaluators/metrics.py b/datafaker/evaluators/metrics.py new file mode 100644 index 0000000..070889d --- /dev/null +++ b/datafaker/evaluators/metrics.py @@ -0,0 +1,178 @@ +"""Metrics for comparing a real column's data against a synthetic sample.""" + +import math +from abc import ABC, abstractmethod +from collections import Counter +from decimal import Decimal + +from datafaker.evaluators.distribution_builders import Distribution + + +def canonicalize_value(value): + """Normalize a value for real-vs-synthetic equality comparison. + + A column's real values may come back as decimal.Decimal (SQLAlchemy's + default for a Numeric column) or as a tuple (a merged/composite + column's row), while a proposer's synthetic values for the same data + are plain float/list. Comparing str(Decimal("123.40")) against + str(123.4), or str((1.2, 3.4)) against str([1.2, 3.4]), would never + match even when the underlying values are identical - normalize both + sides to the same numeric/sequence type first so an exact match survives. + """ + if isinstance(value, (list, tuple)): + return tuple(canonicalize_value(v) for v in value) + if isinstance(value, Decimal): + return float(value) + return value + + +class Metric(ABC): + """A comparison between a real and a synthetic distribution or sample.""" + + @abstractmethod + def compare( + self, + real: Distribution, + synthetic: Distribution, + ) -> float: + """Score how closely ``synthetic`` matches ``real``.""" + + +def _canonicalize_distribution(dist: Distribution) -> Distribution: + """Normalize keys so None and mixed key types are comparable and sortable.""" + normalized = { + ("" if key is None else str(key)): value + for key, value in dist.probabilities.items() + } + return Distribution(normalized) + + +class MeanSquaredError(Metric): + """Mean squared error between two probability distributions.""" + + def compare( + self, + real: Distribution, + synthetic: Distribution, + ) -> float: + """Score the mean squared error between ``real`` and ``synthetic``.""" + real = _canonicalize_distribution(real) + synthetic = _canonicalize_distribution(synthetic) + vocab = sorted(real.vocabulary | synthetic.vocabulary) + + if not vocab: + return 0.0 + + x = real.as_vector(vocab) + y = synthetic.as_vector(vocab) + + squared_error = sum((a - b) ** 2 for a, b in zip(x, y)) + + # x and y are probability vectors (each sums to 1), so their squared + # Euclidean distance is bounded in [0, 2] regardless of vocab size. + # Dividing by that fixed bound (rather than len(vocab)**2) keeps the + # score on a comparable scale to JensenShannon's [0, ln 2] range, so + # pipeline weights combining the two behave as intended. + score = squared_error / 2 + return score + + +class JensenShannon(Metric): + """Jensen-Shannon divergence between two probability distributions.""" + + EPS = 1e-12 + + def compare( + self, + real: Distribution, + synthetic: Distribution, + ) -> float: + """Score the Jensen-Shannon divergence between ``real`` and ``synthetic``.""" + real = _canonicalize_distribution(real) + synthetic = _canonicalize_distribution(synthetic) + vocab = sorted(real.vocabulary | synthetic.vocabulary) + + if not vocab: + return 0.0 + + p = real.as_vector(vocab) + q = synthetic.as_vector(vocab) + + m = [(a + b) / 2 for a, b in zip(p, q)] + + def kl(a, b): + total = 0 + for x, y in zip(a, b): + if x > 0: + total += x * math.log(x / max(y, self.EPS)) + return total + + return (kl(p, m) + kl(q, m)) / 2 + + +class NoveltyMetric(Metric): + """Fraction of synthetic values that don't already appear in the real data.""" + + def compare( + self, + real, + synthetic, + ): + """Score how much of ``synthetic`` is novel relative to ``real``.""" + real_set = { + str(canonicalize_value(v)).strip().lower() for v in real if v is not None + } + + generated = { + str(canonicalize_value(v)).strip().lower() + for v in synthetic + if v is not None + } + + if not generated: + return 0.0 + + overlap = generated & real_set + + return 1.0 - len(overlap) / len(generated) + + +class DiversityMetric(Metric): + """Compare the internal diversity of a real and a synthetic sample. + + Measures how closely the synthetic sample's internal diversity matches + the real sample's diversity, rather than rewarding diversity outright. + + Each sample is scored by its normalized Shannon entropy (entropy divided + by log(k), k = number of distinct values), which is 0 for a single + repeated value and 1 for a uniform distribution over all its distinct + values. The metric returns 1 minus the absolute difference between the + real and synthetic normalized entropies: 1.0 when the synthetic sample + is exactly as diverse (relative to its own value count) as the real + sample, dropping toward 0 the more it over- or under-diversifies + relative to the real data. This avoids favoring generators that spread + probability more uniformly than the real data actually does. + """ + + @staticmethod + def _normalized_entropy(values) -> float: + """Compute the normalized Shannon entropy of ``values``.""" + vals = [str(v).strip().lower() for v in values if v is not None] + if not vals: + return 0.0 + counts = Counter(vals) + k = len(counts) + if k <= 1: + return 0.0 + total = sum(counts.values()) + probs = [c / total for c in counts.values()] + ent = -sum(p * math.log(p) for p in probs if p > 0) + return ent / math.log(k) + + def compare(self, real, synthetic): + """Score how closely ``synthetic``'s diversity matches ``real``'s.""" + real_diversity = self._normalized_entropy(real) + synthetic_diversity = self._normalized_entropy(synthetic) + # clamp against floating-point noise (e.g. two near-1.0 entropies + # differing by a rounding error) so the result stays within [0, 1] + return max(0.0, 1.0 - abs(synthetic_diversity - real_diversity)) diff --git a/datafaker/evaluators/proposal_ranking.py b/datafaker/evaluators/proposal_ranking.py new file mode 100644 index 0000000..8649efb --- /dev/null +++ b/datafaker/evaluators/proposal_ranking.py @@ -0,0 +1,534 @@ +"""Utilities for ranking proposal evaluations against multiple objectives.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any, Sequence + +from datafaker.evaluators.column_evaluator import ProposalEvaluation +from datafaker.evaluators.evaluation_profile import EvaluationProfile +from datafaker.proposers.choice import ChoiceProposer + +# Column-name keywords mapped to substrings of the generator's dotted name +# (e.g. "generic.person.first_name") that they hint at. Column naming +# conventions are a strong, cheap, complementary signal to the statistical +# fidelity/novelty/diversity evaluation: they catch cases the statistics +# genuinely can't discriminate (e.g. a column with a small realistic +# vocabulary where every candidate's exact-value fidelity ties) without +# having to rely on the generator library's own internal naming, which +# rarely matches real-world column names anyway. +# +# This is a soft boost, not a filter: a column name matching nothing here, +# or a generator matching no hint, is entirely unaffected - it never +# excludes a candidate the statistics would otherwise have picked. Keep this +# list small and grow it opportunistically as real columns turn up cases +# worth covering, rather than trying to anticipate every naming convention. +KEYWORD_GENERATOR_HINTS: list[tuple[list[str], list[str]]] = [ + ( + ["first_name", "firstname", "fname", "given_name", "forename"], + ["person.first_name"], + ), + ( + ["last_name", "lastname", "lname", "surname", "family_name"], + ["person.last_name"], + ), + (["full_name", "fullname"], ["person.full_name"]), + (["email"], ["person.email"]), + (["phone", "telephone", "mobile"], ["person.telephone", "address.calling_code"]), + (["country"], ["address.country", "address.country_code"]), + (["city"], ["address.city"]), + ( + ["street"], + ["address.street_name", "address.street_number", "address.street_suffix"], + ), + (["postcode", "postal_code", "zip"], ["address.postal_code"]), + (["gender", "sex"], ["person.gender"]), + (["occupation", "job_title"], ["person.occupation"]), + (["nationality"], ["person.nationality"]), + (["username", "login"], ["person.username"]), +] + +# Small enough not to override a clearly-better statistical result, large +# enough to break the near-ties this is meant to resolve. +KEYWORD_BOOST = 0.15 + +# Below this fraction of distinct values in a candidate's own 4000-value +# synthetic sample, it is producing enough duplicates against itself that it +# could not satisfy a real PRIMARY KEY/UNIQUE constraint once row counts grow +# past a handful - regardless of how well it otherwise fits the real +# distribution. Slack below 1.0 tolerates a rare coincidental collision from +# a genuinely-unique-generating proposer (e.g. a continuous float sampler) +# without treating it the same as a resampler or constant that duplicates +# constantly. +SYNTHETIC_UNIQUENESS_GUARANTEE_THRESHOLD = 0.999 + +# The weak-recommendation warning (below) is only worth showing when the +# winner's own combined score is actually weak - otherwise it's just noise +# on top of a perfectly good pick. +WEAK_WINNER_SCORE_THRESHOLD = 0.5 + + +def keyword_match( + column_name: str | None, proposer_name: str +) -> tuple[float, str | None]: + """Give a soft ranking boost when the column name hints at this generator's kind. + + Returns the boost plus the specific column keyword responsible (or None) + so the caller can show why. + + Uses substring matching (not exact tokenization), so this also catches + compound names like "customer_first_name". + """ + if not column_name: + return 0.0, None + name = column_name.lower() + generator_name = proposer_name.lower() + for column_keywords, generator_keywords in KEYWORD_GENERATOR_HINTS: + matched_keyword = next((kw for kw in column_keywords if kw in name), None) + if matched_keyword is not None and any( + gkw in generator_name for gkw in generator_keywords + ): + return KEYWORD_BOOST, matched_keyword + return 0.0, None + + +@dataclass +class ProposalRankingDisplay: + """Formatted text for a ranked proposal table.""" + + recommendation: str | None + profile_summary: str + rows: list[tuple[str, ...]] + fronts: list[int] + weak_recommendation_warning: str | None = None + no_uniqueness_guarantee: bool = False + + +# pylint: disable=too-many-instance-attributes +@dataclass +class ProposalRanking: + """A ranked set of proposal evaluations.""" + + recommended_index: int | None + recommended_reason: str | None + rows: list[tuple[str, ...]] + profile: EvaluationProfile | None + weights: tuple[float, float, float] + real_uniqueness: float = 0.0 + penalty_applies_to_all: bool = False + fronts: list[int] = field(default_factory=list) + scores: list[float] = field(default_factory=list) + weak_recommendation_warning: str | None = None + no_uniqueness_guarantee: bool = False + + +def normalize_list(vals: Sequence[float]) -> list[float]: + """Normalize values to the range [0, 1] for comparisons.""" + if not vals: + return [] + lo = min(vals) + hi = max(vals) + # Treat near-identical values as a tie rather than an exact one: e.g. + # when every candidate's synthetic sample shares no categories with the + # real data, their Jensen-Shannon divergence all sit at the same + # theoretical maximum (ln 2) but differ in the 10th+ decimal place from + # floating-point rounding. Min-max normalizing that noise would stretch + # it across the full [0, 1] range and manufacture a fake ranking. + if math.isclose(hi, lo, rel_tol=1e-9, abs_tol=1e-12): + return [0.5 for _ in vals] + return [(v - lo) / (hi - lo) for v in vals] + + +def _pareto_fronts(points: Sequence[tuple[float, float, float]]) -> list[list[int]]: + """Compute Pareto fronts for a list of 3D points.""" + remaining = set(range(len(points))) + fronts: list[list[int]] = [] + while remaining: + current_front: set[int] = set() + for i in remaining: + dominated = False + for j in remaining: + if i == j: + continue + if all( + points[j][d] >= points[i][d] for d in range(len(points[i])) + ) and any(points[j][d] > points[i][d] for d in range(len(points[i]))): + dominated = True + break + if not dominated: + current_front.add(i) + fronts.append(sorted(current_front)) + remaining -= current_front + return fronts + + +def _crowding_distances( + points: Sequence[tuple[float, float, float]], + fronts: Sequence[Sequence[int]], + objectives: tuple[Sequence[float], Sequence[float], Sequence[float]], +) -> list[float]: + """Compute NSGA-II style crowding distances.""" + crowding = [0.0 for _ in range(len(points))] + for front in fronts: + if len(front) <= 1: + for idx in front: + crowding[idx] = float("inf") + continue + for dim_vals in objectives: + vals = [(idx, dim_vals[idx]) for idx in front] + vals.sort(key=lambda item: item[1]) + lo = vals[0][1] + hi = vals[-1][1] + crowding[vals[0][0]] = float("inf") + crowding[vals[-1][0]] = float("inf") + if hi == lo: + continue + for k in range(1, len(vals) - 1): + prev_v = vals[k - 1][1] + next_v = vals[k + 1][1] + dist = (next_v - prev_v) / (hi - lo) + idx = vals[k][0] + if crowding[idx] != float("inf"): + crowding[idx] += dist + return crowding + + +# pylint: disable=too-many-arguments too-many-positional-arguments +# pylint: disable=too-many-locals too-many-statements +def rank_proposals( + results: Sequence[ProposalEvaluation], + profile: EvaluationProfile | None, + theme: Any | None = None, + column_name: str | None = None, + real_uniqueness: float = 0.0, + is_numeric_column: bool = False, + is_primary_key: bool = False, +) -> ProposalRanking: + """Rank proposals by fidelity, novelty and diversity using Pareto fronts. + + The ranking logic is deliberately separated from the interactive shell so it can + be tested and reused independently of presentation concerns. + """ + if not results: + return ProposalRanking(None, None, [], profile, (0.5, 0.25, 0.25)) + + # prepare multi-objective ranking: normalize metrics so larger-is-better + # overall_score is lower-is-better (distance/error), so invert after normalization + scores = [result.overall_score for result in results] + novelties = [result.novelty for result in results] + diversities = [result.diversity for result in results] + + norm_scores = normalize_list(scores) + norm_nov = normalize_list(novelties) + norm_div = normalize_list(diversities) + + # fidelity (higher-is-better) is inverted normalized overall_score + fidelity_scores = [1.0 - v for v in norm_scores] + novelty_scores = norm_nov + diversity_scores = norm_div + + # build points for Pareto (higher-is-better in all dims) + points = list(zip(fidelity_scores, novelty_scores, diversity_scores)) + + # non-dominated sort into Pareto fronts (simple O(n^2) algorithm) + fronts = _pareto_fronts(points) + front_of = [0] * len(points) + for front_index, front in enumerate(fronts, start=1): + for idx in front: + front_of[idx] = front_index + + # compute crowding distances per front (NSGA-II style) to prefer diverse solutions + crowding = _crowding_distances( + points, fronts, (fidelity_scores, novelty_scores, diversity_scores) + ) + + # determine profile and weights for scoring + # + # SHORT_TEXT, EMAIL and TEMPORAL all draw from a bounded, structured + # real-world vocabulary (names, addresses, dates), the same way + # CATEGORICAL/IDENTIFIER data does. For that kind of column, novelty and + # fidelity are mechanically in tension for the *best* generators: a + # generator that accurately models the real distribution is more, not + # less, likely to occasionally reproduce a real value by chance. Giving + # novelty a higher weight than fidelity here (as before) let it override + # an even strongly-discriminating fidelity signal and penalize the + # generator that best matched the real data. FREE_TEXT is kept + # novelty-tolerant since open-ended text has no such bounded vocabulary - + # genuinely novel sentences there are a good sign, not a red flag. + profile_weights = { + EvaluationProfile.IDENTIFIER: (0.85, 0.05, 0.10), + EvaluationProfile.CATEGORICAL: (0.7, 0.15, 0.15), + EvaluationProfile.SHORT_TEXT: (0.7, 0.15, 0.15), + EvaluationProfile.EMAIL: (0.7, 0.15, 0.15), + EvaluationProfile.FREE_TEXT: (0.5, 0.3, 0.2), + EvaluationProfile.TEMPORAL: (0.7, 0.15, 0.15), + } + fid_w, nov_w, div_w = profile_weights.get(profile, (0.5, 0.25, 0.25)) + + # compute profile-specific score (higher-is-better) + combined_scores = [fid_w * f + nov_w * n + div_w * d for (f, n, d) in points] + + # ChoiceProposer and its variants (dist_gen.choice/weighted_choice/ + # zipf_choice) resample directly from the column's own observed values - + # by construction, they can only ever emit values already in the real + # data. That's the *correct* thing to do for a column with a small, + # shared vocabulary (a status code, a gender) - reproducing real values + # there is expected and unavoidable - but a privacy problem for a column + # whose real values are meant to be unique per row (an email, a genuine + # ID): it isn't modeling the distribution, it's handing back real + # records. Scale the penalty by how unique the real column actually is + # (near 0 for a low-uniqueness column, where this is fine; near full + # strength for a near-unique one, where it's a leak) and by how much of + # this proposer's own output is actually copied. + # + # Other proposers are, in general, not penalized for copy_fraction: for + # them a high copy_fraction is usually a coincidental side effect of a + # naturally overlapping real-world vocabulary (e.g. common first names), + # not a sign of memorization - penalizing it there wrongly punishes + # exactly the generators that model the real distribution best (verified: + # applying this penalty to person.first_name for a first-name column + # incorrectly hands the win back to a generic word generator). + # + # Numeric columns are the exception, regardless of whether they end up + # profiled IDENTIFIER (high uniqueness, e.g. a primary key) or + # CATEGORICAL (low uniqueness, e.g. a foreign key referencing a small + # set of rows): a plain number has no shared human vocabulary to explain + # a coincidental match. A high copy_fraction there instead means the + # real value range is dense (e.g. a gapless sequence of small integer + # IDs), so *any* generator landing in that range trivially "matches" + # almost every time, independent of whether it resamples. That's the + # same underlying signal as ChoiceProposer's, just from a proposer that + # isn't one - e.g. a generic "weight" generator whose typical output + # range happens to overlap a table's ID range - so it gets the same + # penalty here. A low-cardinality *string* column (a gender, a status) + # doesn't get this: there, high copy_fraction from a well-calibrated + # generator is legitimate/expected, not a coincidental range overlap. + apply_to_every_proposer = is_numeric_column + penalty_multipliers = [1.0] * len(results) + for idx, result in enumerate(results): + resample_penalty = 1.0 + if isinstance(result.proposer, ChoiceProposer) or apply_to_every_proposer: + resample_penalty = 1.0 - real_uniqueness * result.copy_fraction + # A second, separate penalty for duplicating against the candidate's + # OWN output (synthetic_uniqueness) rather than against the real + # data (copy_fraction, above). These catch different failure modes: + # e.g. dist_gen.constant on a column whose fixed value happens not + # to appear in the sampled real data gets copy_fraction=0 - no + # resample penalty at all - despite duplicating every single row + # against itself. Scaled by real_uniqueness for the same reason as + # the resample penalty: a column with a small, legitimate shared + # vocabulary (a gender, a status) is expected to have low + # synthetic_uniqueness too, and shouldn't be penalized for it. + # Unlike the resample penalty, this applies to every candidate + # unconditionally - self-duplication breaks a PRIMARY KEY/UNIQUE + # constraint regardless of column type or whether the proposer is a + # resampler. + self_duplication_penalty = 1.0 - real_uniqueness * ( + 1.0 - result.synthetic_uniqueness + ) + penalty_multipliers[idx] = resample_penalty * self_duplication_penalty + combined_scores[idx] *= penalty_multipliers[idx] + + # soft boost for generators whose kind the column name itself hints at + # (see KEYWORD_GENERATOR_HINTS) - a cheap, complementary signal to the + # statistical evaluation above. Keep the matched keyword per candidate + # so the display can show *why* a boost was applied. + keyword_matches: list[str | None] = [] + for idx, result in enumerate(results): + boost, matched_keyword = keyword_match(column_name, result.proposer.name()) + combined_scores[idx] += boost + keyword_matches.append(matched_keyword) + + # combined_scores is a weighted sum of three [0, 1]-normalized dimensions + # with weights summing to 1.0, so it's bounded to [0, 1] on its own - but + # the keyword boost above is deliberately applied outside that weighted + # sum, and can otherwise push a score above 1. Clamp so "Score" stays a + # comparable, interpretable quantity. (The ChoiceProposer penalty above + # only ever shrinks a score toward 0, so it can't push below the range.) + combined_scores = [max(0.0, min(1.0, s)) for s in combined_scores] + + # A primary key/unique column needs values that are actually unique + # per-row, not just a good statistical fit. If every single candidate + # duplicates enough within its own synthetic sample (see + # synthetic_uniqueness) that it could never satisfy that constraint, + # there is no honest "Recommended: X" to give - naming a winner would + # imply a fitness that doesn't exist and invite a pick that later trips + # a real PRIMARY KEY/UNIQUE violation. This is independent of the + # resample penalty above (which only measures overlap with the *real* + # values): a proposer can have a low copy_fraction and still duplicate + # constantly against itself (e.g. dist_gen.constant). + no_uniqueness_guarantee = ( + is_primary_key + and bool(results) + and all( + result.synthetic_uniqueness < SYNTHETIC_UNIQUENESS_GUARANTEE_THRESHOLD + for result in results + ) + ) + + # pick recommended index: highest combined score across all candidates, + # tiebreaker by crowding distance. This used to restrict the pick to + # Pareto front 1, but the keyword boost and resample penalty above are + # applied after fronts are computed from the raw fidelity/novelty/ + # diversity values - so a technically-dominated candidate can end up + # with a genuinely higher combined_score than a front-1 one (e.g. a + # column-name match earning +0.15). Restricting to front 1 would + # silently override that, defeating the point of adding those signals. + # Front/crowding are still computed and shown per row for transparency + # (which candidates are non-dominated trade-off alternatives vs. + # objectively worse), just no longer used to gate the recommendation. + all_idxs = range(len(combined_scores)) + best_score = max(combined_scores[idx] for idx in all_idxs) + candidates = [idx for idx in all_idxs if combined_scores[idx] == best_score] + if len(candidates) == 1: + recommended_index = candidates[0] + recommended_reason = "best combined score" + else: + best_crowding = max(crowding[idx] for idx in candidates) + selected = [idx for idx in candidates if crowding[idx] == best_crowding] + recommended_index = selected[0] + recommended_reason = "tiebreak by crowding distance" + + # Flag the case where the candidates that actually fit the real data + # best were suppressed by the resample penalty, leaving something that + # was never a good fit to "win" by elimination rather than merit (e.g. + # every high-fidelity resampler on a unique ID column gets zeroed out, + # and dist_gen.constant is left to win despite fitting poorly itself). + # Deliberately narrow: only fires when a *specific* candidate was both + # heavily discounted (< 0.2x) and would otherwise have fit noticeably + # better (> 0.3 higher fidelity) - not just "the winner's score is low", + # which can also just mean a close call among decent options. Also + # requires the winner's own combined score to actually be weak: a + # suppressed resampler's fidelity for a unique-per-row column is + # inflated by the exact memorization the penalty exists to catch, so it + # being nominally higher than a genuinely good winner's (e.g. a keyword- + # matched generic.person.email with strong novelty/diversity) isn't a + # meaningful comparison and shouldn't cast doubt on that pick. Likewise, + # don't fire it when the winner itself is a candidate that guarantees + # fresh values (synthetic_uniqueness >= threshold, e.g. a sequence + # continuing past the observed max): a low combined score there is a + # structural artifact of the fidelity metric penalizing values that + # deliberately fall outside the observed real range, not a sign the + # recommendation is actually shaky - it's exactly the intended, + # principled pick for a column that needs guaranteed-unique values. + weak_recommendation_warning = None + if not no_uniqueness_guarantee and recommended_index is not None: + winner_fidelity = fidelity_scores[recommended_index] + winner_score = combined_scores[recommended_index] + winner_guarantees_uniqueness = ( + results[recommended_index].synthetic_uniqueness + >= SYNTHETIC_UNIQUENESS_GUARANTEE_THRESHOLD + ) + suppressed = [ + idx + for idx in range(len(results)) + if penalty_multipliers[idx] < 0.2 + and fidelity_scores[idx] > winner_fidelity + 0.3 + ] + if ( + suppressed + and winner_score < WEAK_WINNER_SCORE_THRESHOLD + and not winner_guarantees_uniqueness + ): + names = ", ".join(results[idx].proposer.name() for idx in suppressed) + weak_recommendation_warning = ( + f"Note: {names} fit the real data much better but were heavily " + "discounted by the resample penalty (see Penalty column) - reproducing " + "real values almost exactly isn't valid for a unique-per-row column. No " + "remaining candidate is a strong fit here; treat this recommendation as " + "a fallback, not a confident pick." + ) + + # prepare rows with Front and Score, mark Pareto front 1 with coloring. + # Crowding distance is still computed above (used as a tiebreak when + # scores are exactly equal) but isn't shown - it's an NSGA-II internal + # detail (how isolated a candidate is from its front-mates in objective + # space), not a quality signal a user picking a generator can reason + # about, and in practice it's "∞" for most rows anyway. + rows: list[tuple[str, ...]] = [] + for i, result in enumerate(results, start=1): + idx = i - 1 + front = front_of[idx] + cells = [ + str(i), + result.proposer.name(), + profile.name if profile is not None else "UNKNOWN", + str(front), + f"{combined_scores[idx]:.6f}", + keyword_matches[idx] or "", + f"{fidelity_scores[idx]:.6f}", + f"{novelty_scores[idx]:.6f}", + f"{diversity_scores[idx]:.6f}", + f"{real_uniqueness:.3f}", + f"{result.copy_fraction:.3f}", + f"{penalty_multipliers[idx]:.3f}", + ] + # color entire row for Pareto front 1 + if front == 1 and theme is not None: + cells = [f"{theme.function}{cell}{theme.reset}" for cell in cells] + # keep original types compatible with print_table (it will cast to list) + rows.append(tuple(cells)) + + return ProposalRanking( + recommended_index=recommended_index, + recommended_reason=recommended_reason, + rows=rows, + profile=profile, + weights=(fid_w, nov_w, div_w), + real_uniqueness=real_uniqueness, + penalty_applies_to_all=apply_to_every_proposer, + fronts=list(front_of), + scores=list(combined_scores), + weak_recommendation_warning=weak_recommendation_warning, + no_uniqueness_guarantee=no_uniqueness_guarantee, + ) + + +def format_ranking_display( + ranking: ProposalRanking, + results: Sequence[ProposalEvaluation], + profile: EvaluationProfile | None, +) -> ProposalRankingDisplay: + """Prepare user-facing strings for a ranked proposal table.""" + fid_w, nov_w, div_w = ranking.weights + recommendation = None + if ranking.no_uniqueness_guarantee: + recommendation = ( + "Recommended: none — no proposer can guarantee uniqueness for this column\n" + ) + elif ranking.recommended_index is not None: + rec_num = ranking.recommended_index + 1 + rec_name = results[ranking.recommended_index].proposer.name() + recommendation = ( + f"Recommended: {rec_num}. {rec_name} — {ranking.recommended_reason}\n" + ) + if ranking.penalty_applies_to_all: + penalty_scope = ( + "applied to every generator (this is a numeric column, so a high" + " copy_fraction always means the real range is dense, not a coincidental" + " vocabulary match)" + ) + else: + penalty_scope = ( + "applied only to resampling generators (dist_gen.choice/weighted_choice/" + "zipf_choice - see Penalty column); 1.0 (no effect) for everyone else" + ) + profile_summary = ( + f"Profile: {profile.name if profile is not None else 'UNKNOWN'} | " + f"Real uniqueness: {ranking.real_uniqueness:.3f} | " + "Pareto front 1 rows are highlighted\n" + f"Score = clamp( ({fid_w:.2f}*Fidelity + {nov_w:.2f}*Novelty" + f" + {div_w:.2f}*Diversity + Keyword) " + "x Penalty , 0, 1 )\n" + " Keyword: +0.15 if the column name hints at this generator's kind (see Keyword column)\n" + f" Penalty: 1 - real_uniqueness x copy_fraction, {penalty_scope}" + ) + return ProposalRankingDisplay( + recommendation=recommendation, + profile_summary=profile_summary, + rows=ranking.rows, + fronts=ranking.fronts, + weak_recommendation_warning=ranking.weak_recommendation_warning, + no_uniqueness_guarantee=ranking.no_uniqueness_guarantee, + ) diff --git a/datafaker/evaluators/statistical_fidelity.py b/datafaker/evaluators/statistical_fidelity.py new file mode 100644 index 0000000..ea68b2d --- /dev/null +++ b/datafaker/evaluators/statistical_fidelity.py @@ -0,0 +1,297 @@ +"""Per-profile fidelity pipelines that score a proposer against a column.""" + +from dataclasses import dataclass + +from datafaker.evaluators.distribution_builders import ( + CategoryBuilder, + DistributionBuilder, + HistogramBuilder, +) +from datafaker.evaluators.evaluation_profile import EvaluationProfile +from datafaker.evaluators.feature_extractors import ( + CharacterBigramExtractor, + EmailDomainExtractor, + EmailLocalPartExtractor, + EmailTopLevelDomainExtractor, + EmailValidityExtractor, + FeatureExtractor, + FirstLetterExtractor, + IdentityExtractor, + LastLetterExtractor, + LengthExtractor, + SentenceCountExtractor, + TimestampExtractor, + VowelConsonantPatternExtractor, + WeekdayExtractor, + WordCountExtractor, + WordExtractor, +) +from datafaker.evaluators.metrics import JensenShannon, MeanSquaredError, Metric + + +@dataclass +class EvaluationPipeline: + """One named feature-comparison stage within a profile's fidelity score.""" + + name: str + builder: DistributionBuilder + metric: Metric + extractor: FeatureExtractor + weight: float = 1.0 + + +# Short text profile (names, cities, etc.) +SHORT_TEXT_PIPELINE = [ + EvaluationPipeline( + name="length", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.25, + extractor=LengthExtractor(), + ), + EvaluationPipeline( + name="bigrams", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.35, + extractor=CharacterBigramExtractor(), + ), + EvaluationPipeline( + name="first_letter", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.10, + extractor=FirstLetterExtractor(), + ), + EvaluationPipeline( + name="last_letter", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.10, + extractor=LastLetterExtractor(), + ), + EvaluationPipeline( + name="pattern", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.20, + extractor=VowelConsonantPatternExtractor(), + ), +] + +EMAIL_PIPELINE = [ + EvaluationPipeline( + name="length", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.10, + extractor=LengthExtractor(), + ), + EvaluationPipeline( + name="local_part", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.40, + extractor=EmailLocalPartExtractor(), + ), + EvaluationPipeline( + name="domain", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.15, + extractor=EmailDomainExtractor(), + ), + EvaluationPipeline( + name="tld", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.15, + extractor=EmailTopLevelDomainExtractor(), + ), + EvaluationPipeline( + name="format_validity", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.20, + extractor=EmailValidityExtractor(), + ), +] + +# for country, gender, status, category +CATEGORICAL_PIPELINE = [ + EvaluationPipeline( + name="category", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=1.0, + extractor=IdentityExtractor(), + ) +] + +IDENTIFIER_PIPELINE = [ + EvaluationPipeline( + name="identifier", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=1.0, + extractor=IdentityExtractor(), + ) +] + +# Two pipelines instead of five independent year/month/day/hour dimensions: +# "timestamp" scores year+month+day jointly as one continuous quantity (so a +# generator is judged on the real calendar-position distribution as a whole, +# not on marginal components that can each look fine while their +# combination never does), and "day_of_week" is kept separately since it's a +# cyclic pattern a continuous days-since-epoch value can't capture on its +# own. +TEMPORAL_PIPELINE = [ + EvaluationPipeline( + name="timestamp", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.7, + extractor=TimestampExtractor(), + ), + EvaluationPipeline( + name="day_of_week", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.3, + extractor=WeekdayExtractor(), + ), +] + +# for comments, description, review +FREE_TEXT_PIPELINE = [ + EvaluationPipeline( + name="length", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.10, + extractor=LengthExtractor(), + ), + EvaluationPipeline( + name="word_count", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.25, + extractor=WordCountExtractor(), + ), + EvaluationPipeline( + name="sentence_count", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.15, + extractor=SentenceCountExtractor(), + ), + EvaluationPipeline( + name="words", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.30, + extractor=WordExtractor(), + ), + EvaluationPipeline( + name="bigrams", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.20, + extractor=CharacterBigramExtractor(), + ), +] + +PROFILE_PIPELINES = { + EvaluationProfile.IDENTIFIER: IDENTIFIER_PIPELINE, + EvaluationProfile.SHORT_TEXT: SHORT_TEXT_PIPELINE, + EvaluationProfile.EMAIL: EMAIL_PIPELINE, + EvaluationProfile.CATEGORICAL: CATEGORICAL_PIPELINE, + EvaluationProfile.FREE_TEXT: FREE_TEXT_PIPELINE, + EvaluationProfile.TEMPORAL: TEMPORAL_PIPELINE, +} + + +class StatisticalFidelity: + """ + Measure statistical fidelity of the synthetic sample. + + This class evaluates the statistical fidelity of a synthetic sample + against a real sample by comparing the distributions of features extracted + from the data. It uses a set of evaluation pipelines tailored to different + types of data (e.g., short text, categorical, free text, email, identifier, temporal) + to compute a weighted score that reflects how closely the synthetic data matches the real data + in terms of statistical properties. + """ + + def __init__(self, column, engine, sample_size=4000): + """Initialize with no pipelines set; call set_eval_pipelines() first.""" + self.engine = engine + self.column = column + self.table = self.column.table + self.eval_pipelines = {} + # caps how many real rows each pipeline's builder pulls into memory + # when computing the "real" distribution, so this scales to large + # tables instead of scanning the whole column per pipeline. + self.sample_size = sample_size + # (builder, real distribution) per pipeline name, populated lazily + # the first time calculate_scores runs each pipeline. The real + # distribution depends only on the column, not on which proposer is + # being scored, so it only needs computing once per column rather + # than once per calculate_scores() call (one per candidate proposer). + self._real_distributions = {} + + def set_eval_pipelines(self, profile): + """ + Set evaluation pipelines based on the specified profile. + + :param profile: The evaluation profile to use. + """ + self.eval_pipelines = PROFILE_PIPELINES[profile] + self._real_distributions = {} + + def calculate_scores(self, synthetic_samples): + """ + Calculate the overall fidelity score and each pipeline's own score. + + :param synthetic_samples: The synthetic samples to evaluate. + :return: A tuple of (overall score, {pipeline name: pipeline score}). + """ + pipeline_scores = {} + weighted_score = 0.0 + total_weight = 0.0 + + for pipeline in self.eval_pipelines: + cached = self._real_distributions.get(pipeline.name) + if cached is None: + builder = pipeline.builder( + self.engine, + self.table, + self.column, + extractor=pipeline.extractor, + sample_size=self.sample_size, + ) + # Distribution from real table - independent of the + # proposer being scored, so cache it (and the builder, + # which HistogramBuilder.build_from_values needs state + # from - e.g. self.edges - set by this same call). + real = builder.build_from_table() + self._real_distributions[pipeline.name] = (builder, real) + else: + builder, real = cached + + # Distribution from synthetic values + synthetic = builder.build_from_values(synthetic_samples) + + score = pipeline.metric.compare( + real, + synthetic, + ) + + pipeline_scores[pipeline.name] = score + + weighted_score += pipeline.weight * score + total_weight += pipeline.weight + + overall_score = weighted_score / total_weight if total_weight > 0 else 0.0 + + return overall_score, pipeline_scores diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index c6378d8..621f51e 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -9,13 +9,26 @@ from typing import Any, Callable, Optional, cast from sqlalchemy import Column, and_, literal_column, select +from sqlalchemy.types import Integer from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private from datafaker.dialects import Random +from datafaker.evaluators.column_evaluator import ColumnEvaluator +from datafaker.evaluators.proposal_ranking import ( + ProposalRanking, + ProposalRankingDisplay, + format_ranking_display, + rank_proposals, +) from datafaker.interactive.base import DbCmd, TableEntry, fk_column_name, or_default from datafaker.proposers import everything_factory -from datafaker.proposers.base import PredefinedProposer, Proposer -from datafaker.theme import get_active_theme +from datafaker.proposers.base import PredefinedProposer, Proposer, get_column_type +from datafaker.proposers.continuous import ( + GaussianProposer, + LogNormalProposer, + UniformProposer, +) +from datafaker.theme import Theme, get_active_theme from datafaker.utils import ( get_columns_assigned, get_property, @@ -181,6 +194,10 @@ class GeneratorCmd(DbCmd): "{theme_reset}{index}. {theme_func}{name}:" " {theme_fit}{fit} {theme_data}{sample}{theme_reset} ..." ) + RANKED_SAMPLE_TEXT = ( + "{theme_reset}{index}. {theme_func}{name}:" + " {theme_data}{sample}{theme_reset} ..." + ) PRIMARY_PRIVATE_TEXT = "Primary Private" SECONDARY_PRIVATE_TEXT = "Secondary Private on columns {0}" NOT_PRIVATE_TEXT = "Not private" @@ -196,6 +213,11 @@ class GeneratorCmd(DbCmd): ERROR_CANNOT_UNMERGE_ALL = "You cannot unmerge all the generator's columns" PROPOSE_NOTHING = "No proposed generators, sorry." + # Cap on rows shown by default 'propose' (front 1 can otherwise hold more + # candidates than are worth scanning in a terminal table). 'propose all' + # is unaffected. + MAX_PROPOSERS_SHOWN = 10 + SRC_STAT_RE = re.compile( r'\bSRC_STATS\["([^"]+)"\](\["results"\]\[0\]\["([^"]+)"\])?' ) @@ -281,6 +303,9 @@ def __init__( self.proposers: list[Proposer] | None = None self.proposer_index = 0 self.proposers_valid_columns: Optional[tuple[int, list[str]]] = None + # whether the cached self.proposers list currently includes proposers + # that are type-incompatible with the column (e.g. from 'propose all') + self.proposers_include_all: bool = False self.set_prompt() self.roles: dict[str, dict[str, RoleEntry]] = { table_name: { @@ -763,17 +788,61 @@ def _proposers_valid(self) -> bool: self._get_column_names(), ) - def _get_proposer_proposals(self) -> list[Proposer]: - """Get a list of acceptable proposers, sorted by decreasing fit to the actual data.""" - if not self._proposers_valid(): - self.proposers = None - if self.proposers is None: + # Proposers that emit continuous float output (via random.uniform/ + # normalvariate/lognormvariate) with no integer rounding. That's a + # legitimate fit for a Numeric/decimal column, but not for a genuinely + # Integer-typed one: inserting a non-whole float into an Integer column + # is backend-dependent at best (silently stored as-is on SQLite, + # truncated or rejected on MySQL) and raises an error on PostgreSQL. + _CONTINUOUS_FLOAT_PROPOSER_TYPES = ( + GaussianProposer, + UniformProposer, + LogNormalProposer, + ) + + def _is_integer_incompatible( + self, proposer: Proposer, columns: list[Column] + ) -> bool: + """Check whether `proposer` produces float output invalid for an Integer column.""" + if not isinstance(proposer, self._CONTINUOUS_FLOAT_PROPOSER_TYPES): + return False + if len(columns) != 1: + return False + return isinstance(get_column_type(columns[0]), Integer) + + def _get_proposer_proposals( + self, include_all: bool | None = None + ) -> list[Proposer]: + """ + Get a list of acceptable proposers, sorted by decreasing fit to the actual data. + + By default (``include_all=None``), type-incompatible proposers (see + ``_is_integer_incompatible``) are filtered out - this is what + 'propose' shows - and the cached list from the most recent + 'propose'/'propose all' is reused if still valid, so that 'set + ' numbering matches whichever list was last displayed. Pass + an explicit ``True``/``False`` (from 'propose'/'propose all' + themselves) to force that mode, refetching if it differs from what's + cached. + """ + if include_all is None: + include_all = self.proposers_include_all + if ( + not self._proposers_valid() + or self.proposers is None + or include_all != self.proposers_include_all + ): columns = self._column_metadata() props = everything_factory(self.config, self.metadata).get_proposers( columns, self.sync_engine ) + if not include_all: + props = [ + p for p in props if not self._is_integer_incompatible(p, columns) + ] sorted_props = sorted(props, key=lambda g: g.fit(9999)) self.proposers = sorted_props + self.proposers_include_all = include_all self.proposers_valid_columns = ( self.table_index, self._get_column_names().copy(), @@ -978,23 +1047,26 @@ def _get_column_data( result = connection.execute(stmt) return [[to_str(x) for x in xs] for xs in result.all()] - def do_propose(self, _arg: str) -> None: + def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-statements + self, _arg: str + ) -> None: """ Display a list of possible generators for this column. They will be listed in order of fit, the most likely matches first. The results can be compared (against a sample of the real data in the column and against each other) with the 'compare' command. + + By default, generators whose output type is incompatible with the + column (e.g. a continuous-float generator for an Integer column) + are left out. Run 'propose all' to include them anyway. """ theme = get_active_theme() limit = 5 + include_all = _arg.strip().lower() == "all" props = self._get_proposer_proposals() sample = self._get_column_data(limit) - if sample: - rep = [x[0] if len(x) == 1 else ",".join(x) for x in sample] - self.print(self.PROPOSE_SOURCE_SAMPLE_TEXT, "; ".join(rep), theme.data) - else: - self.print(self.PROPOSE_SOURCE_EMPTY_TEXT) + self._print_source_sample(sample, theme) if not props: self.print(self.PROPOSE_NOTHING) for index, prop in enumerate(props): @@ -1017,6 +1089,186 @@ def do_propose(self, _arg: str) -> None: theme_reset=theme.reset, ) + column_evaluator = ColumnEvaluator() + columns = self._column_metadata() + results = [] + column_evaluator.setup(columns, self.sync_engine) + for index, proposer in enumerate(props): + result = column_evaluator.evaluate(proposer) + results.append(result) + + self.print("\n***\n") + # self.print("\nProposal evaluation:\n", results) + + profile = getattr(column_evaluator, "profile", None) + column_name = ( + column_evaluator.column.name + if column_evaluator.column is not None + else None + ) + real_uniqueness = getattr(column_evaluator, "real_uniqueness", 0.0) + is_numeric_column = getattr(column_evaluator, "column_is_numeric", False) + is_primary_key = getattr(column_evaluator, "is_primary_key", False) + ranking = rank_proposals( + results, + profile, + theme, + column_name=column_name, + real_uniqueness=real_uniqueness, + is_numeric_column=is_numeric_column, + is_primary_key=is_primary_key, + ) + display = format_ranking_display(ranking, results, profile) + + if display.recommendation is not None: + if ( + display.no_uniqueness_guarantee + or display.weak_recommendation_warning is not None + ): + # Don't highlight a fallback or non-existent pick - coloring + # it the same way as a solid recommendation would visually + # contradict the caveat that follows (or, for + # no_uniqueness_guarantee, claim confidence in a pick that + # doesn't exist at all). + self.print(display.recommendation) + else: + self.print(f"{theme.column}{display.recommendation}{theme.reset}") + if display.weak_recommendation_warning is not None: + self.print(display.weak_recommendation_warning) + if include_all and ranking.recommended_index is not None: + # 'propose all' deliberately bypasses the type-compatibility filter, + # so a generator whose output type doesn't match the column (e.g. a + # continuous-float generator for an Integer column) can still win - + # the statistical score has no way to detect that mismatch at all, + # which is exactly why the default 'propose' excludes it structurally + # instead of trying to penalize it. + recommended_proposer = results[ranking.recommended_index].proposer + if self._is_integer_incompatible(recommended_proposer, columns): + self.print( + "Warning: {0} produces output whose type doesn't match this" + " column - it's excluded from the default 'propose' list for" + " exactly this reason (run 'propose' without 'all'). The" + " statistics can't detect this mismatch, so a high score here" + " doesn't mean it's safe to use.", + recommended_proposer.name(), + ) + + self.print(display.profile_summary) + + indices_to_show = self._select_indices_to_show(display, ranking, include_all) + rows_to_show = [display.rows[i] for i in indices_to_show] + + self.print_table( + [ + "#", + "Proposer", + "Profile", + "Front", + "Score", + "Keyword", + "Fidelity", + "Novelty", + "Diversity", + "Uniqueness", + "Copies", + "Penalty", + ], + rows_to_show, + ) + + self.print("\n") + self._print_source_sample(sample, theme) + for i in indices_to_show: + prop = props[i] + self.print( + self.RANKED_SAMPLE_TEXT, + index=i + 1, + name=prop.name(), + sample="; ".join(map(repr, prop.generate_data(limit))), + theme_func=theme.function, + theme_data=theme.data, + theme_reset=theme.reset, + ) + + print("\n\n") + + def _print_source_sample(self, sample: list[list[str]], theme: Theme) -> None: + """Print a sample of the real column values, or a note that there are none.""" + if sample: + rep = [x[0] if len(x) == 1 else ",".join(x) for x in sample] + self.print(self.PROPOSE_SOURCE_SAMPLE_TEXT, "; ".join(rep), theme.data) + else: + self.print(self.PROPOSE_SOURCE_EMPTY_TEXT) + + def _select_indices_to_show( + self, + display: ProposalRankingDisplay, + ranking: ProposalRanking, + include_all: bool, + ) -> list[int]: + """Pick which candidate rows to show in the propose table. + + By default, only show Pareto front 1: a front-2+ candidate is, by + definition, strictly worse than some front-1 one on fidelity, + novelty AND diversity simultaneously, so it adds nothing to a + shortlist. 'propose all' bypasses this (and the type-compatibility + filter) to show every candidate. The returned indices let the + sample listing after the table map each shown row back to its + proposer without parsing the (possibly theme-colored) '#' cell text. + """ + indices_to_show = list(range(len(display.rows))) + if include_all: + return indices_to_show + + indices_to_show = [i for i in indices_to_show if display.fronts[i] == 1] + # The recommendation is picked by combined score across *all* + # candidates, not just front 1 (see rank_proposals) - a + # dominated candidate can still win once the resample penalty + # and keyword boost are applied, since those are computed after + # fronts are derived from the raw fidelity/novelty/diversity + # values. Force it into the table even when front-2+, otherwise + # "Recommended: N. x" can point at a row the user never sees. + recommended_outside_front_1 = ( + ranking.recommended_index is not None + and ranking.recommended_index not in indices_to_show + ) + if recommended_outside_front_1: + indices_to_show.append(ranking.recommended_index) + capped = len(indices_to_show) > self.MAX_PROPOSERS_SHOWN + if capped: + # Keep the highest-scoring candidates - front 1 alone can + # still hold more rows than are worth scanning in a + # terminal table (nothing dominates them, but they're not + # all equally worth showing). Always keep the recommendation + # even if its score wouldn't otherwise make the cut, since + # it's referenced by name right above this table. + ranked = sorted( + indices_to_show, key=lambda i: ranking.scores[i], reverse=True + ) + keep = set(ranked[: self.MAX_PROPOSERS_SHOWN]) + if ranking.recommended_index is not None: + keep.add(ranking.recommended_index) + indices_to_show = sorted(keep) + else: + indices_to_show.sort() + if len(indices_to_show) < len(display.rows): + if capped: + qualifier = ( + f" (Pareto front 1, top {self.MAX_PROPOSERS_SHOWN} by score," + " plus the recommendation)" + ) + elif recommended_outside_front_1: + qualifier = " (Pareto front 1, plus the recommendation)" + else: + qualifier = " (Pareto front 1 only)" + self.print( + "Showing {0} of {1} candidates" + qualifier + "." + " Run 'propose all' to see the rest.", + len(indices_to_show), + len(display.rows), + ) + return indices_to_show + def do_p(self, arg: str) -> None: """Synonym for propose.""" self.do_propose(arg) From c9b8eb12ef81fdc24f670fcd197a7f9367a8778a Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Wed, 9 Sep 2026 17:08:00 +0100 Subject: [PATCH 02/23] Add unit tests for column evaluators --- tests/test_evaluators_column_evaluator.py | 233 +++++++++ .../test_evaluators_distribution_builders.py | 200 ++++++++ tests/test_evaluators_feature_extractors.py | 458 ++++++++++++++++++ tests/test_evaluators_metrics.py | 163 +++++++ tests/test_evaluators_proposal_ranking.py | 397 +++++++++++++++ tests/test_evaluators_statistical_fidelity.py | 108 +++++ 6 files changed, 1559 insertions(+) create mode 100644 tests/test_evaluators_column_evaluator.py create mode 100644 tests/test_evaluators_distribution_builders.py create mode 100644 tests/test_evaluators_feature_extractors.py create mode 100644 tests/test_evaluators_metrics.py create mode 100644 tests/test_evaluators_proposal_ranking.py create mode 100644 tests/test_evaluators_statistical_fidelity.py diff --git a/tests/test_evaluators_column_evaluator.py b/tests/test_evaluators_column_evaluator.py new file mode 100644 index 0000000..61e8cab --- /dev/null +++ b/tests/test_evaluators_column_evaluator.py @@ -0,0 +1,233 @@ +"""Unit tests for datafaker.evaluators.column_evaluator.""" +from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine + +from datafaker.evaluators.column_evaluator import ( + ColumnEvaluator, + ColumnStats, + EvaluationProfile, + ProposalEvaluation, + analyse_column, + choose_numeric_profile, + choose_profile, + looks_like_email, +) +from datafaker.proposers.base import ConstantProposer +from tests.utils import DatafakerTestCase + + +class AnalyseColumnTests(DatafakerTestCase): + """Test case for analyse_column.""" + + def test_counts_rows_and_unique_values(self) -> None: + """row_count and unique_count reflect the (non-None) input values.""" + stats = analyse_column(["a", "b", "a"]) + self.assertEqual(3, stats.row_count) + self.assertEqual(2, stats.unique_count) + + def test_none_values_are_excluded(self) -> None: + """None entries don't count towards row_count or unique_count.""" + stats = analyse_column(["a", None, None]) + self.assertEqual(1, stats.row_count) + self.assertEqual(1, stats.unique_count) + + def test_avg_length_and_ratios(self) -> None: + """avg_length, space_ratio, digit_ratio and punctuation_ratio are computed.""" + stats = analyse_column(["ab 1!"]) + self.assertEqual(5, stats.avg_length) + self.assertAlmostEqual(1 / 5, stats.space_ratio) + self.assertAlmostEqual(1 / 5, stats.digit_ratio) + self.assertAlmostEqual(1 / 5, stats.punctuation_ratio) + + def test_empty_input_does_not_divide_by_zero(self) -> None: + """An empty (or all-None) column yields all-zero stats, not an error.""" + stats = analyse_column([None, None]) + self.assertEqual(0, stats.row_count) + self.assertEqual(0, stats.avg_length) + self.assertEqual(0, stats.space_ratio) + self.assertEqual(0, stats.digit_ratio) + self.assertEqual(0, stats.punctuation_ratio) + + def test_non_string_values_are_stringified(self) -> None: + """Non-string values (e.g. ints) are converted to str first.""" + stats = analyse_column([12345]) + self.assertEqual(5, stats.avg_length) + self.assertEqual(1.0, stats.digit_ratio) + + +class ColumnStatsTests(DatafakerTestCase): + """Test case for the ColumnStats dataclass.""" + + def test_uniqueness_is_unique_over_row_count(self) -> None: + """uniqueness divides unique_count by row_count.""" + stats = ColumnStats( + row_count=4, + unique_count=2, + avg_length=0, + space_ratio=0, + digit_ratio=0, + punctuation_ratio=0, + ) + self.assertEqual(0.5, stats.uniqueness) + + def test_uniqueness_with_zero_rows_does_not_divide_by_zero(self) -> None: + """A zero row_count is floored to 1 to avoid a ZeroDivisionError.""" + stats = ColumnStats( + row_count=0, + unique_count=0, + avg_length=0, + space_ratio=0, + digit_ratio=0, + punctuation_ratio=0, + ) + self.assertEqual(0.0, stats.uniqueness) + + +class ChooseProfileTests(DatafakerTestCase): + """Test case for choose_profile.""" + + def _stats( + self, avg_length=10.0, uniqueness_row=(8, 10), space_ratio=0.0 + ) -> ColumnStats: + unique_count, row_count = uniqueness_row + return ColumnStats( + row_count=row_count, + unique_count=unique_count, + avg_length=avg_length, + space_ratio=space_ratio, + digit_ratio=0.0, + punctuation_ratio=0.0, + ) + + def test_short_and_mostly_unique_is_short_text(self) -> None: + """Short, mostly-distinct values look like names/identifiers.""" + stats = self._stats(avg_length=10, uniqueness_row=(9, 10)) + self.assertEqual(EvaluationProfile.SHORT_TEXT, choose_profile(stats)) + + def test_low_uniqueness_is_categorical(self) -> None: + """Values repeated often enough look like a small category set.""" + stats = self._stats(avg_length=10, uniqueness_row=(1, 100)) + self.assertEqual(EvaluationProfile.CATEGORICAL, choose_profile(stats)) + + def test_long_average_length_is_free_text(self) -> None: + """Long values that aren't near-unique look like free text.""" + stats = self._stats(avg_length=60, uniqueness_row=(5, 10)) + self.assertEqual(EvaluationProfile.FREE_TEXT, choose_profile(stats)) + + def test_high_space_ratio_is_free_text(self) -> None: + """Many spaces per character looks like prose even if average length is short.""" + stats = self._stats(avg_length=10, uniqueness_row=(5, 10), space_ratio=0.2) + self.assertEqual(EvaluationProfile.FREE_TEXT, choose_profile(stats)) + + def test_fallback_is_short_text(self) -> None: + """Anything not matching the other rules falls back to SHORT_TEXT.""" + stats = self._stats(avg_length=35, uniqueness_row=(5, 10), space_ratio=0.05) + self.assertEqual(EvaluationProfile.SHORT_TEXT, choose_profile(stats)) + + +class ChooseNumericProfileTests(DatafakerTestCase): + """Test case for choose_numeric_profile.""" + + def test_low_cardinality_is_categorical(self) -> None: + """Few distinct values among many rows looks like a status/flag column.""" + values = [1, 2] * 10 # 2 distinct out of 20: uniqueness 0.1 + self.assertEqual(EvaluationProfile.CATEGORICAL, choose_numeric_profile(values)) + + def test_high_cardinality_is_identifier(self) -> None: + """Mostly-distinct numeric values look like an id/age/salary column.""" + values = list(range(100)) + self.assertEqual(EvaluationProfile.IDENTIFIER, choose_numeric_profile(values)) + + def test_all_none_defaults_to_identifier(self) -> None: + """An empty/all-None column can't be judged, so defaults to IDENTIFIER.""" + self.assertEqual( + EvaluationProfile.IDENTIFIER, choose_numeric_profile([None, None]) + ) + + def test_none_values_are_excluded_from_the_ratio(self) -> None: + """None entries don't count as either distinct or total values.""" + # 2 distinct values among 20 non-null entries: uniqueness 0.1. If the + # Nones were counted in the denominator without being excluded first, + # this would look even more categorical, not less - the point here is + # just that the Nones don't crash or skew the ratio unexpectedly. + values = [1] * 19 + [2] + [None] * 5 + self.assertEqual(EvaluationProfile.CATEGORICAL, choose_numeric_profile(values)) + + +class LooksLikeEmailTests(DatafakerTestCase): + """Test case for looks_like_email.""" + + def test_mostly_email_values_is_true(self) -> None: + """A column where most values look like emails is detected.""" + values = ["a@example.com", "b@example.com", "not-an-email"] + self.assertTrue(looks_like_email(values)) + + def test_mostly_non_email_values_is_false(self) -> None: + """A column where most values don't look like emails is rejected.""" + values = ["a@example.com", "plain text", "another plain value"] + self.assertFalse(looks_like_email(values)) + + def test_empty_input_is_false(self) -> None: + """No values at all can't look like emails.""" + self.assertFalse(looks_like_email([])) + self.assertFalse(looks_like_email([None, " "])) + + def test_value_without_local_part_or_domain_is_not_email_like(self) -> None: + """'@' alone, or with an empty side, doesn't count as email-like.""" + values = ["@example.com", "a@", "@"] + self.assertFalse(looks_like_email(values)) + + +class ProposalEvaluationTests(DatafakerTestCase): + """Test case for the ProposalEvaluation dataclass's display string.""" + + def test_str_includes_key_metrics_and_pipeline_scores(self) -> None: + """The string form surfaces the headline scores and per-pipeline detail.""" + evaluation = ProposalEvaluation( + proposer=ConstantProposer("x"), + novelty=0.5, + diversity=0.25, + overall_score=0.125, + pipeline_scores={"length": 0.1, "words": 0.2}, + copy_fraction=0.0, + synthetic_uniqueness=1.0, + ) + text = str(evaluation) + self.assert_str_in("dist_gen.constant", text) + self.assert_str_in("0.125000", text) + self.assert_str_in("0.500000", text) + self.assert_str_in("0.250000", text) + self.assert_str_in("length", text) + self.assert_str_in("words", text) + + +class ColumnEvaluatorIntegrationTests(DatafakerTestCase): + """End-to-end test of ColumnEvaluator against a real (DuckDB) table.""" + + def test_evaluate_a_constant_proposer_against_a_categorical_column(self) -> None: + """setup() profiles the column and evaluate() scores a real proposer.""" + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table( + "statuses", + metadata, + Column("id", Integer, primary_key=True), + Column("status", String), + ) + metadata.create_all(engine) + with engine.begin() as conn: + conn.execute( + table.insert(), + [{"id": i, "status": ["active", "inactive"][i % 2]} for i in range(20)], + ) + + evaluator = ColumnEvaluator() + evaluator.setup([table.c.status], engine) + self.assertEqual(EvaluationProfile.CATEGORICAL, evaluator.profile) + + result = evaluator.evaluate(ConstantProposer("active")) + self.assertIsInstance(result, ProposalEvaluation) + self.assertEqual(1.0, result.copy_fraction) + # A constant proposer emits the same value every time: only one + # distinct value out of the whole synthetic sample. + self.assertLess(result.synthetic_uniqueness, 0.01) + self.assertIn("category", result.pipeline_scores) diff --git a/tests/test_evaluators_distribution_builders.py b/tests/test_evaluators_distribution_builders.py new file mode 100644 index 0000000..e3a79e2 --- /dev/null +++ b/tests/test_evaluators_distribution_builders.py @@ -0,0 +1,200 @@ +"""Unit tests for datafaker.evaluators.distribution_builders.""" +from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine + +from datafaker.evaluators.distribution_builders import ( + CategoryBuilder, + Distribution, + HistogramBuilder, +) +from datafaker.evaluators.feature_extractors import ( + FirstLetterExtractor, + LengthExtractor, +) +from tests.utils import DatafakerTestCase + + +class DistributionTests(DatafakerTestCase): + """Test case for the Distribution dataclass.""" + + def test_vocabulary_is_the_set_of_keys(self) -> None: + """vocabulary exposes the keys of the probability mapping.""" + dist = Distribution({"a": 0.5, "b": 0.5}) + self.assertEqual({"a", "b"}, dist.vocabulary) + + def test_probability_of_known_key(self) -> None: + """probability() returns the stored value for a known key.""" + dist = Distribution({"a": 0.75}) + self.assertEqual(0.75, dist.probability("a")) + + def test_probability_of_unknown_key_is_zero(self) -> None: + """probability() returns 0.0 for a key that isn't in the distribution.""" + dist = Distribution({"a": 0.75}) + self.assertEqual(0.0, dist.probability("missing")) + + def test_as_vector_orders_by_given_vocabulary(self) -> None: + """as_vector() maps a vocabulary list to matching probabilities, in order.""" + dist = Distribution({"a": 0.2, "b": 0.8}) + self.assertEqual([0.8, 0.2, 0.0], dist.as_vector(["b", "a", "c"])) + + +def _make_duckdb_table(rows: list[dict]) -> tuple: + """Create an in-memory DuckDB engine with one populated table.""" + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table( + "people", + metadata, + Column("age", Integer), + Column("name", String), + ) + metadata.create_all(engine) + with engine.begin() as conn: + conn.execute(table.insert(), rows) + return engine, table + + +class HistogramBuilderBuildFromTableTests(DatafakerTestCase): + """Test case for HistogramBuilder.build_from_table.""" + + def test_builds_a_distribution_that_sums_to_one(self) -> None: + """The resulting histogram is a valid probability distribution.""" + engine, table = _make_duckdb_table( + [{"age": 20 + (i % 10), "name": "x"} for i in range(100)] + ) + builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertIsNotNone(builder.mean) + self.assertIsNotNone(builder.stddev) + + def test_empty_table_yields_empty_distribution(self) -> None: + """An empty table produces an empty distribution rather than an error.""" + engine, table = _make_duckdb_table([]) + builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + dist = builder.build_from_table() + self.assertEqual({}, dist.probabilities) + + def test_constant_column_does_not_crash_on_zero_stddev(self) -> None: + """A column with a single repeated value has zero stddev but still works.""" + engine, table = _make_duckdb_table( + [{"age": 42, "name": "x"} for _ in range(10)] + ) + builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertEqual(0.0, builder.stddev) + + def test_uses_feature_extractor_expression(self) -> None: + """The configured extractor's SQL expression is used, not the raw column.""" + engine, table = _make_duckdb_table( + [{"age": 0, "name": name} for name in ["ab", "cde", "fghi"] * 5] + ) + builder = HistogramBuilder( + engine, table, table.c.name, extractor=LengthExtractor(), sample_size=1000 + ) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + + +class HistogramBuilderBuildFromValuesTests(DatafakerTestCase): + """Test case for HistogramBuilder.build_from_values.""" + + def setUp(self) -> None: + super().setUp() + engine, table = _make_duckdb_table( + [{"age": 20 + (i % 10), "name": "x"} for i in range(100)] + ) + self.builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + # Learn edges from the real data first, as calculate_scores() does. + self.builder.build_from_table() + + def test_result_sums_to_one(self) -> None: + """Bucketed synthetic values form a valid probability distribution.""" + dist = self.builder.build_from_values([20, 21, 22, 23, 24, 25]) + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + + def test_empty_values_yield_empty_distribution(self) -> None: + """No synthetic values produces an empty distribution.""" + dist = self.builder.build_from_values([]) + self.assertEqual({}, dist.probabilities) + + def test_values_outside_edges_are_clamped_to_end_buckets(self) -> None: + """Values far below/above the learned range land in the outer buckets.""" + dist = self.builder.build_from_values([-1000, 1000]) + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + buckets = set(dist.probabilities.keys()) + self.assertTrue(all(0 <= b <= len(self.builder.edges) - 2 for b in buckets)) + + +class CategoryBuilderBuildFromTableTests(DatafakerTestCase): + """Test case for CategoryBuilder.build_from_table.""" + + def test_builds_a_distribution_that_sums_to_one(self) -> None: + """The resulting category histogram sums to (approximately) 1.""" + engine, table = _make_duckdb_table( + [{"age": 0, "name": n} for n in ["alice", "bob", "carol"] * 10] + ) + builder = CategoryBuilder(engine, table, table.c.name, sample_size=1000) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertEqual({"alice", "bob", "carol"}, dist.vocabulary) + + def test_empty_table_yields_empty_distribution(self) -> None: + """An empty table produces an empty distribution.""" + engine, table = _make_duckdb_table([]) + builder = CategoryBuilder(engine, table, table.c.name, sample_size=1000) + dist = builder.build_from_table() + self.assertEqual({}, dist.probabilities) + + def test_uses_feature_extractor(self) -> None: + """The category counts are keyed by extracted features, not raw values.""" + engine, table = _make_duckdb_table( + [{"age": 0, "name": n} for n in ["Alice", "Anna", "Bob"]] + ) + builder = CategoryBuilder( + engine, + table, + table.c.name, + extractor=FirstLetterExtractor(), + sample_size=1000, + ) + dist = builder.build_from_table() + self.assertEqual({"a", "b"}, dist.vocabulary) + self.assertAlmostEqual(2 / 3, dist.probability("a")) + self.assertAlmostEqual(1 / 3, dist.probability("b")) + + +class CategoryBuilderBuildFromValuesTests(DatafakerTestCase): + """Test case for CategoryBuilder.build_from_values.""" + + def setUp(self) -> None: + super().setUp() + self.builder = CategoryBuilder( + engine=None, table=None, column=None, sample_size=1000 + ) + + def test_builds_a_distribution_that_sums_to_one(self) -> None: + """Category counts from a plain Python list sum to one.""" + dist = self.builder.build_from_values(["a", "a", "b"]) + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertAlmostEqual(2 / 3, dist.probability("a")) + self.assertAlmostEqual(1 / 3, dist.probability("b")) + + def test_none_values_are_ignored(self) -> None: + """None entries don't contribute counts.""" + dist = self.builder.build_from_values(["a", None, "a"]) + self.assertEqual({"a": 1.0}, dist.probabilities) + + def test_no_extracted_features_yields_sentinel_missing_distribution(self) -> None: + """An empty result set is represented with an explicit __MISSING__ token.""" + dist = self.builder.build_from_values([None, None]) + self.assertEqual({"__MISSING__": 1.0}, dist.probabilities) + + def test_uses_feature_extractor(self) -> None: + """Values are converted through the configured extractor before counting.""" + builder = CategoryBuilder( + engine=None, table=None, column=None, extractor=LengthExtractor() + ) + dist = builder.build_from_values(["ab", "cd", "efg"]) + self.assertAlmostEqual(2 / 3, dist.probability(2)) + self.assertAlmostEqual(1 / 3, dist.probability(3)) diff --git a/tests/test_evaluators_feature_extractors.py b/tests/test_evaluators_feature_extractors.py new file mode 100644 index 0000000..014b9fa --- /dev/null +++ b/tests/test_evaluators_feature_extractors.py @@ -0,0 +1,458 @@ +"""Unit tests for datafaker.evaluators.feature_extractors.""" +from datetime import date, datetime, timezone + +from datafaker.evaluators.feature_extractors import ( + CharacterBigramExtractor, + CharacterExtractor, + CharacterTrigramExtractor, + EmailDomainExtractor, + EmailLocalPartExtractor, + EmailTopLevelDomainExtractor, + EmailValidityExtractor, + FirstLetterExtractor, + IdentityExtractor, + LastLetterExtractor, + LengthExtractor, + PrefixExtractor, + SentenceCountExtractor, + SuffixExtractor, + TimestampExtractor, + VowelConsonantPatternExtractor, + WeekdayExtractor, + WordCountExtractor, + WordExtractor, +) +from tests.utils import DatafakerTestCase + + +class IdentityExtractorTests(DatafakerTestCase): + """Test case for IdentityExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = IdentityExtractor() + + def test_yields_the_value_unchanged(self) -> None: + """Non-None values pass through untouched.""" + self.assertEqual([42], list(self.extractor.extract(42))) + self.assertEqual(["hello"], list(self.extractor.extract("hello"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_expression_returns_the_column_itself(self) -> None: + """The SQL expression is just the column, unmodified.""" + column = object() + self.assertIs(column, self.extractor.expression(column)) + + +class FirstLetterExtractorTests(DatafakerTestCase): + """Test case for FirstLetterExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = FirstLetterExtractor() + + def test_extracts_lowercased_first_letter(self) -> None: + """The first character is extracted and lowercased.""" + self.assertEqual(["a"], list(self.extractor.extract("Alice"))) + + def test_strips_surrounding_whitespace_first(self) -> None: + """Leading whitespace does not become the 'first letter'.""" + self.assertEqual(["a"], list(self.extractor.extract(" alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_empty_string_yields_nothing(self) -> None: + """A blank/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + def test_expression_does_not_raise(self) -> None: + """A SQL expression is implemented for this extractor.""" + self.extractor.expression("col") + + +class LastLetterExtractorTests(DatafakerTestCase): + """Test case for LastLetterExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = LastLetterExtractor() + + def test_extracts_lowercased_last_letter(self) -> None: + """The last character is extracted and lowercased.""" + self.assertEqual(["e"], list(self.extractor.extract("Alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_empty_string_yields_nothing(self) -> None: + """A blank/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + +class LengthExtractorTests(DatafakerTestCase): + """Test case for LengthExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = LengthExtractor() + + def test_extracts_string_length(self) -> None: + """The length of str(value) is extracted.""" + self.assertEqual([5], list(self.extractor.extract("Alice"))) + + def test_extracts_length_of_non_string_values(self) -> None: + """Non-string values are stringified before measuring length.""" + self.assertEqual([len(str(12345))], list(self.extractor.extract(12345))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + +class WordExtractorTests(DatafakerTestCase): + """Test case for WordExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = WordExtractor() + + def test_tokenizes_and_lowercases_words(self) -> None: + """Words are split on non-word characters and lowercased.""" + self.assertEqual( + ["the", "quick", "fox"], + list(self.extractor.extract("The quick, fox!")), + ) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_no_sql_expression_is_implemented(self) -> None: + """Word extraction has no SQL equivalent and must raise.""" + with self.assertRaises(NotImplementedError): + self.extractor.expression("col") + + +class CharacterExtractorTests(DatafakerTestCase): + """Test case for CharacterExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = CharacterExtractor() + + def test_extracts_each_character(self) -> None: + """Each character of the value is yielded separately.""" + self.assertEqual(["a", "b", "c"], list(self.extractor.extract("abc"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_no_sql_expression_is_implemented(self) -> None: + """Per-character extraction has no SQL equivalent and must raise.""" + with self.assertRaises(NotImplementedError): + self.extractor.expression("col") + + +class CharacterBigramExtractorTests(DatafakerTestCase): + """Test case for CharacterBigramExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = CharacterBigramExtractor() + + def test_extracts_overlapping_bigrams(self) -> None: + """Adjacent character pairs are extracted, lowercased.""" + self.assertEqual(["ab", "bc"], list(self.extractor.extract("ABc"))) + + def test_short_strings_yield_no_bigrams(self) -> None: + """A string shorter than 2 characters has no bigrams.""" + self.assertEqual([], list(self.extractor.extract("a"))) + self.assertEqual([], list(self.extractor.extract(""))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + +class CharacterTrigramExtractorTests(DatafakerTestCase): + """Test case for CharacterTrigramExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = CharacterTrigramExtractor() + + def test_extracts_overlapping_trigrams(self) -> None: + """Adjacent character triples are extracted, lowercased.""" + self.assertEqual(["abc", "bcd"], list(self.extractor.extract("ABCd"))) + + def test_short_strings_yield_no_trigrams(self) -> None: + """A string shorter than 3 characters has no trigrams.""" + self.assertEqual([], list(self.extractor.extract("ab"))) + + +class PrefixExtractorTests(DatafakerTestCase): + """Test case for PrefixExtractor.""" + + def test_default_length_is_two(self) -> None: + """With no explicit length, a 2-character prefix is extracted.""" + extractor = PrefixExtractor() + self.assertEqual(["al"], list(extractor.extract("Alice"))) + + def test_custom_length(self) -> None: + """A custom prefix length is honored.""" + extractor = PrefixExtractor(length=4) + self.assertEqual(["alic"], list(extractor.extract("Alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(PrefixExtractor().extract(None))) + + def test_empty_string_yields_nothing(self) -> None: + """A blank/whitespace-only string produces no features.""" + self.assertEqual([], list(PrefixExtractor().extract(" "))) + + +class SuffixExtractorTests(DatafakerTestCase): + """Test case for SuffixExtractor.""" + + def test_default_length_is_two(self) -> None: + """With no explicit length, a 2-character suffix is extracted.""" + extractor = SuffixExtractor() + self.assertEqual(["ce"], list(extractor.extract("Alice"))) + + def test_custom_length(self) -> None: + """A custom suffix length is honored.""" + extractor = SuffixExtractor(length=3) + self.assertEqual(["ice"], list(extractor.extract("Alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(SuffixExtractor().extract(None))) + + +class VowelConsonantPatternExtractorTests(DatafakerTestCase): + """Test case for VowelConsonantPatternExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = VowelConsonantPatternExtractor() + + def test_extracts_vowel_consonant_pattern(self) -> None: + """Each alphabetic character is classified as V or C.""" + self.assertEqual(["VCVCV"], list(self.extractor.extract("Alice"))) + + def test_non_alphabetic_characters_are_skipped(self) -> None: + """Digits and punctuation don't appear in the pattern.""" + self.assertEqual(["VCC"], list(self.extractor.extract("a1-b2c!"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_all_non_alphabetic_yields_nothing(self) -> None: + """A value with no letters at all produces no pattern.""" + self.assertEqual([], list(self.extractor.extract("123!!"))) + + +class TimestampExtractorTests(DatafakerTestCase): + """Test case for TimestampExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = TimestampExtractor() + + def test_epoch_datetime_extracts_zero(self) -> None: + """The Unix epoch itself extracts to day 0.""" + self.assertEqual([0.0], list(self.extractor.extract(datetime(1970, 1, 1)))) + + def test_extracts_days_since_epoch(self) -> None: + """A later date extracts to the correct day offset.""" + self.assertEqual([1.0], list(self.extractor.extract(datetime(1970, 1, 2)))) + + def test_accepts_date_objects(self) -> None: + """A plain date (no time component) is also supported.""" + self.assertEqual([1.0], list(self.extractor.extract(date(1970, 1, 2)))) + + def test_accepts_iso_format_strings(self) -> None: + """A string that came from a query losing type info is coerced.""" + self.assertEqual([1.0], list(self.extractor.extract("1970-01-02"))) + + def test_invalid_string_yields_nothing(self) -> None: + """A string that isn't a valid date produces no feature.""" + self.assertEqual([], list(self.extractor.extract("not-a-date"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_non_date_non_string_value_yields_nothing(self) -> None: + """A value with no date meaning at all (e.g. an int) is dropped.""" + self.assertEqual([], list(self.extractor.extract(42))) + + def test_timezone_aware_datetime_is_normalized(self) -> None: + """Timezone info is stripped rather than causing a crash or a shift.""" + aware = datetime(1970, 1, 2, tzinfo=timezone.utc) + self.assertEqual([1.0], list(self.extractor.extract(aware))) + + +class WeekdayExtractorTests(DatafakerTestCase): + """Test case for WeekdayExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = WeekdayExtractor() + + def test_extracts_iso_weekday_index(self) -> None: + """1970-01-01 was a Thursday: weekday() == 3.""" + self.assertEqual([3], list(self.extractor.extract(datetime(1970, 1, 1)))) + + def test_accepts_iso_format_strings(self) -> None: + """String dates are coerced the same way as TimestampExtractor.""" + self.assertEqual([3], list(self.extractor.extract("1970-01-01"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_invalid_string_yields_nothing(self) -> None: + """An unparseable string produces no feature.""" + self.assertEqual([], list(self.extractor.extract("nope"))) + + +class EmailLocalPartExtractorTests(DatafakerTestCase): + """Test case for EmailLocalPartExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailLocalPartExtractor() + + def test_extracts_lowercased_local_part(self) -> None: + """The portion before '@' is extracted and lowercased.""" + self.assertEqual( + ["alice.smith"], list(self.extractor.extract("Alice.Smith@Example.com")) + ) + + def test_value_without_at_sign_yields_nothing(self) -> None: + """A non-email string has no local part.""" + self.assertEqual([], list(self.extractor.extract("not-an-email"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + +class EmailDomainExtractorTests(DatafakerTestCase): + """Test case for EmailDomainExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailDomainExtractor() + + def test_extracts_lowercased_domain(self) -> None: + """The portion after '@' is extracted and lowercased.""" + self.assertEqual( + ["example.com"], list(self.extractor.extract("alice@Example.COM")) + ) + + def test_value_without_at_sign_yields_nothing(self) -> None: + """A non-email string has no domain.""" + self.assertEqual([], list(self.extractor.extract("not-an-email"))) + + +class EmailTopLevelDomainExtractorTests(DatafakerTestCase): + """Test case for EmailTopLevelDomainExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailTopLevelDomainExtractor() + + def test_extracts_lowercased_tld(self) -> None: + """The final domain segment is extracted and lowercased.""" + self.assertEqual( + ["com"], list(self.extractor.extract("alice@mail.example.COM")) + ) + + def test_domain_without_dot_yields_nothing(self) -> None: + """A domain with no TLD separator produces no feature.""" + self.assertEqual([], list(self.extractor.extract("alice@localhost"))) + + def test_value_without_at_sign_yields_nothing(self) -> None: + """A non-email string has no TLD.""" + self.assertEqual([], list(self.extractor.extract("not-an-email"))) + + +class EmailValidityExtractorTests(DatafakerTestCase): + """Test case for EmailValidityExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailValidityExtractor() + + def test_well_formed_email_is_valid(self) -> None: + """A syntactically valid email is classified 'valid'.""" + self.assertEqual(["valid"], list(self.extractor.extract("alice@example.com"))) + + def test_malformed_value_is_invalid(self) -> None: + """A string with no '@' or domain is classified 'invalid'.""" + self.assertEqual(["invalid"], list(self.extractor.extract("not-an-email"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_blank_string_yields_nothing(self) -> None: + """An empty/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + +class WordCountExtractorTests(DatafakerTestCase): + """Test case for WordCountExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = WordCountExtractor() + + def test_counts_words(self) -> None: + """Word count matches the number of \\w+ tokens.""" + self.assertEqual([4], list(self.extractor.extract("The quick brown fox."))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_blank_string_yields_nothing(self) -> None: + """An empty/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + +class SentenceCountExtractorTests(DatafakerTestCase): + """Test case for SentenceCountExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = SentenceCountExtractor() + + def test_counts_sentences_by_terminal_punctuation(self) -> None: + """Sentences are split on ./!/? and counted if they contain a word.""" + self.assertEqual( + [3], list(self.extractor.extract("Hi there! How are you? I am fine.")) + ) + + def test_trailing_punctuation_without_words_is_not_a_sentence(self) -> None: + """A run of terminal punctuation with no word content doesn't count.""" + self.assertEqual([1], list(self.extractor.extract("Hello there..."))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_blank_string_yields_nothing(self) -> None: + """An empty/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) diff --git a/tests/test_evaluators_metrics.py b/tests/test_evaluators_metrics.py new file mode 100644 index 0000000..bf4610b --- /dev/null +++ b/tests/test_evaluators_metrics.py @@ -0,0 +1,163 @@ +"""Unit tests for datafaker.evaluators.metrics.""" +import math + +from datafaker.evaluators.distribution_builders import Distribution +from datafaker.evaluators.metrics import ( + DiversityMetric, + JensenShannon, + MeanSquaredError, + NoveltyMetric, +) +from tests.utils import DatafakerTestCase + + +class MeanSquaredErrorTests(DatafakerTestCase): + """Test case for the MeanSquaredError metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = MeanSquaredError() + + def test_identical_distributions_score_zero(self) -> None: + """Two identical distributions should have zero error.""" + dist = Distribution({"a": 0.5, "b": 0.5}) + self.assertEqual(0.0, self.metric.compare(dist, dist)) + + def test_completely_disjoint_distributions(self) -> None: + """Disjoint single-value distributions score the maximum of 1.0.""" + real = Distribution({"a": 1.0}) + synthetic = Distribution({"b": 1.0}) + # squared error = (1-0)**2 + (0-1)**2 = 2, halved = 1.0 + self.assertAlmostEqual(1.0, self.metric.compare(real, synthetic)) + + def test_partial_overlap(self) -> None: + """Score reflects the squared difference in shared probability mass.""" + real = Distribution({"a": 0.6, "b": 0.4}) + synthetic = Distribution({"a": 0.4, "b": 0.6}) + expected = ((0.2) ** 2 + (0.2) ** 2) / 2 + self.assertAlmostEqual(expected, self.metric.compare(real, synthetic)) + + def test_empty_vocabulary_scores_zero(self) -> None: + """Two empty distributions have no vocabulary and score zero.""" + self.assertEqual(0.0, self.metric.compare(Distribution({}), Distribution({}))) + + def test_none_and_mixed_type_keys_are_canonicalized(self) -> None: + """None and non-string keys are normalized so they compare correctly.""" + real = Distribution({None: 1.0}) + synthetic = Distribution({"": 1.0}) + self.assertEqual(0.0, self.metric.compare(real, synthetic)) + + real_int_key = Distribution({3: 1.0}) + synthetic_str_key = Distribution({"3": 1.0}) + self.assertEqual(0.0, self.metric.compare(real_int_key, synthetic_str_key)) + + +class JensenShannonTests(DatafakerTestCase): + """Test case for the JensenShannon metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = JensenShannon() + + def test_identical_distributions_score_zero(self) -> None: + """Two identical distributions have zero divergence.""" + dist = Distribution({"a": 0.3, "b": 0.7}) + self.assertAlmostEqual(0.0, self.metric.compare(dist, dist)) + + def test_completely_disjoint_distributions_score_ln2(self) -> None: + """Disjoint distributions hit the theoretical maximum of ln(2).""" + real = Distribution({"a": 1.0}) + synthetic = Distribution({"b": 1.0}) + self.assertAlmostEqual(math.log(2), self.metric.compare(real, synthetic)) + + def test_partial_overlap_between_zero_and_max(self) -> None: + """A partially-overlapping pair scores strictly between 0 and ln(2).""" + real = Distribution({"a": 0.9, "b": 0.1}) + synthetic = Distribution({"a": 0.1, "b": 0.9}) + score = self.metric.compare(real, synthetic) + self.assertGreater(score, 0.0) + self.assertLess(score, math.log(2)) + + def test_empty_vocabulary_scores_zero(self) -> None: + """Two empty distributions score zero.""" + self.assertEqual(0.0, self.metric.compare(Distribution({}), Distribution({}))) + + def test_is_symmetric(self) -> None: + """Jensen-Shannon divergence is symmetric in its two arguments.""" + real = Distribution({"a": 0.2, "b": 0.8}) + synthetic = Distribution({"a": 0.7, "b": 0.3}) + # pylint: disable-next=arguments-out-of-order + reversed_score = self.metric.compare(synthetic, real) + self.assertAlmostEqual(self.metric.compare(real, synthetic), reversed_score) + + +class NoveltyMetricTests(DatafakerTestCase): + """Test case for the NoveltyMetric metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = NoveltyMetric() + + def test_no_overlap_is_fully_novel(self) -> None: + """Synthetic values sharing nothing with the real data score 1.0.""" + self.assertEqual(1.0, self.metric.compare(["a", "b"], ["c", "d"])) + + def test_full_overlap_is_not_novel(self) -> None: + """Synthetic values that are all copies of real values score 0.0.""" + self.assertEqual(0.0, self.metric.compare(["a", "b"], ["a", "b"])) + + def test_partial_overlap(self) -> None: + """Score is the fraction of distinct synthetic values not seen in real.""" + real = ["a", "b"] + synthetic = ["a", "c"] + self.assertAlmostEqual(0.5, self.metric.compare(real, synthetic)) + + def test_empty_synthetic_scores_zero(self) -> None: + """No synthetic values at all is treated as no novelty.""" + self.assertEqual(0.0, self.metric.compare(["a"], [])) + + def test_comparison_is_case_and_whitespace_insensitive(self) -> None: + """Values are compared after stripping whitespace and lowercasing.""" + self.assertEqual(0.0, self.metric.compare(["Alice"], [" alice "])) + + def test_none_values_are_ignored(self) -> None: + """None entries in either sequence are dropped before comparison.""" + self.assertEqual(0.0, self.metric.compare(["a", None], ["a"])) + + +class DiversityMetricTests(DatafakerTestCase): + """Test case for the DiversityMetric metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = DiversityMetric() + + def test_matching_diversity_scores_one(self) -> None: + """Equally-diverse real and synthetic samples score 1.0.""" + real = ["a", "b", "c", "d"] + synthetic = ["w", "x", "y", "z"] + self.assertAlmostEqual(1.0, self.metric.compare(real, synthetic)) + + def test_single_repeated_value_has_zero_entropy_on_both_sides(self) -> None: + """A single repeated value on both sides is a diversity match.""" + self.assertAlmostEqual(1.0, self.metric.compare(["a", "a", "a"], ["b", "b"])) + + def test_less_diverse_synthetic_scores_below_one(self) -> None: + """A synthetic sample far less diverse than the real one is penalized.""" + real = ["a", "b", "c", "d"] + synthetic = ["x", "x", "x", "x"] + score = self.metric.compare(real, synthetic) + self.assertLess(score, 1.0) + self.assertGreaterEqual(score, 0.0) + + def test_empty_values_treated_as_zero_entropy(self) -> None: + """Empty or all-None sequences have zero normalized entropy.""" + self.assertAlmostEqual(1.0, self.metric.compare([], [])) + self.assertAlmostEqual(1.0, self.metric.compare([None, None], [None])) + + def test_score_is_bounded_below_by_zero(self) -> None: + """The score never goes negative even for maximally different diversity.""" + real = ["a", "a", "a", "a"] + synthetic = ["a", "b", "c", "d"] + score = self.metric.compare(real, synthetic) + self.assertGreaterEqual(score, 0.0) diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py new file mode 100644 index 0000000..5650bd0 --- /dev/null +++ b/tests/test_evaluators_proposal_ranking.py @@ -0,0 +1,397 @@ +"""Unit tests for datafaker.evaluators.proposal_ranking.""" +from typing import Any + +from datafaker.evaluators.column_evaluator import EvaluationProfile, ProposalEvaluation +from datafaker.evaluators.proposal_ranking import ( + ProposalRanking, + _crowding_distances, + _pareto_fronts, + format_ranking_display, + keyword_match, + normalize_list, + rank_proposals, +) +from datafaker.proposers.base import Proposer +from datafaker.proposers.choice import UniformChoiceProposer +from tests.utils import DatafakerTestCase + + +class FakeProposer(Proposer): + """A minimal, DB-free stand-in for a real Proposer, for ranking tests.""" + + def __init__(self, name: str) -> None: + self._name = name + + def function_name(self) -> str: + return self._name + + def nominal_kwargs(self) -> dict[str, str]: + return {} + + def actual_kwargs(self) -> dict[str, Any]: + return {} + + def generate_data(self, count: int) -> list[Any]: + return [None] * count + + +def make_result( # pylint: disable=too-many-arguments,too-many-positional-arguments + name: str = "generic.text", + overall_score: float = 0.5, + novelty: float = 0.5, + diversity: float = 0.5, + copy_fraction: float = 0.0, + synthetic_uniqueness: float = 1.0, + proposer: Proposer | None = None, +) -> ProposalEvaluation: + """Build a ProposalEvaluation for ranking tests.""" + return ProposalEvaluation( + proposer=proposer or FakeProposer(name), + novelty=novelty, + diversity=diversity, + overall_score=overall_score, + pipeline_scores={}, + copy_fraction=copy_fraction, + synthetic_uniqueness=synthetic_uniqueness, + ) + + +class KeywordMatchTests(DatafakerTestCase): + """Test case for keyword_match.""" + + def test_matching_keyword_and_generator_gives_boost(self) -> None: + """A column/generator pair that both mention 'email' gets the boost.""" + boost, keyword = keyword_match("customer_email", "generic.person.email") + self.assertEqual(0.15, boost) + self.assertEqual("email", keyword) + + def test_compound_column_name_matches_by_substring(self) -> None: + """Substring matching catches compound names like 'customer_first_name'.""" + boost, keyword = keyword_match( + "customer_first_name", "generic.person.first_name" + ) + self.assertEqual(0.15, boost) + self.assertEqual("first_name", keyword) + + def test_no_column_name_gives_no_boost(self) -> None: + """A missing column name can't match anything.""" + self.assertEqual((0.0, None), keyword_match(None, "generic.person.email")) + self.assertEqual((0.0, None), keyword_match("", "generic.person.email")) + + def test_column_matches_but_generator_does_not(self) -> None: + """Column name hints at a kind, but this generator isn't of that kind.""" + boost, keyword = keyword_match("email", "generic.text.word") + self.assertEqual(0.0, boost) + self.assertIsNone(keyword) + + def test_is_case_insensitive(self) -> None: + """Matching ignores case in both the column and generator names.""" + boost, keyword = keyword_match("EMAIL", "GENERIC.PERSON.EMAIL") + self.assertEqual(0.15, boost) + self.assertEqual("email", keyword) + + +class NormalizeListTests(DatafakerTestCase): + """Test case for normalize_list.""" + + def test_empty_list(self) -> None: + """An empty input returns an empty output.""" + self.assertEqual([], normalize_list([])) + + def test_spread_out_values_map_to_zero_one(self) -> None: + """Values are linearly rescaled so the min is 0 and the max is 1.""" + self.assertEqual([0.0, 0.5, 1.0], normalize_list([10, 20, 30])) + + def test_all_identical_values_are_treated_as_a_tie(self) -> None: + """Exactly equal values become 0.5 rather than dividing by zero.""" + self.assertEqual([0.5, 0.5, 0.5], normalize_list([3.0, 3.0, 3.0])) + + def test_near_identical_values_are_also_treated_as_a_tie(self) -> None: + """Floating-point noise around a shared value doesn't get stretched out.""" + result = normalize_list([0.6931471805599453, 0.6931471805599454]) + self.assertEqual([0.5, 0.5], result) + + def test_single_value_is_a_tie(self) -> None: + """A single value has no spread, so it's treated as a tie too.""" + self.assertEqual([0.5], normalize_list([42.0])) + + +class ParetoFrontsTests(DatafakerTestCase): + """Test case for the internal _pareto_fronts helper.""" + + def test_strictly_dominated_point_is_in_a_later_front(self) -> None: + """A point beaten or matched on every dimension lands behind the winner.""" + points = [(1.0, 1.0, 1.0), (0.5, 0.5, 0.5)] + fronts = _pareto_fronts(points) + self.assertEqual([[0], [1]], fronts) + + def test_non_dominated_points_share_the_first_front(self) -> None: + """Points that each win on a different dimension are mutually non-dominated.""" + points = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)] + fronts = _pareto_fronts(points) + self.assertEqual([[0, 1, 2]], fronts) + + def test_empty_points(self) -> None: + """No points means no fronts.""" + self.assertEqual([], _pareto_fronts([])) + + +class CrowdingDistancesTests(DatafakerTestCase): + """Test case for the internal _crowding_distances helper.""" + + def test_front_boundary_points_get_infinite_crowding(self) -> None: + """The extreme points of a front are always maximally 'crowded out'.""" + points = [(0.0, 0.0, 0.0), (0.5, 0.5, 0.5), (1.0, 1.0, 1.0)] + fronts = [[0, 1, 2]] + objectives = ( + [p[0] for p in points], + [p[1] for p in points], + [p[2] for p in points], + ) + crowding = _crowding_distances(points, fronts, objectives) + self.assertEqual(float("inf"), crowding[0]) + self.assertEqual(float("inf"), crowding[2]) + self.assertLess(crowding[1], float("inf")) + + def test_singleton_front_gets_infinite_crowding(self) -> None: + """A front with just one point has nothing to be crowded by.""" + points = [(0.5, 0.5, 0.5)] + fronts = [[0]] + objectives = ([0.5], [0.5], [0.5]) + crowding = _crowding_distances(points, fronts, objectives) + self.assertEqual([float("inf")], crowding) + + +class RankProposalsTests(DatafakerTestCase): + """Test case for rank_proposals.""" + + def test_no_results_gives_an_empty_ranking(self) -> None: + """With nothing to rank, no recommendation is made.""" + ranking = rank_proposals([], EvaluationProfile.SHORT_TEXT) + self.assertIsNone(ranking.recommended_index) + self.assertIsNone(ranking.recommended_reason) + self.assertEqual([], ranking.rows) + self.assertEqual((0.5, 0.25, 0.25), ranking.weights) + + def test_single_result_is_trivially_recommended(self) -> None: + """A single candidate is always the recommendation.""" + results = [make_result("only_one")] + ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + self.assertEqual(0, ranking.recommended_index) + self.assertEqual("best combined score", ranking.recommended_reason) + self.assertEqual(1, len(ranking.rows)) + + def test_lower_error_wins_on_fidelity(self) -> None: + """Among otherwise-equal candidates, the one with lower overall_score wins.""" + results = [ + make_result("bad_fit", overall_score=0.9, novelty=0.5, diversity=0.5), + make_result("good_fit", overall_score=0.1, novelty=0.5, diversity=0.5), + ] + ranking = rank_proposals(results, EvaluationProfile.IDENTIFIER) + self.assertEqual(1, ranking.recommended_index) + + def test_identifier_profile_weighs_fidelity_heavily(self) -> None: + """IDENTIFIER profile weights are (0.85, 0.05, 0.10).""" + ranking = rank_proposals([make_result()], EvaluationProfile.IDENTIFIER) + self.assertEqual((0.85, 0.05, 0.10), ranking.weights) + + def test_unknown_profile_falls_back_to_default_weights(self) -> None: + """A profile with no explicit entry uses the (0.5, 0.25, 0.25) default.""" + ranking = rank_proposals([make_result()], None) + self.assertEqual((0.5, 0.25, 0.25), ranking.weights) + + def test_exact_tie_breaks_towards_the_earlier_candidate(self) -> None: + """Identical candidates are both max-crowding; the first one wins.""" + results = [make_result("first"), make_result("second")] + ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + self.assertEqual(0, ranking.recommended_index) + self.assertEqual("tiebreak by crowding distance", ranking.recommended_reason) + + def test_keyword_boost_can_flip_the_recommendation(self) -> None: + """A column-name/generator match can outweigh an otherwise-tied fidelity.""" + results = [ + make_result( + "generic.text.word", overall_score=0.1, novelty=0.5, diversity=0.5 + ), + make_result( + "generic.person.email", overall_score=0.1, novelty=0.5, diversity=0.5 + ), + ] + # Without the keyword boost this is an exact tie, decided by crowding + # distance (which favors the first candidate); the boost should flip it. + ranking = rank_proposals( + results, EvaluationProfile.EMAIL, column_name="user_email" + ) + self.assertEqual(1, ranking.recommended_index) + self.assertEqual("best combined score", ranking.recommended_reason) + + def test_choice_proposer_penalized_by_resampling_on_unique_column(self) -> None: + """ChoiceProposer's score is discounted by real_uniqueness x copy_fraction.""" + choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + results = [ + make_result(overall_score=0.0, copy_fraction=1.0, proposer=choice_proposer), + make_result(overall_score=0.5, copy_fraction=0.0), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, real_uniqueness=1.0 + ) + # The ChoiceProposer perfectly fits but is fully discounted (penalty + # multiplier 0), leaving the other candidate to win despite a worse fit. + self.assertEqual(1, ranking.recommended_index) + + def test_non_choice_proposer_is_not_resample_penalized_on_string_column( + self, + ) -> None: + """Only ChoiceProposer (or a numeric column) is hit by the resample penalty.""" + results = [make_result(overall_score=0.0, copy_fraction=1.0)] + ranking = rank_proposals( + results, EvaluationProfile.SHORT_TEXT, real_uniqueness=1.0 + ) + self.assertFalse(ranking.penalty_applies_to_all) + # combined score should be unaffected by the (non-applicable) penalty: + # a single candidate normalizes every dimension to a 0.5 tie. + fid_w, nov_w, div_w = ranking.weights + expected = (fid_w + nov_w + div_w) * 0.5 + self.assertAlmostEqual(expected, ranking.scores[0]) + + def test_numeric_column_penalizes_every_proposer(self) -> None: + """A numeric column applies the resample-style penalty to all candidates.""" + results = [make_result(overall_score=0.0, copy_fraction=1.0)] + ranking = rank_proposals( + results, + EvaluationProfile.IDENTIFIER, + real_uniqueness=1.0, + is_numeric_column=True, + ) + self.assertTrue(ranking.penalty_applies_to_all) + self.assertEqual(0.0, ranking.scores[0]) + + def test_self_duplication_penalty_applies_regardless_of_proposer_type(self) -> None: + """Low synthetic_uniqueness is penalized even for a non-resampling proposer.""" + results = [ + make_result("full_of_dupes", overall_score=0.0, synthetic_uniqueness=0.0), + make_result("all_unique", overall_score=0.0, synthetic_uniqueness=1.0), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, real_uniqueness=1.0 + ) + self.assertEqual(1, ranking.recommended_index) + + def test_no_uniqueness_guarantee_when_every_candidate_duplicates(self) -> None: + """A primary key column with only duplicate-prone candidates has no safe pick.""" + results = [ + make_result("dupe_a", synthetic_uniqueness=0.5), + make_result("dupe_b", synthetic_uniqueness=0.9), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_primary_key=True + ) + self.assertTrue(ranking.no_uniqueness_guarantee) + + def test_uniqueness_guarantee_satisfied_by_one_candidate(self) -> None: + """If at least one candidate guarantees uniqueness, there's no warning.""" + results = [ + make_result("dupe", synthetic_uniqueness=0.5), + make_result("unique", synthetic_uniqueness=1.0), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_primary_key=True + ) + self.assertFalse(ranking.no_uniqueness_guarantee) + + def test_combined_score_is_clamped_to_one(self) -> None: + """The keyword boost can't push a score above 1.0.""" + results = [make_result("generic.person.email", overall_score=0.0)] + ranking = rank_proposals(results, EvaluationProfile.EMAIL, column_name="email") + self.assertLessEqual(ranking.scores[0], 1.0) + + def test_weak_recommendation_warning_when_best_fit_was_suppressed(self) -> None: + """A heavily-discounted best-fit candidate triggers the fallback warning.""" + choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + results = [ + # Fits almost perfectly but is a resampler on a near-unique column: + # gets discounted to (near) zero. + make_result( + overall_score=0.0, + novelty=0.0, + diversity=0.0, + copy_fraction=1.0, + synthetic_uniqueness=1.0, + proposer=choice_proposer, + ), + # A poor fit that isn't penalized, so it "wins" by elimination. + # synthetic_uniqueness is deliberately below the guarantee + # threshold so the weak-recommendation warning isn't suppressed + # by the "winner guarantees uniqueness" exemption. + make_result( + overall_score=1.0, + novelty=0.0, + diversity=0.0, + copy_fraction=0.0, + synthetic_uniqueness=0.5, + proposer=FakeProposer("generic.text.word"), + ), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, real_uniqueness=1.0 + ) + self.assertEqual(1, ranking.recommended_index) + self.assertIsNotNone(ranking.weak_recommendation_warning) + self.assert_str_in("resample penalty", ranking.weak_recommendation_warning) + + +class FormatRankingDisplayTests(DatafakerTestCase): + """Test case for format_ranking_display.""" + + def test_recommendation_line_names_the_winner(self) -> None: + """The recommendation string names the recommended candidate by number/name.""" + results = [make_result("winner", overall_score=0.0)] + ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + display = format_ranking_display(ranking, results, EvaluationProfile.SHORT_TEXT) + self.assert_str_in("Recommended: 1. winner", display.recommendation) + self.assert_str_in("best combined score", display.recommendation) + + def test_no_uniqueness_guarantee_overrides_recommendation_text(self) -> None: + """When no candidate can guarantee uniqueness, no winner is named.""" + results = [make_result("a", synthetic_uniqueness=0.5)] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_primary_key=True + ) + display = format_ranking_display(ranking, results, EvaluationProfile.IDENTIFIER) + self.assert_str_in("Recommended: none", display.recommendation) + + def test_penalty_scope_text_reflects_numeric_flag(self) -> None: + """The profile summary explains whether the penalty hit every candidate.""" + results = [make_result("a")] + numeric_ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_numeric_column=True + ) + numeric_display = format_ranking_display( + numeric_ranking, results, EvaluationProfile.IDENTIFIER + ) + self.assert_str_in( + "applied to every generator", numeric_display.profile_summary + ) + + string_ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + string_display = format_ranking_display( + string_ranking, results, EvaluationProfile.SHORT_TEXT + ) + self.assert_str_in( + "applied only to resampling generators", string_display.profile_summary + ) + + def test_rows_and_fronts_pass_through_unchanged(self) -> None: + """rows and fronts are carried over verbatim from the ProposalRanking.""" + results = [make_result("winner")] + ranking = ProposalRanking( + recommended_index=0, + recommended_reason="best combined score", + rows=[("1", "x")], + profile=EvaluationProfile.SHORT_TEXT, + weights=(0.5, 0.25, 0.25), + fronts=[1], + ) + display = format_ranking_display(ranking, results, EvaluationProfile.SHORT_TEXT) + self.assertEqual([("1", "x")], display.rows) + self.assertEqual([1], display.fronts) diff --git a/tests/test_evaluators_statistical_fidelity.py b/tests/test_evaluators_statistical_fidelity.py new file mode 100644 index 0000000..fac7aa6 --- /dev/null +++ b/tests/test_evaluators_statistical_fidelity.py @@ -0,0 +1,108 @@ +"""Unit tests for datafaker.evaluators.statistical_fidelity.""" +from datetime import datetime + +from sqlalchemy import Column, DateTime, Integer, MetaData, String, Table, create_engine + +from datafaker.evaluators.column_evaluator import EvaluationProfile +from datafaker.evaluators.statistical_fidelity import ( + EMAIL_PIPELINE, + PROFILE_PIPELINES, + StatisticalFidelity, +) +from tests.utils import DatafakerTestCase + + +class ProfilePipelinesTests(DatafakerTestCase): + """Structural sanity checks for the built-in evaluation pipelines.""" + + def test_every_evaluation_profile_has_a_pipeline(self) -> None: + """Each EvaluationProfile enum member maps to a configured pipeline.""" + for profile in EvaluationProfile: + self.assertIn(profile, PROFILE_PIPELINES) + self.assertTrue(PROFILE_PIPELINES[profile]) + + def test_pipeline_weights_sum_to_one(self) -> None: + """Each profile's pipeline weights add up to 1.0, for a sane weighted average.""" + for profile, pipelines in PROFILE_PIPELINES.items(): + with self.subTest(profile=profile.name): + total_weight = sum(p.weight for p in pipelines) + self.assertAlmostEqual(1.0, total_weight) + + def test_pipeline_names_are_unique_within_a_profile(self) -> None: + """Pipeline names double as dict keys in calculate_scores, so must be unique.""" + for profile, pipelines in PROFILE_PIPELINES.items(): + with self.subTest(profile=profile.name): + names = [p.name for p in pipelines] + self.assertEqual(len(names), len(set(names))) + + +def _make_table(columns: list[Column], rows: list[dict]) -> tuple: + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table("data", metadata, *columns) + metadata.create_all(engine) + if rows: + with engine.begin() as conn: + conn.execute(table.insert(), rows) + return engine, table + + +class StatisticalFidelityTests(DatafakerTestCase): + """Test case for StatisticalFidelity.""" + + def test_identical_real_and_synthetic_data_scores_perfectly(self) -> None: + """A synthetic sample identical to the real data has zero divergence.""" + engine, table = _make_table( + [Column("email", String)], + [{"email": f"user{i}@example.com"} for i in range(30)], + ) + fidelity = StatisticalFidelity(table.c.email, engine, sample_size=1000) + fidelity.set_eval_pipelines(EvaluationProfile.EMAIL) + + synthetic = [f"user{i}@example.com" for i in range(30)] + overall_score, pipeline_scores = fidelity.calculate_scores(synthetic) + + self.assertAlmostEqual(0.0, overall_score) + self.assertEqual({p.name for p in EMAIL_PIPELINE}, set(pipeline_scores)) + for name, score in pipeline_scores.items(): + with self.subTest(pipeline=name): + self.assertAlmostEqual(0.0, score) + + def test_completely_different_synthetic_data_scores_worse(self) -> None: + """A synthetic sample sharing nothing with the real data scores worse.""" + engine, table = _make_table( + [Column("category", String)], + [{"category": "alpha"} for _ in range(30)], + ) + fidelity = StatisticalFidelity(table.c.category, engine, sample_size=1000) + fidelity.set_eval_pipelines(EvaluationProfile.CATEGORICAL) + + matching_score, _ = fidelity.calculate_scores(["alpha"] * 30) + different_score, _ = fidelity.calculate_scores(["zzz_never_seen"] * 30) + + self.assertGreater(different_score, matching_score) + + def test_temporal_profile_scores_a_matching_date_column(self) -> None: + """The TEMPORAL profile's two pipelines run cleanly against a date column.""" + engine, table = _make_table( + [Column("ts", DateTime)], + [{"ts": datetime(2020, 1, (i % 27) + 1)} for i in range(40)], + ) + fidelity = StatisticalFidelity(table.c.ts, engine, sample_size=1000) + fidelity.set_eval_pipelines(EvaluationProfile.TEMPORAL) + + synthetic = [datetime(2020, 1, (i % 27) + 1) for i in range(40)] + overall_score, pipeline_scores = fidelity.calculate_scores(synthetic) + + self.assertAlmostEqual(0.0, overall_score) + self.assertEqual({"timestamp", "day_of_week"}, set(pipeline_scores)) + + def test_no_pipelines_set_scores_zero(self) -> None: + """Without calling set_eval_pipelines, there's nothing to score.""" + engine, table = _make_table( + [Column("n", Integer)], [{"n": i} for i in range(10)] + ) + fidelity = StatisticalFidelity(table.c.n, engine, sample_size=1000) + overall_score, pipeline_scores = fidelity.calculate_scores([1, 2, 3]) + self.assertEqual(0.0, overall_score) + self.assertEqual({}, pipeline_scores) From 619d0cc58c5bea49386e6f9f7542d5fc31f4e9c3 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Wed, 9 Sep 2026 17:10:16 +0100 Subject: [PATCH 03/23] Add IncrementProposer --- datafaker/proposers/__init__.py | 2 + datafaker/proposers/sequence.py | 91 +++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 datafaker/proposers/sequence.py diff --git a/datafaker/proposers/__init__.py b/datafaker/proposers/__init__.py index 1a333c3..bee4c53 100644 --- a/datafaker/proposers/__init__.py +++ b/datafaker/proposers/__init__.py @@ -30,6 +30,7 @@ NullPartitionedLogNormalProposerFactory, NullPartitionedNormalProposerFactory, ) +from datafaker.proposers.sequence import IncrementProposerFactory def everything_factory(config: Mapping, metadata: MetaData) -> ProposerFactory: @@ -57,4 +58,5 @@ def everything_factory(config: Mapping, metadata: MetaData) -> ProposerFactory: NullPartitionedLogNormalProposerFactory(config, metadata), DateAfterProposerFactory(config, metadata), DateComponentExtractProposerFactory(config, metadata), + IncrementProposerFactory(), ) diff --git a/datafaker/proposers/sequence.py b/datafaker/proposers/sequence.py new file mode 100644 index 0000000..99beb0e --- /dev/null +++ b/datafaker/proposers/sequence.py @@ -0,0 +1,91 @@ +"""Proposer for a monotonically-incrementing sequence past the observed max. + +This is the only kind of candidate that can actually guarantee fresh, unique +values for a genuine integer primary key: it can never collide with the real +data (every value it emits is strictly greater than any observed value) and +it can never collide with itself (it only ever increments). It deliberately +reuses ``generic.column_value_provider.increment`` - the same mechanism +``make.py``'s own default generator assignment already uses for integer +primary keys (see ``_integer_generator``) - rather than inventing a new +runtime function, so a column left on this proposer's recommendation behaves +identically to the framework's own established default. +""" + +from collections.abc import Sequence +from typing import Any + +from sqlalchemy import Column, Engine, func, select +from sqlalchemy.types import Integer + +from datafaker.proposers.base import Proposer, ProposerFactory, get_column_type + + +class IncrementProposer(Proposer): + """Generator continuing a sequence past the real data's observed maximum.""" + + def __init__(self, engine: Engine, column: Column): + """Initialise an IncrementProposer.""" + super().__init__() + self.engine = engine + self.column = column + self.table = column.table + + def function_name(self) -> str: + """Get the name of the generator function to call.""" + return "generic.column_value_provider.increment" + + def nominal_kwargs(self) -> dict[str, str]: + """Get the arguments to be entered into ``config.yaml``. + + These match the expressions ``make.py``'s own default integer + primary key generator emits (see ``_integer_generator``): not + SRC_STATS references, but Python expressions the generation + pipeline evaluates directly against the destination database and + its metadata at generation time. + """ + return { + "db_connection": "dst_db_conn", + "column": f'metadata.tables["{self.table.name}"].columns["{self.column.name}"]', + } + + def actual_kwargs(self) -> dict[str, Any]: + """Get the kwargs (summary statistics) this generator was instantiated with.""" + return {"column": f"{self.table.name}.{self.column.name}"} + + def generate_data(self, count: int) -> list[Any]: + """Generate ``count`` random data points for this column. + + For interactive preview purposes only: continues past the *source* + database's observed maximum. At actual generation time, the + ``nominal_kwargs`` above instead continue past whatever's already in + the *destination* database, via ``db_connection``/``column``. + """ + with self.engine.connect() as connection: + row = connection.execute(select(func.max(self.column))).first() + start = 0 if row is None or row[0] is None else row[0] + return [start + i + 1 for i in range(count)] + + +class IncrementProposerFactory(ProposerFactory): + """Propose an incrementing-sequence generator for integer primary keys. + + Deliberately narrow: only a genuine integer primary key (and one that + isn't also a foreign key - those should copy the referenced table's + values, not synthesize their own, matching ``make.py``'s own default + generator logic) gets this proposer. A non-key numeric column doesn't + need guaranteed-fresh values, and a non-integer key (a string/UUID + primary key) needs a different mechanism entirely - out of scope here. + """ + + def get_proposers( + self, columns: list[Column], engine: Engine + ) -> Sequence[Proposer]: + """Get the generators appropriate to these columns.""" + if len(columns) != 1: + return [] + column = columns[0] + if not column.primary_key or column.foreign_keys: + return [] + if not isinstance(get_column_type(column), Integer): + return [] + return [IncrementProposer(engine, column)] From 833d38bc8b18204aecee0934b5156a331836e258 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Wed, 9 Sep 2026 17:16:28 +0100 Subject: [PATCH 04/23] Update documentation --- docs/source/builtin_generators.rst | 653 +++++++++++++++++++++++++++++ docs/source/glossary.rst | 6 + docs/source/index.rst | 1 + docs/source/tutorial_parquet.rst | 9 +- 4 files changed, 666 insertions(+), 3 deletions(-) create mode 100644 docs/source/builtin_generators.rst diff --git a/docs/source/builtin_generators.rst b/docs/source/builtin_generators.rst new file mode 100644 index 0000000..f95a0bc --- /dev/null +++ b/docs/source/builtin_generators.rst @@ -0,0 +1,653 @@ +Built-in generators and proposers +================================== + +This page is a reference for the generator functions that ship with ``datafaker``, +and for the *proposers* that suggest which of those functions (and which arguments) +to use for a given column. If you want to write your own generators instead, +see :doc:`custom_generators`. + +Generators vs. proposers +------------------------- + +A **generator** is the actual callable that produces fake values, referenced by name +in ``config.yaml`` (for example ``dist_gen.normal`` or ``generic.person.first_name``). +Generators are simple: given some arguments (often summary statistics pulled from +``src-stats.yaml``), they return one random value per call. + +A **proposer** is an internal ``datafaker`` object, not something you reference +directly in ``config.yaml``. When you run ``configure-generators`` and use the +``propose`` command on a column, ``datafaker`` runs every applicable proposer against +that column (and, for multi-column proposers, against that group of columns). Each +proposer: + +* decides whether it applies at all, based on the column's SQL type and (for some + proposers) properties of the source data or ``config.yaml``, +* works out what summary queries need to be added to ``src-stats.yaml`` (via + ``select_aggregate_clauses`` or ``custom_queries``), +* works out the generator function name and keyword arguments to write into + ``config.yaml`` if you ``set`` it. + +``set`` simply writes the winning proposer's generator name and arguments into +``config.yaml``; from then on, only the underlying generator function is used --- +the proposer itself is not needed again for that column. + +``propose`` now shows two different views of the same candidates: first the +plain, per-proposer ``(fit: ...)`` list described above, and then a second, +ranked table produced by a separate statistical evaluation pipeline (see +:ref:`evaluating-and-ranking-proposals` below) that scores every candidate on +fidelity, novelty and diversity and names a ``Recommended`` generator. The two +scores are independent and can disagree; the ranked table and its +recommendation are the more reliable of the two for most columns. + +Default generators assigned automatically +------------------------------------------ + +Before you run ``propose``/``set`` at all, ``configure-generators`` (via +``make.py``) already assigns a default generator to every column, purely from its +SQL type (and whether it is a foreign key or primary key). ``propose`` lets you +replace this default with something that better matches the real data. + +.. list-table:: Default generator by SQL column type + :widths: 25 45 30 + :header-rows: 1 + + * - SQL type + - Default generator + - Notes + * - Foreign key column (any type) + - ``generic.column_value_provider.column_value`` + - Picks a random existing value from the referenced column in the destination + database. + * - Integer primary key + - ``generic.column_value_provider.increment`` + - Continues counting up from the highest existing value in the destination + table; see :ref:`the increment proposer ` below for how + this is also offered as an explicit proposal. + * - Other ``Integer`` + - ``generic.numeric.integer_number`` + - + * - ``Numeric`` (decimal/float) + - ``generic.numeric.float_number`` + - If the column has a fixed ``scale``, the range is set to fit within that + many decimal digits. + * - ``String`` + - ``generic.person.password``, or ``generic.text.color`` if the column has no + maximum length + - The password length is set to the column's maximum length so the value + always fits. + * - ``Boolean`` + - ``generic.development.boolean`` + - + * - ``Date`` + - ``generic.datetime.date`` + - + * - ``DateTime`` + - ``generic.datetime.datetime`` + - + * - ``LargeBinary`` + - ``generic.bytes_provider.bytes`` + - + * - ``Uuid`` / PostgreSQL ``UUID`` / MS-SQL ``UNIQUEIDENTIFIER`` + - ``generic.cryptographic.uuid`` + - + * - Anything else unsupported + - ``generic.null_provider.null`` + - ``datafaker`` logs a warning; you should configure a real generator for + this column by hand. + +Built-in proposers +------------------- + +Everything below is registered in ``datafaker.proposers.everything_factory`` and is +tried automatically for every column (or, for the multi-column proposers, every +group of columns you select) when you run ``propose``. + +Single-column value proposers (Mimesis) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These proposers wrap `Mimesis `_ generator functions +(``generic.*``, using the ``en_GB`` locale) that produce a plausible but +unrelated fake value, with no reference to the real data's actual distribution. +Fit is estimated by comparing the length (for strings) or value (for numbers) of +generated samples against buckets built from the real column. + +.. list-table:: + :widths: 20 60 20 + :header-rows: 1 + + * - Applies to + - Generators proposed + - Notes + * - ``String`` + - Every name in :ref:`the Mimesis string generator list ` + (``generic.``) + - If the column has a maximum length, each candidate is wrapped in + ``dist_gen.truncated_string`` so the output never overflows the column. + * - ``Numeric`` + - ``generic.person.height`` + - + * - ``Numeric`` or ``Integer`` + - ``generic.person.weight`` + - + * - ``Date`` + - ``generic.datetime.date`` + - Range (``start``/``end`` years) is taken from the earliest/latest years + actually found in the column. + * - ``DateTime`` + - ``generic.datetime.datetime`` + - Same year-range behaviour as ``Date``. + * - ``Time`` + - ``generic.datetime.time`` + - + +.. _mimesis-string-list: + +The full list of Mimesis string generators tried for ``String`` columns: +``address.calling_code``, ``address.city``, ``address.continent``, +``address.country``, ``address.country_code``, ``address.postal_code``, +``address.province``, ``address.street_number``, ``address.street_name``, +``address.street_suffix``, ``person.blood_type``, ``person.email``, +``person.first_name``, ``person.last_name``, ``person.full_name``, +``person.gender``, ``person.language``, ``person.nationality``, +``person.occupation``, ``person.password``, ``person.title``, +``person.university``, ``person.username``, ``person.worldview``, +``text.answer``, ``text.color``, ``text.level``, ``text.quote``, +``text.sentence``, ``text.text``, ``text.word``. + +Continuous distribution proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These proposers fit a single numeric column to a well-known distribution, using +the column's actual mean and standard deviation (queried into ``src-stats.yaml`` +as ``mean__`` / ``stddev__``, or their log equivalents). + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - ``Numeric`` or ``Integer`` + - ``dist_gen.normal`` + - Gaussian distribution with the observed mean and standard deviation. + * - ``Numeric`` or ``Integer`` + - ``dist_gen.uniform_ms`` + - Uniform distribution with the same mean and standard deviation as the real + data (rather than explicit min/max bounds). + * - ``Numeric`` or ``Integer``, values ``> 0`` + - ``dist_gen.lognormal`` + - Log-normal distribution, fitted to the mean/standard deviation of the logs + of the (positive) values. + +Choice proposers +^^^^^^^^^^^^^^^^^ + +Used for columns with a manageable number of distinct values (up to 500). Each of +the three distribution shapes below is proposed both for **all** distinct values +and, separately, for a version with rare values (seen 7 or fewer times) suppressed +--- and again for both of those computed from a random sample of up to 500 rows, +for large tables where scanning every row would be too slow. + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Generator + - Function name + - Distribution + * - Uniform choice + - ``dist_gen.choice`` + - Every distinct value equally likely. + * - Zipf choice + - ``dist_gen.zipf_choice`` + - Values ranked by real-world frequency; the *n*\ th most common value is + chosen ``1/n`` as often as the most common one (a Zipf/power-law shape, + without needing to store every count). + * - Weighted choice + - ``dist_gen.weighted_choice`` + - Reproduces the real frequency of each value directly (stores a count + alongside each value in ``src-stats.yaml``). + +Constant proposer +^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - Any single nullable column, or ``String``/``Numeric``/``Integer`` + - ``dist_gen.constant`` + - Always returns the same value: ``None`` if the column is nullable, + otherwise ``""``, ``0.0`` or ``0`` depending on type. Useful as a + placeholder, or a baseline to compare fit against. + +.. _increment-proposer: + +Sequence proposer +^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - Integer primary key that is **not** also a foreign key + - ``generic.column_value_provider.increment`` + - Counts up past the highest value already present, guaranteeing fresh, + unique values. This is the same mechanism used as the automatic default + for integer primary keys (see above); proposing it explicitly just makes + it visible and comparable alongside other candidates. + +Multivariate proposers +^^^^^^^^^^^^^^^^^^^^^^^^ + +Offered when you select **two or more** numeric columns together (so that +``datafaker`` can preserve correlations between them, rather than generating each +column independently). A covariate (means/covariance) matrix is queried from the +source data. + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - 2+ ``Numeric``/``Integer`` columns + - ``dist_gen.multivariate_normal`` + - Multivariate Gaussian distribution over all the selected columns. + * - 2+ ``Numeric``/``Integer`` columns, values ``> 0`` + - ``dist_gen.multivariate_lognormal`` + - Multivariate log-normal distribution (covariates computed on the logs of + the values). + +Null-partitioned multivariate proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A more powerful variant of the multivariate proposers above, for a group of +columns (numeric and/or categorical) that don't all follow the same pattern of +missing (``NULL``) values. The source data is split into partitions, one per +distinct combination of which columns in the group are ``NULL``; each partition +gets its own covariate matrix (or, if it has too few rows, is treated as a +suppressed group), and at generation time a partition is picked at random +(weighted by how common it was in the source data) before that partition's +distribution is sampled. + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - Group of columns with mixed nullability + - ``dist_gen.alternatives`` wrapping ``dist_gen.grouped_multivariate_normal`` + - Shown as *"null-partitioned grouped_multivariate_normal"* in ``propose``. + * - Group of columns with mixed nullability, values ``> 0`` + - ``dist_gen.alternatives`` wrapping ``dist_gen.grouped_multivariate_lognormal`` + - Shown as *"null-partitioned grouped_multivariate_lognormal"*. + * - (both of the above) + - *[sampled and suppressed]* variants + - Computed from a random sample of the source table with rare partitions + suppressed, for large tables. + +Date interval ("anchored date") proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Propose a date/datetime as an offset from another date, rather than independently. +This only appears once at least one ``Date``/``DateTime`` column in the same +table (or a directly related table) has been given the ``start`` role --- for +example an admission date. Every *other* date/datetime column in scope (an +implicit "end", such as a discharge date) is then offered a proposal anchored +to it; the anchor column itself is not offered a self-referential proposal. + +Roles are set per column and stored in ``config.yaml`` under +``tables..columns..roles``. The easiest way to set one is the +``role`` command inside ``configure-generators``: + +.. code-block:: shell + + (admission.start_date) role set start + (admission.start_date) role list + start + (discharge.end_date) role on admission.start_date list + +``role list`` shows the roles on the current column, ``role set `` / +``role delete `` add or remove one, and adding ``on `` (or +just ``on `` for a column in the current table) targets a different +column without navigating to it first. Like generator changes, role changes +are only written to ``config.yaml`` when you ``quit`` and confirm --- ``quit`` +will list any pending role changes alongside pending generator changes before +asking you to save. + +Two roles currently exist: ``start`` (consumed by this proposer, as described +above) and ``source`` (reserved for future use --- no built-in proposer reads +it yet). You can still set either role by hand-editing ``config.yaml`` instead +of using ``role``, if you prefer; both routes update the same field. + +.. list-table:: + :widths: 20 30 50 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - ``Date``/``DateTime`` column with an anchor column in the *same* table + - ``generic.anchored_provider.normal_date`` + - Adds a clamped, normally-distributed number of seconds (mean/standard + deviation taken from the real interval lengths) to the anchor column's + generated value; never earlier than the anchor. The sample values shown + by ``propose``/``compare`` are built from real anchor values sampled + from the source database (recycled if fewer than requested), rather + than a single fixed dummy anchor, so the preview reflects realistic + intervals. + * - ``Date``/``DateTime`` column anchored to a column in a *related* table + (via a foreign key) + - ``generic.anchored_provider.normal_date_fk`` + - Same idea, but looks up the anchor value from the related row in the + destination database at generation time. + +Date component extraction proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Propose deriving a column's value from a ``DateTime`` column already generated +elsewhere in the same row, rather than generating it independently -- useful when, +for example, a ``year_of_birth`` column should always agree with a +``date_of_birth`` column. + +.. list-table:: + :widths: 20 30 50 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - ``Date``, based on a ``DateTime`` column in the same table + - ``generic.extract_provider.date`` + - Takes just the date part of the referenced ``DateTime`` value. + * - ``Integer``, based on a ``DateTime`` column in the same table + - ``generic.extract_provider.year``, ``generic.extract_provider.month``, + ``generic.extract_provider.day`` + - One proposal per component, per ``DateTime`` column found in the table. + +.. _evaluating-and-ranking-proposals: + +Evaluating and ranking proposals +---------------------------------- + +For a single-column ``propose``, every candidate proposer that survives the +type-compatibility filter (see below) is additionally scored by +``datafaker.evaluators.ColumnEvaluator`` and ranked by +``datafaker.evaluators.proposal_ranking.rank_proposals``. This produces a + table shown by ``propose``, with a ``Recommended: N. `` line +above it. + +Evaluation profile +^^^^^^^^^^^^^^^^^^^ + +The column is first classified into one ``EvaluationProfile``, based on a +sample of up to 4000 real rows: + +.. list-table:: + :widths: 25 75 + :header-rows: 1 + + * - Profile + - How it's chosen + * - ``TEMPORAL`` + - Column is ``Date``/``DateTime``/``Time``. + * - ``CATEGORICAL`` or ``IDENTIFIER`` + - Column is ``Numeric``/``Integer``: ``CATEGORICAL`` if fewer than 20% of + the sampled values are distinct (a status code, a small foreign key + range), otherwise ``IDENTIFIER``. + * - ``EMAIL`` + - Column is a string and more than half the sampled non-empty values + look like an email address (contain ``@``, with a non-empty local part + and a domain containing a ``.``). + * - ``SHORT_TEXT`` + - Column is a string, not email-like, average length under 30 + characters and more than 80% of sampled values are distinct. + * - ``CATEGORICAL`` + - Column is a string, not email-like or short-text, and fewer than 20% + of sampled values are distinct. + * - ``FREE_TEXT`` + - Remaining string columns with an average length over 50 characters or + a high space ratio (multi-word content). + * - ``SHORT_TEXT`` + - Any remaining string column (the fallback). + +Three scoring dimensions +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Each candidate is scored, from a 4000-value synthetic sample, on three +independent axes, each normalized to ``[0, 1]`` across the candidates being +compared (so these are *relative* scores, not absolute ones): + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Axis + - Meaning + * - **Fidelity** + - How closely the synthetic sample's distribution matches the real + column's, measured by a profile-specific pipeline of feature + extractors run through ``MeanSquaredError`` or ``JensenShannon`` + (see :ref:`statistical-fidelity-pipelines` below), then inverted so + higher is better. + * - **Novelty** + - The fraction of synthetic values (case/whitespace-normalized) that do + *not* already appear among the real sampled values. 1.0 means every + synthetic value is new. + * - **Diversity** + - How closely the synthetic sample's own internal variety (normalized + Shannon entropy over distinct values) matches the real sample's --- not + "more diverse is better", but "as diverse as the real data". + +.. _statistical-fidelity-pipelines: + +Fidelity pipelines by profile +"""""""""""""""""""""""""""""" + +Fidelity is computed by one of six fixed pipelines, chosen by profile, each a +weighted combination of feature extractors compared via ``MeanSquaredError`` +(on a length/count histogram) or ``JensenShannon`` divergence (on a category +distribution): + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Profile + - Pipeline (feature: weight) + * - ``IDENTIFIER`` + - identifier (value histogram): 1.0 + * - ``CATEGORICAL`` + - category (exact value): 1.0 + * - ``SHORT_TEXT`` + - length: 0.25, character bigrams: 0.35, first letter: 0.10, + last letter: 0.10, vowel/consonant pattern: 0.20 + * - ``EMAIL`` + - length: 0.10, local part: 0.40, domain: 0.15, top-level domain: 0.15, + address format validity: 0.20 + * - ``FREE_TEXT`` + - length: 0.10, word count: 0.25, sentence count: 0.15, words: 0.30, + character bigrams: 0.20 + * - ``TEMPORAL`` + - year+month+day, as one joint quantity: 0.7, day-of-week: 0.3 + +Combining the scores, penalties and a keyword hint +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The three normalized scores are combined into one ``Score`` per candidate as + +.. code-block:: text + + Score = clamp( (fid_w * Fidelity + nov_w * Novelty + div_w * Diversity + Keyword) * Penalty, 0, 1 ) + +* **Weights** (``fid_w``/``nov_w``/``div_w``) depend on the profile: 0.85 / + 0.05 / 0.10 for ``IDENTIFIER`` (fidelity dominates); 0.7 / 0.15 / 0.15 for + ``CATEGORICAL``, ``SHORT_TEXT``, ``EMAIL`` and ``TEMPORAL``; 0.5 / 0.3 / 0.2 + for ``FREE_TEXT`` (open-ended text tolerates, even rewards, novelty). +* **Keyword** is a flat ``+0.15`` boost when the column's own name hints at a + particular kind of value (e.g. a column named ``first_name`` or + ``customer_first_name`` boosts any ``person.first_name``-based candidate; + see ``KEYWORD_GENERATOR_HINTS`` in ``proposal_ranking.py`` for the full + list) -- a cheap, complementary signal used only to break near-ties, never + to override a clearly better statistical result. +* **Penalty** (``1 - real_uniqueness * copy_fraction``, further reduced by how + much a candidate duplicates against its *own* output) discounts a + choice-style proposer (``dist_gen.choice``/``weighted_choice``/ + ``zipf_choice``) -- or, for a numeric column, *any* proposer -- for + reproducing real values verbatim, scaled by how unique the real column + actually is. This is a privacy safeguard: resampling real values is + expected and harmless for a low-uniqueness column (a gender, a status) but + a genuine leak for a near-unique one (an email, a real ID). It has no + effect (``1.0``) on other proposers for non-numeric columns, where an + overlap with real values is usually just a naturally shared vocabulary + (e.g. common first names), not memorization. + +Candidates are also grouped into Pareto fronts (front 1 = not +strictly dominated on fidelity, novelty *and* diversity simultaneously by any +other candidate); ``propose`` (without ``all``) shows only front 1, capped at +10 rows by ``Score`` (always keeping the recommended candidate even if it +would otherwise be cut), and tells you how many candidates were hidden. Run +``propose all`` to see every candidate --- including proposers whose output +type doesn't actually match the column (for example a continuous-float +proposer against an ``Integer`` column, invalid on most databases), which the +plain ``propose`` excludes structurally rather than relying on the score to +catch. + +The **recommendation** is simply the candidate with the highest ``Score`` +(ties broken by NSGA-II crowding distance, i.e. preferring a candidate that's +less redundant with its front-mates). Two caveats are surfaced when they +apply: + +* For a primary key or unique column, if *every* candidate's own synthetic + sample is duplicated enough that it couldn't satisfy a uniqueness + constraint, ``propose`` recommends nothing at all rather than a false + positive --- look for the sequence proposer instead (see + :ref:`increment-proposer` above). +* If the candidates that fit the real data best were heavily discounted by + the resample penalty above, leaving a poorer-fitting candidate to "win" by + elimination, ``propose`` prints a warning naming the discounted candidates + and flags the recommendation as a fallback rather than a confident pick. + +Generator function reference +------------------------------ + +The tables above name the generator functions that proposers can select. This +section documents each one, grouped by the Mimesis provider it belongs to, for +when you want to reference or combine them by hand in ``config.yaml`` (see +:doc:`custom_generators`). + +``dist_gen`` -- ``datafaker.providers.DistributionProvider`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Function + - Description + * - ``uniform(low, high)`` + - Uniform distribution between explicit bounds. + * - ``uniform_ms(mean, sd)`` + - Uniform distribution with a given mean and standard deviation. + * - ``normal(mean, sd)`` + - Gaussian (normal) distribution. + * - ``lognormal(logmean, logsd)`` + - Log-normal distribution. + * - ``choice(a)`` / ``choice_direct(a)`` + - Uniform choice between a list of values. ``choice`` takes + ``{"value": ...}`` dicts (as stored in ``src-stats.yaml``); + ``choice_direct`` takes plain values. + * - ``zipf_choice(a, n=None)`` / ``zipf_choice_direct(a, n=None)`` + - Choice following a Zipf distribution over values ranked most-to-least + frequent. + * - ``weighted_choice(a)`` + - Choice weighted by an explicit ``count`` stored alongside each value. + * - ``constant(value)`` + - Always returns ``value``. + * - ``multivariate_normal(cov)`` / ``multivariate_lognormal(cov)`` + - Draws a list of correlated values from a covariate matrix (means and + covariances, keyed as ``mN``/``cN_M``). + * - ``grouped_multivariate_normal(covs)`` / ``grouped_multivariate_lognormal(covs)`` + - As above, but first picks one covariate matrix from a list, weighted by + each group's ``count``. Used for the null-partitioned proposers. + * - ``alternatives(alternative_configs, counts=None)`` + - Picks between other named generators, weighted by count; this is how the + null-partitioned proposers choose a missingness pattern before delegating + to ``grouped_multivariate_normal``/``grouped_multivariate_lognormal``. + * - ``with_constants_at(constants_at, subgen, params)`` + - Runs another generator and splices fixed values into the result list at + given positions (used to reinsert ``NULL``/category columns alongside + generated numeric ones). + * - ``truncated_string(subgen_fn, params, length)`` + - Runs a string-producing generator and truncates the result to ``length`` + characters. + +``generic`` -- Mimesis providers (locale ``en_GB``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Most of ``generic.address.*``, ``generic.person.*``, ``generic.text.*``, +``generic.datetime.*`` and ``generic.numeric.*`` come straight from the +`Mimesis library `_ and aren't +listed again here; see :ref:`the Mimesis string generator list +` above for the ones ``propose`` tries automatically, +plus ``person.height``, ``person.weight``, ``datetime.date``, +``datetime.datetime`` and ``datetime.time``. + +``datafaker`` adds the following extra Mimesis providers of its own +(``datafaker/providers.py``): + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Function + - Description + * - ``column_value_provider.column_value(db_connection, orm_class, column_name)`` + - Returns a random existing value from another column -- the default + generator for foreign key columns. + * - ``column_value_provider.increment(db_connection, column)`` + - Returns a number one higher than the previous call for this column, + starting one above the highest existing value in the destination + database -- the default generator for integer primary keys. + * - ``bytes_provider.bytes()`` + - Random binary data. + * - ``timedelta_provider.timedelta(...)`` + - Random ``datetime.timedelta`` value. + * - ``timespan_provider.timespan(...)`` + - Random pair of dates/times forming a span. + * - ``weighted_boolean_provider.bool(probability)`` + - ``True`` with the given probability. + * - ``sql_group_by_provider.sample(...)`` + - Samples rows grouped and filtered according to SQL-like criteria. + * - ``null_provider.null()`` + - Always returns ``None``. Used as the last-resort default for column + types ``datafaker`` doesn't otherwise recognise. + * - ``extract_provider.year(extract_from)`` / ``month(...)`` / ``day(...)`` / ``date(...)`` + - Pulls a component out of a ``datetime`` produced elsewhere in the same + row (see the date component extraction proposers above). + * - ``anchored_provider.normal_date(mean_seconds, sd_seconds, anchor)`` + - A date offset from ``anchor`` in the same table by a clamped normally + distributed number of seconds. + * - ``anchored_provider.normal_date_fk(dst_db_conn, mean_seconds, sd_seconds, table, on_column, anchor_row, anchor_column)`` + - As above, but ``anchor`` is looked up from a related table via a foreign + key at generation time. + +See also +--------- + +* :doc:`custom_generators` -- writing your own row generators and story + generators. +* :doc:`quickstart` and :doc:`introduction` -- walkthroughs of + ``configure-generators``, ``propose``, ``compare`` and ``set`` in action. +* :doc:`glossary` diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst index 87b9889..a80edb6 100644 --- a/docs/source/glossary.rst +++ b/docs/source/glossary.rst @@ -11,6 +11,12 @@ Glossary - A user-defined Python function which will provide one or more random column values for a single table when called. * - Story generator - A user-defined Python generator function that ``yields`` rows, possibly multiple rows for multiple tables. + * - Generator (function) + - A callable, referenced by name in ``config.yaml`` (e.g. ``dist_gen.normal``), that produces one random value (or tuple of values) per call. See :doc:`builtin_generators` for the ones built into `datafaker`, or :doc:`custom_generators` to write your own. + * - Proposer + - An internal `datafaker` object, used by the ``propose`` command of ``configure-generators``, that suggests a generator function and arguments for a column (or group of columns) and estimates how well it fits the real data. See :doc:`builtin_generators`. + * - Role + - A tag (``start`` or ``source``) set on a column, via the ``role`` command in ``configure-generators`` or by hand in ``config.yaml``, that some proposers use to find related columns -- for example, the ``start`` role marks an anchor date that other dates in the same table can be generated as an offset from. See :doc:`builtin_generators`. * - Destination database and destination schema - A database and a schema within that database where `datafaker` creates the synthetic data tables and inserts the synthetic data it generates. * - Source database and source schema diff --git a/docs/source/index.rst b/docs/source/index.rst index f411c60..4c87a00 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -37,6 +37,7 @@ Contents: configuration health_data Custom Generators + Built-in Generators and Proposers api .. toctree:: diff --git a/docs/source/tutorial_parquet.rst b/docs/source/tutorial_parquet.rst index d28d951..205e4ae 100644 --- a/docs/source/tutorial_parquet.rst +++ b/docs/source/tutorial_parquet.rst @@ -219,7 +219,7 @@ and source statistics: datafaker configure-tables datafaker configure-generators - datafaker configure-missingness + datafaker configure-missing datafaker make-stats This creates: @@ -328,7 +328,7 @@ Generate configuration: datafaker configure-tables datafaker configure-generators - datafaker configure-missingness + datafaker configure-missing datafaker make-stats Create schema and generate data: @@ -356,7 +356,10 @@ For a minimal end-to-end workflow: export DST_DSN=duckdb:///./fake.db datafaker make-tables --parquet-dir ./input_parquet - + datafaker configure-tables + datafaker configure-generators + datafaker configure-missing + datafaker make-stats datafaker create-tables datafaker create-data --num-passes 10 From 33ba1a8e7afcb75bde6900ba00184a1bce6efefd Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Thu, 10 Sep 2026 15:13:36 +0100 Subject: [PATCH 05/23] Update column evaluator module --- datafaker/evaluators/column_evaluator.py | 8 +++- datafaker/evaluators/distribution_builders.py | 44 ++++++++++--------- datafaker/evaluators/proposal_ranking.py | 4 +- datafaker/evaluators/statistical_fidelity.py | 2 +- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/datafaker/evaluators/column_evaluator.py b/datafaker/evaluators/column_evaluator.py index 4b02df9..a6cbbb5 100644 --- a/datafaker/evaluators/column_evaluator.py +++ b/datafaker/evaluators/column_evaluator.py @@ -2,6 +2,7 @@ import string from dataclasses import dataclass +from typing import Any from sqlalchemy import Column, Engine, select from sqlalchemy.types import Date, DateTime, Integer, Numeric, Time @@ -66,7 +67,7 @@ def __str__(self): __repr__ = __str__ -def analyse_column(values: list[str]) -> ColumnStats: +def analyse_column(values: list[Any]) -> ColumnStats: """Compute length/space/digit/punctuation statistics over ``values``.""" values = [str(v) for v in values if v is not None] @@ -198,7 +199,7 @@ def setup(self, columns: list[Column], engine: Engine): # which are computed from this sample. rows = conn.execute( select(columns[0]) - .select_from(self.table) + .select_from(columns[0].table) .order_by(Random()) .limit(self.sample_size) ) @@ -259,6 +260,9 @@ def setup(self, columns: list[Column], engine: Engine): def evaluate(self, proposer) -> ProposalEvaluation: """Score one proposer's synthetic data against the sampled real data.""" + assert ( + self.novelty_metric is not None and self.diversity_metric is not None + ), "setup() must be called before evaluate()" synthetic_samples = proposer.generate_data(self.sample_size) # calculate novelty diff --git a/datafaker/evaluators/distribution_builders.py b/datafaker/evaluators/distribution_builders.py index 868f072..f9298ac 100644 --- a/datafaker/evaluators/distribution_builders.py +++ b/datafaker/evaluators/distribution_builders.py @@ -36,6 +36,22 @@ def as_vector(self, vocabulary: list[Any]) -> list[float]: class DistributionBuilder(ABC): """Build a real or synthetic value's ``Distribution`` for one feature.""" + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + engine, + table, + column, + extractor: FeatureExtractor = IdentityExtractor(), + sample_size=4000, + ): + """Initialize the engine/table/column/extractor/sample_size shared by every builder.""" + self.engine = engine + self.table = table + self.column = column + self.extractor = extractor + self.sample_size = sample_size + @abstractmethod def build_from_table(self) -> Distribution: """Build the distribution of this feature over the real column.""" @@ -60,12 +76,8 @@ def __init__( sample_size=4000, ): """Initialize a histogram builder for one column feature.""" - self.engine = engine - self.table = table - self.column = column + super().__init__(engine, table, column, extractor, sample_size) self.bins = bins - self.extractor = extractor - self.sample_size = sample_size # Learned from the real data self.mean = None @@ -189,6 +201,12 @@ def build_from_table(self) -> Distribution: def build_from_values(self, values) -> Distribution: """Build the synthetic sample's histogram distribution.""" + if self.edges is None: + # build_from_table() never learned edges - either it hasn't run + # yet, or the real column had no usable values. Either way there + # is nothing to bucket synthetic values against. + return Distribution({}) + counts = Counter() total = 0 @@ -209,22 +227,6 @@ def build_from_values(self, values) -> Distribution: class CategoryBuilder(DistributionBuilder): """Build a categorical distribution over a discrete feature's values.""" - # pylint: disable=too-many-arguments too-many-positional-arguments - def __init__( - self, - engine, - table, - column, - extractor: FeatureExtractor = IdentityExtractor(), - sample_size=4000, - ): - """Initialize a category builder for one column feature.""" - self.engine = engine - self.table = table - self.column = column - self.extractor = extractor - self.sample_size = sample_size - def build_from_table(self): """Build the real column's categorical distribution.""" counter = Counter() diff --git a/datafaker/evaluators/proposal_ranking.py b/datafaker/evaluators/proposal_ranking.py index 8649efb..f7558e7 100644 --- a/datafaker/evaluators/proposal_ranking.py +++ b/datafaker/evaluators/proposal_ranking.py @@ -264,7 +264,9 @@ def rank_proposals( EvaluationProfile.FREE_TEXT: (0.5, 0.3, 0.2), EvaluationProfile.TEMPORAL: (0.7, 0.15, 0.15), } - fid_w, nov_w, div_w = profile_weights.get(profile, (0.5, 0.25, 0.25)) + fid_w, nov_w, div_w = ( + profile_weights[profile] if profile in profile_weights else (0.5, 0.25, 0.25) + ) # compute profile-specific score (higher-is-better) combined_scores = [fid_w * f + nov_w * n + div_w * d for (f, n, d) in points] diff --git a/datafaker/evaluators/statistical_fidelity.py b/datafaker/evaluators/statistical_fidelity.py index ea68b2d..d40ab49 100644 --- a/datafaker/evaluators/statistical_fidelity.py +++ b/datafaker/evaluators/statistical_fidelity.py @@ -34,7 +34,7 @@ class EvaluationPipeline: """One named feature-comparison stage within a profile's fidelity score.""" name: str - builder: DistributionBuilder + builder: type[DistributionBuilder] metric: Metric extractor: FeatureExtractor weight: float = 1.0 From 3f02598923e6088d8fa96a4b124888d0bab9c7ed Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Thu, 10 Sep 2026 15:14:33 +0100 Subject: [PATCH 06/23] Update documentation --- docs/source/builtin_generators.rst | 8 +- docs/source/choosing_a_generator.rst | 123 +++++++++++++++++++++++++++ docs/source/index.rst | 1 + 3 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 docs/source/choosing_a_generator.rst diff --git a/docs/source/builtin_generators.rst b/docs/source/builtin_generators.rst index f95a0bc..0020e88 100644 --- a/docs/source/builtin_generators.rst +++ b/docs/source/builtin_generators.rst @@ -3,8 +3,10 @@ Built-in generators and proposers This page is a reference for the generator functions that ship with ``datafaker``, and for the *proposers* that suggest which of those functions (and which arguments) -to use for a given column. If you want to write your own generators instead, -see :doc:`custom_generators`. +to use for a given column. If you just want a quick pointer from "my column +looks like this" to "try this generator", see :doc:`choosing_a_generator` +instead. If you want to write your own generators, see +:doc:`custom_generators`. Generators vs. proposers ------------------------- @@ -646,6 +648,8 @@ plus ``person.height``, ``person.weight``, ``datetime.date``, See also --------- +* :doc:`choosing_a_generator` -- a quick lookup from column shape to + generator, for when you want more than the ``propose`` recommendation. * :doc:`custom_generators` -- writing your own row generators and story generators. * :doc:`quickstart` and :doc:`introduction` -- walkthroughs of diff --git a/docs/source/choosing_a_generator.rst b/docs/source/choosing_a_generator.rst new file mode 100644 index 0000000..b7dcc18 --- /dev/null +++ b/docs/source/choosing_a_generator.rst @@ -0,0 +1,123 @@ +Choosing a generator for a column +================================== + +This page is a quick decision aid for the ``propose`` command: "my column looks +like *this* -- which generator should I expect, and why?" For the full list of +generator functions and proposers, and how ``propose`` scores and ranks them, +see :doc:`builtin_generators`. + +Start here +---------- + +In almost all cases, just run ``propose`` on the column and take the +``Recommended`` generator -- it already scores every applicable candidate on +fidelity, novelty and diversity against your real data, which is more +reliable than guessing from the column's type alone. The table below is for +the two situations where you need more than the recommendation: + +* you want to sanity-check *why* a particular generator was recommended, or +* the recommendation looks wrong (or none was given) and you want to know + what else is worth trying. + +.. list-table:: + :widths: 30 40 30 + :header-rows: 1 + + * - Your column looks like... + - Try this generator + - Why + * - A unique ID, e.g. an integer primary key + - ``generic.column_value_provider.increment`` + - Guarantees fresh, unique values by counting up past the highest + existing one; ``propose`` offers this explicitly and it's also the + automatic default for integer primary keys. See + :ref:`the sequence proposer `. + * - A foreign key to another table + - ``generic.column_value_provider.column_value`` + - Picks a random existing value from the referenced column, so + referential integrity is preserved. This is the automatic default for + any foreign key column. + * - A near-unique value that must look real but not *be* real, e.g. an + email address or free-text ID + - Whichever ``generic.*`` value proposer ``propose`` recommends + - ``propose`` applies an extra privacy penalty here: a candidate that + reproduces real values verbatim is scored down in proportion to how + unique the real column is, so the recommendation already accounts for + leak risk. See :ref:`statistical-fidelity-pipelines`. + * - A small set of repeating values, e.g. a status code, category or flag + - ``dist_gen.weighted_choice`` (matches real frequencies exactly), or + ``dist_gen.zipf_choice`` / ``dist_gen.choice`` for a looser fit + - These are the *choice proposers*, offered for any column with up to + 500 distinct values; they differ in how closely they copy the real + frequency of each value. ``weighted_choice`` stores and reproduces + the exact real frequency of every value -- the closest fit, but the + most exposing of how common each value really is. ``zipf_choice`` + instead only ranks values from most- to least-common and assigns + each a Zipf/power-law frequency (the *n*\ th most common value is + chosen ``1/n`` as often as the most common one), which approximates + a typical real-world skew without storing per-value counts. + ``choice`` drops frequency information entirely and picks uniformly + at random -- the loosest fit, but the simplest and most private + option if flattening the distribution is acceptable. + * - A continuous measurement, e.g. a length, weight or price + - ``dist_gen.normal``, ``dist_gen.lognormal`` (if all values are + positive and skewed), or ``dist_gen.uniform_ms`` + - The *continuous distribution proposers* fit directly to the real + column's mean and standard deviation. + * - Two or more numeric columns that vary together, e.g. width and height + - ``dist_gen.multivariate_normal`` / ``dist_gen.multivariate_lognormal`` + - Select all the columns together (``merge``) before running + ``propose`` so it can offer these -- they preserve the correlation + between columns rather than generating each independently. + * - The same, but some rows have one column ``NULL`` and others don't + - the *null-partitioned* generators, e.g. ``dist_gen.alternatives`` + wrapping ``dist_gen.grouped_multivariate_normal`` + - Real data is split into partitions by which columns are ``NULL``; + each partition keeps its own correlation, and one is picked at + generation time weighted by how common it was in the source data. + * - A recognisable human-ish string, e.g. a first name, city or username + - The matching ``generic.person.*`` / ``generic.address.*`` / + ``generic.text.*`` generator + - ``propose`` gives a flat ``+0.15`` boost when the column's *name* + hints at its content (e.g. a column called ``first_name`` boosts + ``generic.person.first_name``) -- a tie-breaker, not an override of a + clearly better statistical fit. + * - A date/time that should stay consistent with another date on the same + row, e.g. a ``discharge_date`` after an ``admission_date`` + - ``generic.anchored_provider.normal_date`` (same table) or + ``.normal_date_fk`` (related table via foreign key) + - Only offered once the earlier date has been given the ``start`` role + (``role set start``). The offset is a clamped, normally distributed + number of seconds taken from the real interval lengths, so the + generated date is never earlier than its anchor. + * - A column that should just agree with a ``datetime`` column elsewhere + in the row, e.g. ``year_of_birth`` alongside ``date_of_birth`` + - ``generic.extract_provider.date`` / ``.year`` / ``.month`` / ``.day`` + - Derives the value from the other column instead of generating it + independently, so the two can never disagree. + * - Nothing else fits, or you just want a placeholder + - ``dist_gen.constant`` + - Always returns the same value (``None`` if the column is nullable). + Also useful as a baseline to compare other candidates' fit against. + +If a column doesn't match any row above, it's still worth running +``propose`` -- the ranking in :doc:`builtin_generators` covers cases (and +combinations of ``String``/``Numeric``/``Date`` type with real-data shape) +that are awkward to summarise as a simple lookup table. + +When ``propose`` recommends nothing +------------------------------------ + +For a primary key or other unique column, ``propose`` will withhold a +recommendation entirely if every candidate's synthetic sample duplicates too +often to satisfy uniqueness -- rather than confidently suggesting something +that would break a constraint. Look for the sequence proposer +(``generic.column_value_provider.increment``) instead. + +See also +--------- + +* :doc:`builtin_generators` -- full generator and proposer reference, plus + how the ``Recommended`` score is computed. +* :doc:`quickstart` -- walkthrough of ``propose``, ``compare`` and ``set`` in + the ``configure-generators`` CLI. diff --git a/docs/source/index.rst b/docs/source/index.rst index 4c87a00..1233728 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -36,6 +36,7 @@ Contents: orm configuration health_data + Choosing a Generator Custom Generators Built-in Generators and Proposers api From f90ce81070f73ff9a96c568120ca7ddffdb32949 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Thu, 10 Sep 2026 15:15:05 +0100 Subject: [PATCH 07/23] Update unit tests --- tests/test_evaluators_distribution_builders.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_evaluators_distribution_builders.py b/tests/test_evaluators_distribution_builders.py index e3a79e2..4b79e82 100644 --- a/tests/test_evaluators_distribution_builders.py +++ b/tests/test_evaluators_distribution_builders.py @@ -123,6 +123,7 @@ def test_values_outside_edges_are_clamped_to_end_buckets(self) -> None: dist = self.builder.build_from_values([-1000, 1000]) self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) buckets = set(dist.probabilities.keys()) + assert self.builder.edges is not None self.assertTrue(all(0 <= b <= len(self.builder.edges) - 2 for b in buckets)) From 6ae3a5eca20f93a4b19a2bd9b998977935e394ca Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Thu, 10 Sep 2026 16:44:25 +0100 Subject: [PATCH 08/23] Fix unit tests failure and errors --- datafaker/interactive/generators.py | 2 +- docs/source/builtin_generators.rst | 2 +- tests/test_interactive_generators.py | 12 ++++++++++-- tests/test_rst.py | 5 +++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index 621f51e..60dda6d 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -1064,7 +1064,7 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st theme = get_active_theme() limit = 5 include_all = _arg.strip().lower() == "all" - props = self._get_proposer_proposals() + props = self._get_proposer_proposals(include_all) sample = self._get_column_data(limit) self._print_source_sample(sample, theme) if not props: diff --git a/docs/source/builtin_generators.rst b/docs/source/builtin_generators.rst index 0020e88..953094b 100644 --- a/docs/source/builtin_generators.rst +++ b/docs/source/builtin_generators.rst @@ -387,7 +387,7 @@ For a single-column ``propose``, every candidate proposer that survives the type-compatibility filter (see below) is additionally scored by ``datafaker.evaluators.ColumnEvaluator`` and ranked by ``datafaker.evaluators.proposal_ranking.rank_proposals``. This produces a - table shown by ``propose``, with a ``Recommended: N. `` line +table shown by ``propose``, with a ``Recommended: N. `` line above it. Evaluation profile diff --git a/tests/test_interactive_generators.py b/tests/test_interactive_generators.py index 08d4587..1a59e4c 100644 --- a/tests/test_interactive_generators.py +++ b/tests/test_interactive_generators.py @@ -427,7 +427,11 @@ def test_aggregate_queries_merge(self) -> None: column = "position" generator = "dist_gen.uniform_ms" gc.do_next(f"string.{column}") - gc.do_propose("") + # "position" is an Integer column, and dist_gen.uniform_ms produces + # float output that default 'propose' now excludes as type- + # incompatible (see GeneratorCmd._is_integer_incompatible) - 'all' + # bypasses that filter so this generator is still selectable here. + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[f"{generator}"][0])) gc.do_quit("") @@ -604,7 +608,11 @@ def test_existing_configuration_remains(self) -> None: column = "position" generator = "dist_gen.uniform_ms" gc.do_next(f"string.{column}") - gc.do_propose("") + # "position" is an Integer column, and dist_gen.uniform_ms produces + # float output that default 'propose' now excludes as type- + # incompatible (see GeneratorCmd._is_integer_incompatible) - 'all' + # bypasses that filter so this generator is still selectable here. + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[generator][0])) gc.do_quit("") diff --git a/tests/test_rst.py b/tests/test_rst.py index ee89b5b..edd87ac 100644 --- a/tests/test_rst.py +++ b/tests/test_rst.py @@ -40,6 +40,7 @@ def test_dir(self) -> None: # Ignore errors if they contain any of these strings allowed_errors = [ 'No role entry for "ref" in module', + 'No role entry for "doc" in module', 'No directive entry for "toctree"', 'No directive entry for "automodule"', 'No directive entry for "literalinclude"', @@ -51,6 +52,10 @@ def test_dir(self) -> None: 'Hyperlink target "page-quickstart" is not referenced.', 'Hyperlink target "page-installation" is not referenced.', 'Hyperlink target "story-generators" is not referenced.', + 'Hyperlink target "mimesis-string-list" is not referenced', + 'Hyperlink target "increment-proposer" is not referenced', + 'Hyperlink target "evaluating-and-ranking-proposals" is not referenced', + 'Hyperlink target "statistical-fidelity-pipelines" is not referenced', ] filtered_errors = [ file_error From e906ab2a4ab3cd66e4fc13a276d8522780d0c2e0 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Thu, 10 Sep 2026 17:13:41 +0100 Subject: [PATCH 09/23] Fix unit tests error --- datafaker/evaluators/distribution_builders.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/datafaker/evaluators/distribution_builders.py b/datafaker/evaluators/distribution_builders.py index f9298ac..d6afabe 100644 --- a/datafaker/evaluators/distribution_builders.py +++ b/datafaker/evaluators/distribution_builders.py @@ -164,17 +164,23 @@ def build_from_table(self) -> Distribution: # raw-value pulls above). # with self.engine.connect() as conn: - # Group by the bucket expression itself, not its output alias: - # MSSQL (unlike Postgres/DuckDB) rejects a SELECT-list alias - # referenced from GROUP BY. + # Compute the bucket expression once, in an inner subquery, and + # group by the resulting materialized column in the outer query. + # MSSQL rejects both `GROUP BY ` (a SELECT-list alias + # referenced from GROUP BY) and `GROUP BY ` + # (its query planner does not recognize two separately-bound + # occurrences of the same parameterized expression as + # equivalent) - grouping by an actual column of a subquery is + # the one form every dialect (Postgres/DuckDB/MSSQL) accepts. bucket_expr = func.floor((feature_expr - bottom) / width) + inner = ( + select(bucket_expr.label("bucket")).select_from(self.table).subquery() + ) rows = conn.execute( select( - bucket_expr.label("bucket"), + inner.c.bucket, func.count().label("count"), # pylint: disable=not-callable - ) - .select_from(self.table) - .group_by(bucket_expr) + ).group_by(inner.c.bucket) ).all() # total must come from this same full-table query, not from the From 1b98cf70d92dae4fe40c502b13023377bc076792 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Fri, 11 Sep 2026 12:59:19 +0100 Subject: [PATCH 10/23] Refine penalty calculation --- datafaker/evaluators/column_evaluator.py | 29 ++++++++++- datafaker/evaluators/proposal_ranking.py | 60 +++++++++++++++++------ datafaker/interactive/generators.py | 5 ++ docs/source/builtin_generators.rst | 44 ++++++++++++----- docs/source/choosing_a_generator.rst | 2 +- tests/test_evaluators_proposal_ranking.py | 5 +- 6 files changed, 114 insertions(+), 31 deletions(-) diff --git a/datafaker/evaluators/column_evaluator.py b/datafaker/evaluators/column_evaluator.py index a6cbbb5..1f60538 100644 --- a/datafaker/evaluators/column_evaluator.py +++ b/datafaker/evaluators/column_evaluator.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import Any -from sqlalchemy import Column, Engine, select +from sqlalchemy import Column, Engine, UniqueConstraint, select from sqlalchemy.types import Date, DateTime, Integer, Numeric, Time from datafaker.dialects import Random @@ -167,6 +167,7 @@ def __init__(self): self.column = None self.table = None self.is_primary_key = False + self.is_unique_constrained = False self.real_values = [] self.column_is_numeric = False self.real_uniqueness = 0.0 @@ -185,6 +186,14 @@ def setup(self, columns: list[Column], engine: Engine): # A composite key needs every constituent column marked primary_key; # this reduces to the single column's own flag in the common case. self.is_primary_key = bool(columns) and all(c.primary_key for c in columns) + # A real UNIQUE constraint/index needs the same uniqueness guarantee + # as a primary key, even though it isn't one - e.g. a UNIQUE email + # column. Distinct from is_primary_key so the two can be OR'd + # together (needs_uniqueness) without conflating "is the key" with + # "must be unique". + self.is_unique_constrained = self.is_primary_key or self._has_unique_constraint( + columns + ) with engine.connect() as conn: if len(columns) == 1: @@ -258,6 +267,24 @@ def setup(self, columns: list[Column], engine: Engine): self.statistical_fidelity = None self.profile = EvaluationProfile.IDENTIFIER + @staticmethod + def _has_unique_constraint(columns: list[Column]) -> bool: + """Whether exactly this column set is covered by a UNIQUE constraint/index.""" + if not columns: + return False + table = columns[0].table + column_set = set(columns) + for constraint in table.constraints: + if ( + isinstance(constraint, UniqueConstraint) + and set(constraint.columns) == column_set + ): + return True + for index in table.indexes: + if index.unique and set(index.columns) == column_set: + return True + return False + def evaluate(self, proposer) -> ProposalEvaluation: """Score one proposer's synthetic data against the sampled real data.""" assert ( diff --git a/datafaker/evaluators/proposal_ranking.py b/datafaker/evaluators/proposal_ranking.py index f7558e7..9a3b907 100644 --- a/datafaker/evaluators/proposal_ranking.py +++ b/datafaker/evaluators/proposal_ranking.py @@ -68,6 +68,15 @@ # on top of a perfectly good pick. WEAK_WINNER_SCORE_THRESHOLD = 0.5 +# Floor applied to the self-duplication penalty (see rank_proposals) for a +# column that doesn't actually need guaranteed-unique values (no PRIMARY +# KEY/UNIQUE constraint). Self-duplication still matters there - a +# generator whose synthetic sample is mostly repeats of a tiny fixed pool +# (e.g. a canned-quote generator) makes for an obviously repetitive column +# even without breaking a constraint - but it shouldn't be crushed as hard +# as it would be for a column where duplicates are a correctness bug. +SELF_DUPLICATION_PENALTY_FLOOR = 0.4 + def keyword_match( column_name: str | None, proposer_name: str @@ -117,6 +126,7 @@ class ProposalRanking: weights: tuple[float, float, float] real_uniqueness: float = 0.0 penalty_applies_to_all: bool = False + self_duplication_floored: bool = False fronts: list[int] = field(default_factory=list) scores: list[float] = field(default_factory=list) weak_recommendation_warning: str | None = None @@ -204,6 +214,7 @@ def rank_proposals( real_uniqueness: float = 0.0, is_numeric_column: bool = False, is_primary_key: bool = False, + is_unique_constrained: bool = False, ) -> ProposalRanking: """Rank proposals by fidelity, novelty and diversity using Pareto fronts. @@ -323,12 +334,24 @@ def rank_proposals( # vocabulary (a gender, a status) is expected to have low # synthetic_uniqueness too, and shouldn't be penalized for it. # Unlike the resample penalty, this applies to every candidate - # unconditionally - self-duplication breaks a PRIMARY KEY/UNIQUE - # constraint regardless of column type or whether the proposer is a - # resampler. + # regardless of column type or whether the proposer is a resampler - + # but its full strength is only warranted when the column actually + # needs unique values (a PRIMARY KEY/UNIQUE constraint); it's floored + # below (SELF_DUPLICATION_PENALTY_FLOOR) otherwise, since a merely + # observed-unique column (e.g. free-text descriptions that happen to + # all differ) doesn't have a correctness reason to punish a small, + # repetitive-but-otherwise-plausible pool as hard as a real key would. self_duplication_penalty = 1.0 - real_uniqueness * ( 1.0 - result.synthetic_uniqueness ) + if not (is_primary_key or is_unique_constrained): + # No constraint actually requires unique values here, so don't + # let this penalty alone crush the score - just discount it + # relative to a candidate that can sustain the real column's + # observed distinctness. + self_duplication_penalty = max( + self_duplication_penalty, SELF_DUPLICATION_PENALTY_FLOOR + ) penalty_multipliers[idx] = resample_penalty * self_duplication_penalty combined_scores[idx] *= penalty_multipliers[idx] @@ -464,6 +487,7 @@ def rank_proposals( f"{diversity_scores[idx]:.6f}", f"{real_uniqueness:.3f}", f"{result.copy_fraction:.3f}", + f"{result.synthetic_uniqueness:.3f}", f"{penalty_multipliers[idx]:.3f}", ] # color entire row for Pareto front 1 @@ -480,6 +504,7 @@ def rank_proposals( weights=(fid_w, nov_w, div_w), real_uniqueness=real_uniqueness, penalty_applies_to_all=apply_to_every_proposer, + self_duplication_floored=not (is_primary_key or is_unique_constrained), fronts=list(front_of), scores=list(combined_scores), weak_recommendation_warning=weak_recommendation_warning, @@ -505,17 +530,16 @@ def format_ranking_display( recommendation = ( f"Recommended: {rec_num}. {rec_name} — {ranking.recommended_reason}\n" ) - if ranking.penalty_applies_to_all: - penalty_scope = ( - "applied to every generator (this is a numeric column, so a high" - " copy_fraction always means the real range is dense, not a coincidental" - " vocabulary match)" - ) - else: - penalty_scope = ( - "applied only to resampling generators (dist_gen.choice/weighted_choice/" - "zipf_choice - see Penalty column); 1.0 (no effect) for everyone else" - ) + resample_scope = ( + "every generator (numeric column)" + if ranking.penalty_applies_to_all + else "resamplers only (dist_gen.choice/weighted_choice/zipf_choice)" + ) + self_dup_scope = ( + f"floored at {SELF_DUPLICATION_PENALTY_FLOOR:.1f} (no PK/UNIQUE constraint)" + if ranking.self_duplication_floored + else "full strength (PK/UNIQUE constrained)" + ) profile_summary = ( f"Profile: {profile.name if profile is not None else 'UNKNOWN'} | " f"Real uniqueness: {ranking.real_uniqueness:.3f} | " @@ -523,8 +547,12 @@ def format_ranking_display( f"Score = clamp( ({fid_w:.2f}*Fidelity + {nov_w:.2f}*Novelty" f" + {div_w:.2f}*Diversity + Keyword) " "x Penalty , 0, 1 )\n" - " Keyword: +0.15 if the column name hints at this generator's kind (see Keyword column)\n" - f" Penalty: 1 - real_uniqueness x copy_fraction, {penalty_scope}" + " Keyword: +0.15 for a column-name hint (see Keyword column)\n" + " Penalty = resample_penalty x self_duplication_penalty" + " (see Copies / Synth.Uniq columns)\n" + f" resample_penalty (1 - real_uniqueness x copy_fraction): {resample_scope}\n" + " self_duplication_penalty (1 - real_uniqueness x (1 - synthetic_uniqueness)):" + f" {self_dup_scope}" ) return ProposalRankingDisplay( recommendation=recommendation, diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index 60dda6d..a2ca912 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -1109,6 +1109,9 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st real_uniqueness = getattr(column_evaluator, "real_uniqueness", 0.0) is_numeric_column = getattr(column_evaluator, "column_is_numeric", False) is_primary_key = getattr(column_evaluator, "is_primary_key", False) + is_unique_constrained = getattr( + column_evaluator, "is_unique_constrained", False + ) ranking = rank_proposals( results, profile, @@ -1117,6 +1120,7 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st real_uniqueness=real_uniqueness, is_numeric_column=is_numeric_column, is_primary_key=is_primary_key, + is_unique_constrained=is_unique_constrained, ) display = format_ranking_display(ranking, results, profile) @@ -1171,6 +1175,7 @@ def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-st "Diversity", "Uniqueness", "Copies", + "Synth.Uniq", "Penalty", ], rows_to_show, diff --git a/docs/source/builtin_generators.rst b/docs/source/builtin_generators.rst index 953094b..91ddf3b 100644 --- a/docs/source/builtin_generators.rst +++ b/docs/source/builtin_generators.rst @@ -503,17 +503,39 @@ The three normalized scores are combined into one ``Score`` per candidate as see ``KEYWORD_GENERATOR_HINTS`` in ``proposal_ranking.py`` for the full list) -- a cheap, complementary signal used only to break near-ties, never to override a clearly better statistical result. -* **Penalty** (``1 - real_uniqueness * copy_fraction``, further reduced by how - much a candidate duplicates against its *own* output) discounts a - choice-style proposer (``dist_gen.choice``/``weighted_choice``/ - ``zipf_choice``) -- or, for a numeric column, *any* proposer -- for - reproducing real values verbatim, scaled by how unique the real column - actually is. This is a privacy safeguard: resampling real values is - expected and harmless for a low-uniqueness column (a gender, a status) but - a genuine leak for a near-unique one (an email, a real ID). It has no - effect (``1.0``) on other proposers for non-numeric columns, where an - overlap with real values is usually just a naturally shared vocabulary - (e.g. common first names), not memorization. +* **Penalty** is the product of two independent factors, each scaled by how + unique the real column actually is: + + .. code-block:: text + + Penalty = resample_penalty * self_duplication_penalty + + * ``resample_penalty = 1 - real_uniqueness * copy_fraction`` is a privacy + safeguard against reproducing real values verbatim. ``copy_fraction`` is + the fraction of a candidate's synthetic sample that exactly matches a + real value. It only applies to choice-style proposers + (``dist_gen.choice``/``weighted_choice``/``zipf_choice``) -- or, for a + numeric column, *any* proposer -- since resampling real values is + expected and harmless for a low-uniqueness column (a gender, a status) + but a genuine leak for a near-unique one (an email, a real ID). It has no + effect (``1.0``) on other proposers for non-numeric columns, where an + overlap with real values is usually just a naturally shared vocabulary + (e.g. common first names), not memorization. + * ``self_duplication_penalty = 1 - real_uniqueness * (1 - synthetic_uniqueness)`` + guards against a candidate duplicating against its *own* output -- + ``synthetic_uniqueness`` is the fraction of distinct values in its 4000 + synthetic samples. This is a different failure mode from + ``resample_penalty``: a candidate can have a tiny fixed pool of possible + outputs (e.g. a canned-quote generator with only a few dozen distinct + strings) without ever coincidentally matching a *real* value, yet still + be unusable for a column that needs many distinct rows. It's computed + for every candidate, but only applied at full strength when the column + actually needs unique values (a primary key or a ``UNIQUE``-constrained + column) -- otherwise it's floored at ``0.4``, discounting a + self-duplicating-but-otherwise-plausible candidate rather than crushing + its score to near zero over a constraint that doesn't apply here. The + ``Synth.Uniq`` column in the ``propose`` table shows the raw value this + factor is computed from. Candidates are also grouped into Pareto fronts (front 1 = not strictly dominated on fidelity, novelty *and* diversity simultaneously by any diff --git a/docs/source/choosing_a_generator.rst b/docs/source/choosing_a_generator.rst index b7dcc18..a47c2dc 100644 --- a/docs/source/choosing_a_generator.rst +++ b/docs/source/choosing_a_generator.rst @@ -43,7 +43,7 @@ the two situations where you need more than the recommendation: - ``propose`` applies an extra privacy penalty here: a candidate that reproduces real values verbatim is scored down in proportion to how unique the real column is, so the recommendation already accounts for - leak risk. See :ref:`statistical-fidelity-pipelines`. + leak risk. See :ref:`evaluating-and-ranking-proposals`. * - A small set of repeating values, e.g. a status code, category or flag - ``dist_gen.weighted_choice`` (matches real frequencies exactly), or ``dist_gen.zipf_choice`` / ``dist_gen.choice`` for a looser fit diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py index 5650bd0..2e8b73c 100644 --- a/tests/test_evaluators_proposal_ranking.py +++ b/tests/test_evaluators_proposal_ranking.py @@ -370,7 +370,7 @@ def test_penalty_scope_text_reflects_numeric_flag(self) -> None: numeric_ranking, results, EvaluationProfile.IDENTIFIER ) self.assert_str_in( - "applied to every generator", numeric_display.profile_summary + "every generator (numeric column)", numeric_display.profile_summary ) string_ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) @@ -378,7 +378,8 @@ def test_penalty_scope_text_reflects_numeric_flag(self) -> None: string_ranking, results, EvaluationProfile.SHORT_TEXT ) self.assert_str_in( - "applied only to resampling generators", string_display.profile_summary + "resamplers only (dist_gen.choice/weighted_choice/zipf_choice)", + string_display.profile_summary, ) def test_rows_and_fronts_pass_through_unchanged(self) -> None: From 0f6d61757f41e57107461b3f9c92c950fc5b1c76 Mon Sep 17 00:00:00 2001 From: ucl_arc Date: Fri, 11 Sep 2026 13:22:35 +0100 Subject: [PATCH 11/23] Refine penalty calculation --- .github/workflows/docs.yml | 2 +- datafaker/evaluators/proposal_ranking.py | 123 ++++++++++------------- 2 files changed, 55 insertions(+), 70 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d496a39..0b53c6d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,7 +4,7 @@ on: push: branches: # need this to be able to run the workflow until it has been merged into main - - yhong123/131-update_documentation + - yhong123/112-column_evaluators workflow_dispatch: env: PYTHON_VERSION: "3.12" diff --git a/datafaker/evaluators/proposal_ranking.py b/datafaker/evaluators/proposal_ranking.py index 9a3b907..e9b06e1 100644 --- a/datafaker/evaluators/proposal_ranking.py +++ b/datafaker/evaluators/proposal_ranking.py @@ -204,6 +204,51 @@ def _crowding_distances( return crowding +def _penalty_for_candidate( + result: ProposalEvaluation, + real_uniqueness: float, + apply_to_every_proposer: bool, + needs_uniqueness: bool, +) -> float: + """Combine the resample and self-duplication penalties for one candidate. + + ChoiceProposer and its variants (dist_gen.choice/weighted_choice/ + zipf_choice) resample directly from the column's own observed values - + by construction, they can only ever emit values already in the real + data. That's the *correct* thing to do for a column with a small, + shared vocabulary (a status code, a gender) - reproducing real values + there is expected and unavoidable - but a privacy problem for a column + whose real values are meant to be unique per row (an email, a genuine + ID). Numeric columns get the same treatment regardless of proposer + (apply_to_every_proposer): a plain number has no shared human + vocabulary to explain a coincidental match, so a high copy_fraction + there means the real value range is dense, not a legitimate overlap. + + Separately, self_duplication_penalty catches a candidate duplicating + against its OWN output (synthetic_uniqueness) rather than against the + real data (copy_fraction, above) - e.g. dist_gen.constant on a column + whose fixed value happens not to appear in the sampled real data gets + copy_fraction=0 (no resample penalty) despite duplicating every row + against itself. It applies to every candidate, but its full strength + is only warranted when the column actually needs unique values (a + PRIMARY KEY/UNIQUE constraint); it's floored otherwise, since a merely + observed-unique column (e.g. free-text descriptions that happen to all + differ) doesn't have a correctness reason to punish a small, + repetitive-but-otherwise-plausible pool as hard as a real key would. + """ + resample_penalty = 1.0 + if isinstance(result.proposer, ChoiceProposer) or apply_to_every_proposer: + resample_penalty = 1.0 - real_uniqueness * result.copy_fraction + self_duplication_penalty = 1.0 - real_uniqueness * ( + 1.0 - result.synthetic_uniqueness + ) + if not needs_uniqueness: + self_duplication_penalty = max( + self_duplication_penalty, SELF_DUPLICATION_PENALTY_FLOOR + ) + return resample_penalty * self_duplication_penalty + + # pylint: disable=too-many-arguments too-many-positional-arguments # pylint: disable=too-many-locals too-many-statements def rank_proposals( @@ -282,77 +327,17 @@ def rank_proposals( # compute profile-specific score (higher-is-better) combined_scores = [fid_w * f + nov_w * n + div_w * d for (f, n, d) in points] - # ChoiceProposer and its variants (dist_gen.choice/weighted_choice/ - # zipf_choice) resample directly from the column's own observed values - - # by construction, they can only ever emit values already in the real - # data. That's the *correct* thing to do for a column with a small, - # shared vocabulary (a status code, a gender) - reproducing real values - # there is expected and unavoidable - but a privacy problem for a column - # whose real values are meant to be unique per row (an email, a genuine - # ID): it isn't modeling the distribution, it's handing back real - # records. Scale the penalty by how unique the real column actually is - # (near 0 for a low-uniqueness column, where this is fine; near full - # strength for a near-unique one, where it's a leak) and by how much of - # this proposer's own output is actually copied. - # - # Other proposers are, in general, not penalized for copy_fraction: for - # them a high copy_fraction is usually a coincidental side effect of a - # naturally overlapping real-world vocabulary (e.g. common first names), - # not a sign of memorization - penalizing it there wrongly punishes - # exactly the generators that model the real distribution best (verified: - # applying this penalty to person.first_name for a first-name column - # incorrectly hands the win back to a generic word generator). - # - # Numeric columns are the exception, regardless of whether they end up - # profiled IDENTIFIER (high uniqueness, e.g. a primary key) or - # CATEGORICAL (low uniqueness, e.g. a foreign key referencing a small - # set of rows): a plain number has no shared human vocabulary to explain - # a coincidental match. A high copy_fraction there instead means the - # real value range is dense (e.g. a gapless sequence of small integer - # IDs), so *any* generator landing in that range trivially "matches" - # almost every time, independent of whether it resamples. That's the - # same underlying signal as ChoiceProposer's, just from a proposer that - # isn't one - e.g. a generic "weight" generator whose typical output - # range happens to overlap a table's ID range - so it gets the same - # penalty here. A low-cardinality *string* column (a gender, a status) - # doesn't get this: there, high copy_fraction from a well-calibrated - # generator is legitimate/expected, not a coincidental range overlap. + # See _penalty_for_candidate for what resample_penalty and + # self_duplication_penalty actually guard against and why. apply_to_every_proposer = is_numeric_column - penalty_multipliers = [1.0] * len(results) - for idx, result in enumerate(results): - resample_penalty = 1.0 - if isinstance(result.proposer, ChoiceProposer) or apply_to_every_proposer: - resample_penalty = 1.0 - real_uniqueness * result.copy_fraction - # A second, separate penalty for duplicating against the candidate's - # OWN output (synthetic_uniqueness) rather than against the real - # data (copy_fraction, above). These catch different failure modes: - # e.g. dist_gen.constant on a column whose fixed value happens not - # to appear in the sampled real data gets copy_fraction=0 - no - # resample penalty at all - despite duplicating every single row - # against itself. Scaled by real_uniqueness for the same reason as - # the resample penalty: a column with a small, legitimate shared - # vocabulary (a gender, a status) is expected to have low - # synthetic_uniqueness too, and shouldn't be penalized for it. - # Unlike the resample penalty, this applies to every candidate - # regardless of column type or whether the proposer is a resampler - - # but its full strength is only warranted when the column actually - # needs unique values (a PRIMARY KEY/UNIQUE constraint); it's floored - # below (SELF_DUPLICATION_PENALTY_FLOOR) otherwise, since a merely - # observed-unique column (e.g. free-text descriptions that happen to - # all differ) doesn't have a correctness reason to punish a small, - # repetitive-but-otherwise-plausible pool as hard as a real key would. - self_duplication_penalty = 1.0 - real_uniqueness * ( - 1.0 - result.synthetic_uniqueness + needs_uniqueness = is_primary_key or is_unique_constrained + penalty_multipliers = [ + _penalty_for_candidate( + result, real_uniqueness, apply_to_every_proposer, needs_uniqueness ) - if not (is_primary_key or is_unique_constrained): - # No constraint actually requires unique values here, so don't - # let this penalty alone crush the score - just discount it - # relative to a candidate that can sustain the real column's - # observed distinctness. - self_duplication_penalty = max( - self_duplication_penalty, SELF_DUPLICATION_PENALTY_FLOOR - ) - penalty_multipliers[idx] = resample_penalty * self_duplication_penalty + for result in results + ] + for idx in range(len(results)): combined_scores[idx] *= penalty_multipliers[idx] # soft boost for generators whose kind the column name itself hints at From f0c89164809abf2087f9457f3c085e48e32553e7 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Wed, 23 Sep 2026 14:43:07 +0100 Subject: [PATCH 12/23] Added CopyProposer --- datafaker/db_utils.py | 29 +-- datafaker/interactive/generators.py | 5 +- datafaker/proposers/__init__.py | 3 +- datafaker/proposers/base.py | 76 +++++- datafaker/proposers/intervals.py | 296 +++++++++++++++++----- datafaker/providers.py | 35 ++- tests/examples/date.sql | 14 +- tests/test_generators_dialect.py | 11 +- tests/test_interactive_copy_generators.py | 168 ++++++++++++ tests/test_interactive_date_generators.py | 19 +- 10 files changed, 529 insertions(+), 127 deletions(-) create mode 100644 tests/test_interactive_copy_generators.py diff --git a/datafaker/db_utils.py b/datafaker/db_utils.py index 7ed5f17..e7e18a9 100644 --- a/datafaker/db_utils.py +++ b/datafaker/db_utils.py @@ -12,15 +12,7 @@ # pylint: disable=no-name-in-module from psycopg2.errors import UndefinedObject # ty: ignore[unresolved-import] -from sqlalchemy import ( - Column, - Connection, - Engine, - ForeignKey, - create_engine, - event, - select, -) +from sqlalchemy import Connection, Engine, ForeignKey, create_engine, event, select from sqlalchemy.engine.interfaces import DBAPIConnection from sqlalchemy.exc import ( IntegrityError, @@ -280,25 +272,6 @@ def get_orm_metadata( return metadata -def get_fk_column_between( - from_: Table, to_: Table -) -> tuple[Column, ForeignKey] | tuple[None, None]: - """ - Get a foreign key between two tables. - - :param from_: The table on which the foreign key should be defined. - :param to_: The table to which the foreign key should refer. - :return: A pair (column on from_, foreign key to to_) if such a key exists, - or (None, None) if not. - """ - for c in from_.columns: - fk: ForeignKey - for fk in c.foreign_keys: - if fk.column.table == to_: - return (c, fk) - return (None, None) - - def fk_refers_to_ignored_table(fk: ForeignKey) -> bool: """ Test if this foreign key refers to an ignored table. diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index a2ca912..3789afb 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -201,6 +201,9 @@ class GeneratorCmd(DbCmd): PRIMARY_PRIVATE_TEXT = "Primary Private" SECONDARY_PRIVATE_TEXT = "Secondary Private on columns {0}" NOT_PRIVATE_TEXT = "Not private" + REQUIRES_NO_SOURCE_DATA_TEXT = ( + "{0}. {2}{1}{3} requires no data from the source database." + ) REQUIRES_SOURCE_DATA_TEXT = ( "{0}. {2}{1}{3} requires the following data from the source database:" ) @@ -910,7 +913,7 @@ def _print_values_queried(self, table_name: str, n: int, prop: Proposer) -> None theme = get_active_theme() if not prop.select_aggregate_clauses() and not prop.custom_queries(): self.print( - "{0}. {2}{1}{3} requires no data from the source database.", + self.REQUIRES_NO_SOURCE_DATA_TEXT, n, prop.name(), theme.function, diff --git a/datafaker/proposers/__init__.py b/datafaker/proposers/__init__.py index bee4c53..98c969a 100644 --- a/datafaker/proposers/__init__.py +++ b/datafaker/proposers/__init__.py @@ -17,7 +17,7 @@ MultivariateNormalProposerFactory, ) from datafaker.proposers.extract import DateComponentExtractProposerFactory -from datafaker.proposers.intervals import DateAfterProposerFactory +from datafaker.proposers.intervals import CopyProposerFactory, DateAfterProposerFactory from datafaker.proposers.mimesis import ( MimesisDateProposerFactory, MimesisDateTimeProposerFactory, @@ -57,6 +57,7 @@ def everything_factory(config: Mapping, metadata: MetaData) -> ProposerFactory: NullPartitionedNormalProposerFactory(config, metadata), NullPartitionedLogNormalProposerFactory(config, metadata), DateAfterProposerFactory(config, metadata), + CopyProposerFactory(config, metadata), DateComponentExtractProposerFactory(config, metadata), IncrementProposerFactory(), ) diff --git a/datafaker/proposers/base.py b/datafaker/proposers/base.py index 058b7b1..4da4d09 100644 --- a/datafaker/proposers/base.py +++ b/datafaker/proposers/base.py @@ -2,7 +2,8 @@ import re from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass from typing import Any, Union import mimesis @@ -14,7 +15,7 @@ from datafaker.dialects import StdDev from datafaker.providers import DistributionProvider -from datafaker.utils import logger +from datafaker.utils import T, logger NumericType = Union[int, float] @@ -291,6 +292,57 @@ def fit_from_buckets(xs: Sequence[NumericType], ys: Sequence[NumericType]) -> fl return sum_diff_squared / (count * count) +@dataclass +class ForeignKeyRelationship: + """How a table is related.""" + + fk_column: Column # the "from" column + target_column: Column # the "to" column that fk_column matches + + def target_table(self) -> Table: + """Get the related table.""" + return self.target_column.table + + +def _get_one(seq: Iterable[T]) -> T | None: + """Get an arbitrary member of the sequence, or None if it is empty.""" + for elt in seq: + return elt + return None + + +def make_foreign_key_relationship(fk_column: Column) -> ForeignKeyRelationship | None: + """Make a ``ForeignKeyRelationship`` from a foreign key column.""" + if (elt := _get_one(fk_column.foreign_keys)) is None: + return None + return ForeignKeyRelationship(fk_column, elt.column) + + +@dataclass +class RelatedColumn: + """A column in a related table, and how it is related.""" + + target: Column + relationship: ForeignKeyRelationship | None + + def add_from_to_query(self, query: Any) -> Any: + """Take a query and add a ``select_from`` and maybe ``join`` as appropriate.""" + if self.relationship is None: + return query.select_from(self.target.table) + query = query.select_from(self.relationship.fk_column.table) + return self.add_join_to_query(query) + + def add_join_to_query(self, query: Any) -> Any: + """Take a query and maybe add ``join`` as appropriate.""" + rel = self.relationship + if rel is None: + return query + return query.join( + rel.target_column.table, + onclause=rel.fk_column == rel.target_column, + ) + + class Buckets: """ Measured buckets for a real distribution. @@ -308,12 +360,12 @@ def __init__( mean: float, stddev: float, count: int, - join_tables: list[Table] | None = None, + anchor_relation: RelatedColumn | None = None, ): """Initialise a Buckets object.""" self.mean = mean self.stddev = stddev - query = self._get_bucket_query(table, column, join_tables) + query = self._get_bucket_query(table, column, anchor_relation) with engine.connect() as connection: raw_buckets = connection.execute(query.group_by("b")) self.buckets: Sequence[int] = [0] * 10 @@ -335,7 +387,7 @@ def _get_bucket_query( self, table: Table | Join, column: Any, - join_tables: list[Table] | None, + anchor_relation: RelatedColumn | None = None, ) -> Any: """Make the query for half-sd buckets.""" bottom = self.mean - 2 * self.stddev @@ -344,8 +396,8 @@ def _get_bucket_query( func.count(column).label("f"), # pylint: disable=not-callable ((func.floor(column) - bottom) / width).label("b"), ).select_from(table) - for jt in join_tables or []: - query = query.join(jt) + if anchor_relation is not None: + query = anchor_relation.add_join_to_query(query) return query @classmethod @@ -354,7 +406,7 @@ def make_buckets( engine: Engine, table: Table | Join, column: Any, - join_tables: list[Table] | None = None, + anchor_relation: RelatedColumn | None = None, ) -> Self | None: """ Construct a Buckets object. @@ -368,14 +420,16 @@ def make_buckets( :param engine: SQLAlchemy engine. :param table: SQLAlchemy table (or joined tables) to pull data from. :param column: SQLAlchemy column or expression to measure. + :param anchor_relation: A relationship to another column whose table + needs to be joined. """ query = select( func.avg(column).label("mean"), StdDev(column).label("stddev"), func.count(column).label("count"), # pylint: disable=not-callable ).select_from(table) - for jt in join_tables or []: - query = query.join(jt) + if anchor_relation is not None: + query = anchor_relation.add_join_to_query(query) with engine.connect() as connection: result = connection.execute(query).first() if result is None or result.stddev is None or getattr(result, "count") < 2: @@ -388,7 +442,7 @@ def make_buckets( result.mean, result.stddev, getattr(result, "count"), - join_tables, + anchor_relation, ) except DatabaseError as exc: logger.debug("Failed to instantiate Buckets object: %s", exc) diff --git a/datafaker/proposers/intervals.py b/datafaker/proposers/intervals.py index e6bb11f..689a0c0 100644 --- a/datafaker/proposers/intervals.py +++ b/datafaker/proposers/intervals.py @@ -3,23 +3,28 @@ from collections.abc import Mapping, Sequence from typing import Any -from sqlalchemy import Column, Engine, ForeignKey, MetaData, func, select +from sqlalchemy import Column, Engine, MetaData, func, select from sqlalchemy.types import Date, DateTime -from datafaker.db_utils import get_fk_column_between from datafaker.dialects import Random, SecondsDifference, StdDev -from datafaker.proposers.base import Buckets, Proposer, ProposerFactory, get_column_type +from datafaker.proposers.base import ( + Buckets, + ForeignKeyRelationship, + Proposer, + ProposerFactory, + RelatedColumn, + get_column_type, + make_foreign_key_relationship, +) from datafaker.providers import AnchoredProvider from datafaker.utils import get_property, procrustes -RelatedColumn = tuple[ForeignKey | None, Column] - def _set_roles_for_column( out: dict[str, list[RelatedColumn]], - fk: ForeignKey | None, + fk: ForeignKeyRelationship | None, column: Column, - column_config: Mapping, + columns_config: Mapping, ) -> None: """ Set new entries in ``out`` based on the roles ``column`` has. @@ -29,19 +34,19 @@ def _set_roles_for_column( that role and ``fk`` is the foreign key in the table ``column`` appears in that points to the table ``fcol`` appears in, or None if ``column`` and ``fcol`` are in the same table. - :param fk: Foreign key to the table ``column`` appears in (appears in - new entries set in ``out``). + :param fk: Foreign key relationship to the table ``column`` appears in + (appears in new entries set in ``out``). :param column: The column to be checked for roles. - :param column_config: The ``tables: : columns:`` section + :param columns_config: The ``tables: : columns:`` section of the ``config.yaml`` file. """ - roles: list[Any] = get_property(column_config, [column.name, "roles"], []) + roles: list[Any] = get_property(columns_config, [column.name, "roles"], []) for role in roles: - pair: RelatedColumn = (fk, column) + rc = RelatedColumn(column, fk) if role not in out: - out[role] = [pair] + out[role] = [rc] else: - out[role].append(pair) + out[role].append(rc) def _get_roles( @@ -60,21 +65,22 @@ def _get_roles( """ table = column.table tables_config: dict[str, Any] = get_property(config, "tables", {}) - table_conf: dict[str, Any] = get_property( + columns_config: dict[str, Any] = get_property( tables_config, [str(table.name), "columns"], {} ) role_to_fk_columns: dict[str, list[RelatedColumn]] = {} for col in table.columns: - _set_roles_for_column(role_to_fk_columns, None, col, table_conf) + _set_roles_for_column(role_to_fk_columns, None, col, columns_config) # look for roles in related tables - if col.foreign_keys: - fk = list(col.foreign_keys)[0] - target_table = fk.column.table + if (fk_relationship := make_foreign_key_relationship(col)) is not None: + target_table = fk_relationship.target_table() ft_conf: dict[str, Any] = get_property( tables_config, [str(target_table.name), "columns"], {} ) for fcol in target_table.columns: - _set_roles_for_column(role_to_fk_columns, fk, fcol, ft_conf) + _set_roles_for_column( + role_to_fk_columns, fk_relationship, fcol, ft_conf + ) return role_to_fk_columns @@ -103,7 +109,7 @@ def __init__( sd: float, mean: float, column: Column, - anchor: Column, + anchor_related: RelatedColumn, engine: Engine, buckets: Buckets | None = None, ): @@ -116,13 +122,13 @@ def __init__( :param column: The column to generate. :param anchor: The anchor column, which must be in the same table as column or a related table. - :param dialect: The dialect of the source database. + :param engine: The source database. :param buckets: The buckets for the intervals in the source database. """ super().__init__() self._sd = sd self._mean = mean - self._anchor = anchor + self._anchor_related = anchor_related self._column = column self._engine = engine self._provider = AnchoredProvider(metadata=metadata) @@ -134,27 +140,44 @@ def __init__( [(b - coerce_to_datetime(a, b)).total_seconds() for (a, b) in intervals] ) + @classmethod + def get_query( + cls, + column: Column, + anchor_related: RelatedColumn, + ) -> Any: + """Create the query for the summary of interval lengths.""" + anchor = anchor_related.target + return anchor_related.add_from_to_query( + select( + func.avg(SecondsDifference(column, anchor)).label("mean"), + StdDev(SecondsDifference(column, anchor)).label("sd"), + ) + ) + def function_name(self) -> str: """Get the name of the generator function to call.""" - if self._column.table == self._anchor.table: + if self._anchor_related.relationship is None: return "generic.anchored_provider.normal_date" return "generic.anchored_provider.normal_date_fk" def name(self) -> str: """Get the name of the generator.""" fname = self.function_name() - aname = self._anchor.name - atable = self._anchor.table - if atable == self._column.table: + anchor = self._anchor_related.target + aname = anchor.name + atable = anchor.table + if self._anchor_related.relationship is None: return f"{fname} [anchored to {aname}]" - return f"{fname} [anchored to {aname} of table {atable.name}]" + atable = anchor.table.name + rel = self._anchor_related.relationship.fk_column.name + return f"{fname} [anchored to {aname} of table {atable} (via {rel})]" def nominal_kwargs(self) -> dict[str, Any]: """Get the arguments to be entered into ``config.yaml``.""" column = self._column - anchor = self._anchor - (fk_col, fk) = get_fk_column_between(column.table, anchor.table) - if fk_col is None or fk is None: + anchor = self._anchor_related.target + if self._anchor_related.relationship is None: return { "mean_seconds": ( f'SRC_STATS["auto__{column.table.name}"]' @@ -167,33 +190,34 @@ def nominal_kwargs(self) -> dict[str, Any]: "anchor": f'GENERATED_ROW["{anchor.name}"]', } key = f"auto__interval__{column.table.name}__{column.name}" + fk_col = self._anchor_related.relationship.fk_column.name + on_col = self._anchor_related.relationship.target_column.name return { "dst_db_conn": "dst_db_conn", "anchor_column": f'"{anchor.name}"', "table": f'"{anchor.table.name}"', "mean_seconds": (f'SRC_STATS["{key}"]["results"][0]["mean"]'), "sd_seconds": (f'SRC_STATS["{key}"]["results"][0]["sd"]'), - "anchor_row": f'GENERATED_ROW["{fk_col.name}"]', - "on_column": f'"{fk.column.name}"', + "anchor_row": f'GENERATED_ROW["{fk_col}"]', + "on_column": f'"{on_col}"', } def actual_kwargs(self) -> dict[str, Any]: """Get the kwargs (summary statistics) this generator was instantiated with.""" - column = self._column - anchor = self._anchor - (_, fk) = get_fk_column_between(column.table, anchor.table) - if fk is None: + if self._anchor_related.relationship is None: return { "mean_seconds": self._sd, "sd_seconds": self._mean, "anchor": "1970-01-01", } + anchor = self._anchor_related.target + on_col = self._anchor_related.relationship.target_column.name return { "anchor_column": anchor.name, "table": anchor.table.name, "mean_seconds": self._sd, "sd_seconds": self._mean, - "on_column": fk.column.name, + "on_column": on_col, } def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: @@ -202,10 +226,10 @@ def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: This is for anchors in the same table. """ - column = self._column - anchor = self._anchor - if column.table != anchor.table: + if self._anchor_related.relationship is not None: return {} + column = self._column + anchor = self._anchor_related.target mean_q = func.avg(SecondsDifference(column, anchor)) sd_q = StdDev(SecondsDifference(column, anchor)) @@ -243,23 +267,17 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: This is for anchors in a related table. """ column = self._column - anchor = self._anchor - if column.table == anchor.table: + rel = self._anchor_related + if rel.relationship is None: return {} - query = ( - select( - func.avg(SecondsDifference(column, anchor)).label("mean"), - StdDev(SecondsDifference(column, anchor)).label("sd"), - ) - .select_from(column.table) - .join(anchor.table) - ) + query = DateAfterProposer.get_query(column, rel) return { f"auto__interval__{column.table.name}__{column.name}": { "comments": [ "Mean and standard deviation of the length of time between" - f" column {anchor.name} of table {anchor.table.name}" - f" and column {column.name} of table {column.table.name}." + f" column {rel.target.name} of table {rel.target.table.name}" + f" and column {column.name} of table {column.table.name}" + f" (joined through {rel.relationship.fk_column.name})" ], "query": str( query.compile( @@ -288,7 +306,8 @@ def generate_intervals( :param count: The number of intervals to generate. :return: Each pair consists of an anchor and the generated datetime. """ - query = select(self._anchor).select_from(self._anchor.table) + anchor = self._anchor_related.target + query = select(anchor).select_from(anchor.table) query = query.order_by(Random()).limit(count) with self._engine.connect() as conn: anchors = conn.execute(query).scalars().all() @@ -312,18 +331,14 @@ def __init__(self, config: Mapping, metadata: MetaData): self._config = config self._metadata = metadata - def make_date_after_proposers( - self, engine: Engine, column: Column, anchor: Column + def _make_date_after_proposers( + self, + engine: Engine, + column: Column, + anchor_related: RelatedColumn, ) -> list[DateAfterProposer]: """Create a ``DateAfterProposer`` object.""" - query = select( - func.avg(SecondsDifference(column, anchor)).label("mean"), - StdDev(SecondsDifference(column, anchor)).label("sd"), - ).select_from(column.table) - join_tables = [] - if anchor.table != column.table: - join_tables = [anchor.table] - query = query.join(anchor.table) + query = DateAfterProposer.get_query(column, anchor_related) with engine.connect() as connection: result = connection.execute(query).first() if result is None or result.sd is None: @@ -331,8 +346,8 @@ def make_date_after_proposers( buckets = Buckets.make_buckets( engine, column.table, - SecondsDifference(column, anchor), - join_tables, + SecondsDifference(column, anchor_related.target), + anchor_related, ) return [ DateAfterProposer( @@ -340,7 +355,7 @@ def make_date_after_proposers( result.sd, result.mean, column, - anchor, + anchor_related, engine=engine, buckets=buckets, ) @@ -362,10 +377,153 @@ def get_proposers( if "start" not in roles: return [] other_start_columns = [ - fk_col for fk_col in roles["start"] if fk_col[1] != column + fk_col for fk_col in roles["start"] if fk_col.target != column ] return [ prop for anchor in other_start_columns - for prop in self.make_date_after_proposers(engine, column, anchor[1]) + for prop in self._make_date_after_proposers(engine, column, anchor) + ] + + +class CopyProposer(Proposer): + """Proposer that proposes copying another column.""" + + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + metadata: MetaData, + column: Column, + anchor_related: RelatedColumn, + engine: Engine, + ): + """ + Initialise a date after proposer. + + :param metadata: The metadata of the source database. + :param column: The column to generate. + :param anchor: The anchor column, which must be in the same table as column + or a related table. + :param engine: The source database. + :param buckets: The buckets for the intervals in the source database. + """ + super().__init__() + self._anchor_related = anchor_related + self._column = column + self._engine = engine + self._provider = AnchoredProvider(metadata=metadata) + + def function_name(self) -> str: + """Get the name of the generator function to call.""" + if self._anchor_related.relationship is None: + return "generic.anchored_provider.copy" + return "generic.anchored_provider.copy_fk" + + def name(self) -> str: + """Get the name of the generator.""" + fname = self.function_name() + aname = self._anchor_related.target.name + if self._anchor_related.relationship is None: + return f"{fname} [from {aname}]" + atable = self._anchor_related.target.table + on_col = self._anchor_related.relationship.fk_column.name + return f"{fname} [from {aname} of table {atable.name} (via {on_col})]" + + def nominal_kwargs(self) -> dict[str, Any]: + """Get the arguments to be entered into ``config.yaml``.""" + anchor = self._anchor_related.target + if self._anchor_related.relationship is None: + return { + "anchor": f'GENERATED_ROW["{anchor.name}"]', + } + fk_col = self._anchor_related.relationship.fk_column + on_col = self._anchor_related.relationship.target_column + return { + "dst_db_conn": "dst_db_conn", + "anchor_column": f'"{anchor.name}"', + "table": f'"{anchor.table.name}"', + "anchor_row": f'GENERATED_ROW["{fk_col.name}"]', + "on_column": f'"{on_col.name}"', + } + + def actual_kwargs(self) -> dict[str, Any]: + """Get the kwargs (summary statistics) this generator was instantiated with.""" + anchor = self._anchor_related.target + if self._anchor_related.relationship is None: + return { + "anchor": "1970-01-01", + } + on_col = self._anchor_related.relationship.target_column + return { + "anchor_column": anchor.name, + "table": anchor.table.name, + "on_column": on_col.name, + } + + def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: + """No source data required.""" + return {} + + def custom_queries(self) -> dict[str, dict[str, Any]]: + """No source data required.""" + return {} + + def fit(self, default: float = -1) -> float: + """Get this generator's fit against the real data.""" + return default + + def generate_data(self, count: int) -> list[Any]: + """Generate ``count`` random data points for this column.""" + anchor = self._anchor_related.target + query = select(anchor).select_from(anchor.table) + query = query.order_by(Random()).limit(count) + with self._engine.connect() as conn: + return list(conn.execute(query).scalars().all()) + + +class CopyProposerFactory(ProposerFactory): + """Proposer that proposes copying another column.""" + + def __init__(self, config: Mapping, metadata: MetaData): + """Initialize ``CopyProposerFactory``.""" + super().__init__() + self._config = config + self._metadata = metadata + + def make_copy_fk_proposers( + self, engine: Engine, column: Column, anchor: RelatedColumn + ) -> list[CopyProposer]: + """Create a ``CopyProposer`` object.""" + return [ + CopyProposer( + self._metadata, + column, + anchor, + engine=engine, + ) + ] + + def get_proposers( + self, + columns: list[Column], + engine: Engine, + ) -> Sequence[Proposer]: + """Get all proposers of dates that might be anchored to another column.""" + if len(columns) != 1: + return [] + column = columns[0] + roles = _get_roles(self._config, column) + if "source" not in roles: + return [] + ct = get_column_type(column) + other_source_columns = [ + fk_col + for fk_col in roles["source"] + if fk_col.target != column + and type(get_column_type(fk_col.target)) is type(ct) + ] + return [ + prop + for anchor in other_source_columns + for prop in self.make_copy_fk_proposers(engine, column, anchor) ] diff --git a/datafaker/providers.py b/datafaker/providers.py index 0263ef5..55faabf 100644 --- a/datafaker/providers.py +++ b/datafaker/providers.py @@ -693,8 +693,8 @@ def normal_date_fk( :param sd_seconds: Standard deviation of the intervals' lengths in seconds. :param table: The name of the table for the anchor column. :param on_column: The name of the column in the foreign table - that ``row`` must match. - :param row: The value in the ``on_column`` column in table ``table`` + that ``anchor_row`` must match. + :param anchor_row: The value in the ``on_column`` column in table ``table`` for the row providing the start of the interval. :param anchor_column: The name of the column in ``table`` providing the start of the interval. @@ -711,3 +711,34 @@ def normal_date_fk( if not isinstance(out, (str, dt.date, dt.datetime)): return None return self.normal_date(mean_seconds, sd_seconds, out) + + def copy(self, anchor: Any) -> Any: + """Copy another column from the same table.""" + return anchor + + def copy_fk( + self, + dst_db_conn: Connection, + table: str, + on_column: str, + anchor_row: Any, + anchor_column: str, + ) -> Any: + """ + Copy a column from a different table. + + :param dst_db_conn: Connection to the destination database. + :param table: The name of the table for the anchor column. + :param on_column: The name of the column in the foreign table + that ``anchor_row`` must match. + :param anchor_row: The value in the ``on_column`` column in table ``table`` + for the row providing the source to copy. + :param anchor_column: The name of the column in ``table`` to copy. + :return: The value to be copied. + """ + mt = self._metadata.tables[table] + query = select(mt.c[anchor_column].label("out")).where( + mt.c[on_column] == anchor_row, + ) + anchor = dst_db_conn.execute(query).first() + return getattr(anchor, "out", None) diff --git a/tests/examples/date.sql b/tests/examples/date.sql index a0155a9..f24b87b 100644 --- a/tests/examples/date.sql +++ b/tests/examples/date.sql @@ -23,16 +23,20 @@ CREATE TABLE public.happening ( name TEXT NOT NULL, person_id INTEGER NOT NULL, at_time TIMESTAMP WITH TIME ZONE NOT NULL, - at_date DATE NOT NULL + at_date DATE NOT NULL, + other_person_id INTEGER, + comment TEXT ); ALTER TABLE ONLY public.happening ADD CONSTRAINT happening_pkey PRIMARY KEY (id); ALTER TABLE ONLY public.happening ADD CONSTRAINT person_id_fkey FOREIGN KEY (person_id) REFERENCES public.person(id); +ALTER TABLE ONLY public.happening + ADD CONSTRAINT other_person_id_fkey FOREIGN KEY (other_person_id) REFERENCES public.person(id); ALTER TABLE public.happening OWNER TO postgres; -INSERT INTO public.happening VALUES (1, 'stepped on a tack', 1, '1997-04-20 04:05:06+00:00', '1997-04-20'); -INSERT INTO public.happening VALUES (2, 'had a dream', 2, '1997-04-20 04:04:16+00:00', '1997-04-20'); -INSERT INTO public.happening VALUES (3, 'kicked a can', 1, '2001-12-23 07:05:06+00:00', '2001-12-23'); -INSERT INTO public.happening VALUES (4, 'steppen in gum', 2, '2003-11-03 04:15:26+00:00', '2003-11-03'); +INSERT INTO public.happening VALUES (1, 'stepped on a tack', 1, '1997-04-20 04:05:06+00:00', '1997-04-20', NULL, NULL); +INSERT INTO public.happening VALUES (2, 'had a dream', 2, '1997-04-20 04:04:16+00:00', '1997-04-20', NULL, NULL); +INSERT INTO public.happening VALUES (3, 'kicked a can', 1, '2001-12-23 07:05:06+00:00', '2001-12-23', NULL, NULL); +INSERT INTO public.happening VALUES (4, 'shared a cake', 2, '2003-11-03 04:15:26+00:00', '2003-11-03', 1, NULL); diff --git a/tests/test_generators_dialect.py b/tests/test_generators_dialect.py index ec5e174..4ddc21b 100644 --- a/tests/test_generators_dialect.py +++ b/tests/test_generators_dialect.py @@ -11,7 +11,12 @@ from datafaker.dialects import SecondsDifference from datafaker.interactive.generators import get_aggregate_query from datafaker.interactive.missingness import MissingnessType -from datafaker.proposers.base import Buckets, PredefinedProposer, Proposer +from datafaker.proposers.base import ( + Buckets, + PredefinedProposer, + Proposer, + RelatedColumn, +) from datafaker.proposers.choice import ChoiceProposerFactory, ZipfChoiceProposer from datafaker.proposers.continuous import ( ContinuousLogDistributionProposerFactory, @@ -592,7 +597,7 @@ def test_date_after_proposer_stddev_clause_mssql(self) -> None: sd=1.0, mean=86400.0, column=tbl.c.end_date, - anchor=tbl.c.start_date, + anchor_related=RelatedColumn(tbl.c.start_date, None), engine=engine, ) clauses = proposer.select_aggregate_clauses() @@ -620,7 +625,7 @@ def test_date_after_proposer_stddev_clause_postgresql(self) -> None: sd=1.0, mean=86400.0, column=tbl.c.end_date, - anchor=tbl.c.start_date, + anchor_related=RelatedColumn(tbl.c.start_date, None), engine=engine, ) clauses = proposer.select_aggregate_clauses() diff --git a/tests/test_interactive_copy_generators.py b/tests/test_interactive_copy_generators.py new file mode 100644 index 0000000..d6ea5b2 --- /dev/null +++ b/tests/test_interactive_copy_generators.py @@ -0,0 +1,168 @@ +""" Tests for the configure-generators command. """ +from collections.abc import MutableMapping +from typing import Any + +from sqlalchemy import select +from sqlalchemy.orm import aliased + +from datafaker.interactive.base import DbCmd +from tests.test_interactive_generators import MockGeneratorCmd +from tests.utils import GeneratesDBTestCase, MsSqlTestDb + + +class ConfigureCopyGeneratorsWithDateTests(GeneratesDBTestCase): + """Test `configure-generators` with the `instrument.sql` database.""" + + dump_file_path = "date.sql" + database_name = "date_tables" + schema_name = "public" + use_temporary_cwd = True + + def _get_cmd(self, config: MutableMapping[str, Any]) -> MockGeneratorCmd: + """Get the command we are using for this test case.""" + return MockGeneratorCmd( + DbCmd.Settings(self.dsn, self.schema_name, config, self.metadata, None) + ) + + def test_same_table_copy(self) -> None: + """Test that the cross-table copy is proposed and compared.""" + config = { + "tables": { + "happening": { + "columns": { + "name": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + # set up our copy proposer + gc.do_next("happening.comment") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + provider_name = "generic.anchored_provider.copy [from name]" + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + gc.reset() + gc.do_compare(str(prop[0])) + self.assertEqual(gc.messages[0][0], gc.NOT_PRIVATE_TEXT) + self.assertEqual(gc.messages[1][0], gc.REQUIRES_NO_SOURCE_DATA_TEXT) + + def test_cross_table_copy(self) -> None: + """Test that the cross-table copy is proposed and compared.""" + config = { + "tables": { + "person": { + "columns": { + "name": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + # set up our copy proposer + gc.do_next("happening.comment") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + provider_name2 = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via other_person_id)]" + ) + self.assertIn(provider_name2, proposals.keys()) + prop2 = proposals[provider_name2] + gc.reset() + gc.do_compare(f"{prop[0]} {prop2[0]}") + self.assertEqual(gc.messages[0][0], gc.NOT_PRIVATE_TEXT) + self.assertEqual(gc.messages[1][0], gc.REQUIRES_NO_SOURCE_DATA_TEXT) + + def test_cross_table_copy_actually_copies(self) -> None: + """Test that the cross-table copy is copies.""" + config = { + "tables": { + "person": { + "columns": { + "name": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + # set up our copy proposer + gc.do_next("happening.comment") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + gc.do_set(str(prop[0])) + gc.do_next("happening.name") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via other_person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + gc.do_set(str(prop[0])) + gc.do_quit("") + self.generate_data(gc.config, num_passes=50) + assert self.dst_engine is not None + with self.dst_engine.connect() as conn: + pert = self.dst_metadata.tables["person"] + other_pert = aliased(pert) + hapt = self.dst_metadata.tables["happening"] + query = ( + select( + hapt.columns["name"].label("hn"), + hapt.columns["comment"].label("hc"), + pert.columns["name"].label("pn"), + other_pert.columns["name"].label("opn"), + ) + .select_from(hapt) + .join( + pert, + onclause=pert.columns["id"] == hapt.columns["person_id"], + ) + .join( + other_pert, + onclause=other_pert.columns["id"] + == hapt.columns["other_person_id"], + isouter=True, + ) + ) + dst_result = conn.execute(query).fetchall() + saw_other = False + for row in dst_result: + if row.opn: + saw_other = True + self.assertEqual(row.hn, row.opn) + self.assertEqual(row.hc, row.pn) + self.assertTrue(saw_other) + + +# Note that this test won't work with DuckDB because it needs foreign keys to work +class ConfigureCopyGeneratorsWithDateMsSqlTests(ConfigureCopyGeneratorsWithDateTests): + """Test `configure-generators` with `instrument.sql` with MS SQL.""" + + database_type = MsSqlTestDb + schema_name = None diff --git a/tests/test_interactive_date_generators.py b/tests/test_interactive_date_generators.py index 978347e..3addac8 100644 --- a/tests/test_interactive_date_generators.py +++ b/tests/test_interactive_date_generators.py @@ -166,7 +166,8 @@ def test_cross_table_interval_end_to_end(self) -> None: proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor} of table {atable}]" + f" [anchored to {anchor} of table {atable}" + " (via manufacturer_id)]" ) self.assertIn(provider_name, proposals.keys()) prop = proposals[provider_name] @@ -216,7 +217,7 @@ class ConfigureGeneratorsWithInstrumentsMsSqlTests( class ConfigureGeneratorsWithDateTests(GeneratesDBTestCase): - """Test `configure-generators` with the `instrument.sql` database.""" + """Test `configure-generators` with the `date.sql` database.""" dump_file_path = "date.sql" database_name = "date_tables" @@ -274,13 +275,15 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor} of table {atable}]" + f" [anchored to {anchor} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name, proposals.keys()) prop = proposals[provider_name] provider_name2 = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor2} of table {atable}]" + f" [anchored to {anchor2} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name2, proposals.keys()) prop2 = proposals[provider_name2] @@ -295,13 +298,15 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor} of table {atable}]" + f" [anchored to {anchor} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name, proposals.keys()) prop = proposals[provider_name] provider_name2 = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor2} of table {atable}]" + f" [anchored to {anchor2} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name2, proposals.keys()) prop2 = proposals[provider_name2] @@ -314,7 +319,7 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: # Note that this test won't work with DuckDB because it needs foreign keys to work class ConfigureGeneratorsWithDateMsSqlTests(ConfigureGeneratorsWithDateTests): - """Test `configure-generators` with `instrument.sql` with MS SQL.""" + """Test `configure-generators` with `date.sql` with MS SQL.""" database_type = MsSqlTestDb schema_name = None From 9d60a6c22732abf0c83ce6c9ba9982439fd402bd Mon Sep 17 00:00:00 2001 From: Tim Band Date: Fri, 25 Sep 2026 13:15:22 +0100 Subject: [PATCH 13/23] Fixed crash in DateAfterProposerFactory Also: - fixed role change text - fixed remove all roles - allow intervals and copying from same column in aliased table --- datafaker/interactive/generators.py | 10 ++-- datafaker/proposers/base.py | 36 ++++++++++++-- datafaker/proposers/intervals.py | 59 +++++++++++++---------- datafaker/proposers/partitioned.py | 19 +------- datafaker/utils.py | 17 +++++++ docs/source/docker.rst | 2 +- pyproject.toml | 2 +- tests/examples/date.sql | 13 +++-- tests/test_interactive_date_generators.py | 25 ++++++++++ 9 files changed, 126 insertions(+), 57 deletions(-) diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index 3789afb..1c3f85b 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -37,6 +37,7 @@ schema_qualified_name, set_property, split_column_full_name, + text_list, ) @@ -160,7 +161,8 @@ def do(self, column_roles: RoleEntry | None) -> RoleEntry | None: """Perform the command.""" if column_roles is None: return None - column_roles.new.remove(self.role) + if self.role in column_roles.new: + column_roles.new.remove(self.role) return column_roles @@ -481,7 +483,7 @@ def _copy_roles(self) -> None: """Set role information in the configuration.""" for table_name, table_roles in self.roles.items(): for column_name, entry in table_roles.items(): - if entry.new: + if entry.new or entry.old: set_property( self.config, ["tables", table_name, "columns", column_name, "roles"], @@ -543,8 +545,8 @@ def _print_role_changes(self) -> int: "Changing role set of column {0} of table {1} from {2} to {3}", column_name, table_name, - ", ".join(str(oe) for oe in entry.old), - ", ".join(str(ne) for ne in entry.new), + text_list((e.value for e in entry.old), "empty"), + text_list((e.value for e in entry.new), "empty"), ) return count diff --git a/datafaker/proposers/base.py b/datafaker/proposers/base.py index 4da4d09..2c152b4 100644 --- a/datafaker/proposers/base.py +++ b/datafaker/proposers/base.py @@ -10,6 +10,7 @@ import mimesis.locales from sqlalchemy import Column, Engine, Join, Table, func, select from sqlalchemy.exc import DatabaseError +from sqlalchemy.orm import aliased from sqlalchemy.types import Integer, Numeric, String, TypeEngine from typing_extensions import Self @@ -292,17 +293,34 @@ def fit_from_buckets(xs: Sequence[NumericType], ys: Sequence[NumericType]) -> fl return sum_diff_squared / (count * count) -@dataclass class ForeignKeyRelationship: """How a table is related.""" - fk_column: Column # the "from" column - target_column: Column # the "to" column that fk_column matches + def __init__(self, fk_column: Column, target_column: Column) -> None: + """ + Initialise a ``ForeignKeyRelationship``. + + :param fk_column: The column containing the foreign key (the source). + :param target_column: The column that ``fk_column`` matches in + the target table. + """ + self.fk_column = fk_column + table = aliased(target_column.table) + self.target_column = table.columns[target_column.name] def target_table(self) -> Table: """Get the related table.""" return self.target_column.table + def target_table_name(self) -> str: + """ + Get the name of the target table. + + Don't use ``target_table().name`` because this will give the + name of the aliased table. + """ + return str(self.target_column.table.original.name) + def _get_one(seq: Iterable[T]) -> T | None: """Get an arbitrary member of the sequence, or None if it is empty.""" @@ -342,6 +360,18 @@ def add_join_to_query(self, query: Any) -> Any: onclause=rel.fk_column == rel.target_column, ) + def target_table(self) -> Table | None: + """Get the related table, or None if it's the same table.""" + if self.relationship is None: + return None + return self.relationship.target_table() + + def target_table_name(self) -> str | None: + """Get the name of the target table, or None if it's the same table.""" + if self.relationship is None: + return None + return self.relationship.target_table_name() + class Buckets: """ diff --git a/datafaker/proposers/intervals.py b/datafaker/proposers/intervals.py index 689a0c0..3b22611 100644 --- a/datafaker/proposers/intervals.py +++ b/datafaker/proposers/intervals.py @@ -73,11 +73,10 @@ def _get_roles( _set_roles_for_column(role_to_fk_columns, None, col, columns_config) # look for roles in related tables if (fk_relationship := make_foreign_key_relationship(col)) is not None: - target_table = fk_relationship.target_table() ft_conf: dict[str, Any] = get_property( - tables_config, [str(target_table.name), "columns"], {} + tables_config, [fk_relationship.target_table_name(), "columns"], {} ) - for fcol in target_table.columns: + for fcol in fk_relationship.target_table().columns: _set_roles_for_column( role_to_fk_columns, fk_relationship, fcol, ft_conf ) @@ -164,20 +163,20 @@ def function_name(self) -> str: def name(self) -> str: """Get the name of the generator.""" fname = self.function_name() - anchor = self._anchor_related.target - aname = anchor.name - atable = anchor.table - if self._anchor_related.relationship is None: + aname = self._anchor_related.target.name + rel = self._anchor_related.relationship + if rel is None: return f"{fname} [anchored to {aname}]" - atable = anchor.table.name - rel = self._anchor_related.relationship.fk_column.name - return f"{fname} [anchored to {aname} of table {atable} (via {rel})]" + atable = rel.target_table_name() + fk_name = rel.fk_column.name + return f"{fname} [anchored to {aname} of table {atable} (via {fk_name})]" def nominal_kwargs(self) -> dict[str, Any]: """Get the arguments to be entered into ``config.yaml``.""" column = self._column anchor = self._anchor_related.target - if self._anchor_related.relationship is None: + rel = self._anchor_related.relationship + if rel is None: return { "mean_seconds": ( f'SRC_STATS["auto__{column.table.name}"]' @@ -190,12 +189,12 @@ def nominal_kwargs(self) -> dict[str, Any]: "anchor": f'GENERATED_ROW["{anchor.name}"]', } key = f"auto__interval__{column.table.name}__{column.name}" - fk_col = self._anchor_related.relationship.fk_column.name - on_col = self._anchor_related.relationship.target_column.name + fk_col = rel.fk_column.name + on_col = rel.target_column.name return { "dst_db_conn": "dst_db_conn", "anchor_column": f'"{anchor.name}"', - "table": f'"{anchor.table.name}"', + "table": f'"{rel.target_table_name()}"', "mean_seconds": (f'SRC_STATS["{key}"]["results"][0]["mean"]'), "sd_seconds": (f'SRC_STATS["{key}"]["results"][0]["sd"]'), "anchor_row": f'GENERATED_ROW["{fk_col}"]', @@ -377,7 +376,10 @@ def get_proposers( if "start" not in roles: return [] other_start_columns = [ - fk_col for fk_col in roles["start"] if fk_col.target != column + fk_col + for fk_col in roles["start"] + # Don't anchor on ourselves + if not (fk_col.relationship is None and fk_col.target == column) ] return [ prop @@ -423,25 +425,27 @@ def name(self) -> str: """Get the name of the generator.""" fname = self.function_name() aname = self._anchor_related.target.name - if self._anchor_related.relationship is None: + rel = self._anchor_related.relationship + if rel is None: return f"{fname} [from {aname}]" - atable = self._anchor_related.target.table - on_col = self._anchor_related.relationship.fk_column.name - return f"{fname} [from {aname} of table {atable.name} (via {on_col})]" + atable = rel.target_table_name() + on_col = rel.fk_column.name + return f"{fname} [from {aname} of table {atable} (via {on_col})]" def nominal_kwargs(self) -> dict[str, Any]: """Get the arguments to be entered into ``config.yaml``.""" anchor = self._anchor_related.target - if self._anchor_related.relationship is None: + rel = self._anchor_related.relationship + if rel is None: return { "anchor": f'GENERATED_ROW["{anchor.name}"]', } - fk_col = self._anchor_related.relationship.fk_column - on_col = self._anchor_related.relationship.target_column + fk_col = rel.fk_column + on_col = rel.target_column return { "dst_db_conn": "dst_db_conn", "anchor_column": f'"{anchor.name}"', - "table": f'"{anchor.table.name}"', + "table": f'"{rel.target_table_name()}"', "anchor_row": f'GENERATED_ROW["{fk_col.name}"]', "on_column": f'"{on_col.name}"', } @@ -503,6 +507,9 @@ def make_copy_fk_proposers( ) ] + def _can_copy_type(self, copy_from, copy_to) -> bool: + return type(copy_from) is type(copy_to) + def get_proposers( self, columns: list[Column], @@ -519,8 +526,10 @@ def get_proposers( other_source_columns = [ fk_col for fk_col in roles["source"] - if fk_col.target != column - and type(get_column_type(fk_col.target)) is type(ct) + # Don't copy from ourself + if not (fk_col.relationship is None and fk_col.target == column) + # Only copy if the types are compatible + and self._can_copy_type(get_column_type(fk_col.target), ct) ] return [ prop diff --git a/datafaker/proposers/partitioned.py b/datafaker/proposers/partitioned.py index 8db2ee9..f52dc85 100644 --- a/datafaker/proposers/partitioned.py +++ b/datafaker/proposers/partitioned.py @@ -28,7 +28,7 @@ CovariateQuery, MultivariateNormalProposerFactory, ) -from datafaker.utils import T, get_property, logger +from datafaker.utils import T, get_property, logger, text_list NumericType = Union[int, float] @@ -37,23 +37,6 @@ MAXIMUM_CHOICES = 500 -def text_list(items: Iterable[str]) -> str: - """Concatenate the items with commas and one "and".""" - item_i = iter(items) - try: - last_item = next(item_i) - except StopIteration: - return "" - try: - so_far = next(item_i) - except StopIteration: - return last_item - for item in item_i: - so_far += ", " + last_item - last_item = item - return so_far + " and " + last_item - - @dataclass class RowPartition: """A partition where all the rows have the same pattern of NULLs.""" diff --git a/datafaker/utils.py b/datafaker/utils.py index a6a1532..e7ce255 100644 --- a/datafaker/utils.py +++ b/datafaker/utils.py @@ -457,6 +457,23 @@ def underline_error(e: SyntaxError) -> str: return "\n" + " " * start + "^" * (end - start) +def text_list(items: Iterable[str], empty="") -> str: + """Concatenate the items with commas and one "and".""" + item_i = iter(items) + try: + last_item = next(item_i) + except StopIteration: + return empty + try: + so_far = next(item_i) + except StopIteration: + return last_item + for item in item_i: + so_far += ", " + last_item + last_item = item + return so_far + " and " + last_item + + def gather_from_ast( errors: MutableSequence[tuple], name: str, diff --git a/docs/source/docker.rst b/docs/source/docker.rst index 62fcfe0..9c21bb2 100644 --- a/docs/source/docker.rst +++ b/docs/source/docker.rst @@ -23,7 +23,7 @@ and stores the files produced in a directory called ``output``: .. code-block:: shell mkdir output - docker run --rm --user $(id -u):$(id -g) --network host -e SRC_SCHEMA=public -e SRC_DSN=postgresql://tim:tim@localhost:5432/pagila -itv ./output:data datafaker + docker run --rm --user $(id -u):$(id -g) --network host -e SRC_SCHEMA=public -e SRC_DSN=postgresql://tim:tim@localhost:5432/pagila -itv ./output:/data datafaker You do need to create the output folder first. diff --git a/pyproject.toml b/pyproject.toml index 0909605..aa31415 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datafaker" -version = "0.5.3" +version = "0.5.4" description = "Generates fake SQL data" authors = ["Tim Band <3266052+tim-band@users.noreply.github.com>"] license = "MIT" diff --git a/tests/examples/date.sql b/tests/examples/date.sql index f24b87b..037b564 100644 --- a/tests/examples/date.sql +++ b/tests/examples/date.sql @@ -25,7 +25,8 @@ CREATE TABLE public.happening ( at_time TIMESTAMP WITH TIME ZONE NOT NULL, at_date DATE NOT NULL, other_person_id INTEGER, - comment TEXT + comment TEXT, + previous_happening_id INTEGER ); ALTER TABLE ONLY public.happening ADD CONSTRAINT happening_pkey PRIMARY KEY (id); @@ -33,10 +34,12 @@ ALTER TABLE ONLY public.happening ADD CONSTRAINT person_id_fkey FOREIGN KEY (person_id) REFERENCES public.person(id); ALTER TABLE ONLY public.happening ADD CONSTRAINT other_person_id_fkey FOREIGN KEY (other_person_id) REFERENCES public.person(id); +ALTER TABLE ONLY public.happening + ADD CONSTRAINT previous_happening_id_fkey FOREIGN KEY (previous_happening_id) REFERENCES public.happening(id); ALTER TABLE public.happening OWNER TO postgres; -INSERT INTO public.happening VALUES (1, 'stepped on a tack', 1, '1997-04-20 04:05:06+00:00', '1997-04-20', NULL, NULL); -INSERT INTO public.happening VALUES (2, 'had a dream', 2, '1997-04-20 04:04:16+00:00', '1997-04-20', NULL, NULL); -INSERT INTO public.happening VALUES (3, 'kicked a can', 1, '2001-12-23 07:05:06+00:00', '2001-12-23', NULL, NULL); -INSERT INTO public.happening VALUES (4, 'shared a cake', 2, '2003-11-03 04:15:26+00:00', '2003-11-03', 1, NULL); +INSERT INTO public.happening VALUES (1, 'stepped on a tack', 1, '1997-04-20 04:05:06+00:00', '1997-04-20', NULL, NULL, NULL); +INSERT INTO public.happening VALUES (2, 'had a dream', 2, '1997-04-20 04:04:16+00:00', '1997-04-20', NULL, NULL, 1); +INSERT INTO public.happening VALUES (3, 'kicked a can', 1, '2001-12-23 07:05:06+00:00', '2001-12-23', NULL, NULL, 1); +INSERT INTO public.happening VALUES (4, 'shared a cake', 2, '2003-11-03 04:15:26+00:00', '2003-11-03', 1, NULL, NULL); diff --git a/tests/test_interactive_date_generators.py b/tests/test_interactive_date_generators.py index 3addac8..9fcd450 100644 --- a/tests/test_interactive_date_generators.py +++ b/tests/test_interactive_date_generators.py @@ -316,6 +316,31 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: self.assertEqual(gc.messages[1][0], gc.REQUIRES_SOURCE_DATA_TEXT) self.assertEqual(gc.messages[2][0], gc.PROVIDING_VALUES_TEXT) + def test_regression_date_start_crash(self) -> None: + """Test for regression where ``start`` role on a date leads to crash.""" + config = { + "tables": { + "happening": { + "columns": { + "at_date": { + "roles": ["start"], + }, + }, + }, + } + } + with self._get_cmd(config) as gc: + gc.do_next("happening.at_date") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.normal_date_fk" + " [anchored to at_date of table happening" + " (via previous_happening_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + # Note that this test won't work with DuckDB because it needs foreign keys to work class ConfigureGeneratorsWithDateMsSqlTests(ConfigureGeneratorsWithDateTests): From 37248af5a0930a0f06d6c28b04c7c40e54162281 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Tue, 29 Sep 2026 11:20:30 +0100 Subject: [PATCH 14/23] Copies are much more restricted You can't copy or anchor from nullable (or via nullable) to non-nullable. You can't copy from an FK to an FK that points to a different table. Bugs fixed. --- datafaker/proposers/base.py | 7 ++ datafaker/proposers/intervals.py | 95 +++++++++++++++---- tests/examples/date.sql | 10 +- tests/test_interactive_copy_generators.py | 106 ++++++++++++++++++++-- tests/test_interactive_date_generators.py | 8 +- 5 files changed, 193 insertions(+), 33 deletions(-) diff --git a/datafaker/proposers/base.py b/datafaker/proposers/base.py index 2c152b4..b51f86f 100644 --- a/datafaker/proposers/base.py +++ b/datafaker/proposers/base.py @@ -308,6 +308,13 @@ def __init__(self, fk_column: Column, target_column: Column) -> None: table = aliased(target_column.table) self.target_column = table.columns[target_column.name] + def __repr__(self): + """Render the relationship as a JOIN.""" + return ( + f"" + ) + def target_table(self) -> Table: """Get the related table.""" return self.target_column.table diff --git a/datafaker/proposers/intervals.py b/datafaker/proposers/intervals.py index 3b22611..c6142cb 100644 --- a/datafaker/proposers/intervals.py +++ b/datafaker/proposers/intervals.py @@ -49,7 +49,7 @@ def _set_roles_for_column( out[role].append(rc) -def _get_roles( +def _get_all_relative_roles( config: Mapping, column: Column, ) -> dict[str, list[RelatedColumn]]: @@ -83,6 +83,77 @@ def _get_roles( return role_to_fk_columns +def _not_anchored_on_ourselves(column: Column, fk_col: RelatedColumn) -> bool: + """ + Return whether these columns are the exact same thing. + + :param column: The column we are wanting to generate for. + :param fk_col: The column we are considering using as an anchor. + :return: False if ``fk_col`` targets the same column as ``column`` + not through a foreign key, so True if ``fk_col`` is through a FK + or if it doesn't target the same column. + """ + return not (fk_col.relationship is None and fk_col.target == column) + + +def _not_anchoring_nonnullable_to_nullable( + column: Column, fk_col: RelatedColumn +) -> bool: + """ + Return True if ``column`` is nullable or ``fk_col`` is not. + + We are counting ``fk_col`` as nullabe if either the target is nullable + or the foreign key itself. + + This is required if ``fk_col`` can be copied to ``col``. + """ + fk_nullable = ( + fk_col.relationship is not None + and fk_col.relationship.fk_column.nullable is not False + ) + fk_col_nullable = fk_nullable or fk_col.target.nullable + return column.nullable or not fk_col_nullable + + +def _not_copying_incompatible_fk(column: Column, fk_col: RelatedColumn) -> bool: + """Return True if ``column`` and ``fk_col`` are incompatible foreign key types.""" + target = fk_col.target + if not column.foreign_keys: + return not target.foreign_keys + if not target.foreign_keys: + return False + tfks = {c.target_fullname for c in target.foreign_keys} + cfks = {c.target_fullname for c in column.foreign_keys} + return tfks == cfks + + +def _get_roles( + config: Mapping, + column: Column, + role_name: str, +) -> list[RelatedColumn]: + """ + Get all the ``role_name`` roles in this table or a related table. + + :param role_name: The role to return. + :return: The list of all columns in the same table as ``column`` or in + tables directly related to that table that have the named role. This + list has the following columns filtered out: Nullable columns (if + ``column`` is nonnullable), foreign keys if ``column`` is not a + foreign key, non-foreign keys if ``column`` is a foreign key + or foreign keys if ``column`` is a foreign key to a different table. + """ + roles = _get_all_relative_roles(config, column) + if role_name not in roles: + return [] + return [ + fk_col + for fk_col in roles[role_name] + if _not_anchored_on_ourselves(column, fk_col) + and _not_anchoring_nonnullable_to_nullable(column, fk_col) + ] + + def coerce_to_datetime( d: datetime.date | datetime.datetime, tzdonor: datetime.datetime, @@ -372,15 +443,7 @@ def get_proposers( ct = get_column_type(column) if not isinstance(ct, (Date, DateTime)): return [] - roles = _get_roles(self._config, column) - if "start" not in roles: - return [] - other_start_columns = [ - fk_col - for fk_col in roles["start"] - # Don't anchor on ourselves - if not (fk_col.relationship is None and fk_col.target == column) - ] + other_start_columns = _get_roles(self._config, column, "start") return [ prop for anchor in other_start_columns @@ -519,17 +582,9 @@ def get_proposers( if len(columns) != 1: return [] column = columns[0] - roles = _get_roles(self._config, column) - if "source" not in roles: - return [] - ct = get_column_type(column) + fk_cols = _get_roles(self._config, column, "source") other_source_columns = [ - fk_col - for fk_col in roles["source"] - # Don't copy from ourself - if not (fk_col.relationship is None and fk_col.target == column) - # Only copy if the types are compatible - and self._can_copy_type(get_column_type(fk_col.target), ct) + fk_col for fk_col in fk_cols if _not_copying_incompatible_fk(column, fk_col) ] return [ prop diff --git a/tests/examples/date.sql b/tests/examples/date.sql index 037b564..b8e99be 100644 --- a/tests/examples/date.sql +++ b/tests/examples/date.sql @@ -8,15 +8,19 @@ CREATE TABLE public.person ( id INTEGER NOT NULL, name TEXT NOT NULL, timestamp_of_birth TIMESTAMP WITH TIME ZONE NOT NULL, - date_of_birth DATE NOT NULL + date_of_birth DATE NOT NULL, + parent_of INTEGER ); ALTER TABLE ONLY public.person ADD CONSTRAINT person_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.person + ADD CONSTRAINT parent_of_fkey FOREIGN KEY (parent_of) REFERENCES public.person(id); ALTER TABLE public.person OWNER TO postgres; -INSERT INTO public.person VALUES (1, 'Bobby', '1991-01-08 11:12:13+00:00', '1991-01-08'); -INSERT INTO public.person VALUES (2, 'Mary', '1989-03-04 20:19:18+00:00', '1989-03-04'); +INSERT INTO public.person VALUES (1, 'Bobby', '1991-01-08 11:12:13+00:00', '1991-01-08', NULL); +INSERT INTO public.person VALUES (2, 'Mary', '1989-03-04 20:19:18+00:00', '1989-03-04', NULL); +INSERT INTO public.person VALUES (3, 'Elizabeth', '1951-10-14 00:12:12+00:00', '1951-10-14', 2); CREATE TABLE public.happening ( id INTEGER NOT NULL, diff --git a/tests/test_interactive_copy_generators.py b/tests/test_interactive_copy_generators.py index d6ea5b2..f620a31 100644 --- a/tests/test_interactive_copy_generators.py +++ b/tests/test_interactive_copy_generators.py @@ -1,6 +1,7 @@ """ Tests for the configure-generators command. """ +import re from collections.abc import MutableMapping -from typing import Any +from typing import Any, Iterable from sqlalchemy import select from sqlalchemy.orm import aliased @@ -25,7 +26,7 @@ def _get_cmd(self, config: MutableMapping[str, Any]) -> MockGeneratorCmd: ) def test_same_table_copy(self) -> None: - """Test that the cross-table copy is proposed and compared.""" + """Test that the same-table copy is proposed and compared.""" config = { "tables": { "happening": { @@ -102,7 +103,7 @@ def test_cross_table_copy_actually_copies(self) -> None: } with self._get_cmd(config) as gc: # set up our copy proposer - gc.do_next("happening.comment") + gc.do_next("happening.name") gc.reset() gc.do_propose("") proposals = gc.get_proposals() @@ -113,7 +114,7 @@ def test_cross_table_copy_actually_copies(self) -> None: self.assertIn(provider_name, proposals.keys()) prop = proposals[provider_name] gc.do_set(str(prop[0])) - gc.do_next("happening.name") + gc.do_next("happening.comment") gc.reset() gc.do_propose("") proposals = gc.get_proposals() @@ -155,10 +156,103 @@ def test_cross_table_copy_actually_copies(self) -> None: for row in dst_result: if row.opn: saw_other = True - self.assertEqual(row.hn, row.opn) - self.assertEqual(row.hc, row.pn) + self.assertEqual(row.hc, row.opn) + self.assertEqual(row.hn, row.pn) self.assertTrue(saw_other) + def test_cross_table_fk_copy(self) -> None: + """Test that the cross-table foreign key copy is proposed.""" + config = { + "tables": { + "person": { + "columns": { + "parent_of": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + gc.do_next("happening.other_person_id") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from parent_of of table person (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + + COPY_FROM_RE = re.compile(r"generic.anchored_provider.copy \[from ([^\]]+)\]") + + def get_same_table_copy_proposals(self, proposals: Iterable[str]) -> set[str]: + """Return a set of columns proposed to copy from.""" + return { + p.group(1) + for p in (self.COPY_FROM_RE.match(prop) for prop in proposals) + if p is not None + } + + def test_different_target_fk_copy_is_not_proposed(self) -> None: + """Test a foreign key copy with a different target table is not proposed.""" + config = { + "tables": { + "happening": { + "columns": { + "previous_happening_id": { + "roles": ["source"], + }, + "person_id": { + "roles": ["source"], + }, + } + } + } + } + with self._get_cmd(config) as gc: + gc.do_next("happening.other_person_id") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + provider_name = "generic.anchored_provider.copy [from person_id]" + self.assertIn(provider_name, proposals.keys()) + all_copies = self.get_same_table_copy_proposals(proposals.keys()) + self.assertIn("person_id", all_copies) + self.assertNotIn("previous_happening_id", all_copies) + + def test_nullable_copy_nonnullable_is_not_proposed(self) -> None: + """Test a copy from a nullable column to a nonnullable column is not proposed.""" + config = { + "tables": { + "happening": { + "columns": { + "other_person_id": { + "roles": ["source"], + }, + "person_id": { + "roles": ["source"], + }, + } + } + } + } + with self._get_cmd(config) as gc: + # person -> other_person is allowed + gc.do_next("happening.other_person_id") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + all_copies = self.get_same_table_copy_proposals(proposals.keys()) + self.assertIn("person_id", all_copies) + # other_person -> person is allowed + gc.do_next("happening.person_id") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + all_copies = self.get_same_table_copy_proposals(proposals.keys()) + self.assertNotIn("other_person_id", all_copies) + # Note that this test won't work with DuckDB because it needs foreign keys to work class ConfigureCopyGeneratorsWithDateMsSqlTests(ConfigureCopyGeneratorsWithDateTests): diff --git a/tests/test_interactive_date_generators.py b/tests/test_interactive_date_generators.py index 9fcd450..aba9b82 100644 --- a/tests/test_interactive_date_generators.py +++ b/tests/test_interactive_date_generators.py @@ -320,9 +320,9 @@ def test_regression_date_start_crash(self) -> None: """Test for regression where ``start`` role on a date leads to crash.""" config = { "tables": { - "happening": { + "person": { "columns": { - "at_date": { + "date_of_birth": { "roles": ["start"], }, }, @@ -336,8 +336,8 @@ def test_regression_date_start_crash(self) -> None: proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" - " [anchored to at_date of table happening" - " (via previous_happening_id)]" + " [anchored to date_of_birth of table person" + " (via person_id)]" ) self.assertIn(provider_name, proposals.keys()) From f85fd4086c100df7163b238625d9e48e2f517053 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Tue, 6 Oct 2026 09:39:51 +0100 Subject: [PATCH 15/23] Initial Snowflake support --- datafaker/db_utils.py | 39 ++-- examples/omop-mssql/README.md | 2 - poetry.lock | 398 +++++++++++++++++++++++++++++++++- pyproject.toml | 1 + tests/test_utils_mssql.py | 5 +- 5 files changed, 425 insertions(+), 20 deletions(-) diff --git a/datafaker/db_utils.py b/datafaker/db_utils.py index e7e18a9..27cb76d 100644 --- a/datafaker/db_utils.py +++ b/datafaker/db_utils.py @@ -160,18 +160,29 @@ def create_db_engine( logger.error("DSN %s is malformed: %s", db_dsn, exc) raise Exit(1) from exc - settings = {} + settings: list[str] = [] + dialect_name = get_sync_engine(engine).dialect.name if schema_name is not None: - if get_sync_engine(engine).dialect.name == "mssql": - engine = engine.execution_options(schema_translate_map={None: schema_name}) - else: - settings["search_path"] = schema_name + match dialect_name: + case "mssql": + engine = engine.execution_options( + schema_translate_map={None: schema_name} + ) + case "snowflake": + settings.append(f"USE SCHEMA {schema_name};") + case _: + settings.append(f"SET search_path TO {schema_name};") if parquet_dir is not None: - joined = ",".join(_find_parquet_directories(parquet_dir)) - # double up single quotes - dj = joined.replace("'", "''") - # enclose in single quotes - settings["file_search_path"] = f"'{dj}'" + match dialect_name: + case "duckdb": + joined = ",".join(_find_parquet_directories(parquet_dir)) + # double up single quotes + dj = joined.replace("'", "''") + settings.append(f"SET file_search_path TO '{dj}';") + case _: + logger.error( + "--parquet-dir cannot be used with %s, only duckdb.", dialect_name + ) if settings: event_engine = get_sync_engine(engine) @@ -240,16 +251,14 @@ def _names_include_parquet(path: Path, file_names: Iterable[str]) -> bool: return False -def set_db_settings(connection: DBAPIConnection, settings: Mapping[str, str]) -> None: - """Set the SEARCH_PATH for a PostgreSQL connection.""" +def set_db_settings(connection: DBAPIConnection, settings: list[str]) -> None: + """Run commands.""" # https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#remote-schema-table-introspection-and-postgresql-search-path existing_autocommit = connection.autocommit connection.autocommit = True cursor = connection.cursor() - # Parametrised queries don't work with asyncpg, hence the f-string. - sql = "".join(f"SET {k} TO {v};" for k, v in settings.items()) - cursor.execute(sql) + cursor.execute("\n".join(settings)) cursor.close() connection.autocommit = existing_autocommit diff --git a/examples/omop-mssql/README.md b/examples/omop-mssql/README.md index 7c0bbfe..64662a3 100644 --- a/examples/omop-mssql/README.md +++ b/examples/omop-mssql/README.md @@ -51,8 +51,6 @@ PostgreSQL uses `SERIAL` for autoincrement columns. The code already strips `SER [datafaker/utils.py:297-309](../../datafaker/utils.py) toggles `connection.autocommit` directly on the DBAPI connection before executing `SET` commands. The availability and behaviour of `autocommit` differs between `psycopg2`, `pyodbc`, and `pymssql`, so this would need testing or an abstraction per driver. -**Deferred.** `set_db_settings` is only ever called for DuckDB connections (`parquet_dir` source feature); MS-SQL connections never reach it. Fixing the `autocommit` toggle in isolation would also leave the `SET {k} TO {v}` SQL syntax broken for MS-SQL (which requires `SET {k} {v}`). Both issues should be addressed together if `set_db_settings` is ever extended to MS-SQL. - ## Setup ### 1. Install the MS-SQL Python extras diff --git a/poetry.lock b/poetry.lock index a34332f..e1a3576 100644 --- a/poetry.lock +++ b/poetry.lock @@ -251,6 +251,44 @@ d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "boto3" +version = "1.43.106" +description = "The AWS SDK for Python (Boto3)" +optional = false +python-versions = ">=3.10" +files = [ + {file = "boto3-1.43.106-py3-none-any.whl", hash = "sha256:fdf2e304f9e8864b8613591b18b36ee995560c9666033622eda1cb9057866880"}, + {file = "boto3-1.43.106.tar.gz", hash = "sha256:c11ad4c429a983493ba10014c7af9831a455c2c0eea91c1cefff74530e480277"}, +] + +[package.dependencies] +botocore = ">=1.43.106,<1.44.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.19.0,<0.20.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.43.106" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.10" +files = [ + {file = "botocore-1.43.106-py3-none-any.whl", hash = "sha256:c1fb8818f9957cb5037f73db34ac1a12ba43562e31d6e7f9cb7e5fc64e1dba78"}, + {file = "botocore-1.43.106.tar.gz", hash = "sha256:006870b3b4e40547232ad12c3bb4faec91bbbe0659aafaa3b7fa48a112c4ee97"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<2.2.0 || >2.2.0,<3" + +[package.extras] +crt = ["awscrt (==0.36.0)"] + [[package]] name = "certifi" version = "2026.7.22" @@ -262,6 +300,118 @@ files = [ {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] +[[package]] +name = "cffi" +version = "2.1.1" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.10" +files = [ + {file = "cffi-2.1.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be"}, + {file = "cffi-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9"}, + {file = "cffi-2.1.1-cp310-cp310-win32.whl", hash = "sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41"}, + {file = "cffi-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1"}, + {file = "cffi-2.1.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12"}, + {file = "cffi-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa"}, + {file = "cffi-2.1.1-cp311-cp311-win32.whl", hash = "sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3"}, + {file = "cffi-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0"}, + {file = "cffi-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455"}, + {file = "cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0"}, + {file = "cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf"}, + {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517"}, + {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735"}, + {file = "cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e"}, + {file = "cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a"}, + {file = "cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80"}, + {file = "cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e"}, + {file = "cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c"}, + {file = "cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6"}, + {file = "cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2"}, + {file = "cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b"}, + {file = "cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7"}, + {file = "cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac"}, + {file = "cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d"}, + {file = "cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973"}, + {file = "cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c"}, + {file = "cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb"}, + {file = "cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54"}, + {file = "cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96"}, + {file = "cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527"}, + {file = "cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13"}, + {file = "cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c"}, + {file = "cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48"}, + {file = "cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836"}, + {file = "cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3"}, + {file = "cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676"}, + {file = "cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e"}, + {file = "cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f"}, + {file = "cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4"}, + {file = "cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e"}, + {file = "cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5"}, + {file = "cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d"}, + {file = "cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b"}, + {file = "cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4"}, + {file = "cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399"}, + {file = "cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688"}, + {file = "cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7"}, + {file = "cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac"}, + {file = "cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960"}, + {file = "cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1"}, + {file = "cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc"}, + {file = "cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6"}, + {file = "cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94"}, + {file = "cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5"}, + {file = "cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66"}, + {file = "cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3"}, + {file = "cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692"}, + {file = "cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be"}, +] + +[package.dependencies] +pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} + [[package]] name = "cfgv" version = "3.5.0" @@ -545,6 +695,81 @@ files = [ [package.extras] dev = ["black (==22.3.0)", "hypothesis (==6.60.0)", "numpy", "pytest (>=5.30)", "pytest-benchmark", "pytest-xdist"] +[[package]] +name = "cryptography" +version = "50.0.2" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = "!=3.9.0,!=3.9.1,>=3.9" +files = [ + {file = "cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93"}, + {file = "cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c"}, + {file = "cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e"}, + {file = "cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c"}, + {file = "cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94"}, + {file = "cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452"}, + {file = "cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5"}, +] + +[package.dependencies] +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""} +typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11\""} + +[package.extras] +ssh = ["bcrypt (>=3.1.5)"] + [[package]] name = "dataclasses-json" version = "0.6.7" @@ -997,6 +1222,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.1.0" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.9" +files = [ + {file = "jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64"}, + {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"}, +] + [[package]] name = "json-schema-for-humans" version = "1.5.1" @@ -1832,6 +2068,17 @@ files = [ {file = "psycopg2_binary-2.9.12.tar.gz", hash = "sha256:5ac9444edc768c02a6b6a591f070b8aae28ff3a99be57560ac996001580f294c"}, ] +[[package]] +name = "pycparser" +version = "3.0" +description = "C parser in Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, + {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, +] + [[package]] name = "pydantic" version = "1.10.26" @@ -1917,6 +2164,23 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pyjwt" +version = "2.15.1" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193"}, + {file = "pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8"}, +] + +[package.dependencies] +typing_extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] + [[package]] name = "pylint" version = "3.3.9" @@ -2034,6 +2298,25 @@ files = [ {file = "pyodbc-5.3.0.tar.gz", hash = "sha256:2fe0e063d8fb66efd0ac6dc39236c4de1a45f17c33eaded0d553d21c199f4d05"}, ] +[[package]] +name = "pyopenssl" +version = "26.4.0" +description = "Python wrapper module around the OpenSSL library" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c"}, + {file = "pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7"}, +] + +[package.dependencies] +cryptography = ">=49.0.0,<51" +typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\""} + +[package.extras] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] +test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] + [[package]] name = "pytest" version = "9.1.1" @@ -2408,6 +2691,23 @@ sphinx = ["sphinx (>=4.0,<6.0)"] testing = ["coverage-conditional-plugin (>=0.5)", "coverage[toml] (>=6.0)", "pytest (>=6.0)", "pytest-cov (>=3.0)", "pytest-mock (>=3.7)", "pytest-randomly (>=3.0)", "pytest-sugar (>=0.9.5)"] toml = ["tomli (>=2.0,<3.0)"] +[[package]] +name = "s3transfer" +version = "0.19.2" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.10" +files = [ + {file = "s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25"}, + {file = "s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993"}, +] + +[package.dependencies] +botocore = ">=1.37.4,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] + [[package]] name = "scramp" version = "1.4.12" @@ -2474,6 +2774,102 @@ files = [ {file = "snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260"}, ] +[[package]] +name = "snowflake-connector-python" +version = "4.8.0" +description = "Snowflake Connector for Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "snowflake_connector_python-4.8.0-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:5ac7214337e2ca4797a63b12e2d6778b85b95bacbb1e8892e8ef5cdb2696bcdb"}, + {file = "snowflake_connector_python-4.8.0-cp310-cp310-macosx_15_0_x86_64.whl", hash = "sha256:e3de20df18a0279cc913ecd073b79eae860fa00e3d0e8cb9cfb276ecf85fe319"}, + {file = "snowflake_connector_python-4.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a887838e59afa9d6d5f0fb52f55fcdb8726987acce646daa9aa3c698a3c99690"}, + {file = "snowflake_connector_python-4.8.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b95cec73e136a3a373bf19bc19ecc5707557ae9df02674665240cfd46ea78820"}, + {file = "snowflake_connector_python-4.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:3566b6884073dd1c36905c875aa3013991ae814a75f539c6ea1f2e8a1f72dd41"}, + {file = "snowflake_connector_python-4.8.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:3b38204c9c8a9c985cdf36df4eaa16f7d0b8a14f86410f39ab7b49794df4cf03"}, + {file = "snowflake_connector_python-4.8.0-cp311-cp311-macosx_15_0_x86_64.whl", hash = "sha256:3fdea6d45bd22142293ece1224f2ad3a3c169a2f4a768d4be828faf09a640b2b"}, + {file = "snowflake_connector_python-4.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bf2304eedfd94694291a71c96cddb4490e474d2c232625f528813f5caa79c098"}, + {file = "snowflake_connector_python-4.8.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf8d819fd2c943d9be9d521fb6259b28dca05844e6a4d585a9f6e287141ae6dd"}, + {file = "snowflake_connector_python-4.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:1ac5bc235c9f1124bfa1b14413f340ef2bdf7c28746af37280eae3ad7877afa8"}, + {file = "snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:2fde1216c4c1e4015ad7e39b7b39b6a44a29551f4d7a3dfc64b0905e66dc24ac"}, + {file = "snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:c40a058f87dfbca1e65122932afb41c42cb9c3b7cb12826204527b41dd0f39ef"}, + {file = "snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f2a3cbcd3e6c4a449165f78bb331e1570ce6026cc21a2673313b4912d80745c0"}, + {file = "snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1c80df2fc5e16f3a49e6d16b1cdca0358308c1db2eae22d82aa7c038c80cc913"}, + {file = "snowflake_connector_python-4.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:b9bddddc713aaa20dc5ff5635896f66e50c0fed88b6a905b088e727d046e361d"}, + {file = "snowflake_connector_python-4.8.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ebcbad6b16f8228f28bf62b443b6f111d11bceea2612efa2004eccb827923e6b"}, + {file = "snowflake_connector_python-4.8.0-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:1cc888b81b709781eb855bbee062f06a9f52427ab8f9a39a31f39de475e39465"}, + {file = "snowflake_connector_python-4.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2a59396eade5b7b18cad0ac1fa2882a5ddfa5153e56e16c9d3d4a9d5e2d92332"}, + {file = "snowflake_connector_python-4.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f978f897f85fd39cc8f768971c4667f8ca0278ea369373dfc459ae008c9472a5"}, + {file = "snowflake_connector_python-4.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:131db582e383b57af0b345f212c9afba87637e8ef68494b9b4282e6713030dc9"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:7a1566b0fcae33f2c7ec1bbd09965d8045a9388e94d333a88d03ddcfb6d4550a"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:8db3eea888247e4b41b55b8b419613eb1c4c76ec4984341543f849bf9532ae1b"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:39fe1794c63508fb25fda1a57cdf00293dc0630b32d4cda86d5bed2da09eb256"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:43583fb372a3bb698c2ce4717d2598e5452b40543a59b8095146a98f8ecd7408"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:6ba170f3584a425a1a5eff0e53ed656d45db9ef513d9232789cf58c5254b8194"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314t-macosx_15_0_arm64.whl", hash = "sha256:03e248f510ae82a3b190146c060e865905e5e07df6cf3e365ede66c67de136b2"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314t-macosx_15_0_x86_64.whl", hash = "sha256:7612ef5b4a93438fec6fff0e0d2541302f29001d3e77b861140d00d9085c0dce"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bccf23cd780c85bfa74982f286d2aacd663ba0135ba7441a13e98cabdb2def1e"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:da28999b3d6cfc4860c8c537cfb02968e9784f3ec8fe50d92a7348133ea75362"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6c0d7be79b333c1fa85d2860928f286a11abe5789d007dbc6d940313a744f8af"}, + {file = "snowflake_connector_python-4.8.0.tar.gz", hash = "sha256:1bf40aa8684eb0769fedeb53a702b274472571812835d5a03fb9dcf9254333b9"}, +] + +[package.dependencies] +asn1crypto = ">0.24.0,<2.0.0" +boto3 = ">=1.24" +botocore = ">=1.24" +certifi = ">=2024.7.4" +charset_normalizer = ">=2,<4" +cryptography = ">=46.0.5" +filelock = ">=3.5,<4" +idna = ">=3.7,<4" +packaging = "*" +platformdirs = ">=2.6.0,<5.0.0" +pyjwt = ">=2.10.1,<3.0.0" +pyOpenSSL = ">=25.3.0" +pytz = "*" +requests = ">=2.32.4,<3.0.0" +sortedcontainers = ">=2.4.0" +tomlkit = "*" +typing_extensions = ">=4.3,<5" + +[package.extras] +azure = ["azure-identity (>=1.16)"] +boto = ["boto3 (>=1.24)", "botocore (>=1.24)"] +development = ["Cython (>=3.1)", "azure-identity (>=1.16)", "coverage", "mitmproxy (>=12.0.0)", "more-itertools", "numpy (<=2.4.3)", "pendulum (!=2.1.1)", "pexpect", "pytest (>=8.4,<9)", "pytest-asyncio (>=1.4.0,<2)", "pytest-cov", "pytest-rerunfailures (<16.0)", "pytest-timeout", "pytest-xdist", "pytzdata", "responses"] +pandas = ["pandas (>=1.0.0,<3.0.0)", "pandas (>=2.1.2,<3.0.0)", "pyarrow (>=14.0.1)", "pyarrow (>=14.0.1,<24)"] +secure-local-storage = ["keyring (>=23.1.0,<26.0.0)"] + +[[package]] +name = "snowflake-sqlalchemy" +version = "1.11.1" +description = "Snowflake SQLAlchemy Dialect" +optional = false +python-versions = ">=3.8" +files = [ + {file = "snowflake_sqlalchemy-1.11.1-py3-none-any.whl", hash = "sha256:b64ab947469bf25b3df3f88c5b92f17a938c9d47b5b09035551614f3ff1f44d9"}, + {file = "snowflake_sqlalchemy-1.11.1.tar.gz", hash = "sha256:942b203de0404b79550c4098ca851aee2b0b22f695426b073e58d71772e38d51"}, +] + +[package.dependencies] +snowflake-connector-python = "<5.0.0" +sqlalchemy = ">=1.4.19" + +[package.extras] +development = ["alembic", "numpy", "pre-commit", "pytest", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytz", "setuptools", "syrupy"] +pandas = ["snowflake-connector-python[pandas]"] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +optional = false +python-versions = "*" +files = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] + [[package]] name = "sphinx" version = "5.3.0" @@ -3174,4 +3570,4 @@ docs = ["sphinx-rtd-theme", "sphinxcontrib-napoleon"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.14" -content-hash = "32e39aeac70fde68398a8a5d467c29499cd74d177beda122a19f4641689f71c3" +content-hash = "f85dcb1f81a16c742fb5a9a8dfd73f02b028aa7a762f05d1e31a9c46b2286d0c" diff --git a/pyproject.toml b/pyproject.toml index aa31415..38e2ad9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ fastparquet = "^2024.11.0" duckdb-sqlalchemy = "^1.5.2.2" ty = "^0.0.43" colorama = "^0.4.6" +snowflake-sqlalchemy = "^1.11.1" [tool.poetry.group.dev.dependencies] isort = "^5.10.1" diff --git a/tests/test_utils_mssql.py b/tests/test_utils_mssql.py index 5e27cd8..f831667 100644 --- a/tests/test_utils_mssql.py +++ b/tests/test_utils_mssql.py @@ -107,8 +107,9 @@ def test_duckdb_parquet_dir_sets_search_path(self) -> None: if len(calls[0].args) > 1 else calls[0].kwargs.get("settings", {}) ) - self.assertIn("file_search_path", settings_passed) - self.assertEqual(settings_passed["file_search_path"], f"'{parq_dir}'") + all_settings = "\n".join(settings_passed) + self.assertIn("file_search_path", all_settings) + self.assertIn(f"SET file_search_path TO '{parq_dir}';", all_settings) def test_mssql_dsn_schema_sets_translate_map(self) -> None: """schema_translate_map is set even for an MS-SQL DSN (engine creation, no connect).""" From 7c68c026ff3bb9d621e5e7bacc759c7e3b964a12 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Tue, 6 Oct 2026 18:36:22 +0100 Subject: [PATCH 16/23] make-tables gains --views-to-tables flag --- datafaker/db_utils.py | 8 +++- datafaker/main.py | 7 ++- datafaker/make.py | 5 +- datafaker/remove.py | 6 ++- tests/examples/date.sql | 5 ++ tests/test_functional_mssql.py | 4 +- tests/test_make.py | 84 +++++++++++++++++++++++++++++++++- tests/test_utils_mssql.py | 4 +- tests/utils.py | 4 +- 9 files changed, 116 insertions(+), 11 deletions(-) diff --git a/datafaker/db_utils.py b/datafaker/db_utils.py index 27cb76d..2506209 100644 --- a/datafaker/db_utils.py +++ b/datafaker/db_utils.py @@ -223,11 +223,15 @@ def create_db_engine_dst( return create_db_engine(db_dsn, schema_name, use_asyncio) -def get_metadata(engine: Engine, schema_name: Optional[str] = None) -> MetaData: +def get_metadata( + engine: Engine, + schema_name: Optional[str], + views_to_tables: bool, +) -> MetaData: """Get the MetaData object associated with the engine passed.""" md = MetaData() try: - md.reflect(engine, schema=schema_name) + md.reflect(engine, schema=schema_name, views=views_to_tables) except OperationalError as exc: logger.error("Cannot connect to database: %s", exc) raise Exit(1) from exc diff --git a/datafaker/main.py b/datafaker/main.py index b180b97..096ae30 100644 --- a/datafaker/main.py +++ b/datafaker/main.py @@ -177,6 +177,10 @@ def make_tables( file_okay=False, dir_okay=True, ), + views_to_tables: bool = Option( + False, + help=("Generate all views as tables in the synthetic version."), + ), ) -> None: """Make a YAML file representing the tables in the schema. @@ -192,7 +196,8 @@ def make_tables( content = make_tables_file( get_source_dsn(), get_source_schema(), - parquet_dir, + views_to_tables=views_to_tables, + parquet_dir=parquet_dir, ) orm_file_path.write_text(content, encoding="utf-8") logger.debug("%s created.", orm_file) diff --git a/datafaker/make.py b/datafaker/make.py index d9ddf25..0c74a52 100644 --- a/datafaker/make.py +++ b/datafaker/make.py @@ -697,6 +697,7 @@ def _generate_vocabulary_table( def make_tables_file( db_dsn: str, schema_name: Optional[str], + views_to_tables: bool, parquet_dir: Optional[Path] = None, engine: Optional[Engine] = None, ) -> str: @@ -704,7 +705,9 @@ def make_tables_file( if engine is None: engine = get_sync_engine(create_db_engine(db_dsn, schema_name=schema_name)) - metadata = get_metadata(engine, schema_name=schema_name) + metadata = get_metadata( + engine, schema_name=schema_name, views_to_tables=views_to_tables + ) meta_dict = metadata_to_dict(metadata, schema_name, engine, parquet_dir) if parquet_dir is not None: diff --git a/datafaker/remove.py b/datafaker/remove.py index 0949e04..a6eae71 100644 --- a/datafaker/remove.py +++ b/datafaker/remove.py @@ -68,5 +68,9 @@ def remove_db_tables(metadata: Optional[MetaData]) -> None: ) ) if metadata is None: - metadata = get_metadata(dst_engine, schema_name=schema_name) + metadata = get_metadata( + dst_engine, + schema_name=schema_name, + views_to_tables=False, + ) metadata.drop_all(dst_engine) diff --git a/tests/examples/date.sql b/tests/examples/date.sql index b8e99be..2f53171 100644 --- a/tests/examples/date.sql +++ b/tests/examples/date.sql @@ -47,3 +47,8 @@ INSERT INTO public.happening VALUES (1, 'stepped on a tack', 1, '1997-04-20 04:0 INSERT INTO public.happening VALUES (2, 'had a dream', 2, '1997-04-20 04:04:16+00:00', '1997-04-20', NULL, NULL, 1); INSERT INTO public.happening VALUES (3, 'kicked a can', 1, '2001-12-23 07:05:06+00:00', '2001-12-23', NULL, NULL, 1); INSERT INTO public.happening VALUES (4, 'shared a cake', 2, '2003-11-03 04:15:26+00:00', '2003-11-03', 1, NULL, NULL); + +CREATE VIEW public.happening_view (id, name, person_id) + AS SELECT id, name, person_id FROM public.happening; + +ALTER VIEW public.happening_view OWNER TO postgres; diff --git a/tests/test_functional_mssql.py b/tests/test_functional_mssql.py index dbd6415..e0f35b7 100644 --- a/tests/test_functional_mssql.py +++ b/tests/test_functional_mssql.py @@ -52,7 +52,9 @@ def setUp(self) -> None: (self.orm_fd, self.orm_file_path) = mkstemp(".yaml", "orm_", text=True) with os.fdopen(self.orm_fd, "w", encoding="utf-8") as fh: fh.write( - make_tables_file(self.dsn, self.schema_name, engine=self.sync_engine) + make_tables_file( + self.dsn, self.schema_name, False, engine=self.sync_engine + ) ) def tearDown(self) -> None: diff --git a/tests/test_make.py b/tests/test_make.py index 9327ea6..5578ad7 100644 --- a/tests/test_make.py +++ b/tests/test_make.py @@ -3,6 +3,7 @@ import tempfile from pathlib import Path from typing import Any +from unittest import SkipTest from unittest.mock import MagicMock, patch import pandas as pd @@ -20,12 +21,93 @@ from sqlalchemy.dialects.mysql.types import INTEGER from sqlalchemy.dialects.postgresql import UUID +from datafaker.db_utils import create_db_engine, get_sync_engine from datafaker.make import ( _get_default_generator, _get_provider_for_column, make_src_stats, + make_tables_file, ) -from tests.utils import DatafakerTestCase, GeneratesDBTestCase, RequiresDBTestCase +from tests.utils import ( + DatafakerTestCase, + GeneratesDBTestCase, + MsSqlTestDb, + PostgresTestDb, + RequiresDBTestCase, +) + + +class TestSerialize(DatafakerTestCase): + """Unit tests for serialize_metadata.""" + + database_type = PostgresTestDb + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + skip_msg = cls.database_type.skip() + if skip_msg: + raise SkipTest(skip_msg) + cls.database_type.setup() + + def setUp(self) -> None: + super().setUp() + self.database = self.database_type() + self.database.open() + self.database.run_sql(self.get_abs_example_dir() / "date.sql") + self.engine = create_db_engine(self.database.get_dsn("date_tables"), None) + + def test_metadata_to_dict(self) -> None: + """Test metadata_to_dict.""" + orm_yaml = make_tables_file( + self.database.get_dsn("date_tables"), + schema_name=None, + views_to_tables=False, + parquet_dir=None, + engine=get_sync_engine(self.engine), + ) + orm = yaml.load(orm_yaml, yaml.Loader) + table_names = set(orm["tables"].keys()) + self.assert_subset({"person", "happening"}, table_names) + self.assertNotIn("happening_view", table_names) + happening_columns = orm["tables"]["happening"]["columns"] + happening_column_names = set(happening_columns.keys()) + self.assert_subset( + {"id", "name", "comment", "person_id"}, + happening_column_names, + ) + self.assertEqual(happening_columns["person_id"]["type"], "INTEGER") + self.assertFalse(happening_columns["name"]["primary"]) + self.assertTrue(happening_columns["id"]["primary"]) + + def test_views_to_dict(self) -> None: + """Test views_to_tables.""" + orm_yaml = make_tables_file( + self.database.get_dsn("date_tables"), + schema_name=None, + views_to_tables=True, + parquet_dir=None, + engine=get_sync_engine(self.engine), + ) + orm = yaml.load(orm_yaml, yaml.Loader) + table_names = set(orm["tables"].keys()) + self.assert_subset({"person", "happening", "happening_view"}, table_names) + happening_columns = orm["tables"]["happening_view"]["columns"] + happening_column_names = set(happening_columns.keys()) + self.assert_subset( + {"id", "name", "person_id"}, + happening_column_names, + ) + self.assertEqual(happening_columns["person_id"]["type"], "INTEGER") + self.assertFalse(happening_columns["name"]["primary"]) + # FK and PK constraints are not visible through views + # self.assertTrue(happening_columns["id"]["primary"]) + + +class TestSerializeMsSql(TestSerialize): + """Unit tests for serialize_metadata with an MS SQL database.""" + + database_type = MsSqlTestDb class TestGetDefaultGenerator(DatafakerTestCase): diff --git a/tests/test_utils_mssql.py b/tests/test_utils_mssql.py index f831667..a8abccf 100644 --- a/tests/test_utils_mssql.py +++ b/tests/test_utils_mssql.py @@ -145,7 +145,7 @@ def test_reflect_called_with_schema(self) -> None: mock_meta_data.return_value = mock_md mock_md.reflect.return_value = None - get_metadata(mock_engine, schema_name="myschema") + get_metadata(mock_engine, schema_name="myschema", views_to_tables=False) mock_md.reflect.assert_called_once_with(mock_engine, schema="myschema") @@ -159,6 +159,6 @@ def test_reflect_called_without_schema_when_none(self) -> None: mock_meta_data.return_value = mock_md mock_md.reflect.return_value = None - get_metadata(mock_engine) + get_metadata(mock_engine, schema_name=None, views_to_tables=False) mock_md.reflect.assert_called_once_with(mock_engine, schema=None) diff --git a/tests/utils.py b/tests/utils.py index 8bdf505..4e51d9f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -284,7 +284,7 @@ class MsSqlTestDb(TestDatabaseBase): _EOL_RE = re.compile(r";$", re.MULTILINE) _TIMESTAMP_RE = re.compile(r"(TIMESTAMP)\s+WITH\s+TIME\s+ZONE", re.IGNORECASE) _ALTER_OWNER_RE = re.compile( - r'ALTER\s+(TABLE|DATABASE)\s+["A-Za-z0-9_.]+\s+OWNER\s+TO\s+[A-Za-z0-9_]+\s*;' + r'ALTER\s+(TABLE|DATABASE|VIEW)\s+["A-Za-z0-9_.]+\s+OWNER\s+TO\s+[A-Za-z0-9_]+\s*;' ) _ALTER_ONLY_RE = re.compile(r"ALTER\s+TABLE\s+ONLY\s+") _CREATE_INDEX_RE = re.compile( @@ -764,7 +764,7 @@ def setUp(self) -> None: # Generate the `orm.yaml` from the database (self.orm_fd, self.orm_file_path) = mkstemp(".yaml", "orm_", text=True) with os.fdopen(self.orm_fd, "w", encoding="utf-8") as orm_fh: - orm_fh.write(make_tables_file(self.dsn, self.schema_name)) + orm_fh.write(make_tables_file(self.dsn, self.schema_name, False)) # Create a separate empty destination database self.make_destination_database("dst") From 2fc309471a4643c5579de223008e93eab02fa0a1 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Wed, 7 Oct 2026 13:45:26 +0100 Subject: [PATCH 17/23] Snowflake timestamp types added --- datafaker/serialize_metadata.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datafaker/serialize_metadata.py b/datafaker/serialize_metadata.py index 35a066b..b818918 100644 --- a/datafaker/serialize_metadata.py +++ b/datafaker/serialize_metadata.py @@ -6,6 +6,7 @@ from typing import Any import parsy +import snowflake.sqlalchemy.custom_types as snowflake_types from sqlalchemy import Column, Dialect, Engine, ForeignKey, MetaData, Table from sqlalchemy.dialects import mssql, oracle, postgresql from sqlalchemy.sql import schema, sqltypes @@ -204,6 +205,12 @@ def _mssql_varbinary_parser() -> typing.Generator[ParserType, None, typing.Any]: string_type(sqltypes.VARCHAR), string_type(sqltypes.NVARCHAR), string_type(sqltypes.TEXT), + parsy.string("TIMESTAMP_NTZ").result(snowflake_types.TIMESTAMP_NTZ), + parsy.string("TIMESTAMPNTZ").result(snowflake_types.TIMESTAMP_NTZ), + parsy.string("TIMESTAMP_LTZ").result(snowflake_types.TIMESTAMP_LTZ), + parsy.string("TIMESTAMPLTZ").result(snowflake_types.TIMESTAMP_LTZ), + parsy.string("TIMESTAMP_TZ").result(snowflake_types.TIMESTAMP_TZ), + parsy.string("TIMESTAMPTZ").result(snowflake_types.TIMESTAMP_TZ), time_type(sqltypes.TIMESTAMP, postgresql.types.TIMESTAMP), time_type(sqltypes.TIME, postgresql.types.TIME), ) From 55bfcd8b8cbe6a71e592c77d3736c54b0865aa85 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Tue, 13 Oct 2026 15:19:51 +0100 Subject: [PATCH 18/23] Remove one literal_column call These don't work in Snowflake --- datafaker/proposers/choice.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py index acf0e1f..0bc6504 100644 --- a/datafaker/proposers/choice.py +++ b/datafaker/proposers/choice.py @@ -355,18 +355,17 @@ def get_proposers( # pylint: disable=too-many-locals table_sql = schema_qualified_name(table_name, engine) src_table = column.table dialect = engine.dialect - col = literal_column(f'"{column_name}"') src_table = column.table # preserves schema for schema-qualified databases generators = [] with engine.connect() as connection: stmt_count = ( select( - col.label("v"), - func.count(col).label("f"), # pylint: disable=E1102 + column.label("v"), + func.count(column).label("f"), # pylint: disable=E1102 ) .select_from(src_table) - .group_by(col) - .order_by(desc(func.count(col))) # pylint: disable=E1102 + .group_by(column) + .order_by(desc(func.count(column))) # pylint: disable=E1102 .limit(MAXIMUM_CHOICES + 1) ) results = connection.execute(stmt_count) @@ -430,7 +429,7 @@ def get_proposers( # pylint: disable=too-many-locals ), ] inner = ( - select(col.label("v")) + select(column.label("v")) .select_from(src_table) .order_by(Random()) .limit(self.SAMPLE_COUNT) From 8d278b63e8126578c956b5b91e1efa09feb7f69b Mon Sep 17 00:00:00 2001 From: Tim Band Date: Thu, 15 Oct 2026 13:14:02 +0100 Subject: [PATCH 19/23] Removed another literal_column Simplified the code as a result --- datafaker/proposers/choice.py | 115 +++++++--------------- tests/test_evaluators_proposal_ranking.py | 8 +- tests/test_functional_mssql.py | 5 +- tests/test_generators_dialect.py | 10 +- tests/test_interactive_generators.py | 4 +- tests/test_main.py | 13 ++- tests/test_utils_mssql.py | 8 +- 7 files changed, 65 insertions(+), 98 deletions(-) diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py index 0bc6504..38e475f 100644 --- a/datafaker/proposers/choice.py +++ b/datafaker/proposers/choice.py @@ -6,17 +6,7 @@ from abc import abstractmethod from typing import Any, Sequence, Union -from sqlalchemy import ( - Column, - CursorResult, - Engine, - desc, - func, - literal_column, - select, - table, - text, -) +from sqlalchemy import Column, CursorResult, Engine, desc, func, select from datafaker.dialects import Random from datafaker.proposers.base import ( @@ -25,7 +15,6 @@ dist_gen, fit_from_buckets, ) -from datafaker.utils import schema_qualified_name NumericType = Union[int, float] @@ -57,12 +46,10 @@ def zipf_distribution(total: int, bins: int) -> typing.Generator[int, None, None def _choice_stmt( # pylint: disable=R0913,R0917 - column_name: str, - table_name: str, + column: Column, store_counts: bool, sample_count: int | None, suppress_count: int, - table_sql: str | None = None, ) -> Any: """Build a SQLAlchemy SELECT for gathering choice value distributions. @@ -70,13 +57,10 @@ def _choice_stmt( # pylint: disable=R0913,R0917 TOP/newid() on MS-SQL. MS-SQL also forbids ORDER BY inside a subquery without TOP; this function never emits such a clause. """ - col = literal_column(f'"{column_name}"') - tbl = text(table_sql) if table_sql else table(table_name) if sample_count is not None: sample_sub = ( - select(col.label("value")) - .where(col.isnot(None)) - .select_from(tbl) + select(column.label("value")) + .where(column.isnot(None)) .order_by(Random()) .limit(sample_count) .subquery("_inner") @@ -92,12 +76,11 @@ def _choice_stmt( # pylint: disable=R0913,R0917 else: counted_sub = ( select( - col.label("value"), - func.count(col).label("count"), # pylint: disable=E1102 + column.label("value"), + func.count(column).label("count"), # pylint: disable=E1102 ) - .where(col.isnot(None)) - .select_from(tbl) - .group_by(col) + .where(column.isnot(None)) + .group_by(column) .subquery("_counted") ) out_cols = [counted_sub.c.value] @@ -119,31 +102,26 @@ class ChoiceProposer(Proposer): # pylint: disable=too-many-arguments too-many-positional-arguments def __init__( self, - table_name: str, - column_name: str, + column: Column, values: list[Any], counts: list[int], sample_count: int | None = None, suppress_count: int = 0, dialect: Any = None, - table_sql: str | None = None, ) -> None: """Initialise a ChoiceProposer.""" super().__init__() - self.table_name = table_name - self.column_name = column_name + self.column = column self.values = values estimated_counts = self.get_estimated_counts(counts) self._fit = fit_from_buckets(counts, estimated_counts) extra_comment = " and their counts" if self.STORE_COUNTS else "" stmt = _choice_stmt( - column_name, - table_name, + column, self.STORE_COUNTS, sample_count, suppress_count, - table_sql=table_sql, ) compile_opts: dict[str, Any] = {"compile_kwargs": {"literal_binds": True}} if dialect is not None: @@ -153,28 +131,28 @@ def __init__( if suppress_count == 0: if sample_count is None: self._comment = ( - f"All the values{extra_comment} that appear in column {column_name}" - f" of table {table_name}" + f"All the values{extra_comment} that appear in column {column.name}" + f" of table {column.table.name}" ) self._annotation = None else: self._comment = ( - f"The values{extra_comment} that appear in column {column_name}" - f" of a random sample of {sample_count} rows of table {table_name}" + f"The values{extra_comment} that appear in column {column.name}" + f" of a random sample of {sample_count} rows of table {column.table.name}" ) self._annotation = "sampled" else: if sample_count is None: self._comment = ( - f"All the values{extra_comment} that appear in column {column_name}" - f" of table {table_name} more than {suppress_count} times" + f"All the values{extra_comment} that appear in column {column.name}" + f" of table {column.table.name} more than {suppress_count} times" ) self._annotation = "suppressed" else: self._comment = ( f"The values{extra_comment} that appear more than {suppress_count} times" - f" in column {column_name}, out of a random sample of {sample_count} rows" - f" of table {table_name}" + f" in column {column.name}, out of a random sample of {sample_count} rows" + f" of table {column.table.name}" ) self._annotation = "sampled and suppressed" @@ -185,7 +163,7 @@ def get_estimated_counts(self, counts: list[int]) -> list[int]: def nominal_kwargs(self) -> dict[str, Any]: """Get the arguments to be entered into ``config.yaml``.""" return { - "a": f'SRC_STATS["auto__{self.table_name}__{self.column_name}"]["results"]', + "a": f'SRC_STATS["auto__{self.column.table.name}__{self.column.name}"]["results"]', } def name(self) -> str: @@ -206,7 +184,7 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: qs = super().custom_queries() return { **qs, - f"auto__{self.table_name}__{self.column_name}": { + f"auto__{self.column.table.name}__{self.column.name}": { "query": self._query, "comments": [self._comment], }, @@ -350,9 +328,6 @@ def get_proposers( # pylint: disable=too-many-locals if len(columns) != 1: return [] column = columns[0] - column_name = column.name - table_name = column.table.name - table_sql = schema_qualified_name(table_name, engine) src_table = column.table dialect = engine.dialect src_table = column.table # preserves schema for schema-qualified databases @@ -374,58 +349,46 @@ def get_proposers( # pylint: disable=too-many-locals if vg.counts: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name, - column_name, + column, vg.cvs, vg.counts, dialect=dialect, - table_sql=table_sql, ), ] if vg.counts_not_suppressed: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name=table_name, - column_name=column_name, + column=column, values=vg.cvs_not_suppressed, counts=vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), ] inner = ( @@ -449,64 +412,52 @@ def get_proposers( # pylint: disable=too-many-locals if vg.counts: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name, - column_name, + column, vg.cvs, vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, - table_sql=table_sql, ), ] if vg.counts_not_suppressed: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name=table_name, - column_name=column_name, + column=column, values=vg.cvs_not_suppressed, counts=vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), ] return generators diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py index 2e8b73c..b598fb5 100644 --- a/tests/test_evaluators_proposal_ranking.py +++ b/tests/test_evaluators_proposal_ranking.py @@ -1,6 +1,8 @@ """Unit tests for datafaker.evaluators.proposal_ranking.""" from typing import Any +from sqlalchemy import Column, MetaData, Table, Text + from datafaker.evaluators.column_evaluator import EvaluationProfile, ProposalEvaluation from datafaker.evaluators.proposal_ranking import ( ProposalRanking, @@ -227,7 +229,8 @@ def test_keyword_boost_can_flip_the_recommendation(self) -> None: def test_choice_proposer_penalized_by_resampling_on_unique_column(self) -> None: """ChoiceProposer's score is discounted by real_uniqueness x copy_fraction.""" - choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + table = Table("t", MetaData(), Column("c", Text())) + choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"], [1, 1]) results = [ make_result(overall_score=0.0, copy_fraction=1.0, proposer=choice_proposer), make_result(overall_score=0.5, copy_fraction=0.0), @@ -307,7 +310,8 @@ def test_combined_score_is_clamped_to_one(self) -> None: def test_weak_recommendation_warning_when_best_fit_was_suppressed(self) -> None: """A heavily-discounted best-fit candidate triggers the fallback warning.""" - choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + table = Table("t", MetaData(), Column("c", Text())) + choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"], [1, 1]) results = [ # Fits almost perfectly but is a resampler on a near-unique column: # gets discounted to (near) zero. diff --git a/tests/test_functional_mssql.py b/tests/test_functional_mssql.py index e0f35b7..8919186 100644 --- a/tests/test_functional_mssql.py +++ b/tests/test_functional_mssql.py @@ -17,6 +17,7 @@ from tempfile import mkstemp import yaml +from sqlalchemy import Column, MetaData, Table, Text from sqlalchemy import create_engine as sa_create_engine from sqlalchemy import text from sqlalchemy.dialects import mssql as mssql_dialect # noqa: PLC0415 @@ -121,9 +122,9 @@ def test_dialect_rand(self) -> None: """ChoiceProposer compiles its query with NEWID() not RANDOM() for mssql.""" dialect = mssql_dialect.dialect() + table = Table("manufacturer", MetaData(), Column("name", Text())) proposer = ZipfChoiceProposer( - table_name="manufacturer", - column_name="name", + column=table.c.name, values=["Blender", "Gibbs"], counts=[5, 5], sample_count=2, diff --git a/tests/test_generators_dialect.py b/tests/test_generators_dialect.py index 4ddc21b..99261c1 100644 --- a/tests/test_generators_dialect.py +++ b/tests/test_generators_dialect.py @@ -4,7 +4,7 @@ import unittest.mock from unittest.mock import MagicMock -from sqlalchemy import Column, Integer, MetaData, Select, Table, literal_column +from sqlalchemy import Column, Integer, MetaData, Select, Table, Text, literal_column from sqlalchemy.dialects import mssql, postgresql from sqlalchemy.types import DateTime @@ -146,9 +146,13 @@ class TestChoiceGeneratorStoredQuery(DatafakerTestCase): """ChoiceGenerator._query is compiled to dialect-correct SQL at construction time.""" def _make_gen(self, dialect, sample_count=None, suppress_count=0): + table = Table( + "patient", + MetaData(), + Column("gender", Text()), + ) return ZipfChoiceProposer( - table_name="patient", - column_name="gender", + column=table.c.gender, values=["M", "F"], counts=[70, 30], sample_count=sample_count, diff --git a/tests/test_interactive_generators.py b/tests/test_interactive_generators.py index 1a59e4c..f742edb 100644 --- a/tests/test_interactive_generators.py +++ b/tests/test_interactive_generators.py @@ -229,9 +229,9 @@ def test_set_generator_choice(self) -> None: gc.config["src-stats"][0]["query"], ( "SELECT _counted.value \n" - f'FROM (SELECT "{column}" AS value, count("{column}") AS count \n' + f"FROM (SELECT {table}.{column} AS value, count({table}.{column}) AS count \n" f"FROM {table} \n" - f'WHERE "{column}" IS NOT NULL GROUP BY "{column}") ' + f"WHERE {table}.{column} IS NOT NULL GROUP BY {table}.{column}) " "AS _counted ORDER BY _counted.count DESC" ), ) diff --git a/tests/test_main.py b/tests/test_main.py index 02878b1..ab9819f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -111,7 +111,10 @@ def test_make_tables( ) mock_make_tables_file.assert_called_once_with( - "postgresql://suser:spassword@shost:5432/sdbname", None, None + "postgresql://suser:spassword@shost:5432/sdbname", + None, + views_to_tables=False, + parquet_dir=None, ) mock_path.return_value.write_text.assert_called_once_with( "some text", encoding="utf-8" @@ -176,6 +179,8 @@ def test_make_tables_with_force_enabled( for force_option in ["--force", "-f"]: with self.subTest(f"Using option {force_option}"): + mock_make_tables.reset_mock() + mock_path.reset_mock() result: Result = runner.invoke( app, [ @@ -188,16 +193,14 @@ def test_make_tables_with_force_enabled( mock_make_tables.assert_called_once_with( mock_get_settings.return_value.src_dsn, mock_get_settings.return_value.src_schema, - None, + views_to_tables=False, + parquet_dir=None, ) mock_path.return_value.write_text.assert_called_once_with( mock_tables_output, encoding="utf-8" ) self.assertSuccess(result) - mock_make_tables.reset_mock() - mock_path.reset_mock() - @patch("datafaker.main.Path") @patch("datafaker.settings.get_settings") def test_incorrect_dialect_causes_nice_error_message( diff --git a/tests/test_utils_mssql.py b/tests/test_utils_mssql.py index a8abccf..44748ed 100644 --- a/tests/test_utils_mssql.py +++ b/tests/test_utils_mssql.py @@ -147,7 +147,9 @@ def test_reflect_called_with_schema(self) -> None: get_metadata(mock_engine, schema_name="myschema", views_to_tables=False) - mock_md.reflect.assert_called_once_with(mock_engine, schema="myschema") + mock_md.reflect.assert_called_once_with( + mock_engine, schema="myschema", views=False + ) def test_reflect_called_without_schema_when_none(self) -> None: """get_metadata passes schema=None to reflect when no schema_name is given.""" @@ -161,4 +163,6 @@ def test_reflect_called_without_schema_when_none(self) -> None: get_metadata(mock_engine, schema_name=None, views_to_tables=False) - mock_md.reflect.assert_called_once_with(mock_engine, schema=None) + mock_md.reflect.assert_called_once_with( + mock_engine, schema=None, views=False + ) From 273df4e8b9940c41d2e004627489d3d22c851ec5 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Thu, 15 Oct 2026 15:18:54 +0100 Subject: [PATCH 20/23] literal_column called on actual columns removed because Snowflake doesn't like this. --- datafaker/interactive/generators.py | 13 +++++++------ datafaker/interactive/table.py | 21 ++++++++------------- datafaker/proposers/mimesis.py | 5 ++--- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index 1c3f85b..ea0bb75 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -8,7 +8,7 @@ from enum import Enum from typing import Any, Callable, Optional, cast -from sqlalchemy import Column, and_, literal_column, select +from sqlalchemy import Column, and_, select from sqlalchemy.types import Integer from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private @@ -1038,12 +1038,13 @@ def _print_select_aggregate_query(self, table_name: str, prop: Proposer) -> None def _get_column_data( self, count: int, to_str: Callable[[Any], str] = repr ) -> list[list[str]]: - columns = self._get_column_names() - col_exprs = [literal_column(col) for col in columns] - nonnull_clauses = [literal_column(col).isnot(None) for col in columns] + column_names = self._get_column_names() + table = self.table_metadata() + columns = [table.columns[c] for c in column_names] + nonnull_clauses = [col.isnot(None) for col in columns] stmt = ( - select(*col_exprs) - .select_from(self.table_metadata()) + select(*columns) + .select_from(table) .where(and_(*nonnull_clauses)) .order_by(Random()) .limit(count) diff --git a/datafaker/interactive/table.py b/datafaker/interactive/table.py index 7a2852c..107a53d 100644 --- a/datafaker/interactive/table.py +++ b/datafaker/interactive/table.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import Any, cast -from sqlalchemy import func, literal_column, select +from sqlalchemy import func, select from datafaker.dialects import Random from datafaker.interactive.base import ( @@ -469,26 +469,21 @@ def complete_data( return [] return self.get_column_completions(text) - def print_column_data(self, column: str, count: int, min_length: int) -> None: + def print_column_data(self, column_name: str, count: int, min_length: int) -> None: """ Print a sample of data from a certain column of the current table. - :param column: The name of the column to report on. + :param column_name: The name of the column to report on. :param count: The number of rows to sample. :param min_length: The minimum length of text to choose from (0 for any text). """ - col_expr = literal_column(column) + table = self.table_metadata() + column = table.columns[column_name] if 0 < min_length: - where_clause = func.length(col_expr) >= min_length + where_clause = func.length(column) >= min_length else: - where_clause = col_expr.isnot(None) - stmt = ( - select(col_expr) - .select_from(self.table_metadata()) - .where(where_clause) - .order_by(Random()) - .limit(count) - ) + where_clause = column.isnot(None) + stmt = select(column).where(where_clause).order_by(Random()).limit(count) with self.sync_engine.connect() as connection: result = connection.execute(stmt) self.columnize([str(x[0]) for x in result.all()]) diff --git a/datafaker/proposers/mimesis.py b/datafaker/proposers/mimesis.py index 451ad79..dbb9282 100644 --- a/datafaker/proposers/mimesis.py +++ b/datafaker/proposers/mimesis.py @@ -5,7 +5,7 @@ import mimesis import mimesis.locales -from sqlalchemy import Column, Engine, cast, extract, func, literal_column, select +from sqlalchemy import Column, Engine, cast, extract, func, select from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.types import Date, DateTime, Integer, Numeric, String, Time @@ -187,8 +187,7 @@ def make_singleton( cls, column: Column, engine: Engine, function_name: str ) -> Sequence[Proposer]: """Make the appropriate generation configuration for this column.""" - col_expr = literal_column(column.name) - year_expr = cast(extract("year", col_expr), Integer()) + year_expr = cast(extract("year", column), Integer()) min_expr = func.min(year_expr) # pylint: disable=E1111 max_expr = func.max(year_expr) # pylint: disable=E1111 stmt = select(min_expr.label("start"), max_expr.label("end")).select_from( From 985c52a56babe6daf828d5766eacda3980e248b3 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Thu, 15 Oct 2026 17:40:13 +0100 Subject: [PATCH 21/23] Fixed missingness configuration --- datafaker/interactive/missingness.py | 2 +- tests/test_interactive_missingness.py | 43 ++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/datafaker/interactive/missingness.py b/datafaker/interactive/missingness.py index 60940a3..bc41557 100644 --- a/datafaker/interactive/missingness.py +++ b/datafaker/interactive/missingness.py @@ -215,7 +215,7 @@ def _copy_entries(self) -> None: "kwargs": { "patterns": f'SRC_STATS["{src_stat_key}"]["results"]' }, - "columns": [c.name for c in entry.new_type.columns], + "columns": [str(c.name) for c in entry.new_type.columns], } ] src_stats.append( diff --git a/tests/test_interactive_missingness.py b/tests/test_interactive_missingness.py index c1da7c5..215517f 100644 --- a/tests/test_interactive_missingness.py +++ b/tests/test_interactive_missingness.py @@ -1,7 +1,8 @@ """ Tests for the configure-missingness command. """ import random from collections.abc import MutableMapping -from typing import Any +from pathlib import Path +from typing import Any, Iterable from sqlalchemy import select @@ -10,6 +11,15 @@ from tests.utils import GeneratesDBTestCase, RequiresDBTestCase, TestDbCmdMixin +def count_nonnull(it: Iterable): + """Count the number of non-null values in an iterable.""" + n = 0 + for v in it: + if v is not None: + n += 1 + return n + + class MockMissingnessCmd(MissingnessCmd, TestDbCmdMixin): """MissingnessCmd but mocked""" @@ -107,3 +117,34 @@ def test_create_with_missingness(self) -> None: patterns.add(p + b) # all pattern possibilities should be present self.assertSetEqual(patterns, {0, 1, 2, 3}) + + +class ConfigureMissingnessEndToEndTests(ConfigureMissingnessTestsWithGeneration): + """Test `configure-missing` end-to-end.""" + + use_temporary_cwd = True + copy_files = ["row_generators.py", "story_generators.py"] + copy_from_directory = Path("examples") + + def test_missingness_end_to_end(self) -> None: + """Test that if an interval end is applicable it gets proposed and works.""" + table = "signature_model" + with self._get_cmd({}) as mc: + mc.do_next(table) + mc.do_sampled("") + mc.reset() + mc.do_quit("") + self.generate_data(mc.config, num_passes=100) + self.assertIs( + type(mc.config["tables"][table]["missingness_generators"][0]["columns"][0]), + str + ) + with self.dst_engine.connect() as conn: + stmt = select(self.metadata.tables[table]) + rows = conn.execute(stmt).mappings().fetchall() + name_count = count_nonnull(r.name for r in rows) + player_id_count = count_nonnull(r.player_id for r in rows) + based_on_count = count_nonnull(r.based_on for r in rows) + self.assertEqual(name_count, 100) + self.assertAlmostEqual(player_id_count, 50, delta=20) + self.assertAlmostEqual(based_on_count, 70, delta=20) From 613e1f42b67c743f0a65e99b1ea62766b221090f Mon Sep 17 00:00:00 2001 From: Tim Band Date: Fri, 16 Oct 2026 15:13:11 +0100 Subject: [PATCH 22/23] Snowflake timestamps cannot be written Fixes SAFEHR-data/datafaker#148 --- datafaker/dialects.py | 17 +++++++++++ datafaker/make.py | 9 ++++++ datafaker/serialize_metadata.py | 13 +++++---- docs/source/faq.rst | 51 +++++++++++++++++++++++++++------ 4 files changed, 76 insertions(+), 14 deletions(-) diff --git a/datafaker/dialects.py b/datafaker/dialects.py index 9d92e7b..2b0e98c 100644 --- a/datafaker/dialects.py +++ b/datafaker/dialects.py @@ -3,6 +3,7 @@ from collections.abc import Mapping from typing import Any, Optional, TypeVar +import snowflake.sqlalchemy.custom_types as snowflake_types from sqlalchemy import Column, Select, Table from sqlalchemy.ext.compiler import compiles from sqlalchemy.schema import CreateSchema, CreateTable @@ -515,3 +516,19 @@ def compile_random_mssql(_element: Random, _compiler: Any, **_kw: Any) -> str: same random number gets chosen for each row. """ return "NEWID()" + + +@compiles(snowflake_types.TIMESTAMP_LTZ) +def compile_timestampltz_generic( + _element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any +) -> str: + """Render TIMESTAMP_LTZ as DateTime.""" + return compiler.process(DateTime(timezone=False), **kw) + + +@compiles(snowflake_types.TIMESTAMP_LTZ, "snowflake") +def compile_timestampltz_ddb( + element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any +) -> str: + """Render TIMESTAMP_LTZ as TIMESTAMP_LTZ.""" + return compiler.process(element, **kw) diff --git a/datafaker/make.py b/datafaker/make.py index 0c74a52..1fdb10e 100644 --- a/datafaker/make.py +++ b/datafaker/make.py @@ -10,6 +10,7 @@ from typing import Any, Final, Optional, Tuple, Type import pandas as pd +import snowflake.sqlalchemy.custom_types as snowflake_types import snsql import typer import yaml @@ -58,6 +59,8 @@ if issubclass(entry, BaseProvider) and entry.__module__ == "datafaker.providers": PROVIDER_IMPORTS.append(entry_name) +DEFAULT_UNIQUE_CONSTRAINT_TRIES: int | None = 500 + @dataclass class VocabularyTableGeneratorInfo: @@ -378,6 +381,10 @@ def get_result_mappings( generator=_string_generator, choice=True, ), + snowflake_types.TIMESTAMP_LTZ: GeneratorInfo( + generator="generic.datetime.datetime", + arg_types={"start": int, "end": int}, + ), } @@ -654,6 +661,8 @@ def get_generation_info( max_unique_constraint_tries = get_property_or_none( config, "max-unique-constraint-tries", int ) + if max_unique_constraint_tries is None: + max_unique_constraint_tries = DEFAULT_UNIQUE_CONSTRAINT_TRIES return GenerationInfo( provider_imports=PROVIDER_IMPORTS, row_generator_module_name=row_generator_module_name, diff --git a/datafaker/serialize_metadata.py b/datafaker/serialize_metadata.py index b818918..fc2b555 100644 --- a/datafaker/serialize_metadata.py +++ b/datafaker/serialize_metadata.py @@ -104,7 +104,7 @@ def time_type(type_: type, tz_type: type) -> ParserType: Make a parser for a SQL date/time type. Parses TYPE_NAME, TYPE_NAME(32), TYPE_NAME WITH TIME ZONE - or TYPE_NAME(32) WITH TIME ZONE + or TYPE_NAME(32) WITH TIME ZONE, also WITHOUT :param type_: The SQLAlchemy type we would like to parse. :param tz_type: The type to instantiate when precision or timezone is @@ -205,12 +205,15 @@ def _mssql_varbinary_parser() -> typing.Generator[ParserType, None, typing.Any]: string_type(sqltypes.VARCHAR), string_type(sqltypes.NVARCHAR), string_type(sqltypes.TEXT), - parsy.string("TIMESTAMP_NTZ").result(snowflake_types.TIMESTAMP_NTZ), - parsy.string("TIMESTAMPNTZ").result(snowflake_types.TIMESTAMP_NTZ), + parsy.string("TIMESTAMP_NTZ").result(sqltypes.DATETIME), + parsy.string("TIMESTAMPNTZ").result(sqltypes.DATETIME), parsy.string("TIMESTAMP_LTZ").result(snowflake_types.TIMESTAMP_LTZ), parsy.string("TIMESTAMPLTZ").result(snowflake_types.TIMESTAMP_LTZ), - parsy.string("TIMESTAMP_TZ").result(snowflake_types.TIMESTAMP_TZ), - parsy.string("TIMESTAMPTZ").result(snowflake_types.TIMESTAMP_TZ), + parsy.string("TIMESTAMP WITH LOCAL TIME ZONE").result( + snowflake_types.TIMESTAMP_LTZ + ), + parsy.string("TIMESTAMP_TZ").result(lambda: sqltypes.DATETIME(timezone=True)), + parsy.string("TIMESTAMPTZ").result(lambda: sqltypes.DATETIME(timezone=True)), time_type(sqltypes.TIMESTAMP, postgresql.types.TIMESTAMP), time_type(sqltypes.TIME, postgresql.types.TIME), ) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index a5700f0..82bc9db 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -1,19 +1,52 @@ FAQ === -Can datafaker work with two different schemas? +Can DataFaker work with two different schemas? ********************************************** -datafaker can only work with a single source schema and a single destination schema at a time. -However, you can choose for the destination schema to have a different name to the source schema by setting the ``DST_SCHEMA`` environment variable. +DataFaker can only work with a single source schema or a single destination schema at a time. +It is possible to use the same database for source and destination by choosing a different schema for source and destination, +for example by setting ``DST_SCHEMA=fake``. Which DBMSs does datafaker support? *********************************** -* datafaker most fully supports **PostgresSQL**, which it uses for its end-to-end functional tests. -* datafaker also supports **MariaDB**, as long as you don't set ``use-asyncio: true`` in your config. -* datafaker *might* work with **SQLite** but this is largely untested. -* datafaker may also work with SQL Server. - To connect to SQL Server, you will need to install `pyodbc `_ and an `ODBC driver `_, after which you should be able to use a DSN setting similar to ``SRC_DSN="mssql+pyodbc://username:password@hostname/dbname?driver=ODBC Driver 18 for SQL Server"``. +PostgreSQL +^^^^^^^^^^ -Please open a GitHub issue if you would like to see support for another DBMS. +Fully supported. + +Microsoft SQL Server +^^^^^^^^^^^^^^^^^^^^ + +Supported. Microsoft Transact SQL probably works as well. +You need to install an `ODBC driver `_. + +The DSN format is similar to: + +``mssql+pyodbc://:@127.0.0.1:1433/?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes`` + +DuckDB +^^^^^^ + +Supported. +If you want to produce Parquet or CSV files as your output format, the easiest way is via a DuckDB database. +You do not need to install DuckDB to do this, simply use a destination DSN like ``duckdb:///./fake.db``, +then use ``datafaker dump-data --output my-csv-folder`` or ``datafaker dump-data --parquet --output my-parquet-folder``. + +An in-memory DuckDB database can be used as an empty database that you can augment with parquet files: + +``datafaker make-tables --parquet-dir my-parquet-input-folder`` + +Snowflake +^^^^^^^^^ + +Currently experimental support. If the input database includes the type ``TIMEZONE_LTZ`` (or equivalent) +then the output database must also be a Snowflake database. + +Other +^^^^^ + +In principle DataFaker can support any database that SqlAlchemy supports. +However, each new database supported so far has required significant effort, +so no guarantees can be made for, for example, MySQL or MariaDB. From 9f271e4dd7cb97787033212fd5e5b92915ee3132 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Fri, 16 Oct 2026 15:43:22 +0100 Subject: [PATCH 23/23] Fixed ty --- tests/test_interactive_missingness.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_interactive_missingness.py b/tests/test_interactive_missingness.py index 215517f..747f9e0 100644 --- a/tests/test_interactive_missingness.py +++ b/tests/test_interactive_missingness.py @@ -136,9 +136,14 @@ def test_missingness_end_to_end(self) -> None: mc.do_quit("") self.generate_data(mc.config, num_passes=100) self.assertIs( - type(mc.config["tables"][table]["missingness_generators"][0]["columns"][0]), - str + type( + mc.config["tables"][table]["missingness_generators"][0]["columns"][ + 0 + ] + ), + str, ) + assert self.dst_engine is not None with self.dst_engine.connect() as conn: stmt = select(self.metadata.tables[table]) rows = conn.execute(stmt).mappings().fetchall()