From baf6ec84d3f7af00b2b16677f2df23760b3c4630 Mon Sep 17 00:00:00 2001 From: Pascal Tomecek <40371786+ptomecek@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:54:31 -0400 Subject: [PATCH] [feature] Make pandas optional and align Frequency with polars pip install ccflow no longer installs pandas. Frequency, the context classes, normalize_date/normalize_datetime and dates_to_chunks are pure Python; PandasResult, XArrayResult, PandasFilePublisher and Frequency.offset import pandas when used, and pandas moves to the full, develop and test extras. Frequency is a value type whose canonical string is a count and a polars unit (ns, us, ms, s, m, h, d, w, mo, q, y) with the polars meaning: ns to h are fixed durations and d, w, mo, q, y are calendar units, so 1d and 24h are distinct. Equal values share one spelling (60m -> 1h, 3mo -> 1q). Frequency parses, validates and converts (n, unit, .timedelta, .offset) and does no date arithmetic: - normalize_date and normalize_datetime shift relative dates by mo/q/y keeping the day of month (polars offset_by); normalize_date accepts only calendar units. - dates_to_chunks returns polars truncate buckets counted from 1970-01-01 (weeks from Monday 1970-01-05). Chunks from 1970 on are unchanged, and multi-period chunks before 1970 now work. The pandas spellings T, min, H, S, L, U, N and D are accepted with a FutureWarning; ISO-8601 durations, timedeltas and pandas offsets are accepted and normalized. M is rejected as ambiguous, and pandas period-end, anchored, period-start, business-day and semi-month spellings and offsets are rejected because they have no equivalent. Frequency.offset returns pandas ticks, Day(n), Week(n) or DateOffset(n, months=k). PandasFilePublisher validates data without importing pandas, and the installation guide notes that pandas is optional. This is a breaking change; the pull request lists every change in behaviour. Signed-off-by: Pascal Tomecek <40371786+ptomecek@users.noreply.github.com> --- ccflow/exttypes/frequency.py | 371 +++++++++++++++--------- ccflow/publishers/file.py | 14 +- ccflow/tests/exttypes/test_frequency.py | 253 +++++++++++----- ccflow/tests/publishers/test_file.py | 3 +- ccflow/tests/test_context.py | 16 +- ccflow/tests/test_import.py | 28 ++ ccflow/tests/test_validators.py | 42 ++- ccflow/tests/utils/test_chunker.py | 61 ++-- ccflow/utils/chunker.py | 65 +++-- ccflow/validators.py | 55 ++-- docs/wiki/how-to/Installation.md | 8 + pyproject.toml | 4 +- 12 files changed, 627 insertions(+), 293 deletions(-) diff --git a/ccflow/exttypes/frequency.py b/ccflow/exttypes/frequency.py index a1f99ea8..b06ad595 100644 --- a/ccflow/exttypes/frequency.py +++ b/ccflow/exttypes/frequency.py @@ -1,163 +1,270 @@ +"""Frequency types for sampling intervals and calendar periods, following the polars duration language. + +The canonical string form is an integer count followed by a lowercase unit, e.g. ``5m`` (five minutes), ``1h``, ``1d``, ``1mo`` (one month), ``1q`` +(one quarter), ``1y`` (one year). The units mean what they mean in polars: ``ns``, ``us``, ``ms``, ``s``, ``m`` and ``h`` are fixed durations, while +``d``, ``w``, ``mo``, ``q`` and ``y`` are calendar units. ``1d`` (a calendar day) and ``24h`` (24 elapsed hours) are therefore distinct values. + +Canonical units are case-sensitive: ``m`` is minute, and ``M`` is rejected as ambiguous (use ``mo`` for month). The pandas spellings ``T``, ``min``, +``H``, ``S``, ``L``, ``U``, ``N`` and ``D`` are accepted in any case with a :class:`FutureWarning`. ISO-8601 durations (``PT5M``, ``P1M``), timedeltas +and pandas offsets are also accepted and normalized to the canonical form. pandas period-end, anchored and business-day spellings (``ME``, ``QE-DEC``, +``W-SUN``, ``B``) are rejected because they have no equivalent here. +""" + +from __future__ import annotations + +import numbers +import os import re +import sys import warnings from datetime import timedelta from functools import cache, cached_property -from packaging.version import Version +import pydantic from pydantic import TypeAdapter from pydantic_core import core_schema +__all__ = ("CalendarFrequency", "Frequency", "IntradayFrequency") + +# Each canonical unit as (kind, size, pandas offset class). Fixed sizes are in nanoseconds, day sizes in days and month sizes in months. Within a +# kind the order is coarse to fine, so ``_reduce`` can pick the coarsest unit that divides a value exactly. +_UNITS: dict[str, tuple[str, int, str | None]] = { + "y": ("month", 12, None), + "q": ("month", 3, None), + "mo": ("month", 1, None), + "w": ("day", 7, "Week"), + "d": ("day", 1, "Day"), + "h": ("fixed", 3_600_000_000_000, "Hour"), + "m": ("fixed", 60_000_000_000, "Minute"), + "s": ("fixed", 1_000_000_000, "Second"), + "ms": ("fixed", 1_000_000, "Milli"), + "us": ("fixed", 1_000, "Micro"), + "ns": ("fixed", 1, "Nano"), +} +_DAY_NS = 86_400_000_000_000 + +# pandas spellings accepted with a warning, keyed by the lower-cased token. ``m``, ``ms``, ``us`` and ``ns`` are left out so only their exact +# lowercase forms match: ``M`` is ambiguous and ``MS`` is pandas month-start. +_ALIASES: dict[str, str] = {"t": "m", "min": "m", "h": "h", "s": "s", "d": "d", "l": "ms", "u": "us", "n": "ns"} +# pandas units with no equivalent here (period ends, anchored weeks, business days and hours, period starts, semi-months). They are rejected like +# any unknown unit, and listed only so that ``normalize_date`` raises for them instead of reading them as dates. +_PANDAS_ONLY_UNITS: frozenset[str] = frozenset( + {"ME", "QE", "YE", "A", "Q", "Y", "W", "B", "C", "BH", "CBH", "BM", "BME", "CBM", "CBME", "MS", "BMS", "CBMS", "SM", "SME", "SMS", "BQ", "BQE", + "QS", "BQS", "BA", "BY", "BYE", "AS", "YS", "BAS", "BYS"} +) # fmt: skip +_KNOWN_TOKENS: frozenset[str] = frozenset(_UNITS) | frozenset(_ALIASES) | frozenset(u.lower() for u in _PANDAS_ONLY_UNITS) + +_OFFSET_RE = re.compile(r"(?P[+-]?\d+)?\s*(?P[A-Za-z]+)(?:-(?P[A-Za-z]{1,4}))?", re.ASCII) +# Several count-and-unit pairs (``1h30min``): not a valid frequency, but recognised so it is reported rather than read as a date. +_COMPOUND_RE = re.compile(r"[+-]?(?:\d+\s*[A-Za-z]+\s*)+", re.ASCII) +_ISO_SHAPE_RE = re.compile(r"[+-]?P(?=[0-9T])", re.ASCII) +# ISO-8601 durations with integer components: either one date part (``P1D``) or time parts only (``PT1H30M``). +_ISO_DATE_RE = re.compile(r"([+-]?)P(\d+)([YMWD])", re.ASCII) +_ISO_TIME_RE = re.compile(r"([+-]?)PT(?=\d)(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.ASCII) +_ISO_DATE_UNITS: dict[str, str] = {"Y": "y", "M": "mo", "W": "w", "D": "d"} + +_DATEOFFSET_UNITS: dict[str, str] = {"days": "d", "weeks": "w", "months": "mo", "years": "y"} +_PANDAS_OFFSET_UNITS: dict[str, str] = {name: unit for unit, (_, _, name) in _UNITS.items() if name} + +# Directories skipped when attributing a warning, so it points at the caller: ccflow itself (apart from its tests) and pydantic's validation. +_CCFLOW_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + os.sep +_INTERNAL_DIRS = (_CCFLOW_DIR, os.path.dirname(pydantic.__file__) + os.sep) +_CCFLOW_TESTS_DIR = os.path.join(_CCFLOW_DIR, "tests") + os.sep + + +def _looks_like_offset(value: str) -> bool: + """Whether a string is a frequency unit with an optional count (or an ISO-8601 duration), as opposed to a date-like string such as ``15Mar``.""" + value = value.strip() + if _ISO_SHAPE_RE.match(value): + return True + if match := _OFFSET_RE.fullmatch(value): + return match.group("unit").lower() in _KNOWN_TOKENS + return _COMPOUND_RE.fullmatch(value) is not None and all(t.lower() in _KNOWN_TOKENS for t in re.findall("[A-Za-z]+", value)) + + +def _reduce(n: int, unit: str) -> tuple[int, str]: + """Collapse to one canonical spelling so equal values serialize identically: ``60m`` becomes ``1h`` and ``3mo`` becomes ``1q``. + + Fixed units reduce up to hours, never into days, and month-based units reduce among ``mo``/``q``/``y``. Days and weeks are kept as written because + they bucket differently (``7d`` and ``1w`` chunk on different boundaries). Zero is ``0s`` whatever its unit. + """ + if n == 0: + return 0, "s" + kind, size, _ = _UNITS[unit] + if kind == "day": + return n, unit + total = n * size + return next((total // s, u) for u, (k, s, _) in _UNITS.items() if k == kind and total % s == 0) + + +def _from_iso(value: str) -> tuple[int, str]: + if match := _ISO_DATE_RE.fullmatch(value): + sign, n, letter = match.groups() + return _reduce(int(sign + n), _ISO_DATE_UNITS[letter]) + if match := _ISO_TIME_RE.fullmatch(value): + sign, *counts = match.groups() + total = sum(int(count or 0) * _UNITS[unit][1] for count, unit in zip(counts, ("h", "m", "s"))) + return _reduce(-total if sign == "-" else total, "ns") + raise ValueError(f"invalid ISO-8601 duration {value!r}; use one date part such as P1D or time parts only such as PT1H30M") + + +def _parse(value: str) -> tuple[int, str]: + """Parse a duration literal or ISO-8601 duration into ``(count, canonical_unit)``.""" + text = value.strip() + if _ISO_SHAPE_RE.match(text): + return _from_iso(text) + match = _OFFSET_RE.fullmatch(text) + if match and not match.group("anchor"): + token = match.group("unit") + unit = token if token in _UNITS else _ALIASES.get(token.lower()) + if unit is not None: + return _reduce(int(match.group("n") or 1), unit) + raise ValueError(f"cannot parse frequency {value!r}; write a count and one of ns, us, ms, s, m, h, d, w, mo, q or y (m is minute, mo is month)") + + +def _from_timedelta(value: timedelta) -> tuple[int, str]: + """Whole days become ``d``; anything else is a fixed duration such as ``36h``.""" + # A pandas Timedelta subclasses timedelta and adds a sub-microsecond ``nanoseconds`` component. + total_ns = value // timedelta(microseconds=1) * 1000 + getattr(value, "nanoseconds", 0) + days, remainder = divmod(total_ns, _DAY_NS) + return _reduce(days, "d") if remainder == 0 else _reduce(total_ns, "ns") + + +def _from_pandas_offset(value) -> tuple[int, str]: + name = type(value).__name__ + if value.normalize: + raise ValueError("pandas offsets with normalize=True are not supported") + if name == "DateOffset" and len(value.kwds) == 1: + [(key, count)] = value.kwds.items() + if key in _DATEOFFSET_UNITS and isinstance(count, numbers.Integral) and not isinstance(count, bool): + return _reduce(value.n * int(count), _DATEOFFSET_UNITS[key]) + elif name in _PANDAS_OFFSET_UNITS and value.kwds.get("weekday") is None: + return _reduce(value.n, _PANDAS_OFFSET_UNITS[name]) + raise ValueError( + f"unsupported pandas offset {value!r}; use Nano through Hour, Day, Week() or DateOffset with one of days, weeks, months or years" + ) + + +def _warn(message: str) -> None: + """Emit a ``FutureWarning`` attributed to the first frame outside ccflow and pydantic.""" + frame, level = sys._getframe(1), 2 + while frame is not None and frame.f_code.co_filename.startswith(_INTERNAL_DIRS) and not frame.f_code.co_filename.startswith(_CCFLOW_TESTS_DIR): + frame, level = frame.f_back, level + 1 + warnings.warn(message, FutureWarning, stacklevel=level) + + +def _coerce(value: object) -> Frequency: + """Coerce an arbitrary input to a canonical ``Frequency``, warning on pandas spellings.""" + if isinstance(value, Frequency): + n, unit = value._parts + elif isinstance(value, timedelta): + n, unit = _from_timedelta(value) + elif (pd := sys.modules.get("pandas")) is not None and isinstance(value, pd.offsets.BaseOffset): # no pandas offset exists unless it's imported + n, unit = _from_pandas_offset(value) + elif isinstance(value, str): + n, unit = _parse(value) + # Only a pandas unit spelling warns; signs, leading zeros, padding and reductions are coerced silently. + if (match := _OFFSET_RE.fullmatch(value.strip())) and match.group("unit") not in _UNITS: + _warn(f"{value!r} is a deprecated pandas frequency spelling; ccflow uses polars duration units, so write {f'{n}{unit}'!r}") + else: + raise ValueError(f"cannot convert {value!r} to a frequency") + return Frequency(f"{n}{unit}") + class Frequency(str): - """Represents a frequency string that can be converted to a pandas offset.""" + """A sampling frequency such as ``5m``, ``1h`` or ``1mo``. + + Accepts every unit; use :class:`IntradayFrequency` or :class:`CalendarFrequency` to restrict a field to sub-day or calendar frequencies. + """ - validate_always = True + def __reduce__(self): + # Pickle only the string, not the cached properties, so the pandas offset does not travel with the value. + return type(self), (str(self),) @cached_property - def offset(self) -> type: - """Return the underlying pandas DateOffset object.""" - from pandas.tseries.frequencies import to_offset + def _parts(self) -> tuple[int, str]: + return _parse(str(self)) + + @property + def n(self) -> int: + """Number of units, e.g. ``90`` for ``90m``.""" + return self._parts[0] - return to_offset(str(self)) + @property + def unit(self) -> str: + """Canonical unit: one of ``ns``, ``us``, ``ms``, ``s``, ``m``, ``h``, ``d``, ``w``, ``mo``, ``q`` or ``y``.""" + return self._parts[1] @cached_property def timedelta(self) -> timedelta: - import pandas as pd + """Equivalent timedelta; raises for months, quarters, years and nanoseconds. Days and weeks give their nominal 24-hour and 7-day length.""" + n, unit = self._parts + kind, size, _ = _UNITS[unit] + if kind == "month" or unit == "ns": + raise ValueError(f"{str(self)!r} has no fixed datetime.timedelta") + try: + return timedelta(days=n * size) if kind == "day" else timedelta(microseconds=n * size // 1000) + except OverflowError: + raise ValueError(f"{str(self)!r} is too large to represent as a timedelta") from None + + @cached_property + def offset(self): + """Equivalent pandas offset for adding to a timestamp (requires pandas); its ``n`` is the count. - return pd.to_timedelta(self.offset).to_pytimedelta() + ``ns`` to ``h`` give ``Nano`` to ``Hour``, ``d`` gives ``Day`` and ``w`` gives ``Week``; pandas 2 treats ``Day`` as 24 hours, where polars and + pandas 3 use a calendar day. ``mo``, ``q`` and ``y`` give ``DateOffset(n, months=k)``, which is not a pandas frequency: ``resample`` rejects + it, ``date_range`` applies it step by step so month-end dates drift (Jan 31, Feb 29, Mar 29), and pandas 1 also clamps the day at each step + when adding a count above 1 to a single timestamp. + """ + from pandas import offsets + + n, unit = self._parts + _, size, name = _UNITS[unit] + return getattr(offsets, name)(n) if name else offsets.DateOffset(n, months=size) @classmethod def __get_pydantic_core_schema__(cls, source_type, handler): return core_schema.no_info_plain_validator_function(cls._validate) @classmethod - def _validate(cls, value) -> "Frequency": - if isinstance(value, cls): - return cls._validate(str(value)) - - if isinstance(value, timedelta) and value.total_seconds() % 86400 == 0: - # Keep ccflow's day-sized spellings stable when pandas round-trips - # timedeltas through its newer canonical offset machinery. - # Context for the offset alias changes: - # https://pandas.pydata.org/docs/dev/whatsnew/v3.0.0.html#enforced-deprecation-of-aliases-m-q-y-etc-in-favour-of-me-qe-ye-etc-for-offsets - return cls(f"{int(value.total_seconds() // 86400)}D") - - if isinstance(value, str): - # Pandas deprecated legacy aliases like M/Q/Y in 2.2 and enforced the - # new ME/QE/YE forms in 3.0. Normalize old user-facing inputs before - # handing them to pandas. - # Exact sections: - # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#deprecate-aliases-m-q-y-etc-in-favour-of-me-qe-ye-etc-for-offsets - # https://pandas.pydata.org/docs/dev/whatsnew/v3.0.0.html#enforced-deprecation-of-aliases-m-q-y-etc-in-favour-of-me-qe-ye-etc-for-offsets - value = _normalize_frequency_alias(value) - - import pandas as pd - - if isinstance(value, (timedelta, str)): - try: - with warnings.catch_warnings(): - # Because pandas 2.2 deprecated many frequency strings (i.e. "Y", "M", "T" still in common use) - # We should consider switching away from pandas on this and supporting ISO - warnings.simplefilter("ignore", category=FutureWarning) - value = pd.tseries.frequencies.to_offset(value) - except ValueError as e: - raise ValueError(f"ensure this value can be converted to a pandas offset: {e}") - - if isinstance(value, pd.offsets.DateOffset): - return cls(f"{value.n}{value.base.freqstr}") - - raise ValueError(f"ensure this value can be converted to a pandas offset: {value}") + def _check_allowed(cls, n: int, unit: str) -> None: + """Raise ``ValueError`` if this type does not accept ``n`` steps of ``unit``; subclasses narrow the grammar here.""" + + @classmethod + def _validate(cls, value) -> Frequency: + freq = _coerce(value) + cls._check_allowed(freq.n, freq.unit) + return cls(freq) + + @classmethod + def validate(cls, value) -> Frequency: + """Validate/convert an arbitrary value to this frequency type.""" + return _type_adapter(cls).validate_python(value) + + +class IntradayFrequency(Frequency): + """A fixed sub-day frequency: a positive duration shorter than one day.""" @classmethod - def validate(cls, value) -> "Frequency": - """Try to convert/validate an arbitrary value to a Frequency.""" - return _TYPE_ADAPTER.validate_python(value) + def _check_allowed(cls, n: int, unit: str) -> None: + kind, size, _ = _UNITS[unit] + if kind != "fixed": + raise ValueError(f"IntradayFrequency requires a sub-day unit (ns/us/ms/s/m/h), got {n}{unit}") + if not 0 < n * size < _DAY_NS: + raise ValueError(f"IntradayFrequency must be positive and under one day, got {n}{unit}") -_TYPE_ADAPTER = TypeAdapter(Frequency) +class CalendarFrequency(Frequency): + """A calendar frequency: a positive count of days, weeks, months, quarters or years.""" + + @classmethod + def _check_allowed(cls, n: int, unit: str) -> None: + if n <= 0: + raise ValueError(f"CalendarFrequency must be positive, got {n}{unit}") + if _UNITS[unit][0] == "fixed": + raise ValueError(f"CalendarFrequency requires a d, w, mo, q or y unit, got {n}{unit}") @cache -def _pd_ge_22() -> bool: - import pandas as pd - - return Version(pd.__version__) >= Version("2.2") - - -_LEGACY_FREQ_PATTERN = re.compile( - r"^(?P[+-]?\d+)?(?PT|M|A|Y)(?:-(?P[A-Za-z]{3}))?$", - re.IGNORECASE, -) - -_CANONICAL_FREQ_PATTERN = re.compile( - r"^(?P[+-]?\d+)?(?PME|QE|YE)(?:-(?P[A-Za-z]{3}))?$", - re.IGNORECASE, -) - - -def _normalize_frequency_alias(value: str) -> str: - # Pandas 2.2 deprecated, and pandas 3.0 enforced removal of, several - # legacy offset aliases. Keep accepting both old and new forms at the - # ccflow API boundary and translate them to the correct spellings for the - # installed pandas version. - # Exact sections: - # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#deprecate-aliases-m-q-y-etc-in-favour-of-me-qe-ye-etc-for-offsets - # https://pandas.pydata.org/docs/dev/whatsnew/v3.0.0.html#enforced-deprecation-of-aliases-m-q-y-etc-in-favour-of-me-qe-ye-etc-for-offsets - normalized = value.strip() - if not normalized: - return normalized - - if _pd_ge_22(): - # Pandas >= 2.2 deprecated legacy aliases (M, Q, Y, A, T). - # Convert them to the canonical forms (ME, QE, YE, min). - match = _LEGACY_FREQ_PATTERN.fullmatch(normalized) - if not match: - day_match = re.fullmatch(r"(?P[+-]?\d+)?d", normalized, re.IGNORECASE) - if day_match: - return f"{day_match.group('count') or 1}D" - return normalized - - count = match.group("count") or "1" - unit = match.group("unit").upper() - suffix = (match.group("suffix") or "DEC").upper() - replacements = { - "T": f"{count}min", - "M": f"{count}ME", - "A": f"{count}YE-{suffix}", - "Y": f"{count}YE-{suffix}", - } - return replacements[unit] - else: - # Pandas < 2.2 doesn't understand the canonical aliases (ME, QE, YE). - # Convert them back to legacy forms that old pandas accepts. - match = _CANONICAL_FREQ_PATTERN.fullmatch(normalized) - if match: - count = match.group("count") or "1" - unit = match.group("unit").upper() - suffix = match.group("suffix") - if unit == "ME": - return f"{count}M" - elif unit == "QE": - return f"{count}Q" + (f"-{suffix.upper()}" if suffix else "") - elif unit == "YE": - return f"{count}A" + (f"-{suffix.upper()}" if suffix else "-DEC") - - # Also normalize case for legacy aliases. - match = _LEGACY_FREQ_PATTERN.fullmatch(normalized) - if match: - count = match.group("count") or "1" - unit = match.group("unit").upper() - suffix = (match.group("suffix") or "DEC").upper() - replacements = { - "T": f"{count}T", - "M": f"{count}M", - "A": f"{count}A-{suffix}", - "Y": f"{count}A-{suffix}", - } - return replacements[unit] - - day_match = re.fullmatch(r"(?P[+-]?\d+)?d", normalized, re.IGNORECASE) - if day_match: - return f"{day_match.group('count') or 1}D" - - return normalized +def _type_adapter(cls: type) -> TypeAdapter: + return TypeAdapter(cls) diff --git a/ccflow/publishers/file.py b/ccflow/publishers/file.py index 58083ee5..333f96a3 100644 --- a/ccflow/publishers/file.py +++ b/ccflow/publishers/file.py @@ -1,9 +1,9 @@ import pickle +import sys from collections.abc import Callable from typing import IO, Any, Generic, Literal import narwhals.stable.v1 as nw -import pandas as pd from cloudpathlib import AnyPath from pydantic import Field, field_validator from typing_extensions import override @@ -178,14 +178,24 @@ class PandasFilePublisher(BasePublisher): For large-scale exporting (using parquet), see .parquet.PandasParquetPublisher. """ - data: pd.DataFrame = None + data: Any = None # typed Any so importing this module does not import pandas; _check_data enforces the type kwargs: dict[str, Any] = Field(default_factory=dict) func: str = "to_html" # The access function must be able to write to a buffer or file-like object. suffix: str = ".html" mode: Literal["w", "wb"] = "w" + @field_validator("data") + @classmethod + def _check_data(cls, v): + pd = sys.modules.get("pandas") # a DataFrame cannot exist unless pandas is already imported + if v is not None and (pd is None or not isinstance(v, pd.DataFrame)): + raise ValueError(f"data must be a pandas DataFrame, not {type(v).__name__}") + return v + @override def __call__(self) -> AnyPath: + import pandas as pd + return GenericFilePublisher( name=self.name, name_params=self.name_params, diff --git a/ccflow/tests/exttypes/test_frequency.py b/ccflow/tests/exttypes/test_frequency.py index a7c75bc8..853c16f9 100644 --- a/ccflow/tests/exttypes/test_frequency.py +++ b/ccflow/tests/exttypes/test_frequency.py @@ -1,79 +1,194 @@ +import pickle +import warnings from datetime import timedelta from unittest import TestCase -import pandas as pd -from packaging.version import parse -from pandas.tseries.frequencies import to_offset +import pytest -from ccflow.exttypes.frequency import Frequency +from ccflow.exttypes.frequency import CalendarFrequency, Frequency, IntradayFrequency -IS_PD_22 = parse(pd.__version__) >= parse("2.2") + +class TestCanonicalContract(TestCase): + """The canonical string is used as a cache key and path segment, so equal frequencies must serialize identically regardless of how they were + supplied.""" + + def test_cross_representation_canonical(self): + pd = pytest.importorskip("pandas") + groups = { + "1h": ["60m", "3600s", "PT60M", "PT1H", timedelta(hours=1), pd.offsets.Minute(60), pd.offsets.Hour(1)], + "24h": ["1440m", "86400s", "PT24H", pd.offsets.Hour(24)], + "1d": ["P1D", timedelta(days=1), timedelta(hours=24), pd.offsets.Day(1), pd.DateOffset(days=1)], + "1us": ["1000ns", pd.offsets.Nano(1000), pd.offsets.Micro(1)], + "1mo": ["P1M", pd.DateOffset(months=1)], + "1q": ["3mo", "P3M", pd.DateOffset(months=3)], + "1y": ["12mo", "4q", pd.DateOffset(years=1)], + "1w": ["P1W", pd.offsets.Week(1), pd.DateOffset(weeks=1)], + } + for canonical, inputs in groups.items(): + for value in inputs: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self.assertEqual(str(Frequency.validate(value)), canonical, repr(value)) + + def test_zero_is_canonical(self): + self.assertEqual(str(Frequency.validate("0d")), "0s") + self.assertEqual(str(Frequency.validate("0w")), "0s") + self.assertEqual(str(Frequency.validate("0s")), "0s") + + def test_offset_roundtrip(self): + pytest.importorskip("pandas") + for value in ("5m", "1h", "1d", "1w", "1mo", "1q", "1y", "1ns"): + f = Frequency.validate(value) + self.assertEqual(Frequency.validate(f.offset), f) + self.assertNotIn(b"pandas", pickle.dumps(f)) class TestFrequency(TestCase): - def test_basic(self): - f = Frequency("5min") - self.assertIsInstance(f, str) - self.assertEqual(f.offset, to_offset("5min")) - self.assertEqual(f.timedelta, timedelta(minutes=5)) + def test_canonical_roundtrip(self): + # As in polars, 7d and 1w bucket differently and a calendar day differs from 24 elapsed hours. + for value in ("5m", "1h", "30s", "500ms", "1us", "1d", "24h", "7d", "2w", "1mo", "2q", "1y"): + with warnings.catch_warnings(): + warnings.simplefilter("error") + self.assertEqual(str(Frequency.validate(value)), value) + # Canonical units are coerced without a warning: reductions, signs, leading zeros and padding. + with warnings.catch_warnings(): + warnings.simplefilter("error") + self.assertEqual(str(Frequency.validate("60m")), "1h") + self.assertEqual(str(Frequency.validate("6mo")), "2q") + self.assertEqual(str(Frequency.validate(" +01d ")), "1d") + self.assertEqual((Frequency.validate("90m").n, Frequency.validate("90m").unit), (90, "m")) + + def test_legacy_pandas_aliases(self): + cases = { + "5min": "5m", + "5T": "5m", + "5N": "5ns", + "1min": "1m", + "1H": "1h", + "1S": "1s", + "1D": "1d", + } + for value, expected in cases.items(): + with pytest.warns(FutureWarning) as record: + self.assertEqual(str(Frequency.validate(value)), expected) + # The warning points at the caller, not at ccflow or pydantic. + self.assertEqual(record[0].filename, __file__) + + def test_iso8601_input(self): + cases = {"PT15M": "15m", "PT1H30M": "90m", "P3D": "3d", "P1W": "1w", "P1M": "1mo", "P2Y": "2y"} + with warnings.catch_warnings(): + warnings.simplefilter("error") + for value, expected in cases.items(): + self.assertEqual(str(Frequency.validate(value)), expected) + # Compound calendar durations are rejected rather than silently flattened. + self.assertRaises(ValueError, Frequency.validate, "P1Y2M") + self.assertRaises(ValueError, Frequency.validate, "P1WT0H") + for value in ("P1D1D", "P1DT", "PT1.5H", "P1DT12H"): + self.assertRaises(ValueError, Frequency.validate, value) + + def test_signed_iso_duration(self): + self.assertEqual(str(Frequency.validate("-P1D")), "-1d") + self.assertEqual(str(Frequency.validate("-PT15M")), "-15m") + self.assertEqual(str(Frequency.validate("-PT36H")), "-36h") - def test_validate_bad(self): + def test_ambiguous_M_raises(self): + for value in ("M", "1M", "5M", "-1M"): + with self.assertRaises(ValueError): + Frequency.validate(value) + + def test_unsupported_units_and_anchors_rejected(self): + # pandas period-end and anchored spellings have no equivalent, including uppercase Q/Y/W. + for value in ("1ME", "1QE-DEC", "1YE", "1A-DEC", "1Q", "1Y", "1W", "1W-SUN", "1w-SUN", "1me"): + self.assertRaises(ValueError, Frequency.validate, value) + for value in ("1B", "MS", "QS", "1W-MON", "1QE-NOV", "1mon", "2 days", "1d12h", "５m", "foo", ""): + self.assertRaises(ValueError, Frequency.validate, value) + + def test_timedelta_input(self): + self.assertEqual(str(Frequency.validate(timedelta(minutes=5))), "5m") + self.assertEqual(str(Frequency.validate(timedelta(days=14))), "14d") + self.assertEqual(str(Frequency.validate(timedelta(milliseconds=500))), "500ms") + self.assertEqual(str(Frequency.validate(timedelta(hours=36))), "36h") + + def test_pandas_timedelta_nanoseconds(self): + pd = pytest.importorskip("pandas") + self.assertEqual(str(Frequency.validate(pd.Timedelta(5, "ns"))), "5ns") + self.assertEqual(str(Frequency.validate(pd.Timedelta(-1, "ns"))), "-1ns") + + def test_nanoseconds(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + self.assertEqual(str(Frequency.validate("1ns")), "1ns") + self.assertEqual(str(Frequency.validate("100ns")), "100ns") + self.assertEqual(str(Frequency.validate("1000ns")), "1us") + self.assertEqual(str(IntradayFrequency.validate("100ns")), "100ns") + self.assertRaises(ValueError, Frequency.validate, "1NS") + # datetime.timedelta cannot hold nanoseconds. + self.assertRaises(ValueError, lambda: Frequency.validate("1ns").timedelta) + + def test_bad_input(self): self.assertRaises(ValueError, Frequency.validate, None) - self.assertRaises(ValueError, Frequency.validate, "foo") - self.assertRaises(ValueError, Frequency.validate, "") - - def test_validate_1D(self): - f = Frequency("1D") - self.assertEqual(Frequency.validate(f), f) - self.assertEqual(Frequency.validate(str(f)), f) - self.assertEqual(Frequency.validate(f.offset), f) - self.assertEqual(Frequency.validate("1d"), f) - self.assertEqual(Frequency.validate(Frequency("1d")), f) - self.assertEqual(Frequency.validate(timedelta(days=1)), f) - - def test_validate_5T(self): - if IS_PD_22: - f = Frequency("5min") - else: - f = Frequency("5T") - self.assertEqual(Frequency.validate(f), f) - self.assertEqual(Frequency.validate(str(f)), f) - self.assertEqual(Frequency.validate(f.offset), f) - self.assertEqual(Frequency.validate("5T"), f) - self.assertEqual(Frequency.validate("5min"), f) - self.assertEqual(Frequency.validate(Frequency("5T")), f) - self.assertEqual(Frequency.validate(Frequency("5min")), f) - self.assertEqual(Frequency.validate(timedelta(minutes=5)), f) - - def test_validate_1M(self): - if IS_PD_22: - f = Frequency("1ME") - else: - f = Frequency("1M") - self.assertEqual(Frequency.validate(f), f) - self.assertEqual(Frequency.validate(str(f)), f) - self.assertEqual(Frequency.validate(f.offset), f) - self.assertEqual(Frequency.validate("1m"), f) - self.assertEqual(Frequency.validate("1M"), f) - self.assertEqual(Frequency.validate(Frequency("1m")), f) - self.assertEqual(Frequency.validate(Frequency("1M")), f) - - def test_validate_1Y(self): - if IS_PD_22: - f = Frequency("1YE-DEC") - else: - f = Frequency("1A-DEC") - self.assertEqual(Frequency.validate(f), f) - self.assertEqual(Frequency.validate(str(f)), f) - self.assertEqual(Frequency.validate(f.offset), f) - self.assertEqual(Frequency.validate("1A-DEC"), f) - self.assertEqual(Frequency.validate("1y"), f) - self.assertEqual(Frequency.validate(Frequency("1A-DEC")), f) - self.assertEqual(Frequency.validate(Frequency("1y")), f) - - def test_legacy_and_canonical_aliases_normalize_to_same_frequency(self): - self.assertEqual(Frequency.validate("5T"), Frequency.validate("5min")) - self.assertEqual(Frequency.validate("1M"), Frequency.validate("1ME")) - self.assertEqual(Frequency.validate("2M"), Frequency.validate("2ME")) - self.assertEqual(Frequency.validate("1A-DEC"), Frequency.validate("1YE-DEC")) - self.assertEqual(Frequency.validate("1Y"), Frequency.validate("1YE-DEC")) + + def test_directly_constructed_iso_value(self): + self.assertEqual(Frequency("PT5M").timedelta, timedelta(minutes=5)) + + def test_timedelta_property(self): + self.assertEqual(Frequency.validate("5m").timedelta, timedelta(minutes=5)) + self.assertEqual(Frequency.validate("1w").timedelta, timedelta(weeks=1)) + # Months, quarters and years have no fixed length. + for value in ("1mo", "1q", "1y"): + self.assertRaises(ValueError, lambda v=value: Frequency.validate(v).timedelta) + self.assertRaises(ValueError, lambda: Frequency.validate("1000000000d").timedelta) + + def test_offset_matches_pandas(self): + pd = pytest.importorskip("pandas") + to_offset = pd.tseries.frequencies.to_offset + self.assertEqual(Frequency.validate("5m").offset, to_offset("5min")) + self.assertEqual(Frequency.validate("1h").offset, to_offset("h")) + self.assertEqual(Frequency.validate("3d").offset, pd.offsets.Day(3)) + self.assertEqual(Frequency.validate("2w").offset, pd.offsets.Week(2)) + self.assertEqual(Frequency.validate("1mo").offset, pd.DateOffset(months=1)) + self.assertEqual(Frequency.validate("2q").offset.n, 2) + + def test_pandas_offset_input(self): + pd = pytest.importorskip("pandas") + self.assertEqual(str(Frequency.validate(pd.offsets.Minute(5))), "5m") + self.assertEqual(str(Frequency.validate(pd.DateOffset(2, months=1))), "2mo") + # Period-end, anchored and multi-keyword offsets have no equivalent and are rejected. + for value in (pd.offsets.MonthEnd(), pd.offsets.QuarterEnd(), pd.offsets.YearEnd(), pd.offsets.Week(weekday=6), pd.offsets.BusinessDay()): + self.assertRaises(ValueError, Frequency.validate, value) + for value in (pd.DateOffset(months=1, days=1), pd.DateOffset(days=1.5), pd.DateOffset(month=1)): + self.assertRaises(ValueError, Frequency.validate, value) + self.assertRaises(ValueError, Frequency.validate, pd.DateOffset(months=1, normalize=True)) + + +class TestIntradayFrequency(TestCase): + def test_accepts_sub_day(self): + self.assertEqual(str(IntradayFrequency.validate("15m")), "15m") + self.assertEqual(str(IntradayFrequency.validate("23h")), "23h") + self.assertEqual(IntradayFrequency.validate("15m").timedelta, timedelta(minutes=15)) + + def test_rejects_non_intraday(self): + for value in ("1d", "1w", "1mo", "1y", "24h", "1440m", "0m", "-5m"): + self.assertRaises(ValueError, IntradayFrequency.validate, value) + + def test_field_validation(self): + from pydantic import BaseModel, ValidationError + + class M(BaseModel): + freq: IntradayFrequency + + self.assertIsInstance(M(freq="5m").freq, IntradayFrequency) + self.assertIsInstance(pickle.loads(pickle.dumps(M(freq="5m").freq)), IntradayFrequency) + self.assertRaises(ValidationError, M, freq="1mo") + + +class TestCalendarFrequency(TestCase): + def test_accepts_calendar(self): + for value, expected in {"1d": "1d", "2w": "2w", "1mo": "1mo", "1q": "1q", "1y": "1y"}.items(): + self.assertEqual(str(CalendarFrequency.validate(value)), expected) + + def test_rejects_non_calendar(self): + for value in ("15m", "1h", "24h"): + self.assertRaisesRegex(ValueError, "requires a d, w, mo, q or y unit", CalendarFrequency.validate, value) + for value in ("0d", "0mo", "-1mo"): + self.assertRaisesRegex(ValueError, "must be positive", CalendarFrequency.validate, value) diff --git a/ccflow/tests/publishers/test_file.py b/ccflow/tests/publishers/test_file.py index 2c461818..34c6d8a7 100644 --- a/ccflow/tests/publishers/test_file.py +++ b/ccflow/tests/publishers/test_file.py @@ -7,7 +7,7 @@ import narwhals.stable.v1 as nw import pandas as pd -from pydantic import BaseModel as PydanticBaseModel +from pydantic import BaseModel as PydanticBaseModel, ValidationError from ccflow.exttypes import NDArray from ccflow.publishers import ( @@ -125,6 +125,7 @@ def test_pandas_html(self): name_params={"param": "pandas"}, kwargs={"border": 0}, # Remove ugly HTML border! ) + self.assertRaises(ValidationError, setattr, p, "data", {"a": [1, 2, 3]}) p.data = pd.DataFrame({"a": [1, 2, 3], "b": ["foo", "bar", "baz"]}) path = p() self.assertEqual(path, Path("test_pandas.html")) diff --git a/ccflow/tests/test_context.py b/ccflow/tests/test_context.py index c3433fa5..32df5a8f 100644 --- a/ccflow/tests/test_context.py +++ b/ccflow/tests/test_context.py @@ -114,7 +114,7 @@ def test_datetime_validation(self): def test_coercion(self): d = DateContext(date=date(2022, 1, 1)) - f = FreqDateContext(freq="5T", date=date(2022, 1, 1)) + f = FreqDateContext(freq="5m", date=date(2022, 1, 1)) self.assertEqual(DateContext.model_validate(f), f) self.assertRaises(ValidationError, FreqDateContext.model_validate, d) @@ -139,12 +139,12 @@ def test_date_range_from_date(self): def test_freq(self): self.assertEqual( - FreqDateContext.model_validate("5min,2022-01-01"), - FreqDateContext(freq="5T", date=date(2022, 1, 1)), + FreqDateContext.model_validate("5m,2022-01-01"), + FreqDateContext(freq="5m", date=date(2022, 1, 1)), ) self.assertEqual( - FreqDateRangeContext.model_validate("5min,2022-01-01,2022-02-01"), - FreqDateRangeContext(freq="5T", start_date=date(2022, 1, 1), end_date=date(2022, 2, 1)), + FreqDateRangeContext.model_validate("5m,2022-01-01,2022-02-01"), + FreqDateRangeContext(freq="5m", start_date=date(2022, 1, 1), end_date=date(2022, 2, 1)), ) def test_universe(self): @@ -167,10 +167,10 @@ def test_model(self): ModelDateRangeContext(model="EULTS", start_date=date(2022, 1, 1), end_date=date(2022, 2, 1)), ) self.assertEqual( - ModelFreqDateRangeContext.model_validate("EULTS,2 min,2022-01-01,2022-02-01"), + ModelFreqDateRangeContext.model_validate("EULTS,2m,2022-01-01,2022-02-01"), ModelFreqDateRangeContext( model="EULTS", - freq="2T", + freq="2m", start_date=date(2022, 1, 1), end_date=date(2022, 2, 1), ), @@ -188,7 +188,7 @@ def test_model_source(self): def test_list_scalar_consistency(self): """Test that for the contexts with one field, validation from scalar and list of length 1 is consistent.""" - test_cases = [(UniverseContext, "US"), (FreqContext, "1D"), (DateContext, date(2024, 1, 1)), (ModelContext, "USFASTD")] + test_cases = [(UniverseContext, "US"), (FreqContext, "1d"), (DateContext, date(2024, 1, 1)), (ModelContext, "USFASTD")] for context_type, v in test_cases: context_from_scalar = context_type.model_validate(v) context_from_list = context_type.model_validate([v]) diff --git a/ccflow/tests/test_import.py b/ccflow/tests/test_import.py index 92335dcc..af00bfbc 100644 --- a/ccflow/tests/test_import.py +++ b/ccflow/tests/test_import.py @@ -62,6 +62,34 @@ def find_spec(self, name, path=None, target=None): if res.returncode != 0: raise AssertionError(res.stderr) + def test_without_pandas(self): + # pandas is optional; frequencies, relative dates and chunking must work without it. + code = textwrap.dedent( + """ + import sys + from datetime import date + + class _Uninstalled: + def find_spec(self, name, path=None, target=None): + if name.partition(".")[0] == "pandas": + raise ModuleNotFoundError(f"No module named {name!r}", name=name) + + sys.meta_path.insert(0, _Uninstalled()) + + from ccflow import DateContext, Frequency + from ccflow.publishers import PandasFilePublisher # noqa: F401 + from ccflow.publishers.file import PandasFilePublisher as _P # noqa: F401 + from ccflow.utils.chunker import dates_to_chunks + + assert Frequency.validate("3mo") == "1q" + assert isinstance(DateContext(date="-1d").date, date) + assert dates_to_chunks(date(2022, 4, 6), date(2022, 4, 7), "1mo") == [(date(2022, 4, 1), date(2022, 4, 30))] + """ + ) + res = subprocess.run([sys.executable, "-c", code], capture_output=True, check=False, text=True) + if res.returncode != 0: + raise AssertionError(res.stderr) + if __name__ == "__main__": import ccflow diff --git a/ccflow/tests/test_validators.py b/ccflow/tests/test_validators.py index 6f00cd96..a10c0b76 100644 --- a/ccflow/tests/test_validators.py +++ b/ccflow/tests/test_validators.py @@ -1,9 +1,13 @@ import logging -from datetime import datetime, timedelta -from unittest import TestCase +from calendar import monthrange +from datetime import date, datetime, timedelta +from unittest import TestCase, mock from zoneinfo import ZoneInfo -from ccflow.validators import eval_or_load_object, load_object, normalize_date, normalize_datetime, str_to_log_level +import pytest + +from ccflow import Frequency +from ccflow.validators import _shift, eval_or_load_object, load_object, normalize_date, normalize_datetime, str_to_log_level class A: @@ -17,6 +21,8 @@ def test_normalize_date(self): self.assertEqual(normalize_date("0d"), c) c1 = datetime.now().astimezone().date() - timedelta(1) self.assertEqual(normalize_date("-1d"), c1) + with pytest.warns(FutureWarning): + self.assertEqual(normalize_date("D"), normalize_date("1D")) self.assertEqual(normalize_date(datetime.now()), c) self.assertEqual(normalize_date(datetime.now().isoformat()), c) @@ -24,6 +30,14 @@ def test_normalize_date(self): self.assertEqual(normalize_date("foo"), "foo") self.assertEqual(normalize_date(None), None) + def test_normalize_date_frequency_errors_surface(self): + # Frequency-like strings with an invalid or sub-day unit raise instead of falling through to the date parser. + for bad in ("-1M", "-1ME", "-1B", "1MS", "1bh", "1NS", "-6m", "1h", "1h30min", "1000000000d"): + self.assertRaises(ValueError, normalize_date, bad) + # Other strings, including ones that start with a number but have no frequency unit, fall through to the date parser. + self.assertEqual(normalize_date("15Mar"), "15Mar") + self.assertEqual(normalize_date("2022-01-01"), date(2022, 1, 1)) + def test_normalize_datetime(self): today = datetime.today() # noqa: DTZ002 now = datetime.now() @@ -52,13 +66,21 @@ def test_normalize_datetime(self): now.astimezone(tz=ZoneInfo("US/Hawaii")), ) - def test_normalize_date_accepts_legacy_and_canonical_frequency_aliases(self): - self.assertEqual(normalize_date("1M"), normalize_date("1ME")) - self.assertEqual(normalize_date("1A-DEC"), normalize_date("1YE-DEC")) - - def test_normalize_datetime_accepts_legacy_and_canonical_frequency_aliases(self): - self.assertEqual(normalize_datetime("1M"), normalize_datetime("1ME")) - self.assertEqual(normalize_datetime("1A-DEC"), normalize_datetime("1YE-DEC")) + def test_calendar_offsets_keep_day_of_month(self): + with mock.patch("ccflow.validators._local_today", return_value=date(2024, 3, 31)): + self.assertEqual(normalize_date("-1mo"), date(2024, 2, 29)) + self.assertEqual(normalize_date("P1M"), date(2024, 4, 30)) + self.assertEqual(normalize_datetime("-1y"), datetime(2023, 3, 31)) + # Nanoseconds floor to the containing microsecond. + self.assertEqual(normalize_datetime("-1ns"), datetime(2024, 3, 30, 23, 59, 59, 999999)) + + def test_calendar_offsets_match_polars(self): + pl = pytest.importorskip("polars") + dates = [date(y, m, d) for y in (2000, 2023, 2024) for m in (1, 2, 3, 12) for d in (1, 15, 28, 29, 30, 31) if d <= monthrange(y, m)[1]] + for value in ("1mo", "-1mo", "2q", "-1q", "1y", "-3y", "13mo", "3d", "-2w"): + f = Frequency.validate(value) + expected = pl.Series(dates).dt.offset_by(value).to_list() + self.assertEqual([_shift(d, f).date() for d in dates], expected, value) def test_load_object(self): self.assertEqual(load_object("ccflow.tests.test_validators.A"), A) diff --git a/ccflow/tests/utils/test_chunker.py b/ccflow/tests/utils/test_chunker.py index 6fa98c75..d039004c 100644 --- a/ccflow/tests/utils/test_chunker.py +++ b/ccflow/tests/utils/test_chunker.py @@ -6,13 +6,13 @@ class TestChunker(unittest.TestCase): def test_dates_to_chunks_M(self): - out = dates_to_chunks(date(2022, 4, 6), date(2022, 4, 7), "ME") + out = dates_to_chunks(date(2022, 4, 6), date(2022, 4, 7), "1mo") self.assertListEqual(out, [(date(2022, 4, 1), date(2022, 4, 30))]) - out = dates_to_chunks(date(2022, 4, 1), date(2022, 4, 30), "ME") + out = dates_to_chunks(date(2022, 4, 1), date(2022, 4, 30), "1mo") self.assertListEqual(out, [(date(2022, 4, 1), date(2022, 4, 30))]) - out = dates_to_chunks(date(2022, 3, 6), date(2022, 4, 7), "ME") + out = dates_to_chunks(date(2022, 3, 6), date(2022, 4, 7), "1mo") self.assertListEqual( out, [ @@ -20,7 +20,7 @@ def test_dates_to_chunks_M(self): (date(2022, 4, 1), date(2022, 4, 30)), ], ) - out = dates_to_chunks(date(2022, 1, 6), date(2022, 4, 7), "ME") + out = dates_to_chunks(date(2022, 1, 6), date(2022, 4, 7), "1mo") self.assertListEqual( out, [ @@ -31,16 +31,20 @@ def test_dates_to_chunks_M(self): ], ) - # Using 2M chunks, make sure the chunk is the first two months of the year - out = dates_to_chunks(date(2022, 1, 6), date(2022, 1, 7), "2M") + # Using 2mo chunks, make sure the chunk is the first two months of the year + out = dates_to_chunks(date(2022, 1, 6), date(2022, 1, 7), "2mo") self.assertListEqual(out, [(date(2022, 1, 1), date(2022, 2, 28))]) - out = dates_to_chunks(date(2022, 2, 6), date(2022, 2, 7), "2M") + out = dates_to_chunks(date(2022, 2, 6), date(2022, 2, 7), "2mo") self.assertListEqual(out, [(date(2022, 1, 1), date(2022, 2, 28))]) + out = dates_to_chunks(date(1950, 2, 6), date(1950, 2, 7), "2mo") + self.assertListEqual(out, [(date(1950, 1, 1), date(1950, 2, 28))]) + self.assertListEqual(dates_to_chunks(date(2024, 3, 31), date(2024, 3, 31), "13mo"), [(date(2024, 3, 1), date(2025, 3, 31))]) + self.assertListEqual(dates_to_chunks(date(2022, 2, 6), date(2022, 2, 7), "1q"), [(date(2022, 1, 1), date(2022, 3, 31))]) def test_dates_to_chunks_d(self): - out = dates_to_chunks(date(2022, 4, 6), date(2022, 4, 6), "d") + out = dates_to_chunks(date(2022, 4, 6), date(2022, 4, 6), "1d") self.assertListEqual(out, [(date(2022, 4, 6), date(2022, 4, 6))]) - out = dates_to_chunks(date(2022, 4, 1), date(2022, 4, 6), "d") + out = dates_to_chunks(date(2022, 4, 1), date(2022, 4, 6), "1d") self.assertListEqual( out, [ @@ -61,17 +65,17 @@ def test_dates_to_chunks_d(self): out = dates_to_chunks(chunk[1], chunk[1], "2d") self.assertListEqual(out, [chunk]) - out = dates_to_chunks(chunk[1], chunk[0], "d") + out = dates_to_chunks(chunk[1], chunk[0], "1d") self.assertListEqual(out, []) def test_dates_to_chunks_trim(self): - out = dates_to_chunks(date(2022, 4, 6), date(2022, 4, 7), "ME", trim=True) + out = dates_to_chunks(date(2022, 4, 6), date(2022, 4, 7), "1mo", trim=True) self.assertListEqual(out, [(date(2022, 4, 6), date(2022, 4, 7))]) - out = dates_to_chunks(date(2022, 4, 1), date(2022, 4, 30), "ME", trim=True) + out = dates_to_chunks(date(2022, 4, 1), date(2022, 4, 30), "1mo", trim=True) self.assertListEqual(out, [(date(2022, 4, 1), date(2022, 4, 30))]) - out = dates_to_chunks(date(2022, 3, 6), date(2022, 4, 7), "ME", trim=True) + out = dates_to_chunks(date(2022, 3, 6), date(2022, 4, 7), "1mo", trim=True) self.assertListEqual( out, [ @@ -79,7 +83,7 @@ def test_dates_to_chunks_trim(self): (date(2022, 4, 1), date(2022, 4, 7)), ], ) - out = dates_to_chunks(date(2022, 1, 6), date(2022, 4, 7), "ME", trim=True) + out = dates_to_chunks(date(2022, 1, 6), date(2022, 4, 7), "1mo", trim=True) self.assertListEqual( out, [ @@ -90,8 +94,8 @@ def test_dates_to_chunks_trim(self): ], ) - # Using 2M chunks, make sure the chunk is the first two months of the year - out = dates_to_chunks(date(2022, 1, 6), date(2022, 3, 7), "2M", trim=True) + # Using 2mo chunks, make sure the chunk is the first two months of the year + out = dates_to_chunks(date(2022, 1, 6), date(2022, 3, 7), "2mo", trim=True) self.assertListEqual( out, [ @@ -99,7 +103,7 @@ def test_dates_to_chunks_trim(self): (date(2022, 3, 1), date(2022, 3, 7)), ], ) - out = dates_to_chunks(date(2022, 2, 6), date(2022, 3, 7), "2M", trim=True) + out = dates_to_chunks(date(2022, 2, 6), date(2022, 3, 7), "2mo", trim=True) self.assertListEqual( out, [ @@ -108,12 +112,25 @@ def test_dates_to_chunks_trim(self): ], ) - def test_dates_to_chunks_accepts_legacy_and_canonical_month_aliases(self): + def test_dates_to_chunks_rejects_sub_day_and_period_end(self): + for chunk_size in ("1h", "24h", "2ME"): + self.assertRaises(ValueError, dates_to_chunks, date(2024, 1, 1), date(2024, 1, 2), chunk_size) + + def test_dates_to_chunks_weekly_monday_aligned(self): + # Weeks run Monday..Sunday regardless of the query start (ISO weeks). self.assertListEqual( - dates_to_chunks(date(2022, 1, 6), date(2022, 1, 7), "2M"), - dates_to_chunks(date(2022, 1, 6), date(2022, 1, 7), "2ME"), + dates_to_chunks(date(2024, 1, 3), date(2024, 1, 20), "1w"), + [ + (date(2024, 1, 1), date(2024, 1, 7)), + (date(2024, 1, 8), date(2024, 1, 14)), + (date(2024, 1, 15), date(2024, 1, 21)), + ], ) + # Multi-week chunks are counted from Monday 1970-01-05. self.assertListEqual( - dates_to_chunks(date(2022, 1, 6), date(2022, 3, 7), "2M", trim=True), - dates_to_chunks(date(2022, 1, 6), date(2022, 3, 7), "2ME", trim=True), + dates_to_chunks(date(2024, 1, 3), date(2024, 1, 20), "2w"), + [(date(2023, 12, 25), date(2024, 1, 7)), (date(2024, 1, 8), date(2024, 1, 21))], ) + + def test_dates_to_chunks_reversed_within_one_chunk(self): + self.assertListEqual(dates_to_chunks(date(2024, 1, 2), date(2024, 1, 1), "1mo", trim=True), []) diff --git a/ccflow/utils/chunker.py b/ccflow/utils/chunker.py index 7d333ad7..b8404169 100644 --- a/ccflow/utils/chunker.py +++ b/ccflow/utils/chunker.py @@ -6,17 +6,21 @@ small, performance suffers from too many repeated trips to the underlying data store for long-range queries. """ -import warnings -from datetime import date +from datetime import date, timedelta -from ccflow.exttypes.frequency import _normalize_frequency_alias +from ccflow.exttypes.frequency import CalendarFrequency, Frequency -_MIN_END_DATE = date(1969, 12, 31) +# Chunks are polars ``truncate`` buckets: counted from 1970-01-01, and for weeks from the first Monday after it so weeks run Monday..Sunday. +_EPOCH = date(1970, 1, 1) +_WEEK_EPOCH = date(1970, 1, 5) +_ONE_DAY = timedelta(days=1) +# Months in each month-based unit, for counting month buckets. +_MONTHS = {"mo": 1, "q": 3, "y": 12} __all__ = ("dates_to_chunks",) -def dates_to_chunks(start: date, end: date, chunk_size: str = "ME", trim: bool = False) -> list[tuple[date, date]]: +def dates_to_chunks(start: date, end: date, chunk_size: str = "1mo", trim: bool = False) -> list[tuple[date, date]]: """ Chunks a date range in a consistent way (i.e. the same middle chunks will always be generated for overlapping ranges). @@ -24,33 +28,32 @@ def dates_to_chunks(start: date, end: date, chunk_size: str = "ME", trim: bool = Args: start: The start date of the time interval to convert to chunks end: The end date of the time interval to convert to chunks - chunk_size: Any valid Pandas frequency string. i.e. 'D', '2W', 'M'. + chunk_size: A positive calendar frequency, e.g. '1d', '2w', '1mo'. Chunks are counted from 1970-01-01 (weeks from Monday 1970-01-05), so any + two ranges get the same chunks for the dates they share. trim: Whether to trim the ends to match start and end date exactly (versus standard-sized chunks that cover the interval) Returns: - List of tuples of (start date, end date) for each of the chunks + List of tuples of (start date, end date) for each of the chunks; empty if ``start > end`` """ - import pandas as pd - - normalized_chunk_size = _normalize_frequency_alias(chunk_size) - with warnings.catch_warnings(): - # Because pandas 2.2 deprecated many frequency strings (i.e. "Y", "M", "T" still in common use) - # We should consider switching away from pandas on this and supporting ISO - warnings.simplefilter("ignore", category=FutureWarning) - offset = pd.tseries.frequencies.to_offset(normalized_chunk_size) - if offset.n == 1: - end_dates = pd.date_range(start - offset, end + offset, freq=normalized_chunk_size) - else: - # Need to anchor the timeline at some absolute date, because otherwise chunks might depend on the start date - # and end up overlappig each other, i.e. with 2M, would end up with - # i.e. (Jan-Feb) or (Feb,Mar) depending on whether start date was in Jan or Feb, - # instead of always returning (Jan,Feb) for any start date in either of those two months. - end_dates = pd.date_range(_MIN_END_DATE, end + offset, freq=normalized_chunk_size) - start_dates = end_dates + pd.DateOffset(1) - chunks = [(s, e) for s, e in zip(start_dates[:-1].date, end_dates[1:].date) if e >= start and s <= end] - if trim: - if chunks[0][0] < start: - chunks[0] = (start, chunks[0][1]) - if chunks[-1][-1] > end: - chunks[-1] = (chunks[-1][0], end) - return chunks + freq = CalendarFrequency.validate(chunk_size) + if start > end: + return [] + chunks = [(_bucket_start(k, freq), _bucket_start(k + 1, freq) - _ONE_DAY) for k in range(_bucket(start, freq), _bucket(end, freq) + 1)] + if trim: + chunks[0] = (max(chunks[0][0], start), chunks[0][1]) + chunks[-1] = (chunks[-1][0], min(chunks[-1][1], end)) + return chunks + + +def _bucket(d: date, freq: Frequency) -> int: + """Index of the chunk holding ``d``, counted from the epoch.""" + if freq.unit in _MONTHS: + return (12 * (d.year - _EPOCH.year) + d.month - 1) // (freq.n * _MONTHS[freq.unit]) + return (d - (_WEEK_EPOCH if freq.unit == "w" else _EPOCH)) // freq.timedelta + + +def _bucket_start(k: int, freq: Frequency) -> date: + if freq.unit in _MONTHS: + years, month = divmod(k * freq.n * _MONTHS[freq.unit], 12) + return date(_EPOCH.year + years, month + 1, 1) + return (_WEEK_EPOCH if freq.unit == "w" else _EPOCH) + k * freq.timedelta diff --git a/ccflow/validators.py b/ccflow/validators.py index daefbb64..82807d09 100644 --- a/ccflow/validators.py +++ b/ccflow/validators.py @@ -1,16 +1,19 @@ """This module contains common validators.""" +import calendar import logging -from datetime import date, datetime +from datetime import date, datetime, timedelta from typing import Any from zoneinfo import ZoneInfo from pydantic import TypeAdapter, ValidationError from .exttypes import PyObjectPath -from .exttypes.frequency import _normalize_frequency_alias +from .exttypes.frequency import Frequency, _coerce, _looks_like_offset _DatetimeAdapter = TypeAdapter(datetime) +# Months in each month-based unit, for shifting a date. +_MONTHS = {"mo": 1, "q": 3, "y": 12} __all__ = ( "eval_or_load_object", @@ -25,16 +28,36 @@ def _local_today() -> date: return datetime.now().astimezone().date() +def _shift(d: date, freq: Frequency) -> datetime: + """Shift midnight on ``d`` by ``freq`` with polars ``offset_by`` semantics. + + Months, quarters and years keep the day of month, clamped to the end of a shorter month. Nanoseconds round down to the containing microsecond, + because datetime has microsecond resolution. + """ + try: + if freq.unit in _MONTHS: + year, month = divmod(d.year * 12 + d.month - 1 + freq.n * _MONTHS[freq.unit], 12) + return datetime(year, month + 1, min(d.day, calendar.monthrange(year, month + 1)[1])) + midnight = datetime(d.year, d.month, d.day) + if freq.unit == "ns": + return midnight + timedelta(microseconds=freq.n // 1000) + return midnight + freq.timedelta + except (OverflowError, ValueError): + raise ValueError(f"frequency offset is out of range: {freq}") from None + + def normalize_date(v: Any) -> Any: - """Validator that will convert string offsets to date based on today, and convert datetime to date.""" - if isinstance(v, str): # Check case where it's an offset - try: - import pandas as pd + """Convert a relative date such as ``-1d`` or ``1mo`` to a date counted from today, and convert anything datetime-like to a date. - timestamp = pd.tseries.frequencies.to_offset(_normalize_frequency_alias(v)) + _local_today() - return timestamp.date() - except ValueError: - pass + Relative dates use the calendar units of ``CalendarFrequency`` (``d``, ``w``, ``mo``, ``q``, ``y``), with any sign. A string that looks like a + frequency but is invalid, or uses a sub-day unit, raises. + """ + if isinstance(v, str) and _looks_like_offset(v): + # A frequency-like string (-1d, -1M, 1NS, 1h30min, P1D) is parsed as a frequency and raises if invalid; other strings go to the date parser. + freq = _coerce(v) + if freq.n != 0 and freq.unit not in ("d", "w", *_MONTHS): + raise ValueError(f"a relative date needs a calendar unit (d, w, mo, q or y), got {v!r}; m is minutes and mo is months") + return _shift(_local_today(), freq).date() # Convert from anything that can be converted to a datetime to a date via datetime # This is not normally allowed by pydantic. try: @@ -47,14 +70,12 @@ def normalize_date(v: Any) -> Any: def normalize_datetime(v: Any) -> Any: - """Validator that will convert string offsets to datetime based on today, and convert datetime to date.""" - if isinstance(v, str): # Check case where it's an offset - try: - import pandas as pd + """Convert a relative datetime such as ``-1d`` or ``90m`` to a datetime counted from local midnight today, and validate anything datetime-like. - return (pd.tseries.frequencies.to_offset(_normalize_frequency_alias(v)) + _local_today()).to_pydatetime() - except ValueError: - pass + A ``dict`` or ``list`` input holds a datetime and an optional time zone name. A string that looks like a frequency but is invalid raises. + """ + if isinstance(v, str) and _looks_like_offset(v): + return _shift(_local_today(), _coerce(v)) if isinstance(v, dict): # e.g. DatetimeContext object, {"dt": datetime(...)} dt = next(iter(v.values())) diff --git a/docs/wiki/how-to/Installation.md b/docs/wiki/how-to/Installation.md index 1602f676..08dae4ab 100644 --- a/docs/wiki/how-to/Installation.md +++ b/docs/wiki/how-to/Installation.md @@ -22,6 +22,14 @@ Loading registries from YAML configs, the `_target_`-based `ModelRegistry.load_c pip install hydra-core ``` +## Install pandas + +`PandasResult`, `XArrayResult`, `PandasFilePublisher` and `Frequency.offset` (which returns a pandas offset) need [pandas](https://pandas.pydata.org). `ccflow` does not install it by default; install it directly or through the `full` extra: + +```bash +pip install pandas +``` + ## Source installation For other platforms and for development installations, [build `ccflow` from source](Build-from-Source). diff --git a/pyproject.toml b/pyproject.toml index cf9f95fa..94a7d87e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,6 @@ dependencies = [ "numpy<3", "orjson", "packaging", - "pandas", "pyarrow", "pydantic>=2.6,<3", ] @@ -60,6 +59,7 @@ full = [ "hydra-core", "IPython", "opentelemetry-api", + "pandas", "panel", "panel_material_ui", "plotly", @@ -101,6 +101,7 @@ develop = [ "duckdb", "hydra-core", "IPython", + "pandas", "panel", "panel_material_ui", "plotly", @@ -131,6 +132,7 @@ test = [ "httpx2", "hydra-core", "opentelemetry-api", + "pandas", "pytest", "pytest-asyncio", "pytest-cov",