From 1d8e05d62e7f83d08252fda0675d9bdb00cb4e68 Mon Sep 17 00:00:00 2001 From: Will Gordon Date: Fri, 30 Oct 2026 12:40:19 -0400 Subject: [PATCH 1/2] fix(generator): makes generator state fork- and thread-safe --- .github/workflows/test_lint_scan.yml | 2 +- CHANGELOG.md | 3 +- README.md | 6 +- local/tests/conftest.py | 4 +- local/tests/test_cuid.py | 229 +++++++++++++++++++++++++++ local/tests/test_utils.py | 54 +++++++ pyproject.toml | 1 + src/cuid2/generator.py | 78 +++++++-- src/cuid2/utils.py | 14 +- 9 files changed, 368 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test_lint_scan.yml b/.github/workflows/test_lint_scan.yml index 424776f..fdcd81d 100644 --- a/.github/workflows/test_lint_scan.yml +++ b/.github/workflows/test_lint_scan.yml @@ -19,7 +19,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15"] steps: - name: Checkout repository diff --git a/CHANGELOG.md b/CHANGELOG.md index 5140e23..16a229f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ This repository continues the `overflowdigital` `cuid2` package (`b02c8ca`); ver ### Added -- Python 3.15 support in CI and package classifiers. +- Python 3.15 and free-threaded Python 3.14t support in CI and package classifiers (`Programming Language :: Python :: Free Threading :: 3 - Stable`). - `Changelog` and `Issues` project URLs in the package metadata. - CLI flags `-V`/`--version`, `-l`/`--length`, `--slug` (5-character short ID preset), `-n`/`-c`/`--count`, `-f`/`--fingerprint` (used verbatim as the fingerprint, as in `@paralleldrive/cuid2`), and `--check`/`--validate` (with `-` `stdin`, skipping blank lines, and `-q`/`--quiet` support), plus `python -m cuid2` module execution. A count below `1` or an empty fingerprint exits with code `2`. @@ -22,6 +22,7 @@ This repository continues the `overflowdigital` `cuid2` package (`b02c8ca`); ver ### Fixed - CLI out-of-range lengths (`<= 0` or `> 98`) now exit with code `2` and an `argparse` usage error on `stderr` instead of an uncaught `ValueError` traceback. +- `Cuid`, `create_id()`, `cuid_wrapper()`, and `utils.create_counter()` are now thread-safe (synchronizing counter increments, lazy singleton initialization, and per-instance stateful draws across threads, including free-threaded Python `3.14t`) and fork-safe on POSIX via `os.register_at_fork` (resetting the default generator and re-initializing live `Cuid` instances with fresh locks, RNG state, counters, and child-PID fingerprints after `os.fork()`). ## [2.1.0] - 2026-10-25 diff --git a/README.md b/README.md index 79ed6a6..c53c1e6 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ by [Parallel Drive](https://github.com/paralleldrive) to Python 3. ## What is CUID2? -* Secure: It's not possible to guess the next ID. +* Secure: It's not feasible to guess the next ID (unguessability derives from the `secrets.SystemRandom` CSPRNG salt; per [RFC 9562 §8](https://www.rfc-editor.org/rfc/rfc9562.html#section-8), IDs are identifiers, not capability tokens or secrets). * Collision resistant: It's extremely unlikely to generate the same ID twice. * Horizontally scalable: Generate IDs on multiple machines without coordination. * Offline-compatible: Generate IDs without a network connection. @@ -41,7 +41,7 @@ For more information on the improvements of CUID2 over CUID, see the * **123.6-Bit Default Keyspace (`length=24`)**: 1 leading lowercase letter ($26$) followed by 23 base36 characters ($36^{23}$) yields $26 \times 36^{23} \approx \mathbf{1.61 \times 10^{37}}$ possible IDs ($\approx \mathbf{123.6\text{ bits}}$ of entropy)—slightly more than UUIDv4's 122 random bits in a 33% shorter string (24 vs. 36 chars). Reaching a 50% birthday-bound collision probability requires $\sqrt{2 \ln(2) \times 26 \times 36^{23}} \approx \mathbf{4.7 \times 10^{18}}$ IDs (generating **149 million IDs/sec continuously for 1,000 years**). * **Up to 165.0 Bits (`BIG_LENGTH = 32`)**: At `length=32`, the keyspace expands to $26 \times 36^{31} \approx \mathbf{4.54 \times 10^{49}}$ IDs ($\approx \mathbf{165.0\text{ bits}}$; 50% collision bound at $\approx 7.9 \times 10^{24}$ IDs). * **Histogram De-Biasing**: Because $2^{512} \approx 1.34 \times 10^{154}$ is not an exact power of 36, the most-significant base36 digit of a 512-bit SHA3-512 digest is biased toward low digits; CUID2 drops that leading hash character (`[1:]`) and prepends an independent uniform `[a-z]` letter so every character position has a uniform distribution. -* **Pre-Fork & Container Snapshot Resilience**: Combining `os.getpid()`, `socket.gethostname()`, environment variable keys, `time.time_ns()`, a randomly initialized monotonic counter (`INITIAL_COUNT_MAX = 476,782,367`, giving $\approx 25,700$ concurrent hosts before a 50% chance of two hosts even starting at the same counter value), and `secrets.SystemRandom` (`os.urandom`) provides defense-in-depth against PRNG or counter state collisions across forked worker processes (Gunicorn, uWSGI, Celery) and cloned VM/container snapshots. +* **Thread, Pre-Fork, & Container Snapshot Resilience**: `Cuid`, `create_id`, `cuid_wrapper`, and `create_counter` are thread-safe (including on free-threaded Python `3.14t`). On POSIX `os.fork()` (Gunicorn `--preload`, uWSGI, Celery prefork), `os.register_at_fork` automatically resets the default generator and re-initializes live `Cuid` instances with fresh locks, RNG state, random initial counters (`INITIAL_COUNT_MAX = 476,782,367`, giving $\approx 25,700$ concurrent hosts before a 50% chance of two hosts starting at the same counter value), and child-PID fingerprints (`os.getpid()`, `socket.gethostname()`, and environment variable keys). For cloned VM/container snapshots where no `fork()` syscall occurs, per-call `secrets.SystemRandom` (`os.urandom`) salt and `time.time_ns()` provide defense-in-depth against state collisions. * **Polyglot ORM Compatibility (Prisma & Drizzle)**: Generates 24-character base36 primary keys compatible with TypeScript/Node.js services using Prisma (`@default(cuid(2))` / `@onCreate(cuid(2))`) or Drizzle (`@paralleldrive/cuid2`), and replaces the deprecated CUIDv1 (`cuid`) format. | Property | CUID2 (`cuid2`) | UUIDv7 (`uuid.uuid7`) | UUIDv4 (`uuid.uuid4`) | ULID (`python-ulid`) | NanoID (`nanoid`) | @@ -53,7 +53,7 @@ For more information on the improvements of CUID2 over CUID, see the | **Entropy / Random Bits** | **~123.6 bits** (`165.0` at 32) | 74 bits / ms | 122 bits | 80 bits / ms | 126 bits | | **50% Collision Bound** | **~4.7 × 10¹⁸** (`~7.9 × 10²⁴` at 32) | ~1.6 × 10¹¹ / ms | ~2.7 × 10¹⁸ | ~1.3 × 10¹² / ms | ~1.1 × 10¹⁹ | | **Hides Creation Timestamp?** | **Yes (SHA3-512)** | No (leaks 48-bit ms) | **Yes** | No (leaks 48-bit ms) | **Yes** | -| **Fork / Snapshot Safe?** | **Yes (PID + host + CSPRNG)** | Counter can collide | OS CSPRNG only | Counter can collide | OS CSPRNG only | +| **Fork / Snapshot Safe?** | **Yes (`register_at_fork` + CSPRNG)** | Counter can collide | OS CSPRNG only | Counter can collide | OS CSPRNG only | | **B-Tree Index Locality** | Low (uniform hash) | **High (time-ordered)** | Low (uniform random) | **High (time-ordered)** | Low (uniform random) | > **When to choose UUIDv7 instead:** Because CUID2 hashes its inputs via SHA3-512, IDs are uniformly distributed across the key space. For write-heavy relational tables where B-tree insert locality and 16-byte binary `UUID` storage matter more than hiding creation timestamps, prefer Python 3.14's `uuid.uuid7()` or PostgreSQL 18's `uuidv7()`. diff --git a/local/tests/conftest.py b/local/tests/conftest.py index dd370f0..a722140 100644 --- a/local/tests/conftest.py +++ b/local/tests/conftest.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +from weakref import WeakSet import pytest from hypothesis import settings @@ -11,7 +12,8 @@ @pytest.fixture(autouse=True) def _reset_default_generator(monkeypatch: pytest.MonkeyPatch) -> None: - """Restore _default_generator after each test without importing cuid2 before coverage starts.""" + """Restore _default_generator and _instances after each test without importing cuid2 before coverage starts.""" generator_mod = sys.modules.get("cuid2.generator") if generator_mod is not None: monkeypatch.setattr(generator_mod, "_default_generator", generator_mod._default_generator) + monkeypatch.setattr(generator_mod, "_instances", WeakSet(generator_mod._instances)) diff --git a/local/tests/test_cuid.py b/local/tests/test_cuid.py index 9301d44..6d59f31 100644 --- a/local/tests/test_cuid.py +++ b/local/tests/test_cuid.py @@ -1,7 +1,14 @@ from __future__ import annotations +import gc +import multiprocessing +import os import pickle +import random +import threading +import time import warnings +import weakref from functools import partial from math import floor from secrets import SystemRandom @@ -25,6 +32,9 @@ if TYPE_CHECKING: from collections.abc import Callable + from multiprocessing.connection import Connection + from multiprocessing.process import BaseProcess + from typing import Any from unittest.mock import Mock @@ -227,6 +237,51 @@ def test_generate_uses_base36_encoding(self: TestCuid, mocker: Mock) -> None: assert len(result) == DEFAULT_LENGTH assert result == "l9j3ikop1bi8tcvzme3x3yv7" + def test_generate_thread_safety_custom_random(self: TestCuid) -> None: + """Tests that Cuid.generate() serializes calls to a non-reentrant custom random generator across threads.""" + + class _NonReentrantRandom(random.Random): + def __init__(self: _NonReentrantRandom, seed: int) -> None: + super().__init__(seed) + self._busy = False + + def random(self: _NonReentrantRandom) -> float: + if self._busy: + msg = "Concurrent access to non-reentrant random generator" + raise RuntimeError(msg) + self._busy = True + try: + time.sleep(0) + return super().random() + finally: + self._busy = False + + cuid = Cuid(random_generator=lambda: _NonReentrantRandom(42)) + num_threads = 8 + ids_per_thread = 25 + barrier = threading.Barrier(num_threads) + buckets: list[list[str]] = [[] for _ in range(num_threads)] + errors: list[BaseException] = [] + + def worker(index: int) -> None: + try: + barrier.wait() + buckets[index] = [cuid.generate() for _ in range(ids_per_thread)] + except BaseException as exc: # noqa: BLE001 + errors.append(exc) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(num_threads)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert errors == [] + generated = [cid for bucket in buckets for cid in bucket] + assert len(generated) == num_threads * ids_per_thread + assert len(set(generated)) == len(generated) + assert all(is_cuid(cid) for cid in generated) + class TestIsCuid: # Tests that is_cuid accepts well-formed identifiers, including the shortest and longest allowed ones. @@ -556,6 +611,51 @@ def test_create_id_invalid_length( with pytest.raises(ValueError, match=expected_match): create_id(length=invalid_length) + def test_create_id_concurrent_first_call_singleton(self: TestCreateId, mocker: Mock) -> None: + """Tests that concurrent first-time create_id() calls initialize _default_generator once.""" + assert cuid2.generator._default_generator is None + spy = mocker.spy(cuid2.generator, "Cuid") + num_threads = 8 + ids_per_thread = 25 + barrier = threading.Barrier(num_threads) + buckets: list[list[str]] = [[] for _ in range(num_threads)] + + def worker(index: int) -> None: + barrier.wait() + buckets[index] = [create_id() for _ in range(ids_per_thread)] + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(num_threads)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert spy.call_count == 1 + generated = [cid for bucket in buckets for cid in bucket] + assert len(generated) == num_threads * ids_per_thread + assert len(set(generated)) == len(generated) + assert all(is_cuid(cid) for cid in generated) + + def test_create_id_double_checked_locking_inner_branch( + self: TestCreateId, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Deterministically exercises the inner 'generator is not None' branch of double-checked locking.""" + assert cuid2.generator._default_generator is None + pre_created = Cuid() + + class _PrepopulatingLock: + def __enter__(self: _PrepopulatingLock) -> None: + cuid2.generator._default_generator = pre_created + + def __exit__(self: _PrepopulatingLock, *_args: object) -> None: + return None + + monkeypatch.setattr(cuid2.generator, "_default_generator_lock", _PrepopulatingLock()) + generated = create_id() + assert is_cuid(generated) + assert cuid2.generator._default_generator is pre_created + class TestCuidWrapper: def test_cuid_wrapper_is_callable(self: TestCuidWrapper) -> None: @@ -647,3 +747,132 @@ def test_cuid_wrapper_invalid_length( """Tests that cuid_wrapper raises ValueError when length is out of bounds.""" with pytest.raises(ValueError, match=expected_match): cuid_wrapper(length=invalid_length) + + +def _fork_child_worker(custom_cuid: Cuid, conn: Connection[Any, Any]) -> None: + try: + default_ids = [create_id() for _ in range(20)] + custom_ids = [custom_cuid.generate() for _ in range(20)] + conn.send((os.getpid(), custom_cuid._fingerprint, default_ids, custom_ids)) + finally: + conn.close() + + +class TestForkSafety: + def test_register_at_fork_posix_and_non_posix( + self: TestForkSafety, + monkeypatch: pytest.MonkeyPatch, + mocker: Mock, + ) -> None: + """Tests _register_at_fork() both when os.register_at_fork exists and when it is absent.""" + mock_register = mocker.Mock() + monkeypatch.setattr(os, "register_at_fork", mock_register, raising=False) + cuid2.generator._register_at_fork() + mock_register.assert_called_once_with(after_in_child=cuid2.generator._reinit_after_fork) + + monkeypatch.delattr(os, "register_at_fork", raising=False) + cuid2.generator._register_at_fork() + + def test_reinit_after_fork_resets_default_generator_and_live_instances( + self: TestForkSafety, + mocker: Mock, + ) -> None: + """Tests that _reinit_after_fork() resets _default_generator, locks, and live Cuid instances.""" + _ = create_id() + assert cuid2.generator._default_generator is not None + old_default_lock = cuid2.generator._default_generator_lock + old_instances_lock = cuid2.generator._instances_lock + + cuid = Cuid() + old_instance_lock = cuid._lock + old_random = cuid._random + old_counter = cuid._counter + old_fingerprint = cuid._fingerprint + + mocker.patch("os.getpid", return_value=os.getpid() + 10_000) + cuid2.generator._reinit_after_fork() + + assert cuid2.generator._default_generator is None + assert cuid2.generator._default_generator_lock is not old_default_lock + assert cuid2.generator._instances_lock is not old_instances_lock + assert cuid._lock is not old_instance_lock + assert cuid._random is not old_random + assert cuid._counter is not old_counter + assert cuid._fingerprint != old_fingerprint + assert is_cuid(create_id()) + + def test_reinit_after_fork_reseeds_shared_random_and_handles_seedless_rng( + self: TestForkSafety, + mocker: Mock, + ) -> None: + """Tests that _reinit_after_fork() reseeds a reused RNG instance and tolerates RNGs without .seed().""" + shared_rng = random.Random(999) # noqa: S311 + cuid_shared = Cuid(random_generator=lambda: shared_rng) + unreseeded_rng = random.Random() # noqa: S311 + unreseeded_rng.setstate(shared_rng.getstate()) + for _ in range(1 + BIG_LENGTH): + _ = unreseeded_rng.random() + + seed_spy = mocker.spy(shared_rng, "seed") + + class _SeedlessRng: + def random(self: _SeedlessRng) -> float: + return 0.25 + + seedless = _SeedlessRng() + cuid_seedless = Cuid(random_generator=lambda: seedless) # type: ignore[arg-type, return-value] + + cuid2.generator._reinit_after_fork() + + seed_spy.assert_called_once_with() + assert shared_rng.getstate() != unreseeded_rng.getstate() + assert is_cuid(cuid_shared.generate()) + assert is_cuid(cuid_seedless.generate()) + + def test_instances_weakset_allows_garbage_collection(self: TestForkSafety) -> None: + """Tests that unreferenced Cuid instances are garbage-collected and removed from _instances.""" + cuid = Cuid() + ref = weakref.ref(cuid) + assert cuid in cuid2.generator._instances + del cuid + gc.collect() + assert ref() is None + + @pytest.mark.skipif( + "fork" not in multiprocessing.get_all_start_methods(), + reason="POSIX fork start method required", + ) + def test_posix_fork_child_reinitializes_generators(self: TestForkSafety) -> None: + """Tests that forked child processes receive distinct fingerprints, RNG states, and ID streams.""" + shared_rng = random.Random(12345) # noqa: S311 + custom_cuid = Cuid(random_generator=lambda: shared_rng) + _ = create_id() + parent_fingerprint = custom_cuid._fingerprint + + ctx = multiprocessing.get_context("fork") + parent_conns: list[Connection[Any, Any]] = [] + procs: list[BaseProcess] = [] + for _ in range(2): + parent_conn, child_conn = ctx.Pipe(duplex=False) + proc: BaseProcess = ctx.Process(target=_fork_child_worker, args=(custom_cuid, child_conn)) + proc.start() + child_conn.close() + parent_conns.append(parent_conn) + procs.append(proc) + + results = [conn.recv() for conn in parent_conns] + for conn in parent_conns: + conn.close() + for proc in procs: + proc.join(timeout=10) + assert proc.exitcode == 0 + + (pid_a, fp_a, default_a, custom_a), (pid_b, fp_b, default_b, custom_b) = results + assert pid_a != os.getpid() + assert pid_b != os.getpid() + assert pid_a != pid_b + assert fp_a != parent_fingerprint + assert fp_b != parent_fingerprint + assert fp_a != fp_b + assert set(default_a).isdisjoint(set(default_b)) + assert set(custom_a).isdisjoint(set(custom_b)) diff --git a/local/tests/test_utils.py b/local/tests/test_utils.py index 9828253..fb27a05 100644 --- a/local/tests/test_utils.py +++ b/local/tests/test_utils.py @@ -2,6 +2,8 @@ import os import string +import threading +import time from typing import TYPE_CHECKING import pytest @@ -366,3 +368,55 @@ def test_count_variable_not_accessed_outside_function(self: TestCreateCounter) - counter = utils.create_counter(0) with pytest.raises(AttributeError): _ = counter.count # type: ignore[attr-defined] + + def test_counter_thread_safety_barrier(self: TestCreateCounter) -> None: + """Tests that concurrent increments across 8 threads produce a contiguous, duplicate-free sequence.""" + counter = utils.create_counter(0) + num_threads = 8 + increments_per_thread = 500 + barrier = threading.Barrier(num_threads) + buckets: list[list[int]] = [[] for _ in range(num_threads)] + + def worker(index: int) -> None: + barrier.wait() + buckets[index] = [counter() for _ in range(increments_per_thread)] + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(num_threads)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + combined = [val for bucket in buckets for val in bucket] + total = num_threads * increments_per_thread + assert len(combined) == total + assert set(combined) == set(range(1, total + 1)) + assert counter() == total + 1 + + def test_counter_thread_safety_forced_interleaving(self: TestCreateCounter) -> None: + """Tests that the counter lock prevents lost updates even when addition yields the GIL.""" + + class _YieldingInt(int): + def __add__(self: _YieldingInt, other: int) -> _YieldingInt: + time.sleep(0) + return _YieldingInt(super().__add__(other)) + + counter = utils.create_counter(_YieldingInt(0)) + num_threads = 8 + increments_per_thread = 25 + barrier = threading.Barrier(num_threads) + buckets: list[list[int]] = [[] for _ in range(num_threads)] + + def worker(index: int) -> None: + barrier.wait() + buckets[index] = [int(counter()) for _ in range(increments_per_thread)] + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(num_threads)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + combined = [val for bucket in buckets for val in bucket] + total = num_threads * increments_per_thread + assert set(combined) == set(range(1, total + 1)) diff --git a/pyproject.toml b/pyproject.toml index 02be566..29b02a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3.15", + "Programming Language :: Python :: Free Threading :: 3 - Stable", "Topic :: Security :: Cryptography", "Typing :: Typed", ] diff --git a/src/cuid2/generator.py b/src/cuid2/generator.py index e26423a..7111f03 100644 --- a/src/cuid2/generator.py +++ b/src/cuid2/generator.py @@ -3,13 +3,17 @@ from __future__ import annotations import inspect +import os import re +import threading import time import warnings +from contextlib import suppress from functools import partial from math import floor from secrets import SystemRandom from typing import TYPE_CHECKING, ClassVar, Final, Protocol +from weakref import WeakSet from cuid2 import utils from cuid2.utils import BIG_LENGTH, DEFAULT_LENGTH, INITIAL_COUNT_MAX, MAXIMUM_LENGTH, MIN_LENGTH @@ -121,7 +125,7 @@ def is_cuid(cuid_str: str, min_length: int = MIN_LENGTH, max_length: int = MAXIM class Cuid: - """Configurable CUID2 generator instance.""" + """Configurable thread- and fork-safe CUID2 generator instance.""" def __init__( self: Cuid, @@ -135,8 +139,10 @@ def __init__( Parameters ---------- random_generator : Callable[[], "Random"], default=SystemRandom - Used as the base random generator. The default value is `secrets.SystemRandom`, which is a - cryptographically secure random number generator provided by the Python standard library. + Factory callable returning the random number generator instance. Defaults to `secrets.SystemRandom`. + On POSIX `os.fork()`, `random_generator()` is re-invoked in the child process (and reseeded with OS + entropy via `.seed()` if it returns the same generator object as the parent). Fixed-seed factories + (such as `functools.partial(random.Random, 42)`) are intended for deterministic single-process tests. counter : Callable[[int], Callable[[], int]], default=utils.create_counter The `counter` parameter is a callable that creates a counter returning an incremented value each time it is called. The `create_counter` function from the `utils` module is used by default. @@ -160,11 +166,31 @@ def __init__( """ _validate_length(length) + self._random_generator: Callable[[], Random] = random_generator + self._counter_factory: Callable[[int], Callable[[], int]] = counter + self._fingerprint_factory: FingerprintCallable = fingerprint + self._lock: threading.Lock = threading.Lock() + self._random: Random = random_generator() self._counter: Callable[[], int] = counter(floor(self._random.random() * INITIAL_COUNT_MAX)) self._length: int = length self._fingerprint: str = fingerprint(random_generator=self._random) + with _instances_lock: + _instances.add(self) + + def _reinit_after_fork(self: Cuid) -> None: + """Re-initialize lock, random generator, counter, and fingerprint in a forked child process.""" + self._lock = threading.Lock() + prev_random: Random = self._random + new_random: Random = self._random_generator() + if new_random is prev_random: + with suppress(AttributeError, NotImplementedError, TypeError): + new_random.seed() + self._random = new_random + self._counter = self._counter_factory(floor(self._random.random() * INITIAL_COUNT_MAX)) + self._fingerprint = self._fingerprint_factory(random_generator=self._random) + def generate(self: Cuid, length: int | None = None) -> str: """Generate a universally unique, base36-encoded string with a specified length. @@ -197,24 +223,47 @@ def generate(self: Cuid, length: int | None = None) -> str: _validate_length(length) resolved_length: int = self._length if length is None else length - first_letter: str = utils.create_letter(random_generator=self._random) + with self._lock: + first_letter: str = utils.create_letter(random_generator=self._random) + base36_count: str = utils.base36_encode(self._counter()) + salt: str = utils.create_entropy(length=resolved_length, random_generator=self._random) + fingerprint: str = self._fingerprint base36_time: str = utils.base36_encode(time.time_ns()) - base36_count: str = utils.base36_encode(self._counter()) - - salt: str = utils.create_entropy(length=resolved_length, random_generator=self._random) - hash_input: str = base36_time + salt + base36_count + self._fingerprint + hash_input: str = base36_time + salt + base36_count + fingerprint return first_letter + utils.create_hash(hash_input)[1:resolved_length] is_cuid: ClassVar[IsCuidCallable] = staticmethod(is_cuid) +_instances: Final[WeakSet[Cuid]] = WeakSet() +_instances_lock: threading.Lock = threading.Lock() _default_generator: Cuid | None = None +_default_generator_lock: threading.Lock = threading.Lock() + + +def _reinit_after_fork() -> None: + """Reset the lazy default generator and re-initialize live Cuid instances after os.fork().""" + global _default_generator, _default_generator_lock, _instances_lock # noqa: PLW0603 + _default_generator = None + _default_generator_lock = threading.Lock() + _instances_lock = threading.Lock() + for instance in list(_instances): + instance._reinit_after_fork() # noqa: SLF001 # pyright: ignore[reportPrivateUsage] + + +def _register_at_fork() -> None: + """Register the POSIX post-fork callback when supported by the current platform.""" + if hasattr(os, "register_at_fork"): + os.register_at_fork(after_in_child=_reinit_after_fork) + + +_register_at_fork() def create_id(length: int | None = None) -> str: - """Generate a CUID2 string using the module-level lazy default generator. + """Generate a CUID2 string using the thread- and fork-safe module-level lazy default generator. Parameters ---------- @@ -239,9 +288,14 @@ def create_id(length: int | None = None) -> str: If the length parameter is outside `MIN_LENGTH`..`BIG_LENGTH` (2..32 characters). """ global _default_generator # noqa: PLW0603 - if _default_generator is None: - _default_generator = Cuid() - return _default_generator.generate(length=length) + generator = _default_generator + if generator is None: + with _default_generator_lock: + generator = _default_generator + if generator is None: + generator = Cuid() + _default_generator = generator + return generator.generate(length=length) def cuid_wrapper(length: int = DEFAULT_LENGTH) -> Callable[[], str]: diff --git a/src/cuid2/utils.py b/src/cuid2/utils.py index 74ae87c..229b1bb 100644 --- a/src/cuid2/utils.py +++ b/src/cuid2/utils.py @@ -5,11 +5,12 @@ import os import socket import string +import threading from hashlib import sha3_512 from math import floor from typing import TYPE_CHECKING, Final -# ~22k hosts before 50% chance of initial counter collision +# ~25.7k hosts before 50% chance of initial counter collision # with a remaining counter range of 9.0e+15 in JavaScript. INITIAL_COUNT_MAX: Final[int] = 476782367 MIN_LENGTH: Final[int] = 2 @@ -29,7 +30,7 @@ def create_counter(count: int) -> Callable[[], int]: - """Create a counter that returns an incremented value each time it is called. + """Create a thread-safe counter that returns an incremented value each time it is called. Parameters ---------- @@ -41,13 +42,16 @@ def create_counter(count: int) -> Callable[[], int]: Callable[[], int] Closure function `counter` that takes no arguments and returns an integer. The closure function keeps track of a count variable that is initialized with the value passed as an argument to `create_counter`. Each time - the closure function is called, it increments the count variable and returns its new value. + the closure function is called, it atomically increments the count variable under a lock and returns its + new value. """ + lock = threading.Lock() def counter() -> int: nonlocal count - count += 1 - return count + with lock: + count += 1 + return count return counter From 95b6950f124474482acaf9bcd65a0c2990cfc6b4 Mon Sep 17 00:00:00 2001 From: Will Gordon Date: Fri, 30 Oct 2026 14:22:45 -0400 Subject: [PATCH 2/2] fix(generator): re-initializes generator state lazily per process Replaces the os.register_at_fork hook and WeakSet registry with a per-process state bundle that the first generate() call in a new process rebuilds once, covering forks that skip Python's fork hooks (uWSGI default mode). Switches to re-entrant locks, adds Cuid.__deepcopy__, adds 3.15t to CI, and adds free-threading regression guards. --- .github/workflows/test_lint_scan.yml | 2 +- CHANGELOG.md | 5 +- README.md | 4 +- local/tests/conftest.py | 5 +- local/tests/test_cuid.py | 402 ++++++++++++++++++++------- local/tests/test_free_threading.py | 40 +++ local/tests/test_utils.py | 30 +- src/cuid2/generator.py | 150 ++++++---- src/cuid2/utils.py | 2 +- 9 files changed, 479 insertions(+), 161 deletions(-) create mode 100644 local/tests/test_free_threading.py diff --git a/.github/workflows/test_lint_scan.yml b/.github/workflows/test_lint_scan.yml index fdcd81d..9c66e2a 100644 --- a/.github/workflows/test_lint_scan.yml +++ b/.github/workflows/test_lint_scan.yml @@ -19,7 +19,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"] steps: - name: Checkout repository diff --git a/CHANGELOG.md b/CHANGELOG.md index 16a229f..42a941a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ This repository continues the `overflowdigital` `cuid2` package (`b02c8ca`); ver ### Added -- Python 3.15 and free-threaded Python 3.14t support in CI and package classifiers (`Programming Language :: Python :: Free Threading :: 3 - Stable`). +- Python 3.15 and free-threaded Python 3.14t and 3.15t support in CI and package classifiers (`Programming Language :: Python :: Free Threading :: 3 - Stable`). - `Changelog` and `Issues` project URLs in the package metadata. - CLI flags `-V`/`--version`, `-l`/`--length`, `--slug` (5-character short ID preset), `-n`/`-c`/`--count`, `-f`/`--fingerprint` (used verbatim as the fingerprint, as in `@paralleldrive/cuid2`), and `--check`/`--validate` (with `-` `stdin`, skipping blank lines, and `-q`/`--quiet` support), plus `python -m cuid2` module execution. A count below `1` or an empty fingerprint exits with code `2`. @@ -22,7 +22,8 @@ This repository continues the `overflowdigital` `cuid2` package (`b02c8ca`); ver ### Fixed - CLI out-of-range lengths (`<= 0` or `> 98`) now exit with code `2` and an `argparse` usage error on `stderr` instead of an uncaught `ValueError` traceback. -- `Cuid`, `create_id()`, `cuid_wrapper()`, and `utils.create_counter()` are now thread-safe (synchronizing counter increments, lazy singleton initialization, and per-instance stateful draws across threads, including free-threaded Python `3.14t`) and fork-safe on POSIX via `os.register_at_fork` (resetting the default generator and re-initializing live `Cuid` instances with fresh locks, RNG state, counters, and child-PID fingerprints after `os.fork()`). +- `Cuid`, `create_id()`, `cuid_wrapper()`, and `utils.create_counter()` are now thread-safe (synchronizing counter increments, lazy singleton initialization, and per-instance stateful draws across threads, including free-threaded Python, where unsynchronized counters returned roughly 323,000–380,000 unique values from 1,600,000 calls on 8 threads, with re-entrant locks so a signal handler or finalizer that generates an ID cannot deadlock) and fork-safe (the first ID generated in a forked child process, including uWSGI workers that skip Python's fork hooks, rebuilds the lock, RNG state, counter, and child-PID fingerprint once, even when several child threads race; fixed-seed `random_generator` factories still give every child the same stream). +- `copy.deepcopy()` of a `Cuid` now returns an independent generator that shares the factories and builds fresh RNG state, counter, and fingerprint, instead of raising `NotImplementedError` for the default `SystemRandom` or cloning a seeded generator's stream and sharing its counter. ## [2.1.0] - 2026-10-25 diff --git a/README.md b/README.md index c53c1e6..d63ad83 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ For more information on the improvements of CUID2 over CUID, see the * **123.6-Bit Default Keyspace (`length=24`)**: 1 leading lowercase letter ($26$) followed by 23 base36 characters ($36^{23}$) yields $26 \times 36^{23} \approx \mathbf{1.61 \times 10^{37}}$ possible IDs ($\approx \mathbf{123.6\text{ bits}}$ of entropy)—slightly more than UUIDv4's 122 random bits in a 33% shorter string (24 vs. 36 chars). Reaching a 50% birthday-bound collision probability requires $\sqrt{2 \ln(2) \times 26 \times 36^{23}} \approx \mathbf{4.7 \times 10^{18}}$ IDs (generating **149 million IDs/sec continuously for 1,000 years**). * **Up to 165.0 Bits (`BIG_LENGTH = 32`)**: At `length=32`, the keyspace expands to $26 \times 36^{31} \approx \mathbf{4.54 \times 10^{49}}$ IDs ($\approx \mathbf{165.0\text{ bits}}$; 50% collision bound at $\approx 7.9 \times 10^{24}$ IDs). * **Histogram De-Biasing**: Because $2^{512} \approx 1.34 \times 10^{154}$ is not an exact power of 36, the most-significant base36 digit of a 512-bit SHA3-512 digest is biased toward low digits; CUID2 drops that leading hash character (`[1:]`) and prepends an independent uniform `[a-z]` letter so every character position has a uniform distribution. -* **Thread, Pre-Fork, & Container Snapshot Resilience**: `Cuid`, `create_id`, `cuid_wrapper`, and `create_counter` are thread-safe (including on free-threaded Python `3.14t`). On POSIX `os.fork()` (Gunicorn `--preload`, uWSGI, Celery prefork), `os.register_at_fork` automatically resets the default generator and re-initializes live `Cuid` instances with fresh locks, RNG state, random initial counters (`INITIAL_COUNT_MAX = 476,782,367`, giving $\approx 25,700$ concurrent hosts before a 50% chance of two hosts starting at the same counter value), and child-PID fingerprints (`os.getpid()`, `socket.gethostname()`, and environment variable keys). For cloned VM/container snapshots where no `fork()` syscall occurs, per-call `secrets.SystemRandom` (`os.urandom`) salt and `time.time_ns()` provide defense-in-depth against state collisions. +* **Thread, Pre-Fork, & Container Snapshot Resilience**: `Cuid`, `create_id`, `cuid_wrapper`, and `create_counter` are thread-safe (including on free-threaded Python, tested on `3.14t` and `3.15t`). Each generator records the process ID (`os.getpid()`) that built its state; the first ID generated in a forked child process (`multiprocessing`, Gunicorn, Celery prefork, and uWSGI workers, including uWSGI's default mode, which skips Python's `os.register_at_fork` hooks) rebuilds that state with a fresh lock, RNG state, random initial counter (`INITIAL_COUNT_MAX = 476,782,367`, giving $\approx 25,700$ concurrent hosts before a 50% chance of two hosts starting at the same counter value), and child-PID fingerprints (`os.getpid()`, `socket.gethostname()`, and environment variable keys). For cloned VM/container snapshots where no `fork()` syscall occurs, per-call `secrets.SystemRandom` (`os.urandom`) salt and `time.time_ns()` provide defense-in-depth against state collisions. * **Polyglot ORM Compatibility (Prisma & Drizzle)**: Generates 24-character base36 primary keys compatible with TypeScript/Node.js services using Prisma (`@default(cuid(2))` / `@onCreate(cuid(2))`) or Drizzle (`@paralleldrive/cuid2`), and replaces the deprecated CUIDv1 (`cuid`) format. | Property | CUID2 (`cuid2`) | UUIDv7 (`uuid.uuid7`) | UUIDv4 (`uuid.uuid4`) | ULID (`python-ulid`) | NanoID (`nanoid`) | @@ -53,7 +53,7 @@ For more information on the improvements of CUID2 over CUID, see the | **Entropy / Random Bits** | **~123.6 bits** (`165.0` at 32) | 74 bits / ms | 122 bits | 80 bits / ms | 126 bits | | **50% Collision Bound** | **~4.7 × 10¹⁸** (`~7.9 × 10²⁴` at 32) | ~1.6 × 10¹¹ / ms | ~2.7 × 10¹⁸ | ~1.3 × 10¹² / ms | ~1.1 × 10¹⁹ | | **Hides Creation Timestamp?** | **Yes (SHA3-512)** | No (leaks 48-bit ms) | **Yes** | No (leaks 48-bit ms) | **Yes** | -| **Fork / Snapshot Safe?** | **Yes (`register_at_fork` + CSPRNG)** | Counter can collide | OS CSPRNG only | Counter can collide | OS CSPRNG only | +| **Fork / Snapshot Safe?** | **Yes (per-PID state + CSPRNG)** | Counter can collide | OS CSPRNG only | Counter can collide | OS CSPRNG only | | **B-Tree Index Locality** | Low (uniform hash) | **High (time-ordered)** | Low (uniform random) | **High (time-ordered)** | Low (uniform random) | > **When to choose UUIDv7 instead:** Because CUID2 hashes its inputs via SHA3-512, IDs are uniformly distributed across the key space. For write-heavy relational tables where B-tree insert locality and 16-byte binary `UUID` storage matter more than hiding creation timestamps, prefer Python 3.14's `uuid.uuid7()` or PostgreSQL 18's `uuidv7()`. diff --git a/local/tests/conftest.py b/local/tests/conftest.py index a722140..89241a2 100644 --- a/local/tests/conftest.py +++ b/local/tests/conftest.py @@ -1,7 +1,6 @@ from __future__ import annotations import sys -from weakref import WeakSet import pytest from hypothesis import settings @@ -12,8 +11,8 @@ @pytest.fixture(autouse=True) def _reset_default_generator(monkeypatch: pytest.MonkeyPatch) -> None: - """Restore _default_generator and _instances after each test without importing cuid2 before coverage starts.""" + """Restore the default generator and per-process locks after each test without importing cuid2 before coverage.""" generator_mod = sys.modules.get("cuid2.generator") if generator_mod is not None: monkeypatch.setattr(generator_mod, "_default_generator", generator_mod._default_generator) - monkeypatch.setattr(generator_mod, "_instances", WeakSet(generator_mod._instances)) + monkeypatch.setattr(generator_mod, "_process_locks", dict(generator_mod._process_locks)) diff --git a/local/tests/test_cuid.py b/local/tests/test_cuid.py index 6d59f31..98baa59 100644 --- a/local/tests/test_cuid.py +++ b/local/tests/test_cuid.py @@ -1,6 +1,6 @@ from __future__ import annotations -import gc +import copy import multiprocessing import os import pickle @@ -8,7 +8,6 @@ import threading import time import warnings -import weakref from functools import partial from math import floor from secrets import SystemRandom @@ -159,10 +158,10 @@ def test_constructor_default_values(self: TestCuid) -> None: cuid = Cuid() # Assert that the values are set to their default values - assert isinstance(cuid._random, SystemRandom) - assert callable(cuid._counter) + assert isinstance(cuid._state.random, SystemRandom) + assert callable(cuid._state.counter) assert cuid._length == DEFAULT_LENGTH - assert isinstance(cuid._fingerprint, str) + assert isinstance(cuid._state.fingerprint, str) # Tests that the random_generator and fingerprint functions are called correctly when mocked. def test_mock_random_generator_and_fingerprint(self: TestCuid, mocker: Mock) -> None: @@ -184,9 +183,9 @@ def test_mock_random_generator_and_fingerprint(self: TestCuid, mocker: Mock) -> # Generate a Cuid and assert that the mocked functions were called correctly cuid_str = cuid.generate() assert cuid_str == "locked_hash" # first letter is "l" from mocked create_letter function - assert cuid._fingerprint == "mocked_fingerprint" + assert cuid._state.fingerprint == "mocked_fingerprint" assert ( - cuid._counter() == 238391185 + cuid._state.counter() == 238391185 ) # counter is statically assigned since mocked random always returns the same value # Tests that generate() returns a string of specified length. @@ -209,10 +208,10 @@ def test_constructor_initializes_members(self: TestCuid, mocker: Mock) -> None: cuid = Cuid(random_generator=mock_random_class, counter=mock_counter, fingerprint=mock_fingerprint) # Assert - assert cuid._random == mock_random_class() - assert cuid._counter == mock_counter(floor(cuid._random.random() * INITIAL_COUNT_MAX)) + assert cuid._state.random == mock_random_class() + assert cuid._state.counter == mock_counter(floor(cuid._state.random.random() * INITIAL_COUNT_MAX)) assert cuid._length == DEFAULT_LENGTH - assert cuid._fingerprint == mock_fingerprint(random_generator=cuid._random) + assert cuid._state.fingerprint == mock_fingerprint(random_generator=cuid._state.random) # Tests that generate() uses base36 encoding and creates a hash. def test_generate_uses_base36_encoding(self: TestCuid, mocker: Mock) -> None: @@ -282,6 +281,89 @@ def worker(index: int) -> None: assert len(set(generated)) == len(generated) assert all(is_cuid(cid) for cid in generated) + def test_generate_reentrant_call_on_same_thread_does_not_deadlock(self: TestCuid) -> None: + """Tests that generate() re-entered on the same thread (as from a signal handler) completes.""" + + class _ReentrantRandom(random.Random): + def __init__(self: _ReentrantRandom, seed: int) -> None: + super().__init__(seed) + self.cuid: Cuid | None = None + self.entered = False + self.nested: list[str] = [] + + def random(self: _ReentrantRandom) -> float: + if self.cuid is not None and not self.entered: + self.entered = True + self.nested.append(self.cuid.generate()) + return super().random() + + rng = _ReentrantRandom(1) + cuid = Cuid(random_generator=lambda: rng) + rng.cuid = cuid + outer: list[str] = [] + thread = threading.Thread(target=lambda: outer.append(cuid.generate()), daemon=True) + thread.start() + thread.join(timeout=10) + + assert not thread.is_alive(), "re-entrant generate() deadlocked on the instance lock" + assert len(outer) == 1 + assert len(rng.nested) == 1 + assert is_cuid(outer[0]) + assert is_cuid(rng.nested[0]) + assert outer[0] != rng.nested[0] + + def test_deepcopy_returns_independent_generator(self: TestCuid) -> None: + """Tests that deepcopy builds fresh state instead of cloning the lock, RNG stream, or counter.""" + cuid = Cuid(random_generator=random.Random, length=10) + clone = copy.deepcopy(cuid) + + assert type(clone) is Cuid + assert clone._length == 10 + assert clone._state.lock is not cuid._state.lock + assert clone._state.random is not cuid._state.random + assert clone._state.counter is not cuid._state.counter + assert len(clone.generate()) == 10 + assert is_cuid(copy.deepcopy(Cuid()).generate()) + + first, second = copy.deepcopy([cuid, cuid]) + assert first is second + + def test_deepcopy_shares_factories_and_never_clones_their_rng(self: TestCuid) -> None: + """Tests that deepcopy shares factory objects, so an RNG held by a factory is reused rather than cloned.""" + + class _RngSource: + def __init__(self: _RngSource, seed: int) -> None: + self.rng = random.Random(seed) # noqa: S311 + + def __call__(self: _RngSource) -> random.Random: + return self.rng + + source = _RngSource(7) + cuid = Cuid(random_generator=source) + clone = copy.deepcopy(cuid) + + assert clone._random_generator is source + assert clone._state.random is cuid._state.random + assert clone._counter_factory is cuid._counter_factory + assert clone._fingerprint_factory is cuid._fingerprint_factory + + def test_deepcopy_preserves_self_references_and_copies_other_attributes(self: TestCuid) -> None: + """Tests that deepcopy registers the clone in the memo and deep-copies subclass attributes.""" + + class _TaggedCuid(Cuid): + def __init__(self: _TaggedCuid) -> None: + super().__init__() + self.me: _TaggedCuid = self + self.tags: list[str] = ["a"] + + cuid = _TaggedCuid() + clone = copy.deepcopy(cuid) + + assert type(clone) is _TaggedCuid + assert clone.me is clone + assert clone.tags == ["a"] + assert clone.tags is not cuid.tags + class TestIsCuid: # Tests that is_cuid accepts well-formed identifiers, including the shortest and longest allowed ones. @@ -614,7 +696,15 @@ def test_create_id_invalid_length( def test_create_id_concurrent_first_call_singleton(self: TestCreateId, mocker: Mock) -> None: """Tests that concurrent first-time create_id() calls initialize _default_generator once.""" assert cuid2.generator._default_generator is None - spy = mocker.spy(cuid2.generator, "Cuid") + real_cuid = cuid2.generator.Cuid + + def slow_cuid() -> Cuid: + # Widen the race window so every thread reaches the None check while the first is still constructing, + # which makes an unsynchronized singleton fail deterministically on GIL and free-threaded builds alike. + time.sleep(0.05) + return real_cuid() + + spy = mocker.patch.object(cuid2.generator, "Cuid", side_effect=slow_cuid) num_threads = 8 ids_per_thread = 25 barrier = threading.Barrier(num_threads) @@ -651,7 +741,7 @@ def __enter__(self: _PrepopulatingLock) -> None: def __exit__(self: _PrepopulatingLock, *_args: object) -> None: return None - monkeypatch.setattr(cuid2.generator, "_default_generator_lock", _PrepopulatingLock()) + monkeypatch.setattr(cuid2.generator, "_process_locks", {os.getpid(): _PrepopulatingLock()}) generated = create_id() assert is_cuid(generated) assert cuid2.generator._default_generator is pre_created @@ -753,66 +843,44 @@ def _fork_child_worker(custom_cuid: Cuid, conn: Connection[Any, Any]) -> None: try: default_ids = [create_id() for _ in range(20)] custom_ids = [custom_cuid.generate() for _ in range(20)] - conn.send((os.getpid(), custom_cuid._fingerprint, default_ids, custom_ids)) + default_generator = cuid2.generator._default_generator + default_fingerprint = "" if default_generator is None else default_generator._state.fingerprint + conn.send((os.getpid(), default_fingerprint, custom_cuid._state.fingerprint, default_ids, custom_ids)) finally: conn.close() class TestForkSafety: - def test_register_at_fork_posix_and_non_posix( - self: TestForkSafety, - monkeypatch: pytest.MonkeyPatch, - mocker: Mock, - ) -> None: - """Tests _register_at_fork() both when os.register_at_fork exists and when it is absent.""" - mock_register = mocker.Mock() - monkeypatch.setattr(os, "register_at_fork", mock_register, raising=False) - cuid2.generator._register_at_fork() - mock_register.assert_called_once_with(after_in_child=cuid2.generator._reinit_after_fork) - - monkeypatch.delattr(os, "register_at_fork", raising=False) - cuid2.generator._register_at_fork() - - def test_reinit_after_fork_resets_default_generator_and_live_instances( - self: TestForkSafety, - mocker: Mock, - ) -> None: - """Tests that _reinit_after_fork() resets _default_generator, locks, and live Cuid instances.""" - _ = create_id() - assert cuid2.generator._default_generator is not None - old_default_lock = cuid2.generator._default_generator_lock - old_instances_lock = cuid2.generator._instances_lock - + def test_generate_rebuilds_state_when_pid_changes(self: TestForkSafety, mocker: Mock) -> None: + """Tests that generate() replaces the lock, RNG, counter, and fingerprint once per new process ID.""" cuid = Cuid() - old_instance_lock = cuid._lock - old_random = cuid._random - old_counter = cuid._counter - old_fingerprint = cuid._fingerprint - - mocker.patch("os.getpid", return_value=os.getpid() + 10_000) - cuid2.generator._reinit_after_fork() - - assert cuid2.generator._default_generator is None - assert cuid2.generator._default_generator_lock is not old_default_lock - assert cuid2.generator._instances_lock is not old_instances_lock - assert cuid._lock is not old_instance_lock - assert cuid._random is not old_random - assert cuid._counter is not old_counter - assert cuid._fingerprint != old_fingerprint - assert is_cuid(create_id()) - - def test_reinit_after_fork_reseeds_shared_random_and_handles_seedless_rng( + old_state = cuid._state + assert old_state.pid == os.getpid() + + mocker.patch("os.getpid", return_value=old_state.pid + 1) + assert is_cuid(cuid.generate()) + + new_state = cuid._state + assert new_state.pid == old_state.pid + 1 + assert new_state.lock is not old_state.lock + assert new_state.random is not old_state.random + assert new_state.counter is not old_state.counter + assert new_state.fingerprint != old_state.fingerprint + assert is_cuid(cuid.generate()) + assert cuid._state is new_state + + def test_pid_change_reseeds_shared_random_and_tolerates_seedless_rng( self: TestForkSafety, mocker: Mock, ) -> None: - """Tests that _reinit_after_fork() reseeds a reused RNG instance and tolerates RNGs without .seed().""" + """Tests that a reused RNG instance is reseeded after a PID change and that RNGs without .seed() work.""" shared_rng = random.Random(999) # noqa: S311 cuid_shared = Cuid(random_generator=lambda: shared_rng) unreseeded_rng = random.Random() # noqa: S311 unreseeded_rng.setstate(shared_rng.getstate()) - for _ in range(1 + BIG_LENGTH): + # Draws made by the rebuild (counter + fingerprint entropy) and by one generate() (letter + salt). + for _ in range(1 + BIG_LENGTH + 1 + DEFAULT_LENGTH): _ = unreseeded_rng.random() - seed_spy = mocker.spy(shared_rng, "seed") class _SeedlessRng: @@ -822,57 +890,201 @@ def random(self: _SeedlessRng) -> float: seedless = _SeedlessRng() cuid_seedless = Cuid(random_generator=lambda: seedless) # type: ignore[arg-type, return-value] - cuid2.generator._reinit_after_fork() + mocker.patch("os.getpid", return_value=os.getpid() + 1) + assert is_cuid(cuid_shared.generate()) + assert is_cuid(cuid_seedless.generate()) seed_spy.assert_called_once_with() + assert cuid_shared._state.random is shared_rng assert shared_rng.getstate() != unreseeded_rng.getstate() - assert is_cuid(cuid_shared.generate()) - assert is_cuid(cuid_seedless.generate()) + assert cuid_seedless._state.pid == os.getpid() + assert cuid_seedless._state.random.random() == 0.25 + + def test_rebuild_failure_propagates_without_affecting_other_instances( + self: TestForkSafety, + mocker: Mock, + ) -> None: + """Tests that a factory failing in a new process raises from that instance only and is retried later.""" + parent_pid = os.getpid() + fail = {"enabled": True} + + def flaky_fingerprint(random_generator: object) -> str: # noqa: ARG001 + if fail["enabled"] and os.getpid() != parent_pid: + msg = "fingerprint source unavailable" + raise OSError(msg) + return "f" * BIG_LENGTH + + flaky = Cuid(fingerprint=flaky_fingerprint) + healthy = Cuid() + stale_state = flaky._state + + mocker.patch("os.getpid", return_value=parent_pid + 1) + with pytest.raises(OSError, match="fingerprint source unavailable"): + flaky.generate() + assert flaky._state is stale_state + assert is_cuid(healthy.generate()) + assert healthy._state.pid == parent_pid + 1 + + fail["enabled"] = False + assert is_cuid(flaky.generate()) + assert flaky._state.pid == parent_pid + 1 + + def test_pid_change_never_acquires_inherited_locks( + self: TestForkSafety, + monkeypatch: pytest.MonkeyPatch, + mocker: Mock, + ) -> None: + """Tests that a new process never waits on instance or default-generator locks held at fork time.""" + monkeypatch.setattr(cuid2.generator, "_default_generator", None) + cuid = Cuid() + inherited_instance_lock = cuid._state.lock + inherited_process_lock = cuid2.generator._process_lock(os.getpid()) + child_pid = os.getpid() + 1 + holding = threading.Event() + release = threading.Event() + + def hold_locks() -> None: + with inherited_instance_lock, inherited_process_lock: + holding.set() + release.wait() + + holder = threading.Thread(target=hold_locks, daemon=True) + holder.start() + assert holding.wait(timeout=10) + generated: list[str] = [] + try: + mocker.patch("os.getpid", return_value=child_pid) + worker = threading.Thread(target=lambda: generated.extend([cuid.generate(), create_id()]), daemon=True) + worker.start() + worker.join(timeout=10) + assert not worker.is_alive(), "generate()/create_id() blocked on a lock held at fork time" + finally: + release.set() + holder.join(timeout=10) + + assert len(generated) == 2 + assert all(is_cuid(cid) for cid in generated) + assert cuid._state.lock is not inherited_instance_lock + assert cuid2.generator._process_locks[child_pid] is not inherited_process_lock + + def test_child_threads_rebuild_state_and_default_generator_once( + self: TestForkSafety, + monkeypatch: pytest.MonkeyPatch, + mocker: Mock, + ) -> None: + """Tests that threads racing the first calls after a PID change rebuild each instance and the default once.""" + monkeypatch.setattr(cuid2.generator, "_default_generator", None) + factory = mocker.Mock(side_effect=random.Random) + cuid = Cuid(random_generator=factory) + real_cuid = cuid2.generator.Cuid + default_constructor = mocker.patch.object(cuid2.generator, "Cuid", side_effect=real_cuid) + mocker.patch("os.getpid", return_value=os.getpid() + 1) + num_threads = 8 + barrier = threading.Barrier(num_threads) + locks: list[object] = [] + + def worker() -> None: + barrier.wait() + cuid.generate() + create_id() + locks.append(cuid._state.lock) - def test_instances_weakset_allows_garbage_collection(self: TestForkSafety) -> None: - """Tests that unreferenced Cuid instances are garbage-collected and removed from _instances.""" + threads = [threading.Thread(target=worker) for _ in range(num_threads)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert factory.call_count == 2 # construction plus exactly one rebuild for the new process + assert default_constructor.call_count == 1 + assert len({id(lock) for lock in locks}) == 1 + + def test_rebuild_skipped_when_another_thread_already_rebuilt( + self: TestForkSafety, + monkeypatch: pytest.MonkeyPatch, + mocker: Mock, + ) -> None: + """Deterministically exercises the re-check that skips a rebuild finished by another thread.""" cuid = Cuid() - ref = weakref.ref(cuid) - assert cuid in cuid2.generator._instances - del cuid - gc.collect() - assert ref() is None + child_pid = os.getpid() + 1 + rebuilt_elsewhere = cuid._state._replace(pid=child_pid) + + class _RebuildingLock: + def __enter__(self: _RebuildingLock) -> None: + cuid._state = rebuilt_elsewhere + + def __exit__(self: _RebuildingLock, *_args: object) -> None: + return None + + monkeypatch.setattr(cuid2.generator, "_process_locks", {child_pid: _RebuildingLock()}) + mocker.patch("os.getpid", return_value=child_pid) + assert is_cuid(cuid.generate()) + assert cuid._state is rebuilt_elsewhere @pytest.mark.skipif( "fork" not in multiprocessing.get_all_start_methods(), reason="POSIX fork start method required", ) - def test_posix_fork_child_reinitializes_generators(self: TestForkSafety) -> None: - """Tests that forked child processes receive distinct fingerprints, RNG states, and ID streams.""" + @pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning") + def test_posix_fork_children_reinitialize_despite_held_locks( + self: TestForkSafety, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Tests that forked children never block on inherited held locks and get independent RNG streams.""" + monkeypatch.setattr(cuid2.generator, "_default_generator", None) shared_rng = random.Random(12345) # noqa: S311 custom_cuid = Cuid(random_generator=lambda: shared_rng) - _ = create_id() - parent_fingerprint = custom_cuid._fingerprint + parent_fingerprint = custom_cuid._state.fingerprint + inherited_process_lock = cuid2.generator._process_lock(os.getpid()) + holding = threading.Event() + release = threading.Event() + + def hold_locks() -> None: + with custom_cuid._state.lock, inherited_process_lock: + holding.set() + release.wait() + + holder = threading.Thread(target=hold_locks, daemon=True) + holder.start() + assert holding.wait(timeout=10) ctx = multiprocessing.get_context("fork") parent_conns: list[Connection[Any, Any]] = [] procs: list[BaseProcess] = [] - for _ in range(2): - parent_conn, child_conn = ctx.Pipe(duplex=False) - proc: BaseProcess = ctx.Process(target=_fork_child_worker, args=(custom_cuid, child_conn)) - proc.start() - child_conn.close() - parent_conns.append(parent_conn) - procs.append(proc) - - results = [conn.recv() for conn in parent_conns] - for conn in parent_conns: - conn.close() - for proc in procs: - proc.join(timeout=10) - assert proc.exitcode == 0 - - (pid_a, fp_a, default_a, custom_a), (pid_b, fp_b, default_b, custom_b) = results - assert pid_a != os.getpid() - assert pid_b != os.getpid() + results: list[tuple[int, str, str, list[str], list[str]]] = [] + try: + for _ in range(2): + parent_conn, child_conn = ctx.Pipe(duplex=False) + proc: BaseProcess = ctx.Process(target=_fork_child_worker, args=(custom_cuid, child_conn)) + proc.start() + child_conn.close() + parent_conns.append(parent_conn) + procs.append(proc) + for conn in parent_conns: + assert conn.poll(10), "forked child did not respond; it may be blocked on an inherited lock" + results.append(conn.recv()) + finally: + release.set() + holder.join(timeout=10) + for conn in parent_conns: + conn.close() + for proc in procs: + proc.join(timeout=10) + if proc.is_alive(): + proc.kill() + proc.join() + + assert [proc.exitcode for proc in procs] == [0, 0] + ( + (pid_a, default_fp_a, custom_fp_a, default_a, custom_a), + (pid_b, default_fp_b, custom_fp_b, default_b, custom_b), + ) = results + assert os.getpid() not in {pid_a, pid_b} assert pid_a != pid_b - assert fp_a != parent_fingerprint - assert fp_b != parent_fingerprint - assert fp_a != fp_b - assert set(default_a).isdisjoint(set(default_b)) - assert set(custom_a).isdisjoint(set(custom_b)) + assert parent_fingerprint not in {custom_fp_a, custom_fp_b} + assert custom_fp_a != custom_fp_b + assert default_fp_a != default_fp_b + # The leading letter comes from the RNG alone; without a reseed both children replay shared_rng's stream. + assert [cid[0] for cid in custom_a] != [cid[0] for cid in custom_b] + assert set(default_a).isdisjoint(default_b) + assert set(custom_a).isdisjoint(custom_b) diff --git a/local/tests/test_free_threading.py b/local/tests/test_free_threading.py new file mode 100644 index 0000000..cbb14d5 --- /dev/null +++ b/local/tests/test_free_threading.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import ast +import re +import sys +import sysconfig +from pathlib import Path + +import pytest + +TEST_WORKFLOW = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test_lint_scan.yml" +# PEP 779 made free-threaded builds officially supported starting with Python 3.14. +FIRST_SUPPORTED_FREE_THREADED = (3, 14) + + +def _ci_python_versions() -> list[str]: + """Return the `python-version` matrix entries of the CI test job.""" + match = re.search(r"^\s*python-version:\s*(\[.*\])\s*$", TEST_WORKFLOW.read_text(encoding="utf-8"), re.MULTILINE) + assert match is not None, f"python-version matrix not found in {TEST_WORKFLOW}" + versions: list[str] = ast.literal_eval(match.group(1)) + return versions + + +@pytest.mark.skipif(not sysconfig.get_config_var("Py_GIL_DISABLED"), reason="free-threaded build required") +def test_free_threaded_build_runs_without_gil() -> None: + """Tests that no imported module re-enabled the GIL, so free-threaded CI jobs exercise real parallelism.""" + if sys.version_info >= (3, 13): + assert not sys._is_gil_enabled() + + +def test_ci_matrix_tests_free_threaded_build_for_each_supported_python() -> None: + """Tests that every CI Python version from 3.14 on also runs as a free-threaded ("t") build.""" + versions = _ci_python_versions() + expected = { + f"{version}t" + for version in versions + if not version.endswith("t") and tuple(map(int, version.split("."))) >= FIRST_SUPPORTED_FREE_THREADED + } + assert expected + assert expected <= set(versions), f"missing free-threaded CI jobs: {sorted(expected - set(versions))}" diff --git a/local/tests/test_utils.py b/local/tests/test_utils.py index fb27a05..a2a78e9 100644 --- a/local/tests/test_utils.py +++ b/local/tests/test_utils.py @@ -11,6 +11,7 @@ from cuid2 import utils if TYPE_CHECKING: + from collections.abc import Callable from unittest.mock import Mock @@ -370,10 +371,13 @@ def test_count_variable_not_accessed_outside_function(self: TestCreateCounter) - _ = counter.count # type: ignore[attr-defined] def test_counter_thread_safety_barrier(self: TestCreateCounter) -> None: - """Tests that concurrent increments across 8 threads produce a contiguous, duplicate-free sequence.""" + """Tests that 1,600,000 increments across 8 threads produce a contiguous, duplicate-free sequence. + + An unsynchronized counter on free-threaded Python returns roughly 320,000-380,000 unique values here. + """ counter = utils.create_counter(0) num_threads = 8 - increments_per_thread = 500 + increments_per_thread = 200_000 barrier = threading.Barrier(num_threads) buckets: list[list[int]] = [[] for _ in range(num_threads)] @@ -420,3 +424,25 @@ def worker(index: int) -> None: combined = [val for bucket in buckets for val in bucket] total = num_threads * increments_per_thread assert set(combined) == set(range(1, total + 1)) + + def test_counter_reentrant_call_on_same_thread_does_not_deadlock(self: TestCreateCounter) -> None: + """Tests that the counter re-entered on the same thread (as from a signal handler) completes.""" + nested: list[int] = [] + counters: list[Callable[[], int]] = [] + + class _ReentrantInt(int): + def __add__(self: _ReentrantInt, other: int) -> _ReentrantInt: + if not nested: + nested.append(0) + nested[0] = int(counters[0]()) + return _ReentrantInt(super().__add__(other)) + + counters.append(utils.create_counter(_ReentrantInt(0))) + outer: list[int] = [] + thread = threading.Thread(target=lambda: outer.append(int(counters[0]())), daemon=True) + thread.start() + thread.join(timeout=10) + + assert not thread.is_alive(), "re-entrant counter call deadlocked on the counter lock" + assert len(outer) == 1 + assert nested[0] >= 1 diff --git a/src/cuid2/generator.py b/src/cuid2/generator.py index 7111f03..e1d9dbb 100644 --- a/src/cuid2/generator.py +++ b/src/cuid2/generator.py @@ -9,11 +9,11 @@ import time import warnings from contextlib import suppress +from copy import copy, deepcopy from functools import partial from math import floor from secrets import SystemRandom -from typing import TYPE_CHECKING, ClassVar, Final, Protocol -from weakref import WeakSet +from typing import TYPE_CHECKING, ClassVar, Final, NamedTuple, Protocol from cuid2 import utils from cuid2.utils import BIG_LENGTH, DEFAULT_LENGTH, INITIAL_COUNT_MAX, MAXIMUM_LENGTH, MIN_LENGTH @@ -124,6 +124,32 @@ def is_cuid(cuid_str: str, min_length: int = MIN_LENGTH, max_length: int = MAXIM ) +class _GeneratorState(NamedTuple): + """Per-process generator state, replaced as a whole when a forked child process first generates an ID.""" + + pid: int + lock: threading.RLock + random: Random + counter: Callable[[], int] + fingerprint: str + + +# Factories are shared by reference when a Cuid is deep-copied so that RNG objects they hold are never cloned. +_SHARED_ON_DEEPCOPY: Final[frozenset[str]] = frozenset( + {"_random_generator", "_counter_factory", "_fingerprint_factory", "_state"}, +) +_process_locks: Final[dict[int, threading.RLock]] = {} + + +def _process_lock(pid: int) -> threading.RLock: + """Return the lock that serializes state rebuilds and default generator creation within process `pid`. + + Keyed by process ID so a forked child never acquires a lock inherited from its parent, which a parent thread + that does not exist in the child may hold; `dict.setdefault` gives every thread of a process the same lock. + """ + return _process_locks.setdefault(pid, threading.RLock()) + + class Cuid: """Configurable thread- and fork-safe CUID2 generator instance.""" @@ -140,19 +166,19 @@ def __init__( ---------- random_generator : Callable[[], "Random"], default=SystemRandom Factory callable returning the random number generator instance. Defaults to `secrets.SystemRandom`. - On POSIX `os.fork()`, `random_generator()` is re-invoked in the child process (and reseeded with OS - entropy via `.seed()` if it returns the same generator object as the parent). Fixed-seed factories - (such as `functools.partial(random.Random, 42)`) are intended for deterministic single-process tests. + Called again in a forked child process (see Notes). counter : Callable[[int], Callable[[], int]], default=utils.create_counter The `counter` parameter is a callable that creates a counter returning an incremented value each time it - is called. The `create_counter` function from the `utils` module is used by default. + is called. The `create_counter` function from the `utils` module is used by default. Called again with a + new random starting value in a forked child process. length : int, default=DEFAULT_LENGTH (24) The length parameter is an integer that determines the maximum length of the generated string. It has a default value of DEFAULT_LENGTH (24). A length value less than 1 or greater than `MAXIMUM_LENGTH` (98 characters) will raise a ValueError. Lengths outside `MIN_LENGTH`..`BIG_LENGTH` (2..32) emit a DeprecationWarning. fingerprint : "FingerprintCallable", default=utils.create_fingerprint - The "fingerprint" parameter is a callable function that generates a unique identifier. + The "fingerprint" parameter is a callable function that generates a unique identifier. Called again in a + forked child process; the default includes the child's `os.getpid()`. Raises ------ @@ -163,33 +189,69 @@ def __init__( ----- DeprecationWarning If the length parameter is outside `MIN_LENGTH`..`BIG_LENGTH` (2..32 characters). + + Notes + ----- + `generate()` may be called concurrently from multiple threads, including on free-threaded Python. Draws from + the random generator and counter are serialized per instance by a re-entrant lock, so a signal handler or + finalizer that generates an ID on the same thread does not deadlock. + + Each instance records the process ID that built its state. When `generate()` first runs in a different + process, such as any `os.fork()` child (including pre-fork servers that fork without running Python's fork + hooks), it builds a new lock, random generator, counter, and fingerprint once, by calling `random_generator`, + `counter`, and `fingerprint` again before generating. If `random_generator` returns the same object as before + the fork, its `seed()` method is called to reseed it from OS entropy; if it has no `seed()` method or `seed()` + raises `NotImplementedError` or `TypeError`, that generator keeps the parent's stream, and IDs still differ by + timestamp and, with the default `fingerprint`, by process ID. Fixed-seed factories (such as + `functools.partial(random.Random, 42)`) give every process the same stream and are intended for deterministic + single-process tests. + + `copy.deepcopy()` returns an independent generator: the three factories are shared by reference, other + attributes are deep-copied, and the lock, random generator, counter, and fingerprint are built anew. """ _validate_length(length) self._random_generator: Callable[[], Random] = random_generator self._counter_factory: Callable[[int], Callable[[], int]] = counter self._fingerprint_factory: FingerprintCallable = fingerprint - self._lock: threading.Lock = threading.Lock() - - self._random: Random = random_generator() - self._counter: Callable[[], int] = counter(floor(self._random.random() * INITIAL_COUNT_MAX)) self._length: int = length - self._fingerprint: str = fingerprint(random_generator=self._random) - - with _instances_lock: - _instances.add(self) - - def _reinit_after_fork(self: Cuid) -> None: - """Re-initialize lock, random generator, counter, and fingerprint in a forked child process.""" - self._lock = threading.Lock() - prev_random: Random = self._random - new_random: Random = self._random_generator() - if new_random is prev_random: + self._state: _GeneratorState = self._create_state(previous_random=None) + + def __deepcopy__(self: Cuid, memo: dict[int, object]) -> Cuid: + """Return an independent generator that never clones the lock, random generator state, or counter.""" + clone = copy(self) + memo[id(self)] = clone + for name, value in vars(self).items(): + if name not in _SHARED_ON_DEEPCOPY: + setattr(clone, name, deepcopy(value, memo)) + clone._state = clone._create_state(previous_random=None) # noqa: SLF001 + return clone + + def _current_state(self: Cuid) -> _GeneratorState: + """Return the state for the current process, rebuilding it once on the first call after a fork.""" + state: _GeneratorState = self._state + pid: int = os.getpid() + if state.pid == pid: + return state + with _process_lock(pid): + state = self._state + if state.pid != pid: + state = self._state = self._create_state(previous_random=state.random) + return state + + def _create_state(self: Cuid, previous_random: Random | None) -> _GeneratorState: + """Build the lock, random generator, counter, and fingerprint for the current process.""" + random_generator: Random = self._random_generator() + if random_generator is previous_random: with suppress(AttributeError, NotImplementedError, TypeError): - new_random.seed() - self._random = new_random - self._counter = self._counter_factory(floor(self._random.random() * INITIAL_COUNT_MAX)) - self._fingerprint = self._fingerprint_factory(random_generator=self._random) + random_generator.seed() + return _GeneratorState( + pid=os.getpid(), + lock=threading.RLock(), + random=random_generator, + counter=self._counter_factory(floor(random_generator.random() * INITIAL_COUNT_MAX)), + fingerprint=self._fingerprint_factory(random_generator=random_generator), + ) def generate(self: Cuid, length: int | None = None) -> str: """Generate a universally unique, base36-encoded string with a specified length. @@ -223,43 +285,21 @@ def generate(self: Cuid, length: int | None = None) -> str: _validate_length(length) resolved_length: int = self._length if length is None else length - with self._lock: - first_letter: str = utils.create_letter(random_generator=self._random) - base36_count: str = utils.base36_encode(self._counter()) - salt: str = utils.create_entropy(length=resolved_length, random_generator=self._random) - fingerprint: str = self._fingerprint + state: _GeneratorState = self._current_state() + with state.lock: + first_letter: str = utils.create_letter(random_generator=state.random) + base36_count: str = utils.base36_encode(state.counter()) + salt: str = utils.create_entropy(length=resolved_length, random_generator=state.random) base36_time: str = utils.base36_encode(time.time_ns()) - hash_input: str = base36_time + salt + base36_count + fingerprint + hash_input: str = base36_time + salt + base36_count + state.fingerprint return first_letter + utils.create_hash(hash_input)[1:resolved_length] is_cuid: ClassVar[IsCuidCallable] = staticmethod(is_cuid) -_instances: Final[WeakSet[Cuid]] = WeakSet() -_instances_lock: threading.Lock = threading.Lock() _default_generator: Cuid | None = None -_default_generator_lock: threading.Lock = threading.Lock() - - -def _reinit_after_fork() -> None: - """Reset the lazy default generator and re-initialize live Cuid instances after os.fork().""" - global _default_generator, _default_generator_lock, _instances_lock # noqa: PLW0603 - _default_generator = None - _default_generator_lock = threading.Lock() - _instances_lock = threading.Lock() - for instance in list(_instances): - instance._reinit_after_fork() # noqa: SLF001 # pyright: ignore[reportPrivateUsage] - - -def _register_at_fork() -> None: - """Register the POSIX post-fork callback when supported by the current platform.""" - if hasattr(os, "register_at_fork"): - os.register_at_fork(after_in_child=_reinit_after_fork) - - -_register_at_fork() def create_id(length: int | None = None) -> str: @@ -290,7 +330,7 @@ def create_id(length: int | None = None) -> str: global _default_generator # noqa: PLW0603 generator = _default_generator if generator is None: - with _default_generator_lock: + with _process_lock(os.getpid()): generator = _default_generator if generator is None: generator = Cuid() diff --git a/src/cuid2/utils.py b/src/cuid2/utils.py index 229b1bb..2e0cae1 100644 --- a/src/cuid2/utils.py +++ b/src/cuid2/utils.py @@ -45,7 +45,7 @@ def create_counter(count: int) -> Callable[[], int]: the closure function is called, it atomically increments the count variable under a lock and returns its new value. """ - lock = threading.Lock() + lock = threading.RLock() def counter() -> int: nonlocal count