From 8e74eb2e5a7017237b984bbf745344412c222ad7 Mon Sep 17 00:00:00 2001 From: Diego Migotto Date: Fri, 16 Oct 2026 19:15:56 -0300 Subject: [PATCH 1/2] feat: batch options and shared item seeds (4.1.0) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_018Cn4xFiHhbCHDhom8w2Ghq --- .claude/worktrees/v4-json-bool-fix | 1 + .github/workflows/ci.yml | 23 +- Doxyfile | 2 +- README.md | 2 + dasmig/core.hpp | 2 + dasmig/core/batch.hpp | 331 +++++++++++++++++++++ dasmig/core/distributions.hpp | 2 +- dasmig/core/json.hpp | 50 +++- dasmig/core/rng.hpp | 17 +- dasmig/core/sinks.hpp | 443 +++++++++++++++++++++++++++++ dasmig/core/version.hpp | 4 +- dasmig/core/weighted_table.hpp | 19 +- dasmig/entitygen.hpp | 93 +++++- doc/usage.md | 26 ++ tests/tests.cpp | 73 ++++- 15 files changed, 1058 insertions(+), 30 deletions(-) create mode 160000 .claude/worktrees/v4-json-bool-fix create mode 100644 dasmig/core/batch.hpp create mode 100644 dasmig/core/sinks.hpp diff --git a/.claude/worktrees/v4-json-bool-fix b/.claude/worktrees/v4-json-bool-fix new file mode 160000 index 0000000..15d5060 --- /dev/null +++ b/.claude/worktrees/v4-json-bool-fix @@ -0,0 +1 @@ +Subproject commit 15d506018baa5c588194fe8fe68f4b78051c941f diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aee0f23..c912684 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,8 +173,29 @@ jobs: name: bench-report path: build/bench.tsv + # The batch tests run worker threads; ThreadSanitizer reports any data race + # in the getters they call. + tsan: + name: ThreadSanitizer (batch tests) + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Install LLVM 18 + run: | + sudo apt-get update + sudo apt-get install -y clang-18 + + - name: Build and run the batch tests + run: | + mkdir -p build + clang++-18 -std=c++23 -O1 -g -fsanitize=thread -Wall -Wextra -Wpedantic -I. \ + -o build/test_tsan tests/tests.cpp tests/catch_amalgamated.cpp + ./build/test_tsan "[batch]" + deploy-docs: - needs: [build-and-test, compat, msvc, wasm] + needs: [build-and-test, compat, tsan, msvc, wasm] if: github.ref == 'refs/heads/main' && github.event_name == 'push' runs-on: ubuntu-24.04 permissions: diff --git a/Doxyfile b/Doxyfile index 6fbbe3c..bcdc812 100644 --- a/Doxyfile +++ b/Doxyfile @@ -2,7 +2,7 @@ # Run: doxygen Doxyfile PROJECT_NAME = "Entity Generator" -PROJECT_NUMBER = 4.0.0 +PROJECT_NUMBER = 4.1.0 PROJECT_BRIEF = "Composable, deterministic entity generation for C++23" PROJECT_LOGO = doc/entity-generator.png diff --git a/README.md b/README.md index b9613fd..d1289b5 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ - **Core** — Generate entities composed of user-defined components. Each component produces a typed random value accessible via `entity.get(key)`. Register, remove, and generate with fluent chaining. +- **Batch Generation**. Generate thousands of entities from one seed, on several threads with the same output for any thread count, optionally unique by a key, and stream them as JSON Lines or CSV (see the usage guide). + - **Generic Components** — Ready-made templates for common patterns: `constant_component`, `choice_component`, `range_component`, `callback_component`, and `weighted_choice_component`. All accept an optional custom formatter. - **Request-First API** — `entity_request` carries component selection, group, batch count, entity validation and per-call `inputs` (a country, a reference date, ...) for each call. Generation methods accept `(request)` or `(request, seed)`; setup stays on the instance. diff --git a/dasmig/core.hpp b/dasmig/core.hpp index ff6c159..74e9a65 100644 --- a/dasmig/core.hpp +++ b/dasmig/core.hpp @@ -15,6 +15,7 @@ #include "core/version.hpp" +#include "core/batch.hpp" #include "core/dataset.hpp" #include "core/distributions.hpp" #include "core/json.hpp" @@ -22,6 +23,7 @@ #include "core/resources.hpp" #include "core/rng.hpp" #include "core/seed_signature.hpp" +#include "core/sinks.hpp" #include "core/table.hpp" #include "core/tsv.hpp" #include "core/utf8.hpp" diff --git a/dasmig/core/batch.hpp b/dasmig/core/batch.hpp new file mode 100644 index 0000000..45d5343 --- /dev/null +++ b/dasmig/core/batch.hpp @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2020-2026 Diego Dasso Migotto + +#ifndef DASMIG_CORE_BATCH_HPP +#define DASMIG_CORE_BATCH_HPP + +#include "rng.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// @file batch.hpp +/// @brief Batch generation: item seeds, parallel and streaming runs, and +/// uniqueness, shared by every generator library. +/// @author Diego Dasso Migotto (diegomigotto at hotmail dot com) +/// +/// A batch is `count` results generated from one batch seed. Item `i` is +/// exactly what the single-call getter returns for `item_seed(batch_seed, i)`, +/// so any item replays on its own, any range is computed directly, and the +/// result does not depend on how the work is divided between threads. + +namespace dasmig::core +{ + +/// @brief The seed of item @p index in a batch. +/// +/// `rng{batch_seed}.split(index).seed()`, computed without building an engine. +/// +/// @param batch_seed Seed of the whole batch. +/// @param index Index of the item (including `batch_options::first`). +/// @return The seed to give the single-call getter for that item. +[[nodiscard]] constexpr std::uint64_t item_seed(std::uint64_t batch_seed, + std::uint64_t index) noexcept +{ + return rng::child_seed_of(batch_seed, index); +} + +/// @brief The seed of attempt @p attempt of item @p index, for a batch that +/// must be unique. +/// +/// Attempt 0 is the plain item_seed(), so items that do not collide equal the +/// single call. Attempt `k > 0` is `item_seed(item_seed(batch_seed, index), k)`. +/// +/// @param batch_seed Seed of the whole batch. +/// @param index Index of the item. +/// @param attempt Attempt number, starting at 0. +/// @return The seed to generate that attempt with. +[[nodiscard]] constexpr std::uint64_t attempt_seed(std::uint64_t batch_seed, + std::uint64_t index, + std::uint64_t attempt) noexcept +{ + const std::uint64_t base = item_seed(batch_seed, index); + return attempt == 0 ? base : item_seed(base, attempt); +} + +/// @brief Makes a batch unique by a key (an email, a username, ...). +/// +/// Items are generated, then checked in index order; an item whose key was +/// already produced earlier in the same call is generated again with the next +/// attempt seed (see attempt_seed()) until it is new or the attempts run out. +/// The outcome is the same for any thread count. Uniqueness holds within one +/// call: two batches that are concatenated are not checked against each other. +class unique_rule +{ + public: + /// @brief Unique by @p key. + /// @tparam T The item type the batch produces. + /// @tparam Key Callable taking `const T&` and returning something + /// convertible to `std::string`. + /// @param key Extracts the key. + /// @param max_attempts Attempts per item, including the first (at least 1). + /// @return The rule. + template + [[nodiscard]] static unique_rule by(Key key, std::size_t max_attempts = 100) + { + unique_rule rule; + rule._type = std::type_index{typeid(T)}; + rule._max_attempts = max_attempts; + rule._key = [key = std::move(key)](const void* item) + { return std::string{key(*static_cast(item))}; }; + return rule; + } + + /// @brief Attempts per item, including the first. + /// @return The limit. + [[nodiscard]] std::size_t max_attempts() const noexcept { return _max_attempts; } + + /// @brief The item type the rule was made for. + /// @return Its type index. + [[nodiscard]] std::type_index type() const noexcept { return _type; } + + /// @brief The key of an item. + /// @param item Pointer to an item of the rule's type. + /// @return The key. + [[nodiscard]] std::string key_of(const void* item) const { return _key(item); } + + private: + unique_rule() = default; + + std::type_index _type{typeid(void)}; + std::size_t _max_attempts{1}; + std::function _key; +}; + +/// @brief How a batch is generated. Requests keep describing one item. +struct batch_options +{ + std::uint64_t count{1}; ///< Number of items. + std::uint64_t first{0}; ///< Index of the first item; item `i` is item_seed(seed, first + i). + std::size_t threads{1}; ///< Worker threads; 0 means the hardware count, 1 runs on the caller. + std::optional unique{}; // NOLINT(readability-redundant-member-init): keeps -Wextra quiet for partial designated initializers + ///< Make the batch unique by a key. +}; + +namespace detail +{ + +template +using batch_item_t = std::remove_cvref_t>; + +// NOLINTNEXTLINE(bugprone-easily-swappable-parameters) +[[nodiscard]] inline std::size_t resolve_threads(std::size_t requested, std::size_t work) noexcept +{ +#if defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__) + (void)requested; + (void)work; + return 1; +#else + std::size_t threads = requested; + if (threads == 0) + { + threads = std::max(1, std::thread::hardware_concurrency()); + } + return std::max(1, std::min(threads, work)); +#endif +} + +// Fills slots[0, n) with make(item_seed(batch_seed, first + k)), in contiguous +// chunks, one per thread. The failure of the lowest index is rethrown. +template +void fill_window(const Make& make, std::uint64_t batch_seed, std::uint64_t first, + std::size_t n, std::size_t threads, std::vector>& slots) +{ + // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) + const auto run_chunk = [&](std::size_t lo, std::size_t hi, std::exception_ptr& error) + { + try + { + for (std::size_t k = lo; k < hi; ++k) + { + slots[k].emplace(make(item_seed(batch_seed, first + k))); + } + } + catch (...) + { + error = std::current_exception(); + } + }; + + if (threads <= 1) + { + std::exception_ptr error; + run_chunk(0, n, error); + if (error) std::rethrow_exception(error); + return; + } + + std::vector errors(threads); + const auto bound = [&](std::size_t chunk) { return n * chunk / threads; }; + std::vector workers; + workers.reserve(threads - 1); + for (std::size_t c = 1; c < threads; ++c) + { + try + { + workers.emplace_back(run_chunk, bound(c), bound(c + 1), std::ref(errors[c])); + } + catch (const std::system_error&) + { + // No thread available: the caller does this chunk after its own. + run_chunk(bound(c), bound(c + 1), errors[c]); + } + } + run_chunk(bound(0), bound(1), errors[0]); + for (auto& worker : workers) worker.join(); + for (const auto& error : errors) + { + if (error) std::rethrow_exception(error); + } +} + +template +void run_batch_windows(const batch_options& options, std::uint64_t batch_seed, + const Make& make, Sink& sink, std::uint64_t window_items) +{ + using item = batch_item_t; + + if (options.count > std::numeric_limits::max() - options.first) + { + throw std::invalid_argument("batch: first + count overflows the item index"); + } + if (options.unique) + { + if (options.unique->type() != std::type_index{typeid(item)}) + { + throw std::invalid_argument("batch: the unique rule was made for another item type"); + } + if (options.unique->max_attempts() == 0) + { + throw std::invalid_argument("batch: a unique rule needs at least one attempt"); + } + } + + const std::size_t threads = resolve_threads( + options.threads, + static_cast(std::min(options.count, std::numeric_limits::max()))); + std::unordered_set seen; + std::vector> slots; + + for (std::uint64_t done = 0; done < options.count;) + { + const auto n = static_cast(std::min(window_items, options.count - done)); + const std::uint64_t window_first = options.first + done; + slots.clear(); + slots.resize(n); + fill_window(make, batch_seed, window_first, n, threads, slots); + + for (std::size_t k = 0; k < n; ++k) + { + const std::uint64_t index = window_first + k; + if (options.unique) + { + const unique_rule& rule = *options.unique; + std::string key = rule.key_of(&*slots[k]); + for (std::uint64_t attempt = 1; !seen.insert(key).second; ++attempt) + { + if (attempt >= rule.max_attempts()) + { + throw std::runtime_error( + "batch: item " + std::to_string(index) + " has no unique key after " + + std::to_string(rule.max_attempts()) + " attempts"); + } + slots[k].emplace(make(attempt_seed(batch_seed, index, attempt))); + key = rule.key_of(&*slots[k]); + } + } + sink(index, *slots[k]); + } + done += n; + } +} + +} // namespace detail + +/// @brief Generate a batch and return it. +/// +/// Item `i` is `make(item_seed(batch_seed, options.first + i))`. @p make runs +/// on several threads when `options.threads` allows, so it must be callable +/// concurrently (the libraries' const seeded getters are). +/// +/// @code +/// auto names = dasmig::core::run_batch({.count = 1000, .threads = 4}, 7, +/// [&](std::uint64_t seed) { return gen.get_name(request, seed); }); +/// @endcode +/// +/// @tparam Make Callable taking a `std::uint64_t` seed and returning an item. +/// @param options Count, first index, threads and uniqueness. +/// @param batch_seed Seed of the whole batch. +/// @param make Generates one item from its seed. +/// @return The items, in index order. +/// @throws std::invalid_argument If the options are inconsistent. +/// @throws std::runtime_error If a unique batch runs out of attempts. +/// @throws Whatever @p make throws, for the lowest failing index. +template + requires std::invocable +[[nodiscard]] std::vector> run_batch(const batch_options& options, + std::uint64_t batch_seed, + const Make& make) +{ + using item = detail::batch_item_t; + std::vector out; + out.reserve(static_cast(options.count)); + auto sink = [&out](std::uint64_t, item& value) { out.push_back(std::move(value)); }; + detail::run_batch_windows(options, batch_seed, make, sink, + std::max(options.count, 1)); + return out; +} + +/// @brief Generate a batch and pass each item to @p sink, in index order, +/// without holding the whole batch. +/// +/// At most a few thousand items per thread are alive at once (plus the keys of +/// a unique batch). @p sink runs on the calling thread and receives the +/// item's index (including `options.first`) and the item. +/// +/// @tparam Make Callable taking a `std::uint64_t` seed and returning an item. +/// @tparam Sink Callable taking `(std::uint64_t index, Item&)`. +/// @param options Count, first index, threads and uniqueness. +/// @param batch_seed Seed of the whole batch. +/// @param make Generates one item from its seed. +/// @param sink Receives the items. +/// @throws The same errors as run_batch(), and whatever @p sink throws. +template + requires std::invocable +void for_each_batch(const batch_options& options, std::uint64_t batch_seed, const Make& make, + Sink&& sink) // NOLINT(cppcoreguidelines-missing-std-forward): called as an lvalue, never moved +{ + const std::size_t threads = detail::resolve_threads(options.threads, std::numeric_limits::max()); + detail::run_batch_windows(options, batch_seed, make, sink, + static_cast(threads) * 4096U); +} + +} // namespace dasmig::core + +#endif // DASMIG_CORE_BATCH_HPP diff --git a/dasmig/core/distributions.hpp b/dasmig/core/distributions.hpp index babe7aa..afe55f8 100644 --- a/dasmig/core/distributions.hpp +++ b/dasmig/core/distributions.hpp @@ -141,7 +141,7 @@ namespace detail // whose cumulative weight exceeds uniform01(r) * total. Rounding can make the // target reach the total; then the last index with a positive weight. [[nodiscard]] inline std::size_t pick_cumulative(rng& r, - const std::vector& cumulative, + std::span cumulative, std::size_t last_positive) { const double target = uniform01(r) * cumulative.back(); diff --git a/dasmig/core/json.hpp b/dasmig/core/json.hpp index 23a8847..83f1437 100644 --- a/dasmig/core/json.hpp +++ b/dasmig/core/json.hpp @@ -48,6 +48,13 @@ namespace dasmig::core class json_writer { public: + /// @brief An empty writer, with room for a typical document. + json_writer() + { + _out.reserve(initial_capacity_); + _stack.reserve(initial_depth_); + } + /// @brief Open an object. /// @return `*this`. /// @throws std::logic_error Where a value is not allowed. @@ -227,15 +234,29 @@ class json_writer /// @throws std::invalid_argument If @p text is not valid UTF-8. static void escape(std::string& out, std::string_view text) { - if (!is_utf8(text)) + // Most text is printable ASCII without quotes or backslashes: nothing + // to check or escape. An ASCII prefix is valid UTF-8 on its own, so + // only the rest needs the check. + std::size_t plain = 0; + while (plain < text.size() && needs_no_escape(text[plain]) && static_cast(text[plain]) < 0x80U) + { + ++plain; + } + if (plain < text.size() && !is_utf8(text.substr(plain))) { throw std::invalid_argument("json_writer: string is not valid UTF-8"); } constexpr std::string_view hex = "0123456789abcdef"; out += '"'; - for (const char c : text) + // Bytes that need no escape are appended a run at a time. + std::size_t run = 0; + for (std::size_t i = plain; i < text.size(); ++i) { + const char c = text[i]; + if (needs_no_escape(c)) continue; const auto b = static_cast(c); + out.append(text.substr(run, i - run)); + run = i + 1; switch (c) { case '"': out += "\\\""; break; @@ -246,18 +267,12 @@ class json_writer case '\r': out += "\\r"; break; case '\t': out += "\\t"; break; default: - if (b < 0x20U) - { - out += "\\u00"; - out += hex[b >> 4U]; - out += hex[b & 0x0FU]; - } - else - { - out += c; - } + out += "\\u00"; + out += hex[b >> 4U]; + out += hex[b & 0x0FU]; } } + out.append(text.substr(run)); out += '"'; } @@ -268,6 +283,17 @@ class json_writer array }; + // Reserved up front: growing a string from empty reallocates it about ten + // times on the way to a few kilobytes. + static constexpr std::size_t initial_capacity_ = 1024; + static constexpr std::size_t initial_depth_ = 8; + + // A byte written as it is: not a control character, quote or backslash. + [[nodiscard]] static constexpr bool needs_no_escape(char c) noexcept + { + return static_cast(c) >= 0x20U && c != '"' && c != '\\'; + } + struct frame { kind type; diff --git a/dasmig/core/rng.hpp b/dasmig/core/rng.hpp index c843ed9..80011a9 100644 --- a/dasmig/core/rng.hpp +++ b/dasmig/core/rng.hpp @@ -190,8 +190,21 @@ class rng [[nodiscard]] constexpr std::uint64_t child_seed( std::uint64_t index) const noexcept { - return splitmix64_mix( - _seed ^ splitmix64_mix(index ^ 0x6a09e667f3bcc909ULL)); + return child_seed_of(_seed, index); + } + + /// @brief child_seed(index) for a seed, without building an engine. + /// + /// Building an engine expands the seed into 256 bits of state; callers + /// that only need the derived seed (item_seed()) skip that. + /// + /// @param seed Parent seed. + /// @param index Index of the child stream. + /// @return The same value as `rng{seed}.child_seed(index)`. + [[nodiscard]] static constexpr std::uint64_t child_seed_of( + std::uint64_t seed, std::uint64_t index) noexcept + { + return splitmix64_mix(seed ^ splitmix64_mix(index ^ 0x6a09e667f3bcc909ULL)); } /// @brief A child engine for a label, independent of this engine's state. diff --git a/dasmig/core/sinks.hpp b/dasmig/core/sinks.hpp new file mode 100644 index 0000000..c155306 --- /dev/null +++ b/dasmig/core/sinks.hpp @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2020-2026 Diego Dasso Migotto + +#ifndef DASMIG_CORE_SINKS_HPP +#define DASMIG_CORE_SINKS_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// @file sinks.hpp +/// @brief Writers for batches: JSON Lines, and CSV flattened from a result's +/// JSON. Both can be given to for_each_batch() as the sink. +/// @author Diego Dasso Migotto (diegomigotto at hotmail dot com) + +namespace dasmig::core +{ + +namespace detail +{ + +class json_flattener +{ + public: + explicit json_flattener(std::string_view text) : _text{text} {} + + std::vector> run() + { + std::vector> cells; + skip_space(); + value("", cells); + skip_space(); + if (_pos != _text.size()) fail("unexpected text after the value"); + return cells; + } + + private: + // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) + static std::string child_path(const std::string& path, const std::string& key) + { + if (path.empty()) return key; + std::string out = path; + out += '.'; + out += key; + return out; + } + + [[noreturn]] void fail(const char* what) const + { + throw std::invalid_argument("flatten_json: " + std::string{what} + " at offset " + + std::to_string(_pos)); + } + + void skip_space() + { + while (_pos < _text.size() && + (_text[_pos] == ' ' || _text[_pos] == '\t' || _text[_pos] == '\n' || _text[_pos] == '\r')) + { + ++_pos; + } + } + + [[nodiscard]] char peek() const + { + if (_pos >= _text.size()) fail("unexpected end"); + return _text[_pos]; + } + + static void append_utf8(std::string& out, std::uint32_t cp) + { + if (cp < 0x80U) + { + out += static_cast(cp); + } + else if (cp < 0x800U) + { + out += static_cast(0xC0U | (cp >> 6U)); + out += static_cast(0x80U | (cp & 0x3FU)); + } + else if (cp < 0x10000U) + { + out += static_cast(0xE0U | (cp >> 12U)); + out += static_cast(0x80U | ((cp >> 6U) & 0x3FU)); + out += static_cast(0x80U | (cp & 0x3FU)); + } + else + { + out += static_cast(0xF0U | (cp >> 18U)); + out += static_cast(0x80U | ((cp >> 12U) & 0x3FU)); + out += static_cast(0x80U | ((cp >> 6U) & 0x3FU)); + out += static_cast(0x80U | (cp & 0x3FU)); + } + } + + std::uint32_t hex4() + { + if (_pos + 4 > _text.size()) fail("short \\u escape"); + std::uint32_t v = 0; + for (int i = 0; i < 4; ++i) + { + const char c = _text[_pos++]; + v <<= 4U; + if (c >= '0' && c <= '9') v |= static_cast(c - '0'); + else if (c >= 'a' && c <= 'f') v |= static_cast(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') v |= static_cast(c - 'A' + 10); + else fail("bad \\u escape"); + } + return v; + } + + std::string string_value() + { + if (peek() != '"') fail("expected a string"); + ++_pos; + std::string out; + for (;;) + { + const char c = peek(); + ++_pos; + if (c == '"') return out; + if (c != '\\') + { + out += c; + continue; + } + const char e = peek(); + ++_pos; + switch (e) + { + case '"': out += '"'; break; + case '\\': out += '\\'; break; + case '/': out += '/'; break; + case 'b': out += '\b'; break; + case 'f': out += '\f'; break; + case 'n': out += '\n'; break; + case 'r': out += '\r'; break; + case 't': out += '\t'; break; + case 'u': + { + std::uint32_t cp = hex4(); + if (cp >= 0xD800U && cp <= 0xDBFFU && _pos + 1 < _text.size() && + _text[_pos] == '\\' && _text[_pos + 1] == 'u') + { + _pos += 2; + const std::uint32_t lo = hex4(); + if (lo < 0xDC00U || lo > 0xDFFFU) fail("bad surrogate pair"); + cp = 0x10000U + ((cp - 0xD800U) << 10U) + (lo - 0xDC00U); + } + append_utf8(out, cp); + break; + } + default: fail("bad escape"); + } + } + } + + // Moves past one value of any kind without keeping it. + // NOLINTNEXTLINE(readability-function-cognitive-complexity, misc-no-recursion) + void skip_value() + { + skip_space(); + const char c = peek(); + if (c == '"') + { + (void)string_value(); + } + else if (c == '{' || c == '[') + { + const char close = c == '{' ? '}' : ']'; + ++_pos; + skip_space(); + if (peek() == close) + { + ++_pos; + return; + } + for (;;) + { + if (c == '{') + { + skip_space(); + (void)string_value(); + skip_space(); + if (peek() != ':') fail("expected ':'"); + ++_pos; + } + skip_value(); + skip_space(); + const char sep = peek(); + ++_pos; + if (sep == close) return; + if (sep != ',') fail("expected ',' or the end of the container"); + } + } + else + { + const std::size_t start = _pos; + while (_pos < _text.size() && _text[_pos] != ',' && _text[_pos] != '}' && + _text[_pos] != ']' && _text[_pos] != ' ' && _text[_pos] != '\n' && + _text[_pos] != '\r' && _text[_pos] != '\t') + { + ++_pos; + } + if (_pos == start) fail("expected a value"); + } + } + + // NOLINTNEXTLINE(readability-function-cognitive-complexity, misc-no-recursion) + void value(const std::string& path, std::vector>& cells) + { + const char c = peek(); + if (c == '{') + { + const std::size_t start = _pos; + ++_pos; + skip_space(); + if (peek() == '}') + { + ++_pos; + if (!path.empty()) cells.emplace_back(path, std::string{_text.substr(start, 2)}); + return; + } + for (;;) + { + skip_space(); + const std::string key = string_value(); + skip_space(); + if (peek() != ':') fail("expected ':'"); + ++_pos; + skip_space(); + value(child_path(path, key), cells); + skip_space(); + const char sep = peek(); + ++_pos; + if (sep == '}') return; + if (sep != ',') fail("expected ',' or '}'"); + } + } + if (c == '[') + { + const std::size_t start = _pos; + skip_value(); + cells.emplace_back(path, std::string{_text.substr(start, _pos - start)}); + return; + } + if (c == '"') + { + cells.emplace_back(path, string_value()); + return; + } + const std::size_t start = _pos; + skip_value(); + const std::string_view token = _text.substr(start, _pos - start); + cells.emplace_back(path, token == "null" ? std::string{} : std::string{token}); + } + + std::string_view _text; + std::size_t _pos{0}; +}; + +} // namespace detail + +/// @brief Flatten a JSON document into dotted columns. +/// +/// An object's members become `parent.child` cells, strings are unescaped, +/// numbers and booleans keep their text, `null` is an empty cell, and an array +/// (or an empty nested object) is one cell holding its JSON text. A document +/// that is not an object gives one cell with an empty name. +/// +/// @param json A complete JSON document. +/// @return The `(column, cell)` pairs in document order. +/// @throws std::invalid_argument If @p json is not valid JSON. +[[nodiscard]] inline std::vector> flatten_json( + std::string_view json) +{ + return detail::json_flattener{json}.run(); +} + +/// @brief Whether a result has `to_json()` returning text. +template +concept json_result = requires(const T& item) { + { item.to_json() } -> std::convertible_to; +}; + +/// @brief Writes one JSON document per line (JSON Lines / NDJSON). +/// +/// Use it as the sink of for_each_batch(); results need `to_json()`. +/// +/// @code +/// dasmig::core::jsonl_writer out{std::cout}; +/// gen.for_each_name(request, {.count = 1000}, 7, out); +/// @endcode +class jsonl_writer +{ + public: + /// @brief Write to @p out, which must outlive the writer. + /// @param out Destination stream. + explicit jsonl_writer(std::ostream& out) : _out{&out} {} + + /// @brief Write one document as a line. + /// @param json Compact JSON; it must not contain a line break. + /// @throws std::invalid_argument If @p json contains a line break. + void write_line(std::string_view json) + { + if (json.find_first_of("\r\n") != std::string_view::npos) + { + throw std::invalid_argument("jsonl_writer: a JSON Lines record cannot contain a line break"); + } + _out->write(json.data(), static_cast(json.size())); + _out->put('\n'); + ++_lines; + } + + /// @brief Sink call: write the item's `to_json()`. + /// @param item The result. + template + void operator()(std::uint64_t /*index*/, const T& item) + { + write_line(item.to_json()); + } + + /// @brief Lines written so far. + /// @return The count. + [[nodiscard]] std::size_t lines() const noexcept { return _lines; } + + private: + std::ostream* _out; + std::size_t _lines{0}; +}; + +/// @brief Writes results as CSV, one row each, with columns flattened from +/// their JSON (see flatten_json()). +/// +/// The header comes from the first record. Later records are matched to it by +/// column name: a column the record lacks is empty, and a column the header +/// lacks is dropped. Fields are quoted when they hold a comma, a quote or a +/// line break; rows end with `\n`. +class csv_writer +{ + public: + /// @brief Write to @p out, which must outlive the writer. + /// @param out Destination stream. + explicit csv_writer(std::ostream& out) : _out{&out} {} + + /// @brief Write one record given as JSON. + /// @param json A JSON document. + /// @throws std::invalid_argument If @p json is not valid JSON. + void write_json(std::string_view json) + { + const auto cells = flatten_json(json); + if (!_header_written) + { + _columns.reserve(cells.size()); + for (const auto& cell : cells) _columns.push_back(cell.first); + write_row(_columns); + _header_written = true; + } + std::vector row(_columns.size()); + bool in_order = cells.size() == _columns.size(); + for (std::size_t i = 0; in_order && i < cells.size(); ++i) + { + in_order = cells[i].first == _columns[i]; + } + if (in_order) + { + for (std::size_t i = 0; i < cells.size(); ++i) row[i] = cells[i].second; + } + else + { + std::unordered_map position; + position.reserve(_columns.size()); + for (std::size_t i = 0; i < _columns.size(); ++i) position.emplace(_columns[i], i); + for (const auto& cell : cells) + { + if (const auto it = position.find(cell.first); it != position.end()) + { + row[it->second] = cell.second; + } + } + } + write_row(row); + ++_records; + } + + /// @brief Sink call: write the item's `to_json()`. + /// @param item The result. + template + void operator()(std::uint64_t /*index*/, const T& item) + { + write_json(item.to_json()); + } + + /// @brief The columns, known after the first record. + /// @return The column names. + [[nodiscard]] const std::vector& columns() const noexcept { return _columns; } + + /// @brief Records written so far (not counting the header). + /// @return The count. + [[nodiscard]] std::size_t records() const noexcept { return _records; } + + private: + void write_row(const std::vector& fields) + { + for (std::size_t i = 0; i < fields.size(); ++i) + { + if (i > 0) _out->put(','); + write_field(fields[i]); + } + _out->put('\n'); + } + + void write_field(const std::string& field) + { + if (field.find_first_of(",\"\r\n") == std::string::npos) + { + _out->write(field.data(), static_cast(field.size())); + return; + } + _out->put('"'); + for (const char c : field) + { + if (c == '"') _out->put('"'); + _out->put(c); + } + _out->put('"'); + } + + std::ostream* _out; + std::vector _columns; + bool _header_written{false}; + std::size_t _records{0}; +}; + +} // namespace dasmig::core + +#endif // DASMIG_CORE_SINKS_HPP diff --git a/dasmig/core/version.hpp b/dasmig/core/version.hpp index 0da0bb9..1ac3cce 100644 --- a/dasmig/core/version.hpp +++ b/dasmig/core/version.hpp @@ -16,9 +16,9 @@ /// incompatibly. #define DASMIG_CORE_VERSION_MAJOR 1 /// @brief Minor version. Changes when features are added compatibly. -#define DASMIG_CORE_VERSION_MINOR 0 +#define DASMIG_CORE_VERSION_MINOR 1 /// @brief Patch version. Changes for fixes that keep output identical. -#define DASMIG_CORE_VERSION_PATCH 1 +#define DASMIG_CORE_VERSION_PATCH 0 /// @brief Fails compilation unless the core in use is compatible with the /// version a library was built against. diff --git a/dasmig/core/weighted_table.hpp b/dasmig/core/weighted_table.hpp index af53fe0..e8d7eea 100644 --- a/dasmig/core/weighted_table.hpp +++ b/dasmig/core/weighted_table.hpp @@ -7,6 +7,8 @@ #include "distributions.hpp" #include "rng.hpp" +#include +#include #include #include #include @@ -135,8 +137,16 @@ class weighted_table template F> [[nodiscard]] std::optional try_pick_index(rng& r, const F& adjust) const { - std::vector cumulative; - cumulative.reserve(_entries.size()); + // The sums of a small table stay on the stack: an allocation would + // cost more than the pick. + std::array inline_sums; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::vector heap_sums; + std::span cumulative{inline_sums.data(), std::min(_entries.size(), inline_sums_)}; + if (_entries.size() > inline_sums_) + { + heap_sums.resize(_entries.size()); + cumulative = heap_sums; + } std::size_t last_positive = 0; double total = 0.0; for (std::size_t i = 0; i < _entries.size(); ++i) @@ -145,7 +155,7 @@ class weighted_table check_weight(w); total += w; if (w > 0.0) last_positive = i; - cumulative.push_back(total); + cumulative[i] = total; } if (!std::isfinite(total)) { @@ -244,6 +254,9 @@ class weighted_table std::vector _entries; std::vector _cumulative; + + // Entries whose adjusted sums try_pick_index() keeps on the stack. + static constexpr std::size_t inline_sums_ = 64; std::size_t _last_positive{0}; }; diff --git a/dasmig/entitygen.hpp b/dasmig/entitygen.hpp index c2dcea5..db7ee4c 100644 --- a/dasmig/entitygen.hpp +++ b/dasmig/entitygen.hpp @@ -36,7 +36,7 @@ /// @author Diego Dasso Migotto (diegomigotto at hotmail dot com) /// @see See doc/usage.md for the narrative tutorial. -DASMIG_CORE_REQUIRE(1, 0); +DASMIG_CORE_REQUIRE(1, 1); namespace dasmig { @@ -1498,9 +1498,11 @@ class eg /// @brief Generate multiple entities using a seed. /// - /// Entity @c i uses the @c i-th value of `core::rng{call_seed}.split("batch")` - /// as its call seed, so the result equals generate_batch_async(request, - /// call_seed) and each entity replays with generate(request, e.seed()). + /// Entity @c i uses `core::item_seed(call_seed, i)` as its call seed, so + /// the result equals generate_batch_async(request, call_seed) and each + /// entity replays with generate(request, e.seed()). Since 4.1.0 this is + /// the same seed derivation as every other library's batch; before, entity + /// @c i used the @c i-th value of `core::rng{call_seed}.split("batch")`. /// @param request Selection, batch count and entity validation options. /// @param call_seed Seed for reproducible results. /// @return A vector of generated entities. @@ -1513,6 +1515,81 @@ class eg return generate_batch_seeded(resolve_components(request), request, call_seed); } + /// @brief Generate a batch described by core::batch_options, from a seed + /// drawn from the generator's engine. + /// + /// Uses `options.count` (the request's `count` is ignored here; it stays + /// for generate_batch(request) until the next major version). + /// @param request Selection and entity validation options. + /// @param options Count, first index, threads and uniqueness. + /// @return The entities, in index order. + /// @throws std::invalid_argument If both selection fields are set. + /// @throws std::out_of_range If the requested group does not exist. + /// @throws std::runtime_error If component or entity validation is exhausted. + [[nodiscard]] std::vector generate_batch(const entity_request& request, + const core::batch_options& options) + { + return generate_batch(request, options, _engine()); + } + + /// @brief Generate a batch described by core::batch_options, deterministically. + /// + /// Entity @c i is exactly `generate(request, core::item_seed(batch_seed, + /// options.first + i))`, for any `options.threads`. With more than one + /// thread, components, observers and the request's validator (including + /// captured state) must be thread-safe. + /// @param request Selection and entity validation options. + /// @param options Count, first index, threads and uniqueness (see + /// core::unique_rule, made for `entity`). + /// @param batch_seed Seed of the whole batch. + /// @return The entities, in index order. + /// @throws std::invalid_argument If both selection fields are set. + /// @throws std::out_of_range If the requested group does not exist. + /// @throws std::runtime_error If component or entity validation is exhausted, + /// or a unique batch runs out of attempts. + [[nodiscard]] std::vector generate_batch(const entity_request& request, + const core::batch_options& options, + std::uint64_t batch_seed) const + { + const auto refs = resolve_components(request); + return core::run_batch(options, batch_seed, [&](std::uint64_t item_seed) + { return generate_seeded(refs, item_seed, request); }); + } + + /// @brief Stream a batch of entities to a sink without holding the batch. + /// + /// Generates the same entities as generate_batch(request, options, + /// batch_seed) and calls `sink(index, entity&)` in index order on the + /// calling thread; `index` includes `options.first`. + /// @tparam Sink Callable taking `(std::uint64_t, entity&)`; for example + /// core::jsonl_writer. + /// @param request Selection and entity validation options. + /// @param options Count, first index, threads and uniqueness. + /// @param batch_seed Seed of the whole batch. + /// @param sink Receives the entities. + template + void for_each_entity(const entity_request& request, const core::batch_options& options, + std::uint64_t batch_seed, Sink&& sink) const + { + const auto refs = resolve_components(request); + core::for_each_batch(options, batch_seed, + [&](std::uint64_t item_seed) { return generate_seeded(refs, item_seed, request); }, + std::forward(sink)); + } + + /// @brief Stream a batch of entities with a batch seed drawn from the + /// generator's engine. + /// @tparam Sink Callable taking `(std::uint64_t, entity&)`. + /// @param request Selection and entity validation options. + /// @param options Count, first index, threads and uniqueness. + /// @param sink Receives the entities. + template + void for_each_entity(const entity_request& request, const core::batch_options& options, + Sink&& sink) + { + for_each_entity(request, options, _engine(), std::forward(sink)); + } + /// @} /// @name Concurrent Batch Generation /// @{ @@ -1660,13 +1737,15 @@ class eg static constexpr std::launch async_policy = std::launch::async; #endif - // The call seeds of a batch. + // The call seeds of a batch: item i is core::item_seed(call_seed, i). [[nodiscard]] static std::vector batch_seeds( std::size_t count, std::uint64_t call_seed) // NOLINT(bugprone-easily-swappable-parameters) { - auto r = core::rng{call_seed}.split("batch"); std::vector seeds(count); - std::ranges::generate(seeds, [&r] { return r(); }); + for (std::size_t i = 0; i < count; ++i) + { + seeds[i] = core::item_seed(call_seed, i); + } return seeds; } diff --git a/doc/usage.md b/doc/usage.md index ba35422..04ff6d6 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -1070,6 +1070,32 @@ Version 2 moves the library onto [dasmig-core](https://github.com/dasmig/dasmig- - **Files:** replace `random.hpp` with `core.hpp` and the `core/` directory, kept next to `entitygen.hpp`. - **New:** `entity::to_json()`, `entity::signature()`, `component::to_json()` (and `dasmig::json_value`). +## Batch Generation + +`generate_batch` takes the same `core::batch_options` as every other library, next to the older `request.count` form, which keeps working until the next major version. + +```cpp +dasmig::eg gen; +// ... add components ... + +// 1,000 entities from one batch seed, on four threads. +auto entities = gen.generate_batch({}, {.count = 1000, .threads = 4}, 42); + +// Entity 1 replays on its own. +auto again = gen.generate({}, dasmig::core::item_seed(42, 1)); + +// A million entities to a file as JSON Lines, never all in memory. +std::ofstream out{"entities.jsonl"}; +dasmig::core::jsonl_writer lines{out}; +gen.for_each_entity({}, {.count = 1'000'000, .threads = 0}, 42, lines); +``` + +- **Item seeds.** Entity `i` is exactly `generate(request, core::item_seed(batch_seed, first + i))`. A range can be generated directly (`first = 1000`), two batches that follow each other equal one, and the output is the same for any `threads` (0 means the hardware count). `generate_batch_async` gives the same entities. +- **What changed in 4.1.0.** The seeded `generate_batch(request, seed)` and `generate_batch_async(request, seed)` now use `core::item_seed` too, so their entities differ from 4.0.x for the same seed. Each entity still replays with `generate(request, e.seed())`. +- **Threads.** With more than one thread, components, observers and the request's validator (including captured state) must be thread-safe; the CI runs the batch tests under ThreadSanitizer. +- **Uniqueness.** `.unique = core::unique_rule::by(key)` regenerates an entity whose key already appeared in the batch. +- **Sinks.** `core::jsonl_writer` writes one `to_json()` per line and `core::csv_writer` flattens each entity's JSON into dotted columns. + ## Error Reference The library throws standard exceptions. No custom exception types are used. diff --git a/tests/tests.cpp b/tests/tests.cpp index 925909d..50b9acf 100644 --- a/tests/tests.cpp +++ b/tests/tests.cpp @@ -4060,7 +4060,7 @@ TEST_CASE("known answers for fixed seeds", "[kat]") std::uint64_t h = 0xcbf29ce484222325ULL; for (const auto& e : g.generate_batch({.count = 10000}, 7)) h = fnv1a(e.to_json(), h); - REQUIRE(h == 0xff5e8a58727602b3ULL); + REQUIRE(h == 0x6ff4b2ae28ef27aaULL); // item seeds since 4.1.0 (core::item_seed) } // --------------------------------------------------------------------------- @@ -4768,3 +4768,74 @@ TEST_CASE("default conversions cover std::string and the integer types", "[to_st "{\"components\":{\"s\":\"Jo\xC3\xA3o\",\"ll\":-9000000000,\"u\":7,\"ul\":8,\"ull\":9000000000," "\"big\":\"18446744073709551615\"}")); } + +// --------------------------------------------------------------------------- +// Batch generation with core::batch_options +// --------------------------------------------------------------------------- + +TEST_CASE("batch options: items equal the single calls for any thread count", "[batch][options]") +{ + dasmig::eg g; + add_kat_components(g); + const auto batch = g.generate_batch({}, {.count = 200, .threads = 1}, 77); + REQUIRE(batch.size() == 200); + for (std::uint64_t i = 0; i < batch.size(); ++i) + { + REQUIRE(batch[i].to_json() == g.generate({}, dasmig::core::item_seed(77, i)).to_json()); + } + + const auto tail = g.generate_batch({}, {.count = 100, .first = 100, .threads = 4}, 77); + REQUIRE(tail.size() == 100); + for (std::size_t i = 0; i < tail.size(); ++i) + { + REQUIRE(tail[i].to_json() == batch[100 + i].to_json()); + } + for (const std::size_t threads : {2U, 7U, 0U}) + { + const auto other = g.generate_batch({}, {.count = 200, .threads = threads}, 77); + REQUIRE(other.size() == batch.size()); + for (std::size_t i = 0; i < other.size(); ++i) + { + REQUIRE(other[i].to_json() == batch[i].to_json()); + } + } + + // The request's count and the options give the same entities, and the + // async batch too. + const auto by_count = g.generate_batch({.count = 50}, 77); + const auto async = g.generate_batch_async({.count = 50}, 77); + for (std::size_t i = 0; i < by_count.size(); ++i) + { + REQUIRE(by_count[i].to_json() == batch[i].to_json()); + REQUIRE(async[i].to_json() == batch[i].to_json()); + } + REQUIRE(g.generate_batch({}, {.count = 5}).size() == 5); +} + +TEST_CASE("batch options: streaming gives the same entities in order", "[batch][options]") +{ + dasmig::eg g; + add_kat_components(g); + const auto batch = g.generate_batch({}, {.count = 300, .first = 10, .threads = 3}, 5); + std::uint64_t next = 10; + std::size_t seen = 0; + g.for_each_entity({}, {.count = 300, .first = 10, .threads = 3}, 5, + [&](std::uint64_t index, const dasmig::entity& e) + { + REQUIRE(index == next); + REQUIRE(e.to_json() == batch[seen].to_json()); + ++next; + ++seen; + }); + REQUIRE(seen == batch.size()); +} + +TEST_CASE("batch options: known answer of 10,000 entities", "[batch][options][kat]") +{ + dasmig::eg g; + add_kat_components(g); + std::uint64_t h = 0xcbf29ce484222325ULL; + g.for_each_entity({}, {.count = 10000, .threads = 4}, 1, + [&](std::uint64_t, const dasmig::entity& e) { h = fnv1a(e.to_json(), h); }); + REQUIRE(h == 0xa172668aaf7295d7ULL); +} \ No newline at end of file From b40c13a72cd98d8f072f6c0c31f433f85d1666ec Mon Sep 17 00:00:00 2001 From: Diego Migotto Date: Fri, 16 Oct 2026 19:16:26 -0300 Subject: [PATCH 2/2] chore: do not track the local .claude worktree entry Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_018Cn4xFiHhbCHDhom8w2Ghq --- .claude/worktrees/v4-json-bool-fix | 1 - 1 file changed, 1 deletion(-) delete mode 160000 .claude/worktrees/v4-json-bool-fix diff --git a/.claude/worktrees/v4-json-bool-fix b/.claude/worktrees/v4-json-bool-fix deleted file mode 160000 index 15d5060..0000000 --- a/.claude/worktrees/v4-json-bool-fix +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 15d506018baa5c588194fe8fe68f4b78051c941f