From 8d9edc64d9a6c77cd380975af8c46ed3a9368d85 Mon Sep 17 00:00:00 2001 From: HDDen <62592944+HDDen@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:52:15 +0300 Subject: [PATCH] MCOtxt decoder --- .github/workflows/run-unit-tests.yml | 4 + build_as_lib.py | 1 + docs/mcotxt.md | 224 ++++++ examples/companion_radio/MCOtxtDelivery.cpp | 186 +++++ examples/companion_radio/MCOtxtDelivery.h | 30 + examples/companion_radio/MyMesh.cpp | 57 +- examples/companion_radio/MyMesh.h | 4 + examples/companion_radio/ui-new/UITask.cpp | 3 +- examples/companion_radio/ui-orig/UITask.cpp | 7 +- platformio.ini | 4 + src/helpers/Base91.cpp | 110 +++ src/helpers/Base91.h | 20 + src/helpers/BaseChatMesh.h | 3 +- src/helpers/BaseSerialInterface.h | 3 +- src/helpers/MessageLimits.h | 8 + src/helpers/mcotxt/MCOtxtCodec.cpp | 376 +++++++++ src/helpers/mcotxt/MCOtxtCodec.h | 48 ++ src/helpers/mcotxt/MCOtxtDisplay.cpp | 47 ++ src/helpers/mcotxt/MCOtxtDisplay.h | 20 + src/helpers/mcotxt/MCOtxtModels.cpp | 118 +++ src/helpers/mcotxt/MCOtxtModels.h | 84 ++ src/helpers/mcotxt/MCOtxtParts.cpp | 117 +++ src/helpers/mcotxt/MCOtxtParts.h | 41 + src/helpers/mcotxt/MCOtxtStreamFormat.h | 50 ++ src/helpers/mcotxt/MCOtxtText.cpp | 113 +++ src/helpers/mcotxt/MCOtxtText.h | 15 + src/helpers/mcotxt/MCOtxtTransport.cpp | 343 ++++++++ src/helpers/mcotxt/MCOtxtTransport.h | 96 +++ src/helpers/mcotxt/generated/model_be.h | 96 +++ src/helpers/mcotxt/generated/model_de.h | 92 +++ src/helpers/mcotxt/generated/model_en.h | 87 +++ src/helpers/mcotxt/generated/model_fr.h | 108 +++ src/helpers/mcotxt/generated/model_it.h | 99 +++ src/helpers/mcotxt/generated/model_ru.h | 97 +++ src/helpers/mcotxt/generated/model_uk.h | 97 +++ test/test_mcotxt/test_mcotxt.cpp | 730 ++++++++++++++++++ test/test_mcotxt_delivery/fixtures.h | 185 +++++ test/test_mcotxt_delivery/golden.h | 11 + .../test_mcotxt_delivery.cpp | 528 +++++++++++++ tools/test_mcotxt_queue.cpp.in | 140 ++++ tools/test_mcotxt_queue.py | 72 ++ 41 files changed, 4455 insertions(+), 19 deletions(-) create mode 100644 docs/mcotxt.md create mode 100644 examples/companion_radio/MCOtxtDelivery.cpp create mode 100644 examples/companion_radio/MCOtxtDelivery.h create mode 100644 src/helpers/Base91.cpp create mode 100644 src/helpers/Base91.h create mode 100644 src/helpers/MessageLimits.h create mode 100644 src/helpers/mcotxt/MCOtxtCodec.cpp create mode 100644 src/helpers/mcotxt/MCOtxtCodec.h create mode 100644 src/helpers/mcotxt/MCOtxtDisplay.cpp create mode 100644 src/helpers/mcotxt/MCOtxtDisplay.h create mode 100644 src/helpers/mcotxt/MCOtxtModels.cpp create mode 100644 src/helpers/mcotxt/MCOtxtModels.h create mode 100644 src/helpers/mcotxt/MCOtxtParts.cpp create mode 100644 src/helpers/mcotxt/MCOtxtParts.h create mode 100644 src/helpers/mcotxt/MCOtxtStreamFormat.h create mode 100644 src/helpers/mcotxt/MCOtxtText.cpp create mode 100644 src/helpers/mcotxt/MCOtxtText.h create mode 100644 src/helpers/mcotxt/MCOtxtTransport.cpp create mode 100644 src/helpers/mcotxt/MCOtxtTransport.h create mode 100644 src/helpers/mcotxt/generated/model_be.h create mode 100644 src/helpers/mcotxt/generated/model_de.h create mode 100644 src/helpers/mcotxt/generated/model_en.h create mode 100644 src/helpers/mcotxt/generated/model_fr.h create mode 100644 src/helpers/mcotxt/generated/model_it.h create mode 100644 src/helpers/mcotxt/generated/model_ru.h create mode 100644 src/helpers/mcotxt/generated/model_uk.h create mode 100644 test/test_mcotxt/test_mcotxt.cpp create mode 100644 test/test_mcotxt_delivery/fixtures.h create mode 100644 test/test_mcotxt_delivery/golden.h create mode 100644 test/test_mcotxt_delivery/test_mcotxt_delivery.cpp create mode 100644 tools/test_mcotxt_queue.cpp.in create mode 100644 tools/test_mcotxt_queue.py diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 5d48f4c67be..bd4bae99e1f 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -22,6 +22,10 @@ jobs: - name: Run Unit Tests run: pio test -e native -e native_kiss_modem -vv + - name: Run MCOtxt Queue Tests (ASan) + # The native tests above prepare Google Test in .pio/libdeps/native. + run: python tools/test_mcotxt_queue.py --asan + - name: Upload Test Results # Upload test results even if the test step failed. if: always() diff --git a/build_as_lib.py b/build_as_lib.py index fbc7c15a7a8..d0057e7e0d5 100644 --- a/build_as_lib.py +++ b/build_as_lib.py @@ -6,6 +6,7 @@ src_filter = [ '+<*.cpp>', '+', + '+', '+', '+', '+', diff --git a/docs/mcotxt.md b/docs/mcotxt.md new file mode 100644 index 00000000000..5c7570a5be9 --- /dev/null +++ b/docs/mcotxt.md @@ -0,0 +1,224 @@ +# MCOtxt compatibility delivery + +The companion can decode MCOtxt v1 for an app that does not implement the codec. +An app with its own decoder continues to receive the original messages. There +are no CLI commands, persistent settings, or encoder. Companion compatibility +is selected separately for each app session. Local displays decode received +MCOtxt independently of the connected app's capabilities. + +## Negotiation + +An app declares `mctxt` as a complete, case-sensitive item in a `cap=` token of +the existing `CMD_APP_START` application name (after its eight-byte header): + +```text +ExampleApp;cap=mctxt +ExampleApp;cap=other,mctxt,another +``` + +The `cap=` field is optional. When present, its value is split on commas and +each complete item is checked for `mctxt`; the parser does not require the literal +field `cap=mctxt`. An absent or empty list means that the app has not declared +MCOtxt decoding support. Empty items and unknown capabilities are ignored. +The name itself, `mctxt2`, and `MCTXT` do not match. The bounded parser reads the +entire supplied app-name field, without a separate truncated copy. + +This port does not add framed-packet support (`FR01`/`frmfrg1`) or negotiate it. +Splitting decoded text into existing companion messages is independent of that +separate transport feature. + +## Supported transports + +| Received form | App declares `mctxt` | App does not declare `mctxt` | +|---|---|---| +| Text containing `mct:` + canonical Base91 | Original frame | Decoded text frame(s), replacing the encoded text | +| Channel GROUP_DATA type `0x0120`, subtype/version `0x31` | Original binary frame only | Original binary frame first, followed by decoded channel text frame(s) | +| Other text, CLI data, other binary types, malformed/unsupported containers | Original frame | Original frame | + +Text containers can appear inside surrounding prose, including quotes and +parentheses. They must start at a word boundary, not in the middle of a Latin, +Cyrillic, or other supported-script word. The scanner consumes the longest +complete valid container, retaining surrounding text and punctuation; punctuation +cannot simply be stripped because it is also part of the Base91 alphabet. +Candidates must round-trip to the same canonical Base91 spelling. Counted +compressed strings must also have zero padding after their significant bits. +These checks prevent adjacent punctuation from changing the last decoded byte +while still appearing to be a complete container. + +The decoder supports generation-0 EN/RU/FR/DE/IT/UK/BE models and the v1 stream +controls, including raw UTF-8. `generated/model_*.h` are protocol tables: changing +symbol order changes wire meaning. Do not edit them by hand. + +## Queue and delivery lifecycle + +`addToOfflineQueue()` is unchanged apart from resetting the delivery cursor when +overflow evicts the head. One queue entry holds the original **companion frame** +with its encoded MCOtxt payload, not derived text parts. This is the existing +received-message representation, not an encrypted radio packet or a new storage +format. The existing 176-byte frame admission/truncation rules are unchanged. + +`MCOtxtDelivery::next()` runs only when the app requests +`CMD_SYNC_NEXT_MESSAGE`. It renders one response and reports whether the raw +queue head can be consumed. A binary source is returned unchanged on the first +request; subsequent requests render its text. Derived parts are never enqueued, +so one long message cannot occupy several queue slots. On each request the +decoder reuses the bounded shared workspace; no decoded copy is retained per +queue entry. + +After the final part, the raw source is removed. An unsupported binary source +is removed on the request after its raw delivery, and that same request can +continue with the next queue entry. It must not produce an early +`RESP_CODE_NO_MORE_MESSAGES` while another entry remains. + +`APP_START` resets the cursor and re-evaluates capabilities. If a previous app +disconnected partway through delivery, a new app sees the retained source in +its own representation, starting from the beginning. Parts already received +before the reconnect can consequently repeat. This is not an exactly-once +protocol, and the upstream queue remains RAM-only. + +## Text formatting and metadata + +One part has no numbering. Longer text is split within both the 176-byte +companion-frame budget and the 160-byte text budget, accounting for headers, +the sender prefix, leading/trailing continuation markers, and part number: + +```text +Bob: @[Ann] first part of the reply... [1/3] +Bob: ...continuation... [2/3] +Bob: ...last part [3/3] +``` + +Every part except the last ends its text with `...` before the part number. +Every part except the first starts its text with `...`. A single unsplit message +has neither added ellipses nor numbering. + +The reply author comes from the container's reply field, not from heuristics on +the text. The mention is inserted once before splitting, so it stays in the +first part; an identical existing leading mention is not added twice. A reply +inside quoted prose is also promoted to the beginning of the first part. If +several embedded containers have distinct reply anchors, the source is left +raw rather than merging incompatible reply relationships. + +Sender and reply-author names use fixed 32-byte arrays in `DecodedMessage`, +matching the current `NodePrefs::node_name` and `ContactInfo::name` fields. +`kMaxNameBytes` is 31 bytes excluding NUL; raise it if the firmware's name limit +changes. Longer names are clipped to the longest complete UTF-8 prefix that +fits. The decoder validates the entire field before clipping; compressed names +are temporarily decoded in the shared workspace, then copied to the fixed +metadata field. A name that cannot be fully decoded within that workspace falls +back to raw. Metadata owns its names and remains valid when text storage is +reused. The sender is repeated on each numbered binary-text part. + +Reply mentions and conflict checks use these bounded names. If a complete +bounded mention cannot fit in the first part together with the sender and part +marks, the original is preserved. Text is never clipped to hide decode overflow. + +For channel text, the original outer `sender: ` prefix is retained. Direct and +signed-room messages keep their contact/author prefixes without adding a sender +name to their bodies. Binary compatibility text uses the container's sender, +falling back to the envelope's name or `?` if neither exists. + +All text parts preserve the source channel/contact and packed path metadata. +Text transports retain the original packet timestamp. Binary transports use +their container timestamp; an inherited timestamp is captured once when binary +delivery begins and reused for all parts. Generated messages use the current +app's legacy or V3 layout; capable apps retain the original bytes/layout. + +Splits use UTF-8 character boundaries, preferring a nearby space or newline. +Whitespace at a selected split boundary is omitted. The splitter supports up +to 20 parts; the shared decode scratch defaults to 1280 bytes. No heap +allocation or filesystem write is needed. The shared scratch assumes the +existing single companion loop; concurrent callers would need separate buffers +or synchronization. + +Insufficient workspace, malformed UTF-8, embedded NUL that cannot be represented +in companion C-string text, and unknown compressed formats fall back to the raw +source. A truncated decode is never published as a successful complete message. + +## Local display previews + +After the unchanged raw frame is queued, `MyMesh` adapts the text passed to its +synchronous UI listener. Base91 MCOtxt, including containers embedded in prose, +is decoded for channel, direct, and signed-room text. CLI data is passed through. +Binary MCOtxt is still delivered to the binary listener unchanged; a valid +container additionally produces one ordinary channel-text preview with its +sender and optional reply mention. Unsupported/malformed binary creates no text +preview. This does not add text frames to the companion queue. + +The existing `ui-orig` and `ui-new` message previews therefore show decoded text +even when the connected app declares `mctxt`. Their existing buffer sizes are +retained: a long message shows its initial preview, clipped on a UTF-8 character +boundary. BLE part numbers and continuation markers are not added to the local +preview. Unsupported glyphs retain the existing display/font behavior. +`ui-tiny` retains its status/count-only behavior, as it has no ordinary message +preview to adapt. + +Decoded callback text borrows the shared scratch buffer. Listeners must copy it +before returning; the existing UI implementations do so. Queue-count listeners +are also updated when an exhausted binary source is removed without a text +response, so an empty queue clears the unread display state. + +## Code map and tests + +- `examples/companion_radio/MCOtxtDelivery.*`: session capability, one-entry cursor, + companion layouts, binary-first policy, generated text frames. +- `src/helpers/mcotxt/`: pure C++ stream decoder, protocol models, container + parser, embedded-text scanner, reply adaptation, UTF-8-safe part planner. +- `src/helpers/mcotxt/MCOtxtDisplay.*`: local text/binary preview adaptation and + UTF-8-safe copying, independent of companion session capabilities. +- `src/helpers/Base91.*`: canonical basE91, based on Joachim Henke's public-domain + algorithm. +- `src/helpers/MessageLimits.h`: shared `MAX_FRAME_SIZE` and `MAX_TEXT_LEN` + definitions, used by `BaseSerialInterface`, `BaseChatMesh`, and `MCOtxtDelivery`. + Delivery buffers, inline-token/Base91 buffers, and text budgets follow these + limits without numeric copies. +- `MyMesh.*`: app-start/offline-head lifecycle hooks and display-listener + adaptation after raw enqueueing. `Frame` and radio protocol layouts are unchanged. +- `platformio.ini` and `build_as_lib.py`: include the pure decoder sources for + the normal project build, native tests, and `BUILD_EXAMPLE` library builds. + +Run the focused native suites with the project's normal PlatformIO setup: + +```sh +pio test -e native -f test_mcotxt -f test_mcotxt_delivery +``` + +Queue lifecycle checks use the actual `MyMesh.cpp` enqueue/drain/SYNC bodies +with small native hardware doubles. Run them with an already installed compiler +and Google Test sources (a developer shell is required for MSVC): + +```sh +python tools/test_mcotxt_queue.py --gtest /path/to/googletest --asan +``` + +The unit-test CI workflow also runs `python tools/test_mcotxt_queue.py --asan` +after the PlatformIO tests. It reuses Google Test from +`.pio/libdeps/native/googletest`; a build error, test failure, or AddressSanitizer +error fails the CI step. + +The runner never installs dependencies. It covers overflow/eviction of a partial +head, eviction behind a direct-message head, an all-direct full queue, skipping +an exhausted malformed binary, listener count updates, and a new app taking the +retained raw head. This is separate from compiling all of `MyMesh` for a board. +Polling an already-empty queue is also checked: it must not reset the UI screen. + +The decoder suite covers models, bit counts, controls, strict parsing, and text +splitting. Delivery tests cover capability parsing, both companion layouts, +signed-room prefixes, inline containers, reply placement, raw preservation, +multipart/reconnect/eviction cursor handling, malformed/future formats, UTF-8, +bounded mutation/random-input cases, and local display adaptation/copying. +Name regressions cover the 31-byte boundary, clipping raw/compressed/UTF-8 names, +validation beyond the clipped prefix, multiple inline replies, metadata ownership, +full-frame bounds, and preservation of whole bounded reply mentions. Punctuation +regressions cover adjacent Base91 punctuation across token endings, canonical +encoding, and counted-stream padding, including the app and display paths. +`golden.h` contains fixed counted +streams produced by the independent Dart `MCOtxtFrame` encoder with RU/EN. + +The decoder, transport, part planner, and unchanged model tables were ported from +south-edition commit `62cf56c694aed48841a7a37e75304f0c98bfa587`, removing fork-only +feature guards. The companion adapter and focused MCOtxt-only scanner are kept +separate from South's UI, CLI, other codecs, and persistent-queue extensions. +Before release, also check the chosen board's complete firmware build, available +flash/RAM/stack, and real BLE/USB queue draining with both kinds of app. Native +tests and standalone ARM compilation do not establish those hardware properties. diff --git a/examples/companion_radio/MCOtxtDelivery.cpp b/examples/companion_radio/MCOtxtDelivery.cpp new file mode 100644 index 00000000000..b703ff066d8 --- /dev/null +++ b/examples/companion_radio/MCOtxtDelivery.cpp @@ -0,0 +1,186 @@ +#include "MCOtxtDelivery.h" + +#include +#include + +#include +#include +#include + +namespace { + +// Existing companion codes, not new commands. Keep this adapter independent of +// Arduino/MyMesh so queue delivery and legacy/V3 layouts can be tested natively. +const uint8_t kContact = 7, kChannel = 8, kContactV3 = 16, kChannelV3 = 17, kData = 27; +const uint8_t kPlain = 0, kSignedPlain = 2; + +// APP_START may omit cap= entirely. In each semicolon-delimited field, look for +// cap= first, then scan its comma-separated list for the exact mctxt item. +// Empty lists/items and unrelated capabilities do not enable this decoder. +bool hasCapability(const uint8_t* info, size_t length) { + if (!info) return false; + size_t pos = 0; + while (pos < length && info[pos]) { + const size_t start = pos; + while (pos < length && info[pos] && info[pos] != ';') ++pos; + if (pos - start >= 4 && memcmp(info + start, "cap=", 4) == 0) { + size_t item = start + 4; + while (item < pos) { + size_t end = item; + while (end < pos && info[end] != ',') ++end; + if (end - item == 5 && memcmp(info + item, "mctxt", 5) == 0) return true; + item = end + 1; + } + } + if (pos < length && info[pos] == ';') ++pos; + } + return false; +} + +struct Layout { + size_t header; + bool channel; + bool v3; +}; + +bool textLayout(const uint8_t* frame, size_t length, Layout& layout) { + switch (frame[0]) { + case kContact: layout = {13, false, false}; break; + case kContactV3: layout = {16, false, true}; break; + case kChannel: layout = {8, true, false}; break; + case kChannelV3: layout = {11, true, true}; break; + default: return false; + } + if (length < layout.header) return false; + const uint8_t type = frame[layout.header - 5]; + if (type != kPlain && type != kSignedPlain) return false; // never rewrite CLI data + if (!layout.channel && type == kSignedPlain) layout.header += 4; // room author key + return length >= layout.header; +} + +// Only generated text is adapted to the current app version. A capable app gets +// its original frame byte-for-byte, including its original protocol layout. +size_t copyHeader(const uint8_t* source, const Layout& layout, uint8_t app_version, + uint8_t* header) { + const bool v3 = app_version >= 3; + size_t n = 0; + header[n++] = layout.channel ? (v3 ? kChannelV3 : kChannel) : (v3 ? kContactV3 : kContact); + if (v3) { + for (size_t i = 1; i <= 3; ++i) header[n++] = layout.v3 ? source[i] : 0; + } + const size_t start = layout.v3 ? 4 : 1; + memcpy(header + n, source + start, layout.header - start); + return n + layout.header - start; +} + +} // namespace + +void MCOtxtDelivery::resetMessage() { + _original_sent = false; + _part = 0; + _timestamp = 0; +} + +void MCOtxtDelivery::appStart(const uint8_t* app_info, size_t length) { + _supports_mctxt = hasCapability(app_info, length); + resetMessage(); // reconnect/new app restarts the retained raw head from part one +} + +int MCOtxtDelivery::next(const uint8_t* source, size_t length, uint8_t app_version, + uint32_t now, uint8_t* output, bool& consume) { + consume = true; + if (!source || !output || length == 0 || length > kFrameSize) return 0; + auto raw = [&]() -> int { + if (_original_sent) return 0; + memcpy(output, source, length); + return (int)length; + }; + if (_supports_mctxt) return raw(); + + Layout layout; + uint8_t header[20]; // V3 direct header including a signed-room author prefix + size_t header_length = 0, prefix_length = 0, mention_length = 0; + char prefix[mcotxt::kMaxNameBytes + 2]; // bounded sender plus ": " + char* decoded = mcotxt::scratch(); + if (textLayout(source, length, layout)) { + char original[kFrameSize + 1]; + const size_t text_length = length - layout.header; + memcpy(original, source + layout.header, text_length); + original[text_length] = '\0'; + // Do not silently lose trailing bytes hidden after an embedded NUL. + if (strlen(original) != text_length) return raw(); + const char* body = original; + if (layout.channel) { + const char* separator = strstr(original, ": "); + if (separator) { + prefix_length = (size_t)(separator + 2 - original); + if (prefix_length > sizeof(prefix)) return raw(); // prefix is counted, not NUL-terminated + memcpy(prefix, original, prefix_length); + body += prefix_length; + } + } + if (!mcotxt::decodeText(body, decoded, mcotxt::kScratchBytes, &mention_length)) return raw(); + header_length = copyHeader(source, layout, app_version, header); + } else if (source[0] == kData && length >= 9 && length == (size_t)9 + source[8]) { + const uint16_t type = (uint16_t)source[6] | ((uint16_t)source[7] << 8); + if (!mcotxt::isBinaryEnvelope(type, source + 9, source[8])) return raw(); + // Preserve binary delivery for apps that use GROUP_DATA for other purposes. + // Even malformed/future MCOtxt is delivered raw once, without extra notices. + if (!_original_sent) { + _original_sent = true; + _timestamp = now; // stable inherited timestamp across all subsequent parts + memcpy(output, source, length); + consume = false; + return (int)length; + } + mcotxt::DecodedMessage message; + if (mcotxt::decodeBinaryEnvelope(type, source + 9, source[8], decoded, + mcotxt::kScratchBytes, message) != mcotxt::MessageStatus::Ok) + return 0; + bool truncated = false; + if (!mcotxt::ensureReplyMentionPrefix(message, decoded, mcotxt::kScratchBytes, truncated) || + truncated) return 0; + if (message.has_reply && message.reply_author[0]) + mention_length = strlen(message.reply_author) + 3; + header[header_length++] = app_version >= 3 ? kChannelV3 : kChannel; + if (app_version >= 3) { + header[header_length++] = source[1]; + header[header_length++] = 0; + header[header_length++] = 0; + } + header[header_length++] = source[4]; // channel index + header[header_length++] = source[5]; // original packed path metadata + header[header_length++] = kPlain; + const uint32_t timestamp = message.has_timestamp ? message.timestamp : _timestamp; + for (size_t i = 0; i < 4; ++i) header[header_length++] = (uint8_t)(timestamp >> (8 * i)); + const char* sender = message.has_sender ? message.sender : "?"; + const size_t name_prefix = strlen(sender) + 2; + memcpy(prefix, sender, name_prefix - 2); + prefix[name_prefix - 2] = ':'; + prefix[name_prefix - 1] = ' '; + prefix_length = name_prefix; + } else { + return raw(); + } + + const size_t budget = kFrameSize - header_length < kTextSize + ? kFrameSize - header_length : kTextSize; + mcotxt::TextPart parts[mcotxt::kMaxParts]; + size_t count = 0; + if (decoded[0] == '\0') { parts[0] = {0, 0}; count = 1; } + else count = mcotxt::splitForApp(decoded, strlen(decoded), budget, prefix_length, + parts, mcotxt::kMaxParts); + if (count == 0 || _part >= count) return raw(); + // Never split an exact author mention across frames. If it cannot coexist + // with the sender/part marks, preserve the original instead of changing it. + if (parts[0].length < mention_length) return raw(); + const size_t written = mcotxt::formatPart(prefix, prefix_length, decoded, parts[_part], + _part, count, (char*)output + header_length, + kFrameSize + 1 - header_length); + // An empty direct message is valid; formatPart returns zero for it too. + if (written > budget || (written == 0 && (prefix_length != 0 || decoded[0] != '\0'))) + return raw(); + memcpy(output, header, header_length); + consume = ++_part >= count; + return (int)(header_length + written); +} diff --git a/examples/companion_radio/MCOtxtDelivery.h b/examples/companion_radio/MCOtxtDelivery.h new file mode 100644 index 00000000000..77cfc68aaaa --- /dev/null +++ b/examples/companion_radio/MCOtxtDelivery.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include + +// Per-app cursor over ONE unchanged offline-queue entry. Text parts are generated +// during SYNC_NEXT_MESSAGE, never enqueued. No allocation, flash settings or UI +// dependency; the decoder workspace is shared by the single companion loop. +class MCOtxtDelivery { +public: + static const size_t kFrameSize = MAX_FRAME_SIZE; + static const size_t kTextSize = MAX_TEXT_LEN; + void appStart(const uint8_t* app_info, size_t length); + void resetMessage(); // also call when overflow evicts the queue head + + // output has MAX_FRAME_SIZE+1 bytes (including formatter's temporary NUL), + // separate from source; source stays owned by the queue and is never modified. + // consume tells the queue when the source may be removed. A zero return with + // consume=true means binary was already sent and has no decoded text to add; + // the caller may continue with the next queue entry in the same SYNC request. + int next(const uint8_t* source, size_t length, uint8_t app_version, uint32_t now, + uint8_t* output, bool& consume); + +private: + bool _supports_mctxt = false; + bool _original_sent = false; + size_t _part = 0; + uint32_t _timestamp = 0; +}; diff --git a/examples/companion_radio/MyMesh.cpp b/examples/companion_radio/MyMesh.cpp index d74050be7e0..c8b8d27b83b 100644 --- a/examples/companion_radio/MyMesh.cpp +++ b/examples/companion_radio/MyMesh.cpp @@ -5,6 +5,7 @@ #include #endif #include +#include #define CMD_APP_START 1 #define CMD_SEND_TXT_MSG 2 @@ -227,6 +228,7 @@ void MyMesh::addToOfflineQueue(const uint8_t frame[], int len) { int pos = 0; while (pos < offline_queue_len) { if (offline_queue[pos].isChannelMsg()) { + if (pos == 0) mcotxt_delivery.resetMessage(); // evicted a partially delivered source for (int i = pos; i < offline_queue_len - 1; i++) { // delete oldest channel msg from queue offline_queue[i] = offline_queue[i + 1]; } @@ -246,15 +248,27 @@ void MyMesh::addToOfflineQueue(const uint8_t frame[], int len) { } int MyMesh::getFromOfflineQueue(uint8_t frame[]) { - if (offline_queue_len > 0) { // check offline queue - size_t len = offline_queue[0].len; // take from top of queue - memcpy(frame, offline_queue[0].buf, len); - - offline_queue_len--; - for (int i = 0; i < offline_queue_len; i++) { // delete top item from queue - offline_queue[i] = offline_queue[i + 1]; + // The original pop-on-read removed the source after a single response. MCOtxt + // may need several SYNC requests (raw binary first, then decoded text parts), + // so retain the unchanged queue head until next() sets consume. Generated + // parts never occupy queue slots; ordinary/capable-app delivery still pops + // after one response. Loop past an exhausted binary source with no valid text + // so it cannot report NO_MORE_MESSAGES while another queue entry remains. + while (offline_queue_len > 0) { + bool consume; + const int len = mcotxt_delivery.next(offline_queue[0].buf, offline_queue[0].len, + app_target_ver, getRTCClock()->getCurrentTime(), + frame, consume); + if (consume) { + offline_queue_len--; + for (int i = 0; i < offline_queue_len; i++) { + offline_queue[i] = offline_queue[i + 1]; + } + mcotxt_delivery.resetMessage(); } - return len; + if (len > 0) return len; + // Binary was already returned but has no valid text to add. Continue to + // the next source instead of reporting an empty queue prematurely. } return 0; // queue is empty } @@ -473,7 +487,11 @@ void MyMesh::queueMessage(const ContactInfo &from, uint8_t txt_type, mesh::Packe } if (_listener) { - _listener->onMessageRecv(pkt, from, txt_type, sender_timestamp, text); + // Decode only for the synchronous local preview. The queue above retains + // the original frame for whichever app connects later; never rewrite CLI. + const char* preview = (txt_type == TXT_TYPE_PLAIN || txt_type == TXT_TYPE_SIGNED_PLAIN) + ? mcotxt::displayText(text, false) : text; + _listener->onMessageRecv(pkt, from, txt_type, sender_timestamp, preview); _listener->onQueueSizeChanged(offline_queue_len); } } @@ -593,7 +611,7 @@ void MyMesh::onChannelMessageRecv(const mesh::GroupChannel &channel, mesh::Packe if (!getChannel(channel_idx, channel_details)) { strcpy(channel_details.name, "Unknown"); } - _listener->onChannelMessageRecv(pkt, channel_details, text); + _listener->onChannelMessageRecv(pkt, channel_details, mcotxt::displayText(text, true)); _listener->onQueueSizeChanged(offline_queue_len); } } @@ -633,6 +651,17 @@ void MyMesh::onChannelDataRecv(const mesh::GroupChannel &channel, mesh::Packet * } if (_listener) { _listener->onChannelDataRecv(pkt, channel, data_type, data, data_len); + // A display cannot render the binary envelope. Feed one decoded preview to + // its existing channel-text handler, without creating extra queue entries. + const char* preview = mcotxt::displayBinary(data_type, data, data_len); + if (preview) { + ChannelDetails channel_details; + if (!getChannel(channel_idx, channel_details)) { + strcpy(channel_details.name, "Unknown"); + } + _listener->onChannelMessageRecv(pkt, channel_details, preview); + } + _listener->onQueueSizeChanged(offline_queue_len); } } @@ -1078,6 +1107,7 @@ void MyMesh::handleCmdFrame(size_t len) { // cmd_frame[1..7] reserved future char *app_name = (char *)&cmd_frame[8]; cmd_frame[len] = 0; // make app_name null terminated + mcotxt_delivery.appStart(&cmd_frame[8], len - 8); MESH_DEBUG_PRINTLN("App %s connected", app_name); _iter_started = false; // stop any left-over ContactsIterator @@ -1415,14 +1445,19 @@ void MyMesh::handleCmdFrame(size_t len) { writeErrFrame(ERR_CODE_ILLEGAL_ARG); } } else if (cmd_frame[0] == CMD_SYNC_NEXT_MESSAGE) { + const int previous_queue_len = offline_queue_len; int out_len; if ((out_len = getFromOfflineQueue(out_frame)) > 0) { _serial->writeFrame(out_frame, out_len); - if (_listener) _listener->onQueueSizeChanged(offline_queue_len); } else { out_frame[0] = RESP_CODE_NO_MORE_MESSAGES; _serial->writeFrame(out_frame, 1); } + // An exhausted/malformed binary head can be removed without a text frame. + // The display must still clear its unread count/preview when the queue drains. + // A poll of an already-empty queue must not reset the user's current UI screen. + if (_listener && (out_len > 0 || previous_queue_len != offline_queue_len)) + _listener->onQueueSizeChanged(offline_queue_len); } else if (cmd_frame[0] == CMD_SET_RADIO_PARAMS) { int i = 1; uint32_t freq; diff --git a/examples/companion_radio/MyMesh.h b/examples/companion_radio/MyMesh.h index 944790df922..a6839ca23ff 100644 --- a/examples/companion_radio/MyMesh.h +++ b/examples/companion_radio/MyMesh.h @@ -24,6 +24,7 @@ #include "DataStore.h" #include "NodePrefs.h" +#include "MCOtxtDelivery.h" #include #include @@ -91,6 +92,8 @@ class MyMesh : public BaseChatMesh, public DataStoreHost { public: class Listener { public: + // Text is a transient preview (possibly shared decoder scratch). Copy it + // during the callback; the companion queue keeps its original raw frame. virtual void onMessageRecv(mesh::Packet *pkt, const ContactInfo &from, uint8_t txt_type, uint32_t sender_timestamp, const char* text) = 0; virtual void onChannelMessageRecv(mesh::Packet *pkt, ChannelDetails& channel_details, const char* text) = 0; virtual void onQueueSizeChanged(int msgcount) = 0; @@ -276,6 +279,7 @@ class MyMesh : public BaseChatMesh, public DataStoreHost { }; int offline_queue_len; Frame offline_queue[OFFLINE_QUEUE_SIZE]; + MCOtxtDelivery mcotxt_delivery; struct AckTableEntry { unsigned long msg_sent; diff --git a/examples/companion_radio/ui-new/UITask.cpp b/examples/companion_radio/ui-new/UITask.cpp index aced739dc45..381f79b11ba 100644 --- a/examples/companion_radio/ui-new/UITask.cpp +++ b/examples/companion_radio/ui-new/UITask.cpp @@ -1,5 +1,6 @@ #include "UITask.h" #include +#include #include "../MyMesh.h" #include "target.h" #include @@ -664,7 +665,7 @@ class MsgPreviewScreen : public UIScreen { } else { sprintf(p->origin, "(%d) %s:", (uint32_t) path_len, from_name); } - StrHelper::strncpy(p->msg, msg, sizeof(p->msg)); + mcotxt::copyDisplayPreview(p->msg, sizeof(p->msg), msg); } int render(DisplayDriver& display) override { diff --git a/examples/companion_radio/ui-orig/UITask.cpp b/examples/companion_radio/ui-orig/UITask.cpp index db8772f46e6..99b3da84c84 100644 --- a/examples/companion_radio/ui-orig/UITask.cpp +++ b/examples/companion_radio/ui-orig/UITask.cpp @@ -1,6 +1,7 @@ #include "UITask.h" #include #include +#include #include "../MyMesh.h" #define AUTO_OFF_MILLIS 15000 // 15 seconds @@ -153,7 +154,7 @@ void UITask::onMessageRecv(mesh::Packet *pkt, const ContactInfo &from, uint8_t t } else { sprintf(_origin, "(%d) %s", (uint32_t) path_len, from.name); } - StrHelper::strncpy(_msg, text, sizeof(_msg)); + mcotxt::copyDisplayPreview(_msg, sizeof(_msg), text); if (_display != NULL) { if (!_display->isOn() && !hasConnection()) { @@ -180,7 +181,7 @@ void UITask::onChannelMessageRecv(mesh::Packet *pkt, ChannelDetails& channel_det } else { sprintf(_origin, "(%d) %s", (uint32_t) path_len, channel_details.name); } - StrHelper::strncpy(_msg, text, sizeof(_msg)); + mcotxt::copyDisplayPreview(_msg, sizeof(_msg), text); if (_display != NULL) { if (!_display->isOn() && !hasConnection()) { @@ -630,4 +631,4 @@ void UITask::handleButtonLongPress() { } else { shutdown(); } -} \ No newline at end of file +} diff --git a/platformio.ini b/platformio.ini index f296f838213..a9b1b44b4fd 100644 --- a/platformio.ini +++ b/platformio.ini @@ -49,6 +49,7 @@ build_flags = -w -DNDEBUG -DRADIOLIB_STATIC_ONLY=1 -DRADIOLIB_GODMODE=1 build_src_filter = +<*.cpp> + + + + + + @@ -174,6 +175,9 @@ build_src_filter = +<../src/Packet.cpp> +<../src/helpers/ConfigSerializer.cpp> +<../src/helpers/DynamicConfigSerializer.cpp> + +<../src/helpers/Base91.cpp> + +<../src/helpers/mcotxt/*.cpp> + +<../examples/companion_radio/MCOtxtDelivery.cpp> lib_deps = google/googletest @ 1.17.0 diff --git a/src/helpers/Base91.cpp b/src/helpers/Base91.cpp new file mode 100644 index 00000000000..98bba1ba549 --- /dev/null +++ b/src/helpers/Base91.cpp @@ -0,0 +1,110 @@ +// basE91 encode/decode (canonical 91-character alphabet). +// Algorithm by Joachim Henke, public domain (2000-2006). +// Reference: http://base91.sourceforge.net +#include "Base91.h" + +#include + +namespace mesh { +namespace base91 { +namespace { + +constexpr char ALPHABET[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + "!#$%&()*+,./:;<=>?@[]^_`{|}~\""; + +int valueOf(uint8_t ch) { + if (ch == 0) return -1; + const char* match = strchr(ALPHABET, static_cast(ch)); + return match == nullptr ? -1 : static_cast(match - ALPHABET); +} + +bool append(char value, char* output, size_t capacity, size_t& length) { + if (length + 1 >= capacity) return false; + output[length++] = value; + return true; +} + +} // namespace + +bool encode(const uint8_t* input, size_t input_length, char* output, + size_t output_capacity, size_t& output_length) { + output_length = 0; + if ((input == nullptr && input_length != 0) || output == nullptr || output_capacity == 0) + return false; + + uint32_t queue = 0; + unsigned bits = 0; + for (size_t i = 0; i < input_length; ++i) { + queue |= static_cast(input[i]) << bits; + bits += 8; + if (bits > 13) { + uint32_t value = queue & 8191U; + if (value > 88U) { + queue >>= 13; + bits -= 13; + } else { + value = queue & 16383U; + queue >>= 14; + bits -= 14; + } + if (!append(ALPHABET[value % 91U], output, output_capacity, output_length) || + !append(ALPHABET[value / 91U], output, output_capacity, output_length)) { + output_length = 0; + output[0] = '\0'; + return false; + } + } + } + + if (bits != 0) { + if (!append(ALPHABET[queue % 91U], output, output_capacity, output_length) || + ((bits > 7 || queue > 90U) && + !append(ALPHABET[queue / 91U], output, output_capacity, output_length))) { + output_length = 0; + output[0] = '\0'; + return false; + } + } + output[output_length] = '\0'; + return true; +} + +bool decode(const char* input, size_t input_length, uint8_t* output, + size_t output_capacity, size_t& output_length) { + output_length = 0; + if ((input == nullptr && input_length != 0) || output == nullptr) return false; + + uint32_t queue = 0; + unsigned bits = 0; + int pending = -1; + for (size_t i = 0; i < input_length; ++i) { + const int decoded = valueOf(static_cast(input[i])); + if (decoded < 0) return false; + if (pending < 0) { + pending = decoded; + continue; + } + + const uint32_t value = static_cast(pending + decoded * 91); + queue |= value << bits; + bits += (value & 8191U) > 88U ? 13U : 14U; + while (bits >= 8) { + if (output_length >= output_capacity) return false; + output[output_length++] = static_cast(queue); + queue >>= 8; + bits -= 8; + } + pending = -1; + } + + if (pending >= 0) { + queue |= static_cast(pending) << bits; + if (output_length >= output_capacity) return false; + output[output_length++] = static_cast(queue); + } + return true; +} + +} // namespace base91 +} // namespace mesh diff --git a/src/helpers/Base91.h b/src/helpers/Base91.h new file mode 100644 index 00000000000..ae84570a1f9 --- /dev/null +++ b/src/helpers/Base91.h @@ -0,0 +1,20 @@ +// basE91 encode/decode (canonical 91-character alphabet). +// Algorithm by Joachim Henke, public domain (2000-2006). +// Reference: http://base91.sourceforge.net +#pragma once + +#include +#include + +namespace mesh { +namespace base91 { + +// basE91 using the canonical 91-character alphabet. Output capacities include +// the terminating NUL for encode(); decoded binary buffers are not terminated. +bool encode(const uint8_t* input, size_t input_length, char* output, + size_t output_capacity, size_t& output_length); +bool decode(const char* input, size_t input_length, uint8_t* output, + size_t output_capacity, size_t& output_length); + +} // namespace base91 +} // namespace mesh diff --git a/src/helpers/BaseChatMesh.h b/src/helpers/BaseChatMesh.h index 835384b7415..4b262a8d5f6 100644 --- a/src/helpers/BaseChatMesh.h +++ b/src/helpers/BaseChatMesh.h @@ -4,8 +4,7 @@ #include #include #include - -#define MAX_TEXT_LEN (10*CIPHER_BLOCK_SIZE) // must be LESS than (MAX_PACKET_PAYLOAD - 4 - CIPHER_MAC_SIZE - 1) +#include #include "ContactInfo.h" diff --git a/src/helpers/BaseSerialInterface.h b/src/helpers/BaseSerialInterface.h index 23933fcb4b0..190d40fdd67 100644 --- a/src/helpers/BaseSerialInterface.h +++ b/src/helpers/BaseSerialInterface.h @@ -1,8 +1,7 @@ #pragma once #include - -#define MAX_FRAME_SIZE 176 // +4 for transport codes (region scoping) +#include class BaseSerialInterface { protected: diff --git a/src/helpers/MessageLimits.h b/src/helpers/MessageLimits.h new file mode 100644 index 00000000000..79860b15c3e --- /dev/null +++ b/src/helpers/MessageLimits.h @@ -0,0 +1,8 @@ +#pragma once + +#include + +// Shared by the chat/serial interfaces and their delivery adapters. Keep these +// limits here so callers can size buffers without including Arduino interfaces. +#define MAX_FRAME_SIZE 176 // +4 for transport codes (region scoping) +#define MAX_TEXT_LEN (10*CIPHER_BLOCK_SIZE) // must be LESS than (MAX_PACKET_PAYLOAD - 4 - CIPHER_MAC_SIZE - 1) diff --git a/src/helpers/mcotxt/MCOtxtCodec.cpp b/src/helpers/mcotxt/MCOtxtCodec.cpp new file mode 100644 index 00000000000..60b40042860 --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtCodec.cpp @@ -0,0 +1,376 @@ +#include + +#include + +#include +#include + +namespace mcotxt { +namespace { + +struct BitReader { + const uint8_t* data; + uint32_t bit_length; + uint32_t position; + + uint32_t remaining() const { return bit_length - position; } + + bool readBits(uint8_t count, uint32_t& value) { + if (count > remaining()) return false; + value = 0; + for (uint8_t i = 0; i < count; i++) { + const uint32_t byte = position >> 3; + const uint32_t bit = (data[byte] >> (7 - (position & 7))) & 1u; + value = (value << 1) | bit; + position++; + } + return true; + } +}; + +struct Utf8Out { + char* buf; + size_t capacity; + size_t length; + + bool put(uint32_t codepoint) { + uint8_t tmp[4]; + const size_t n = utf8Encode(codepoint, tmp); + if (n == 0 || length + n + 1 > capacity) return false; + memcpy(buf + length, tmp, n); + length += n; + return true; + } +}; + +DecodeResult fail(DecodeStatus status) { + DecodeResult result; + result.status = status; + result.raw_utf8 = false; + result.language_a = kLanguageNone; + result.language_b = kLanguageNone; + return result; +} + +// Version and generation share one shape: 0..6 inline, 7 + eight bits of +// value - 7 above that. Every value has exactly one encoding. +bool readHeaderField(BitReader& reader, uint32_t& value) { + if (!reader.readBits(kHeaderFieldBits, value)) return false; + if (value != kHeaderFieldEscape) return true; + uint32_t extra; + if (!reader.readBits(8, extra)) return false; + value = kHeaderFieldEscape + extra; + return true; +} + +bool validUtf8(const uint8_t* data, size_t length) { + size_t position = 0; + uint32_t codepoint; + while (position < length) { + if (!utf8Decode(data, length, position, codepoint)) return false; + } + return true; +} + +size_t byteLengthForBits(uint32_t bit_length) { + return (size_t)(bit_length / 8U) + (bit_length % 8U != 0U ? 1U : 0U); +} + +} // namespace + +bool utf8Decode(const uint8_t* data, size_t length, size_t& position, uint32_t& codepoint) { + if (position >= length) return false; + const uint8_t b0 = data[position]; + if (b0 < 0x80) { + codepoint = b0; + position++; + return true; + } + size_t need; + uint32_t minimum; + if ((b0 & 0xE0) == 0xC0) { + need = 1; codepoint = b0 & 0x1F; minimum = 0x80; + } else if ((b0 & 0xF0) == 0xE0) { + need = 2; codepoint = b0 & 0x0F; minimum = 0x800; + } else if ((b0 & 0xF8) == 0xF0) { + need = 3; codepoint = b0 & 0x07; minimum = 0x10000; + } else { + return false; + } + if (length - position - 1 < need) return false; + for (size_t i = 1; i <= need; i++) { + const uint8_t b = data[position + i]; + if ((b & 0xC0) != 0x80) return false; + codepoint = (codepoint << 6) | (b & 0x3F); + } + if (codepoint < minimum || codepoint > 0x10FFFF || + (codepoint >= 0xD800 && codepoint <= 0xDFFF)) { + return false; + } + position += need + 1; + return true; +} + +size_t utf8Encode(uint32_t codepoint, uint8_t* out) { + if (codepoint < 0x80) { + out[0] = (uint8_t)codepoint; + return 1; + } + if (codepoint < 0x800) { + out[0] = (uint8_t)(0xC0 | (codepoint >> 6)); + out[1] = (uint8_t)(0x80 | (codepoint & 0x3F)); + return 2; + } + if (codepoint < 0x10000) { + if (codepoint >= 0xD800 && codepoint <= 0xDFFF) return 0; + out[0] = (uint8_t)(0xE0 | (codepoint >> 12)); + out[1] = (uint8_t)(0x80 | ((codepoint >> 6) & 0x3F)); + out[2] = (uint8_t)(0x80 | (codepoint & 0x3F)); + return 3; + } + if (codepoint > 0x10FFFF) return 0; + out[0] = (uint8_t)(0xF0 | (codepoint >> 18)); + out[1] = (uint8_t)(0x80 | ((codepoint >> 12) & 0x3F)); + out[2] = (uint8_t)(0x80 | ((codepoint >> 6) & 0x3F)); + out[3] = (uint8_t)(0x80 | (codepoint & 0x3F)); + return 4; +} + +DecodeResult decodeStream(const uint8_t* data, size_t data_length, uint32_t bit_length, + char* out, size_t out_capacity, size_t& out_length) { + out_length = 0; + if (out == nullptr || out_capacity == 0) return fail(DecodeStatus::OutputTooSmall); + out[0] = '\0'; + if (data == nullptr && bit_length != 0) return fail(DecodeStatus::Malformed); + if (byteLengthForBits(bit_length) > data_length) return fail(DecodeStatus::Malformed); + + BitReader reader = { data, bit_length, 0 }; + Utf8Out output = { out, out_capacity, 0 }; + // Overflow keeps the longest prefix of whole characters that fit, so a + // caller can still show how the text starts. + auto overflow = [&]() { + out[output.length] = '\0'; + out_length = output.length; + return fail(DecodeStatus::OutputTooSmall); + }; + uint32_t value; + + if (!readHeaderField(reader, value)) return fail(DecodeStatus::UnexpectedEnd); + if (value != kCodecVersion) return fail(DecodeStatus::UnknownVersion); + uint32_t generation; + if (!readHeaderField(reader, generation)) return fail(DecodeStatus::UnexpectedEnd); + + uint32_t language_a_field, language_b_field; + if (!reader.readBits(kLanguageBits, language_a_field) || + !reader.readBits(kLanguageBits, language_b_field)) { + return fail(DecodeStatus::UnexpectedEnd); + } + + uint8_t language_a = kLanguageNone; + uint8_t language_b = kLanguageNone; + if (language_a_field == kExtendedHeaderWireId) { + if (language_b_field == kRawUtf8Format) { + // No tables involved: the generation is reported, never checked. + uint32_t padding; + if (!reader.readBits(kRawUtf8PaddingBits, padding)) return fail(DecodeStatus::UnexpectedEnd); + if (padding != 0) return fail(DecodeStatus::InvalidUtf8); + if (reader.remaining() % 8 != 0) return fail(DecodeStatus::InvalidUtf8); + const size_t byte_count = reader.remaining() / 8; + const size_t kept = byte_count + 1 > out_capacity ? out_capacity - 1 : byte_count; + for (size_t i = 0; i < byte_count; i++) { + uint32_t byte; + reader.readBits(8, byte); + if (i < kept) out[i] = (char)byte; + } + if (kept < byte_count) { + // Overflow: keep the longest prefix of whole characters that fit. + size_t prefix = 0, position = 0; + uint32_t codepoint; + while (position < kept && utf8Decode((const uint8_t*)out, kept, position, codepoint)) { + prefix = position; + } + out[prefix] = '\0'; + out_length = prefix; + DecodeResult result = fail(DecodeStatus::OutputTooSmall); + result.raw_utf8 = true; + return result; + } + if (!validUtf8((const uint8_t*)out, byte_count)) return fail(DecodeStatus::InvalidUtf8); + out[byte_count] = '\0'; + out_length = byte_count; + DecodeResult result = fail(DecodeStatus::Ok); + result.raw_utf8 = true; + return result; + } + if (language_b_field != kExtendedPair8Format) return fail(DecodeStatus::UnsupportedHeader); + uint32_t global_a, global_b; + if (!reader.readBits(8, global_a) || !reader.readBits(8, global_b)) { + return fail(DecodeStatus::UnexpectedEnd); + } + if (global_a == kLanguageNone || global_a >= kLanguageCount) return fail(DecodeStatus::UnknownLanguage); + if (global_b != kLanguageNone && global_b >= kLanguageCount) return fail(DecodeStatus::UnknownLanguage); + language_a = (uint8_t)global_a; + language_b = (uint8_t)global_b; + } else { + language_a = (uint8_t)language_a_field; + language_b = language_b_field == kLanguageNoneWireId ? kLanguageNone : (uint8_t)language_b_field; + } + + if (generation != kModelGeneration) return fail(DecodeStatus::UnsupportedGeneration); + if (modelFor(language_a) == nullptr) return fail(DecodeStatus::ModelUnavailable); + if (language_b != kLanguageNone) { + if (language_b == language_a) return fail(DecodeStatus::UnknownLanguage); + if (modelFor(language_b) == nullptr) return fail(DecodeStatus::ModelUnavailable); + } + + uint8_t current = language_a; + Context context = Context::Start; + uint8_t prev = 0; + bool shift = false; + bool caps = false; + + while (reader.remaining() > 0) { + uint32_t bit; + // A language symbol from a TOP4 rank, a PRIMARY or an EXTENSION literal. + int symbol = -1; + + if (!reader.readBits(1, bit)) return fail(DecodeStatus::UnexpectedEnd); + if (bit == 0) { + // TOP4: 00, 010, 0110, 0111. + uint32_t rank = 0; + if (!reader.readBits(1, bit)) return fail(DecodeStatus::UnexpectedEnd); + if (bit != 0) { + if (!reader.readBits(1, bit)) return fail(DecodeStatus::UnexpectedEnd); + if (bit == 0) { + rank = 1; + } else { + if (!reader.readBits(1, bit)) return fail(DecodeStatus::UnexpectedEnd); + rank = bit == 0 ? 2 : 3; + } + } + const Model* model = modelFor(current); + symbol = rowForContext(*model, context, prev)[rank]; + if (symbol >= model->symbolCount()) return fail(DecodeStatus::InvalidToken); + } else { + if (!reader.readBits(1, bit)) return fail(DecodeStatus::UnexpectedEnd); + if (bit == 0) { + // PRIMARY literal. + uint32_t id; + if (!reader.readBits(5, id)) return fail(DecodeStatus::UnexpectedEnd); + const Model* model = modelFor(current); + if (id >= model->primary_count) return fail(DecodeStatus::InvalidToken); + symbol = (int)id; + } else { + if (!reader.readBits(1, bit)) return fail(DecodeStatus::UnexpectedEnd); + if (bit == 0) { + // PUNCTUATION. + if (shift) return fail(DecodeStatus::InvalidShift); + uint32_t id; + if (!reader.readBits(5, id)) return fail(DecodeStatus::UnexpectedEnd); + if (id >= kPunctuationCount) return fail(DecodeStatus::InvalidToken); + if (!output.put(punctuationAt((uint8_t)id))) return overflow(); + contextAfterPunctuation((uint8_t)id, context); + continue; + } + if (!reader.readBits(1, bit)) return fail(DecodeStatus::UnexpectedEnd); + if (bit == 0) { + // EXTENSION literal. + uint32_t id; + if (!reader.readBits(5, id)) return fail(DecodeStatus::UnexpectedEnd); + const Model* model = modelFor(current); + if (id >= model->extension_count) return fail(DecodeStatus::InvalidToken); + symbol = model->primary_count + (int)id; + } else { + if (!reader.readBits(1, bit)) return fail(DecodeStatus::UnexpectedEnd); + if (bit == 0) { + // SHIFT. + if (shift) return fail(DecodeStatus::InvalidShift); + shift = true; + continue; + } + if (!reader.readBits(1, bit)) return fail(DecodeStatus::UnexpectedEnd); + if (shift) return fail(DecodeStatus::InvalidShift); + if (bit == 0) { + // TOGGLE_LANGUAGE. + if (language_b == kLanguageNone) return fail(DecodeStatus::InvalidToken); + if (current == language_a) { + current = language_b; + } else if (current == language_b) { + current = language_a; + } else { + return fail(DecodeStatus::InvalidToken); + } + context = Context::Start; + continue; + } + // Extended control. + uint32_t subopcode; + if (!reader.readBits(3, subopcode)) return fail(DecodeStatus::UnexpectedEnd); + switch (subopcode) { + case kSubopcodeSwitchOther: { + uint32_t id; + if (!reader.readBits(8, id)) return fail(DecodeStatus::UnexpectedEnd); + if (id == kLanguageNone || id >= kLanguageCount) return fail(DecodeStatus::InvalidToken); + if (modelFor((uint8_t)id) == nullptr) return fail(DecodeStatus::ModelUnavailable); + current = (uint8_t)id; + context = Context::Start; + continue; + } + case kSubopcodeResetContext: + context = Context::Start; + continue; + case kSubopcodeUtf8Run: { + uint32_t count; + if (!reader.readBits(5, count)) return fail(DecodeStatus::UnexpectedEnd); + count += 1; + uint8_t bytes[kUtf8RunMaxBytes]; + for (uint32_t i = 0; i < count; i++) { + uint32_t byte; + if (!reader.readBits(8, byte)) return fail(DecodeStatus::UnexpectedEnd); + bytes[i] = (uint8_t)byte; + } + if (!validUtf8(bytes, count)) return fail(DecodeStatus::InvalidUtf8); + size_t position = 0; + while (position < count) { + uint32_t codepoint; + utf8Decode(bytes, count, position, codepoint); // validated above + if (!output.put(codepoint)) return overflow(); + } + context = Context::Start; + continue; + } + case kSubopcodeToggleCase: + // Case mode never touches the prediction context. + caps = !caps; + continue; + default: + return fail(DecodeStatus::InvalidToken); + } + } + } + } + + // Emit a language symbol; the context follows the lowercase table entry + // whatever case was output. + const Model* model = modelFor(current); + const uint32_t lower = model->symbolAt((uint8_t)symbol); + const uint32_t upper = model->uppercaseOf((uint8_t)symbol); + if (shift && upper == 0) return fail(DecodeStatus::InvalidShift); + const bool make_upper = upper != 0 && (caps != shift); + if (!output.put(make_upper ? upper : lower)) return overflow(); + context = Context::Symbol; + prev = (uint8_t)symbol; + shift = false; + } + + if (shift) return fail(DecodeStatus::InvalidShift); + out[output.length] = '\0'; + out_length = output.length; + DecodeResult result = fail(DecodeStatus::Ok); + result.language_a = language_a; + result.language_b = language_b; + return result; +} + +} // namespace mcotxt + diff --git a/src/helpers/mcotxt/MCOtxtCodec.h b/src/helpers/mcotxt/MCOtxtCodec.h new file mode 100644 index 00000000000..9d07a7c9e24 --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtCodec.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include + +// MCOtxt v1 stream codec: the bit stream of docs/MCOTXT_V1_PROTOCOL.md in the +// MeshCore Open Advanced repository (header, token tree, prediction contexts, +// case controls, UTF8_RUN, RAW_UTF8 mode). Pure C++, no allocation: every +// buffer belongs to the caller, and the stream is never self-terminating, so +// the exact bit count travels with it (see MCOtxtTransport for the frame). +namespace mcotxt { + +enum class DecodeStatus : uint8_t { + Ok, + UnknownVersion, + UnsupportedGeneration, + UnknownLanguage, + ModelUnavailable, + UnsupportedHeader, + InvalidToken, + InvalidShift, + InvalidUtf8, + UnexpectedEnd, + Malformed, + OutputTooSmall, +}; + +struct DecodeResult { + DecodeStatus status; + bool raw_utf8; + uint8_t language_a; // kLanguageNone in RAW_UTF8 mode + uint8_t language_b; // kLanguageNone when absent +}; + +// Decodes [bit_length] bits of [data] into NUL-terminated UTF-8. [out_length] +// excludes the terminator; the capacity includes it. On OutputTooSmall the +// output holds the longest prefix of whole characters that fit, terminated, +// so a caller can still show how the text starts. +DecodeResult decodeStream(const uint8_t* data, size_t data_length, uint32_t bit_length, + char* out, size_t out_capacity, size_t& out_length); + +// Strict UTF-8, the way the reference decoder's `utf8.decode(allowMalformed: +// false)` sees it: no overlongs, no surrogates, nothing past U+10FFFF. +bool utf8Decode(const uint8_t* data, size_t length, size_t& position, uint32_t& codepoint); +size_t utf8Encode(uint32_t codepoint, uint8_t* out); + +} // namespace mcotxt + diff --git a/src/helpers/mcotxt/MCOtxtDisplay.cpp b/src/helpers/mcotxt/MCOtxtDisplay.cpp new file mode 100644 index 00000000000..9b02098f06e --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtDisplay.cpp @@ -0,0 +1,47 @@ +#include "MCOtxtDisplay.h" + +#include + +#include +#include "MCOtxtText.h" +#include "MCOtxtTransport.h" + +namespace mcotxt { + +const char* displayText(const char* text, bool channel) { + if (!text) return text; + size_t prefix_length = 0; + if (channel) { + const char* separator = strstr(text, ": "); + if (separator) prefix_length = (size_t)(separator + 2 - text); + } + if (prefix_length >= kScratchBytes) return text; + char* output = scratch(); + if (!decodeText(text + prefix_length, output + prefix_length, + kScratchBytes - prefix_length)) return text; + memcpy(output, text, prefix_length); + return output; +} + +const char* displayBinary(uint16_t data_type, const uint8_t* data, size_t length) { + char* output = scratch(); + DecodedMessage message; + if (decodeBinaryEnvelope(data_type, data, length, output, kScratchBytes, message) + != MessageStatus::Ok) return nullptr; + bool truncated = false; + if (!ensureReplyMentionPrefix(message, output, kScratchBytes, truncated) || truncated) + return nullptr; + + // Reuse the ordinary channel preview: "Name: @[Reply] decoded text". The + // binary callback still receives the original data separately, once. + return prependSenderName(message, output, kScratchBytes) ? output : nullptr; +} + +void copyDisplayPreview(char* output, size_t capacity, const char* text) { + if (!output || capacity == 0) return; + const size_t keep = mesh::validUtf8PrefixLength(text, capacity - 1); + if (keep) memcpy(output, text, keep); + output[keep] = '\0'; +} + +} // namespace mcotxt diff --git a/src/helpers/mcotxt/MCOtxtDisplay.h b/src/helpers/mcotxt/MCOtxtDisplay.h new file mode 100644 index 00000000000..3831fb446e1 --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtDisplay.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace mcotxt { + +// Display-only adaptation, independent of APP_START capabilities. Returned +// decoded text uses scratch() and must be copied synchronously by the listener +// before another decoder call. Neither function modifies or queues its input. +// Channel text keeps its outer "Name: " ahead of any decoded reply mention. +// Unsupported/malformed text is returned unchanged; binary returns nullptr. +const char* displayText(const char* text, bool channel); +const char* displayBinary(uint16_t data_type, const uint8_t* data, size_t length); + +// Keep the UI's existing preview capacity without cutting a UTF-8 character. +// No BLE part numbers or continuation markers belong in a local preview. +void copyDisplayPreview(char* output, size_t capacity, const char* text); + +} // namespace mcotxt diff --git a/src/helpers/mcotxt/MCOtxtModels.cpp b/src/helpers/mcotxt/MCOtxtModels.cpp new file mode 100644 index 00000000000..ba5c156d33c --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtModels.cpp @@ -0,0 +1,118 @@ +#include + +#include + +#include "generated/model_en.h" +#include "generated/model_ru.h" +#include "generated/model_fr.h" +#include "generated/model_de.h" +#include "generated/model_it.h" +#include "generated/model_uk.h" +#include "generated/model_be.h" + +namespace mcotxt { +namespace { + +#define MCOTXT_MODEL(lang) \ + { \ + mcotxt_##lang##_language_id, mcotxt_##lang##_primary_count, \ + mcotxt_##lang##_extension_count, mcotxt_##lang##_uppercase_count, \ + mcotxt_##lang##_primary_symbols, mcotxt_##lang##_extension_symbols, \ + mcotxt_##lang##_start_top4, mcotxt_##lang##_punct_start_top4, \ + mcotxt_##lang##_top4, mcotxt_##lang##_uppercase_map, \ + mcotxt_##lang##_wire_hash \ + } + +// Wire id order. +const Model kModels[kLanguageCount] = { + MCOTXT_MODEL(en), MCOTXT_MODEL(ru), MCOTXT_MODEL(fr), MCOTXT_MODEL(de), + MCOTXT_MODEL(it), MCOTXT_MODEL(uk), MCOTXT_MODEL(be), +}; + +#undef MCOTXT_MODEL + +const char* const kLanguageCodes[kLanguageCount] = { + "en", "ru", "fr", "de", "it", "uk", "be", +}; + +// Fixed for v1: SPACE, ASCII punctuation, typographic quotes and dashes, LF. +const uint16_t kPunctuation[kPunctuationCount] = { + 0x0020, 0x002E, 0x002C, 0x0021, 0x003F, 0x003A, 0x003B, 0x002D, + 0x2014, 0x005F, 0x0027, 0x0022, 0x00AB, 0x00BB, 0x201C, 0x201D, + 0x201E, 0x2018, 0x2019, 0x0028, 0x0029, 0x005B, 0x005D, 0x002F, + 0x005C, 0x0040, 0x0023, 0x0025, 0x0026, 0x002B, 0x003D, 0x000A, +}; + +} // namespace + +uint32_t Model::symbolAt(uint8_t index) const { + if (index < primary_count) return primary[index]; + index = (uint8_t)(index - primary_count); + if (index < extension_count) return extension[index]; + return 0; +} + +int Model::indexOf(uint32_t codepoint) const { + if (codepoint > 0xFFFF) return -1; + for (uint8_t i = 0; i < primary_count; i++) { + if (primary[i] == codepoint) return i; + } + for (uint8_t i = 0; i < extension_count; i++) { + if (extension[i] == codepoint) return primary_count + i; + } + return -1; +} + +int Model::indexOfUppercase(uint32_t codepoint) const { + if (codepoint > 0xFFFF) return -1; + for (uint8_t i = 0; i < uppercase_count; i++) { + if (uppercase[i].uppercase_codepoint == codepoint) { + return uppercase[i].lowercase_symbol_index; + } + } + return -1; +} + +uint32_t Model::uppercaseOf(uint8_t index) const { + for (uint8_t i = 0; i < uppercase_count; i++) { + if (uppercase[i].lowercase_symbol_index == index) { + return uppercase[i].uppercase_codepoint; + } + } + return 0; +} + +const Model* modelFor(uint8_t language_id) { + if (language_id >= kLanguageCount) return nullptr; + return &kModels[language_id]; +} + +const char* languageCode(uint8_t language_id) { + if (language_id >= kLanguageCount) return nullptr; + return kLanguageCodes[language_id]; +} + +int languageIdForCode(const char* code) { + if (code == nullptr || code[0] == '\0' || code[1] == '\0' || code[2] != '\0') return -1; + const char a = (char)(code[0] | 0x20); + const char b = (char)(code[1] | 0x20); + for (uint8_t i = 0; i < kLanguageCount; i++) { + if (kLanguageCodes[i][0] == a && kLanguageCodes[i][1] == b) return i; + } + return -1; +} + +uint32_t punctuationAt(uint8_t id) { + return id < kPunctuationCount ? kPunctuation[id] : 0; +} + +int punctuationIdOf(uint32_t codepoint) { + if (codepoint > 0xFFFF) return -1; + for (uint8_t i = 0; i < kPunctuationCount; i++) { + if (kPunctuation[i] == codepoint) return i; + } + return -1; +} + +} // namespace mcotxt + diff --git a/src/helpers/mcotxt/MCOtxtModels.h b/src/helpers/mcotxt/MCOtxtModels.h new file mode 100644 index 00000000000..d3170d40d90 --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtModels.h @@ -0,0 +1,84 @@ +#pragma once + +#include +#include + +// The generated model headers define this pair type under the same guard; the +// definition here is the one every translation unit shares. +#ifndef MCOTXT_UPPERCASE_PAIR_T_DEFINED +#define MCOTXT_UPPERCASE_PAIR_T_DEFINED +#if defined(__GNUC__) || defined(__clang__) +#define MCOTXT_PACKED __attribute__((packed)) +#else +#define MCOTXT_PACKED +#endif +typedef struct MCOTXT_PACKED { + uint16_t uppercase_codepoint; + uint8_t lowercase_symbol_index; +} mcotxt_uppercase_pair_t; +#endif + +// MCOtxt v1 static language models, generation 0. +// +// The tables are the C headers the MeshCore Open Advanced trainer writes +// (tools/MCOtxt/generated//model_.h in that repository), copied +// verbatim into generated/. Symbol order is wire format: it defines the +// literal ids, so the tables are never edited by hand and a regenerated table +// is a new model generation. The wire hash of every table is embedded and +// listed in the trainer's model_manifest.json. +namespace mcotxt { + +static const uint8_t kCodecVersion = 1; +static const uint8_t kModelGeneration = 0; +static const uint8_t kLanguageCount = 7; +static const uint8_t kLanguageNone = 0xFF; // global id: no language +static const uint8_t kPunctuationCount = 32; +static const uint8_t kPunctuationSpace = 0; +static const uint8_t kPunctuationLineFeed = 31; + +struct Model { + uint8_t language_id; + uint8_t primary_count; + uint8_t extension_count; + uint8_t uppercase_count; + const uint16_t* primary; + const uint16_t* extension; + const uint8_t* start_top4; + const uint8_t* punct_start_top4; + const uint8_t* top4; + const mcotxt_uppercase_pair_t* uppercase; + const char* wire_hash; + + uint8_t symbolCount() const { return (uint8_t)(primary_count + extension_count); } + + // Symbol index -> lowercase code point; indexes run over primary ++ extension. + uint32_t symbolAt(uint8_t index) const; + + // Lowercase code point -> symbol index, or -1 when it is not a model symbol. + int indexOf(uint32_t codepoint) const; + + // Uppercase code point -> the symbol index it lowercases to, or -1. + int indexOfUppercase(uint32_t codepoint) const; + + // The uppercase form of a symbol, or 0 when it has none (digits, SPACE). + uint32_t uppercaseOf(uint8_t index) const; + + // The four-entry TOP4 row of the SYMBOL(prev) context. + const uint8_t* rowFor(uint8_t index) const { return top4 + (size_t)index * 4u; } +}; + +// Wire ids 0..6: en ru fr de it uk be. Null for any other id. +const Model* modelFor(uint8_t language_id); + +// Two-letter code of a wire id, or null. +const char* languageCode(uint8_t language_id); + +// Wire id for a two-letter code (case-insensitive), or -1. +int languageIdForCode(const char* code); + +// The punctuation page shared by every language. +uint32_t punctuationAt(uint8_t id); +int punctuationIdOf(uint32_t codepoint); + +} // namespace mcotxt + diff --git a/src/helpers/mcotxt/MCOtxtParts.cpp b/src/helpers/mcotxt/MCOtxtParts.cpp new file mode 100644 index 00000000000..93f58314cff --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtParts.cpp @@ -0,0 +1,117 @@ +#include + +#include +#include + +#include + +namespace mcotxt { +namespace { + +size_t decimalWidth(size_t value) { + size_t width = 1; + while (value >= 10) { + value /= 10; + width++; + } + return width; +} + +// " [i/n]": a space, a bracket, i, a slash, n, a bracket. +size_t markWidth(size_t digits) { return 4 + 2 * digits; } + +bool isBreak(char c) { return c == ' ' || c == '\n'; } + +// The longest prefix of [text] within [budget] bytes on a character boundary. +size_t fitBytes(const char* text, size_t length, size_t budget) { + size_t position = 0, fit = 0; + uint32_t codepoint; + while (position < length && utf8Decode((const uint8_t*)text, length, position, codepoint) && + position <= budget) { + fit = position; + } + return fit; +} + +// Splits with marks of [digits] wide counts; returns the part count. +size_t splitWith(const char* text, size_t text_length, size_t max_bytes, size_t prefix_length, + size_t digits, TextPart* parts, size_t max_parts) { + size_t count = 0, offset = 0; + while (offset < text_length) { + if (count == max_parts) return 0; + const size_t reserve = prefix_length + (count > 0 ? 3 : 0) + markWidth(digits); + if (reserve >= max_bytes) return 0; + size_t budget = max_bytes - reserve; + const size_t remaining = text_length - offset; + // A final part needs no trailing marker and may use the full budget. + // Otherwise reserve "..." before choosing a UTF-8-safe split point. + if (remaining > budget) { + if (budget <= 3) return 0; + budget -= 3; + } + size_t take = fitBytes(text + offset, remaining, budget); + if (take == 0) return 0; + if (take < remaining) { + // Prefer ending on a space or line feed when that keeps at least half + // the chunk; a long word is cut rather than pushed whole to the next. + size_t cut = take; + while (cut > 0 && !isBreak(text[offset + cut - 1])) cut--; + if (cut >= (take + 1) / 2) take = cut; + } + size_t length = take; + while (length > 0 && isBreak(text[offset + length - 1])) length--; + parts[count].offset = offset; + parts[count].length = length; + count++; + offset += take; + while (offset < text_length && isBreak(text[offset])) offset++; + } + return count; +} + +} // namespace + +size_t splitForApp(const char* text, size_t text_length, size_t max_bytes, + size_t prefix_length, TextPart* parts, size_t max_parts) { + if (text == nullptr || text_length == 0 || parts == nullptr || max_parts == 0) return 0; + // Everything in one message needs no marks. + if (prefix_length < max_bytes && text_length <= max_bytes - prefix_length) { + parts[0].offset = 0; + parts[0].length = text_length; + return 1; + } + // The mark grows with the count, so plan with a width and confirm it. + for (size_t digits = 1; digits <= decimalWidth(max_parts); digits++) { + const size_t count = splitWith(text, text_length, max_bytes, prefix_length, digits, parts, max_parts); + if (count == 0) return 0; + if (decimalWidth(count) <= digits) return count; + } + return 0; +} + +size_t formatPart(const char* prefix, size_t prefix_length, const char* text, + const TextPart& part, size_t index, size_t count, + char* out, size_t out_capacity) { + if (out == nullptr || out_capacity == 0 || count == 0 || index >= count) return 0; + size_t length = 0; + auto append = [&](const char* bytes, size_t n) { + if (length + n + 1 > out_capacity) return false; + memcpy(out + length, bytes, n); + length += n; + return true; + }; + if (prefix_length > 0 && !append(prefix, prefix_length)) return 0; + if (count > 1 && index > 0 && !append("...", 3)) return 0; + if (!append(text + part.offset, part.length)) return 0; + if (count > 1 && index + 1 < count && !append("...", 3)) return 0; + if (count > 1) { + char mark[16]; + const int n = snprintf(mark, sizeof(mark), " [%u/%u]", (unsigned)(index + 1), (unsigned)count); + if (n <= 0 || !append(mark, (size_t)n)) return 0; + } + out[length] = '\0'; + return length; +} + +} // namespace mcotxt + diff --git a/src/helpers/mcotxt/MCOtxtParts.h b/src/helpers/mcotxt/MCOtxtParts.h new file mode 100644 index 00000000000..b76631ecfd1 --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtParts.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include + +// How a decoded MCOtxt text reaches an app that cannot decode it itself: as +// one plain message when it fits, else as numbered parts +// "Name: ... [1/3]", "Name: ...... [2/3]", "Name: ... [3/3]" +// each within the byte budget of one message. The name is the channel's +// outer "Name: " layer and is absent for a direct message. +namespace mcotxt { + +// Deliberate future headroom. Normal radio-sized messages produce 2-3 parts; +// 20 is not a target batch size and should be revisited together with +// MCOTXT_SCRATCH_BYTES and the companion offline queue if either grows. +static const size_t kMaxParts = 20; + +struct TextPart { + size_t offset; // into the text + size_t length; +}; + +// Splits [text] into at most [max_parts] parts so that every formatted part +// fits [max_bytes] once [prefix_length] bytes of "Name: " are in front. A +// chunk breaks after a space or line feed when one is in reach and keeps at +// least half the chunk, else on a character boundary; the break characters +// themselves are dropped. Returns the part count; 0 when the text needs more +// parts than [max_parts] or not even one character fits. +size_t splitForApp(const char* text, size_t text_length, size_t max_bytes, + size_t prefix_length, TextPart* parts, size_t max_parts); + +// Writes part [index] of [count] as "<...><...> [i/n]" into [out], +// NUL-terminated. Leading dots mark a continuation; trailing dots mark another +// part to follow. A single part is "" with no marks. [prefix] +// is the "Name: " layer, or empty. Returns the length, 0 when it does not fit. +size_t formatPart(const char* prefix, size_t prefix_length, const char* text, + const TextPart& part, size_t index, size_t count, + char* out, size_t out_capacity); + +} // namespace mcotxt + diff --git a/src/helpers/mcotxt/MCOtxtStreamFormat.h b/src/helpers/mcotxt/MCOtxtStreamFormat.h new file mode 100644 index 00000000000..064a48022ed --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtStreamFormat.h @@ -0,0 +1,50 @@ +#pragma once + +#include + +#include + +// The bit-level shape of an MCOtxt v1 stream: header field widths and escapes, +// the extended-header formats, +// the extended-control sub-opcodes, and the prediction context rules. Internal +// to the codec. +namespace mcotxt { + +static const uint8_t kHeaderFieldBits = 3; +static const uint8_t kHeaderFieldEscape = 7; +static const uint8_t kLanguageBits = 3; +static const uint8_t kExtendedHeaderWireId = 7; // language A field: extended header +static const uint8_t kLanguageNoneWireId = 7; // language B field: none +static const uint8_t kExtendedPair8Format = 0; +static const uint8_t kRawUtf8Format = 1; +static const uint8_t kRawUtf8PaddingBits = 4; +static const uint8_t kSubopcodeSwitchOther = 0; +static const uint8_t kSubopcodeResetContext = 1; +static const uint8_t kSubopcodeUtf8Run = 2; +static const uint8_t kSubopcodeToggleCase = 3; +static const uint8_t kUtf8RunMaxBytes = 32; + +enum class Context : uint8_t { Start, AfterPunct, Symbol }; + +// SPACE keeps a SYMBOL context and otherwise gives START; LF gives START; +// every other entry gives AFTER_PUNCT. +inline void contextAfterPunctuation(uint8_t id, Context& context) { + if (id == kPunctuationSpace) { + if (context != Context::Symbol) context = Context::Start; + } else if (id == kPunctuationLineFeed) { + context = Context::Start; + } else { + context = Context::AfterPunct; + } +} + +inline const uint8_t* rowForContext(const Model& model, Context context, uint8_t prev) { + switch (context) { + case Context::Start: return model.start_top4; + case Context::AfterPunct: return model.punct_start_top4; + default: return model.rowFor(prev); + } +} + +} // namespace mcotxt + diff --git a/src/helpers/mcotxt/MCOtxtText.cpp b/src/helpers/mcotxt/MCOtxtText.cpp new file mode 100644 index 00000000000..9839a441475 --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtText.cpp @@ -0,0 +1,113 @@ +#include "MCOtxtText.h" + +#include +#include + +#include "MCOtxtTransport.h" + +namespace mcotxt { +namespace { + +const char kBase91Alphabet[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + "!#$%&()*+,./:;<=>?@[]^_`{|}~\""; +const size_t kMaxTokenBytes = MAX_FRAME_SIZE + 1; // input frame plus NUL + +bool isBoundary(const char* input, const char* position) { + if (position == input) return true; + const char* previous = position - 1; + while (previous > input && ((uint8_t)*previous & 0xC0) == 0x80) --previous; + const uint8_t lead = (uint8_t)*previous; + if (lead < 0x80) return !isalnum(lead) && lead != '_'; + + // Do not decode a suffix of a word, including the scripts in the models. + uint32_t cp; + size_t length; + if ((lead & 0xE0) == 0xC0) { cp = lead & 0x1F; length = 2; } + else if ((lead & 0xF0) == 0xE0) { cp = lead & 0x0F; length = 3; } + else if ((lead & 0xF8) == 0xF0) { cp = lead & 0x07; length = 4; } + else return false; + if ((size_t)(position - previous) != length) return false; + for (size_t i = 1; i < length; ++i) cp = (cp << 6) | ((uint8_t)previous[i] & 0x3F); + const bool letter = (cp >= 0xC0 && cp <= 0xFF && cp != 0xD7 && cp != 0xF7) || + (cp >= 0x100 && cp <= 0x24F) || (cp >= 0x370 && cp <= 0x52F); + return !letter; +} + +bool append(char* output, size_t capacity, size_t& length, const char* text, size_t count) { + if (count >= capacity - length) return false; + memcpy(output + length, text, count); + length += count; + output[length] = '\0'; + return true; +} + +} // namespace + +bool decodeText(const char* input, char* output, size_t capacity, size_t* mention_length) { + if (mention_length) *mention_length = 0; + if (!input || !output || capacity == 0) return false; + output[0] = '\0'; + size_t length = 0; + bool changed = false; + DecodedMessage reply; + const char* plain = input; + const char* cursor = input; + while (*cursor) { + if (strncmp(cursor, kTextPrefix, 4) != 0 || !isBoundary(input, cursor)) { + ++cursor; + continue; + } + const char* encoded = cursor + 4; + const char* end = encoded; + while (*end && strchr(kBase91Alphabet, *end)) ++end; + if (end == encoded) { ++cursor; continue; } + + if (!append(output, capacity, length, plain, (size_t)(cursor - plain))) return false; + const char* accepted = nullptr; + char token[kMaxTokenBytes]; + // Quotes/brackets are also Base91 characters. Consume the longest complete + // canonical, structurally valid container and leave punctuation untouched. + for (const char* candidate = end; candidate > encoded; --candidate) { + const size_t token_length = (size_t)(candidate - cursor); + if (token_length >= sizeof(token)) continue; + memcpy(token, cursor, token_length); + token[token_length] = '\0'; + DecodedMessage message; + const MessageStatus status = decodeTextPayload(token, output + length, + capacity - length, message); + if (status == MessageStatus::TooLong) return false; // never publish a clipped decode + if (status != MessageStatus::Ok) continue; + if (message.has_reply) { + // One ordinary companion message has one leading mention. Distinct + // reply anchors in several embedded containers cannot be merged without + // losing their relationship; keep that unusual source raw instead. + if (reply.has_reply && (strcmp(reply.reply_author, message.reply_author) != 0 || + reply.reply_timestamp != message.reply_timestamp)) return false; + if (!reply.has_reply) reply = message; // fixed name arrays own their copies + } + accepted = candidate; + break; + } + if (!accepted) { + // Failed candidates may have written a partial result. Discard it; the + // undecoded token stays in the next plain range. + output[length] = '\0'; + plain = cursor; + ++cursor; + continue; + } + length += strlen(output + length); + changed = true; + plain = cursor = accepted; + } + if (!append(output, capacity, length, plain, strlen(plain))) return false; + bool truncated = false; + if (!changed || !ensureReplyMentionPrefix(reply, output, capacity, truncated) || truncated) + return false; + if (mention_length && reply.has_reply && reply.reply_author[0]) + *mention_length = strlen(reply.reply_author) + 3; + return true; +} + +} // namespace mcotxt diff --git a/src/helpers/mcotxt/MCOtxtText.h b/src/helpers/mcotxt/MCOtxtText.h new file mode 100644 index 00000000000..d49db6fb0a6 --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtText.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace mcotxt { + +// Decode complete mct: containers at word boundaries, including containers in +// surrounding prose. Unknown/malformed containers remain verbatim. False means +// no substitution, or insufficient workspace: the caller must keep the raw frame. +// A reply anchor becomes one leading @[Name] before multipart planning. +// When requested, mention_length reports the complete leading reply mention so +// companion splitting can keep it entirely in the first part. +bool decodeText(const char* input, char* output, size_t capacity, size_t* mention_length = nullptr); + +} // namespace mcotxt diff --git a/src/helpers/mcotxt/MCOtxtTransport.cpp b/src/helpers/mcotxt/MCOtxtTransport.cpp new file mode 100644 index 00000000000..b5d543e4d1e --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtTransport.cpp @@ -0,0 +1,343 @@ +#include + +#include + +#include +#include +#include +#include + +namespace mcotxt { +namespace { + +const uint8_t kKnownFlags = kFlagReply | kFlagSenderName | kFlagTimestampInherited; + +// Longest `mct:` payload a MeshCore text can hold once decoded from Base91. +const size_t kMaxTextPayloadBytes = MAX_FRAME_SIZE; + +struct ByteReader { + const uint8_t* data; + size_t length; + size_t position; + + bool done() const { return position == length; } + + bool readByte(uint8_t& value) { + if (position >= length) return false; + value = data[position++]; + return true; + } + + bool readUint32LE(uint32_t& value) { + if (length - position < 4) return false; + value = (uint32_t)data[position] | ((uint32_t)data[position + 1] << 8) | + ((uint32_t)data[position + 2] << 16) | ((uint32_t)data[position + 3] << 24); + position += 4; + return true; + } + + // Seven value bits per byte, least significant group first; rejected past + // five bytes. + bool readVarUint(uint32_t& value) { + value = 0; + uint8_t shift = 0; + for (uint8_t bytes = 0; bytes < 5; bytes++) { + uint8_t byte; + if (!readByte(byte)) return false; + if (shift == 28 && (byte & 0xF0) != 0) return false; + value |= (uint32_t)(byte & 0x7F) << shift; + if ((byte & 0x80) == 0) return true; + shift = (uint8_t)(shift + 7); + } + return false; + } +}; + +MessageStatus statusForDecode(DecodeStatus status) { + switch (status) { + case DecodeStatus::Ok: return MessageStatus::Ok; + case DecodeStatus::ModelUnavailable: + case DecodeStatus::UnsupportedGeneration: return MessageStatus::ModelUnavailable; + case DecodeStatus::OutputTooSmall: return MessageStatus::TooLong; + default: return MessageStatus::Malformed; + } +} + +size_t byteLengthForBits(uint32_t bit_length) { + return (size_t)(bit_length / 8U) + (bit_length % 8U != 0U ? 1U : 0U); +} + +bool hasNameAndTrailingByte(const ByteReader& reader, uint32_t name_length) { + if (reader.position > reader.length) return false; + const size_t remaining = reader.length - reader.position; + return remaining != 0 && (size_t)name_length <= remaining - 1; +} + +// One string field: a mode byte, then a frame or a counted UTF-8 run. +MessageStatus readString(ByteReader& reader, char* out, size_t out_capacity, size_t& out_length) { + out_length = 0; + if (out_capacity == 0) return MessageStatus::TooLong; + uint8_t mode; + if (!reader.readByte(mode)) return MessageStatus::Malformed; + if (mode == kStringModeMCOtxt) { + uint32_t bit_length; + if (!reader.readVarUint(bit_length)) return MessageStatus::Malformed; + const size_t byte_length = byteLengthForBits(bit_length); + if (reader.length - reader.position < byte_length) return MessageStatus::Malformed; + // A counted stream ends at bit_length, with zero padding in its last byte. + // Reject nonzero padding rather than letting an adjacent Base91 character + // change the last byte into another apparently valid compressed string. + const uint8_t unused = (uint8_t)((8 - bit_length % 8) % 8); + if (byte_length && unused && + (reader.data[reader.position + byte_length - 1] & ((1u << unused) - 1u))) + return MessageStatus::Malformed; + const DecodeResult decoded = decodeStream(reader.data + reader.position, byte_length, + bit_length, out, out_capacity, out_length); + reader.position += byte_length; + // Companion text is C-string based. Do not accept a valid stream whose + // embedded NUL would silently hide the rest of the decoded message. + if (out_length && memchr(out, '\0', out_length)) return MessageStatus::Malformed; + return statusForDecode(decoded.status); + } + if (mode == kStringModeUtf8) { + uint32_t byte_length; + if (!reader.readVarUint(byte_length)) return MessageStatus::Malformed; + if (reader.length - reader.position < byte_length) return MessageStatus::Malformed; + const uint8_t* bytes = reader.data + reader.position; + size_t position = 0; + uint32_t codepoint; + while (position < byte_length) { + if (!utf8Decode(bytes, byte_length, position, codepoint)) return MessageStatus::Malformed; + if (codepoint == 0) return MessageStatus::Malformed; + } + if (byte_length + 1 > out_capacity) { + // Keep the longest prefix of whole characters that fits. + size_t prefix = 0; + position = 0; + while (position < byte_length && utf8Decode(bytes, byte_length, position, codepoint) && + position + 1 <= out_capacity) { + prefix = position; + } + memcpy(out, bytes, prefix); + out[prefix] = '\0'; + out_length = prefix; + reader.position += byte_length; + return MessageStatus::TooLong; + } + memcpy(out, reader.data + reader.position, byte_length); + out[byte_length] = '\0'; + out_length = byte_length; + reader.position += byte_length; + return MessageStatus::Ok; + } + return MessageStatus::Malformed; +} + +// Reuse the caller's workspace to validate/decode the whole wire name, then +// copy only the node-name-sized prefix. The next field can reuse that workspace; +// metadata owns its small fixed arrays and never borrows decoded text storage. +MessageStatus readName(ByteReader& reader, char* output, size_t capacity, char* name) { + size_t length; + const MessageStatus status = readString(reader, output, capacity, length); + if (status != MessageStatus::Ok) return status; + const size_t keep = mesh::validUtf8PrefixLength(output, kMaxNameBytes); + memcpy(name, output, keep); + name[keep] = '\0'; + return MessageStatus::Ok; +} + +MessageStatus decodeContainer(const uint8_t* body, size_t body_length, char* out, + size_t out_capacity, DecodedMessage& message) { + ByteReader reader = { body, body_length, 0 }; + uint8_t flags; + if (!reader.readByte(flags)) return MessageStatus::Malformed; + if ((flags & ~kKnownFlags) != 0) return MessageStatus::Malformed; + + if ((flags & kFlagTimestampInherited) == 0) { + if (!reader.readUint32LE(message.timestamp)) return MessageStatus::Malformed; + message.has_timestamp = true; + } + if ((flags & kFlagSenderName) != 0) { + const MessageStatus status = readName(reader, out, out_capacity, message.sender); + if (status != MessageStatus::Ok) return status; + message.has_sender = true; + } + if ((flags & kFlagReply) != 0) { + const MessageStatus status = readName(reader, out, out_capacity, message.reply_author); + if (status != MessageStatus::Ok) return status; + if (!reader.readUint32LE(message.reply_timestamp)) return MessageStatus::Malformed; + message.has_reply = true; + } + size_t text_length; + const MessageStatus status = readString(reader, out, out_capacity, text_length); + if (status != MessageStatus::Ok) return status; + if (!reader.done()) return MessageStatus::Malformed; + return MessageStatus::Ok; +} + +// subtypeVersion, then the container. NotMCOtxt for another subtype. +MessageStatus decodeSubtypedBody(const uint8_t* data, size_t length, char* out, + size_t out_capacity, DecodedMessage& message) { + if (length == 0) return MessageStatus::Malformed; + const uint8_t subtype_version = data[0]; + if ((subtype_version >> 4) != kSubtypeId) return MessageStatus::NotMCOtxt; + message.declared_version = (uint8_t)(subtype_version & 0x0F); + if (message.declared_version != kWireVersion) return MessageStatus::UnsupportedVersion; + return decodeContainer(data + 1, length - 1, out, out_capacity, message); +} + +const char* skipLeadingWhitespace(const char* text) { + while (*text == ' ' || *text == '\t' || *text == '\r' || *text == '\n') text++; + return text; +} + +bool startsWithReplyMention(const char* text, const char* author) { + if (text == nullptr || author == nullptr || author[0] == '\0') return false; + const size_t author_length = strlen(author); + const size_t mention_length = author_length + 3; // "@[" + name + "]" + if (strlen(text) < mention_length) return false; + return text[0] == '@' && text[1] == '[' && + strncmp(text + 2, author, author_length) == 0 && + text[author_length + 2] == ']'; +} + +} // namespace + +char* scratch() { + static char buffer[kScratchBytes]; + return buffer; +} + +DecodedMessage::DecodedMessage() + : status(MessageStatus::NotMCOtxt), declared_version(0), has_timestamp(false), + timestamp(0), has_sender(false), sender(), has_reply(false), reply_author(), + reply_timestamp(0) { } + +bool isBinaryEnvelope(uint16_t data_type, const uint8_t* data, size_t data_length) { + if (data_type != kChannelAppDataType || data == nullptr) return false; + ByteReader reader = { data, data_length, 0 }; + uint32_t name_length; + if (!reader.readVarUint(name_length)) return false; + if (!hasNameAndTrailingByte(reader, name_length)) return false; + return (data[reader.position + name_length] >> 4) == kSubtypeId; +} + +bool isTextPayload(const char* text) { + if (text == nullptr) return false; + text = skipLeadingWhitespace(text); + const size_t prefix_length = sizeof(kTextPrefix) - 1; + return strncmp(text, kTextPrefix, prefix_length) == 0 && text[prefix_length] != '\0'; +} + +MessageStatus decodeTextPayload(const char* text, char* out, size_t out_capacity, + DecodedMessage& message) { + message = DecodedMessage(); + if (out == nullptr || out_capacity == 0) return message.status = MessageStatus::TooLong; + out[0] = '\0'; + if (!isTextPayload(text)) return message.status = MessageStatus::NotMCOtxt; + text = skipLeadingWhitespace(text) + (sizeof(kTextPrefix) - 1); + + uint8_t payload[kMaxTextPayloadBytes]; + size_t payload_length; + const size_t encoded_length = strlen(text); + if (encoded_length > MAX_FRAME_SIZE || + !mesh::base91::decode(text, encoded_length, payload, sizeof(payload), payload_length)) { + return message.status = MessageStatus::Malformed; + } + // Base91 decoding alone permits noncanonical tail bits. For inline tokens, + // punctuation belongs to the surrounding prose unless the whole candidate + // is exactly the canonical encoding of its bytes. Keep this strictness local + // to MCOtxt, leaving the generic Base91 helper's behavior unchanged. + char canonical[MAX_FRAME_SIZE + 1]; + size_t canonical_length; + if (!mesh::base91::encode(payload, payload_length, canonical, sizeof(canonical), canonical_length) || + canonical_length != encoded_length || memcmp(canonical, text, encoded_length) != 0) { + return message.status = MessageStatus::Malformed; + } + return message.status = decodeSubtypedBody(payload, payload_length, out, out_capacity, message); +} + +bool ensureReplyMentionPrefix(const DecodedMessage& message, char* text, + size_t text_capacity, bool& truncated) { + truncated = false; + if (text == nullptr || text_capacity == 0) return false; + if (!message.has_reply || message.reply_author[0] == '\0') return true; + if (startsWithReplyMention(text, message.reply_author)) return true; + + const size_t author_length = strlen(message.reply_author); + const size_t mention_length = author_length + 3; // "@[" + name + "]" + const size_t text_length = strlen(text); + const size_t separator_length = text_length == 0 ? 0 : 1; + if (mention_length + separator_length + 1 > text_capacity) return false; + + size_t keep = text_length; + if (mention_length + separator_length + text_length + 1 > text_capacity) { + const size_t available = text_capacity - mention_length - separator_length - 1; + keep = mesh::validUtf8PrefixLength(text, available); + truncated = keep < text_length; + } + + memmove(text + mention_length + separator_length, text, keep); + text[0] = '@'; + text[1] = '['; + memcpy(text + 2, message.reply_author, author_length); + text[author_length + 2] = ']'; + if (separator_length != 0) text[mention_length] = ' '; + text[mention_length + separator_length + keep] = '\0'; + return true; +} + +bool prependSenderName(const DecodedMessage& message, char* text, size_t capacity) { + const char* sender = message.has_sender ? message.sender : "?"; + const size_t sender_length = strlen(sender); + const size_t text_length = strlen(text); + if (sender_length + 2 + text_length + 1 > capacity) return false; + memmove(text + sender_length + 2, text, text_length + 1); + memcpy(text, sender, sender_length); + text[sender_length] = ':'; + text[sender_length + 1] = ' '; + return true; +} + +MessageStatus decodeBinaryEnvelope(uint16_t data_type, const uint8_t* data, size_t data_length, + char* out, size_t out_capacity, DecodedMessage& message) { + message = DecodedMessage(); + if (out == nullptr || out_capacity == 0) return message.status = MessageStatus::TooLong; + out[0] = '\0'; + if (data_type != kChannelAppDataType || data == nullptr) { + return message.status = MessageStatus::NotMCOtxt; + } + + // senderNameLength(varuint) | senderName | subtypeVersion | body + ByteReader reader = { data, data_length, 0 }; + uint32_t name_length; + if (!reader.readVarUint(name_length)) return message.status = MessageStatus::NotMCOtxt; + if (!hasNameAndTrailingByte(reader, name_length)) { + return message.status = MessageStatus::NotMCOtxt; + } + const uint8_t* name = data + reader.position; + reader.position += name_length; + const MessageStatus status = decodeSubtypedBody(data + reader.position, + data_length - reader.position, out, + out_capacity, message); + if (status == MessageStatus::Ok && !message.has_sender && + name_length > 0) { + // Unlike a counted string, an envelope name is not NUL-terminated on wire. + // Validate all of it before clipping to the firmware's node-name limit. + size_t position = 0; + size_t keep = 0; + uint32_t codepoint; + while (position < name_length) { + if (!utf8Decode(name, name_length, position, codepoint) || codepoint == 0) + return message.status = MessageStatus::Malformed; + if (position <= kMaxNameBytes) keep = position; + } + memcpy(message.sender, name, keep); + message.sender[keep] = '\0'; + message.has_sender = true; + } + return message.status = status; +} + +} // namespace mcotxt + diff --git a/src/helpers/mcotxt/MCOtxtTransport.h b/src/helpers/mcotxt/MCOtxtTransport.h new file mode 100644 index 00000000000..bc0f29867ba --- /dev/null +++ b/src/helpers/mcotxt/MCOtxtTransport.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include + +// The layers above the MCOtxt stream: the frame (varuint bit count + bytes), +// the application container (flags, timestamp, sender name, reply anchor, +// text) and the two transports MeshCore Open Advanced uses for chat, the +// `mct:` Base91 text and the 0x0120 GROUP_DATA envelope with subtype 0x03. +namespace mcotxt { + +static const uint16_t kChannelAppDataType = 0x0120; +static const uint8_t kSubtypeId = 0x03; +static const uint8_t kWireVersion = 0x01; +static const uint8_t kSubtypeVersion = (uint8_t)((kSubtypeId << 4) | kWireVersion); +static const char kTextPrefix[] = "mct:"; + +// The container: flags, then a mode byte in front of every string. +static const uint8_t kFlagReply = 0x01; +static const uint8_t kFlagSenderName = 0x02; +static const uint8_t kFlagTimestampInherited = 0x04; +static const uint8_t kStringModeMCOtxt = 0x00; +static const uint8_t kStringModeUtf8 = 0x01; + +// One decode scratch for everything that decodes MCOtxt on the loop task: the +// display and the conversion for an app without MCOtxt. +// -D MCOTXT_SCRATCH_BYTES=... resizes it. +#ifndef MCOTXT_SCRATCH_BYTES +#define MCOTXT_SCRATCH_BYTES 1280 +#endif +static const size_t kScratchBytes = MCOTXT_SCRATCH_BYTES; +char* scratch(); + +// NodePrefs::node_name and ContactInfo::name are currently 32-byte C strings. +// Keep at most 31 UTF-8 bytes, plus NUL. Revisit this limit if those fields or +// the protocol change; oversized wire names are clipped on character boundaries. +static const size_t kMaxNameBytes = 31; + +enum class MessageStatus : uint8_t { + Ok, + NotMCOtxt, // not this transport at all; nothing was touched + UnsupportedVersion, // subtype 0x03 with a revision other than 1 + Malformed, // container or stream rejected + ModelUnavailable, // a language this build has no tables for + TooLong, // decode workspace exhausted; incomplete text must not be published +}; + +struct DecodedMessage { + MessageStatus status; + uint8_t declared_version; + bool has_timestamp; + uint32_t timestamp; + bool has_sender; + char sender[kMaxNameBytes + 1]; + bool has_reply; + char reply_author[kMaxNameBytes + 1]; + uint32_t reply_timestamp; + DecodedMessage(); +}; + +// True for text that starts with `mct:` after optional leading whitespace and +// carries something after the prefix. +bool isTextPayload(const char* text); + +// True for a GROUP_DATA payload of type 0x0120 whose subtype nibble is 0x3. +// The envelope name is skipped and nothing else is checked, so a message that +// then fails to decode still counts as MCOtxt; callers preserve its raw frame. +bool isBinaryEnvelope(uint16_t data_type, const uint8_t* data, size_t data_length); + +// Decodes a `mct:` text payload into NUL-terminated UTF-8. The decoded name +// fields, when the container carries them, land in [message]; the caller's +// transport supplies the timestamp when the container inherits it. +// Names are owned by [message], independently of the output buffer. +MessageStatus decodeTextPayload(const char* text, char* out, size_t out_capacity, + DecodedMessage& message); + +// Decodes a GROUP_DATA 0x0120 envelope of subtype 0x03. Any other data type +// or subtype is NotMCOtxt, so other application data keeps its own handler. +// The sender is the container's when present, else the envelope's. +MessageStatus decodeBinaryEnvelope(uint16_t data_type, const uint8_t* data, size_t data_length, + char* out, size_t out_capacity, DecodedMessage& message); + +// Compatibility helper for apps that receive decoded MCOtxt instead of the +// original container: preserve the exact-reply anchor as the familiar +// "@[Name]" prefix. The prefix is added only when the decoded text does not +// already start with the same mention. [truncated] is raised if the tail had +// to be shortened to keep the result NUL-terminated. +bool ensureReplyMentionPrefix(const DecodedMessage& message, char* text, + size_t text_capacity, bool& truncated); + +// Prepend the bounded sender (or "?") to the decoded display text. +bool prependSenderName(const DecodedMessage& message, char* text, size_t capacity); + +} // namespace mcotxt + diff --git a/src/helpers/mcotxt/generated/model_be.h b/src/helpers/mcotxt/generated/model_be.h new file mode 100644 index 00000000000..cc7ef6d1318 --- /dev/null +++ b/src/helpers/mcotxt/generated/model_be.h @@ -0,0 +1,96 @@ +/* GENERATED FILE - DO NOT EDIT BY HAND. */ +/* MCOtxt v1 static TOP-4 model (START + AFTER_PUNCT): BE (wire id 6). */ +#ifndef MCOTXT_MODEL_BE_H +#define MCOTXT_MODEL_BE_H + +#include + +#ifndef MCOTXT_UPPERCASE_PAIR_T_DEFINED +#define MCOTXT_UPPERCASE_PAIR_T_DEFINED +#if defined(__GNUC__) || defined(__clang__) +#define MCOTXT_PACKED __attribute__((packed)) +#else +#define MCOTXT_PACKED +#endif +typedef struct MCOTXT_PACKED { + uint16_t uppercase_codepoint; + uint8_t lowercase_symbol_index; +} mcotxt_uppercase_pair_t; +#endif + +static const uint8_t mcotxt_be_available = 1u; +static const char mcotxt_be_wire_hash[] = "dd23c2b82288e0a4452f237ccd905055974779e3cc5cc6f3d03be1b24e5bb1fd"; +static const uint8_t mcotxt_be_language_id = 6u; +static const uint8_t mcotxt_be_primary_count = 32u; +static const uint8_t mcotxt_be_extension_count = 11u; +static const uint8_t mcotxt_be_symbol_count = 43u; +static const uint8_t mcotxt_be_uppercase_count = 32u; + +static const uint16_t mcotxt_be_primary_symbols[32] = { + 0x0020u, 0x043Au, 0x043Bu, 0x043Cu, 0x0441u, 0x0435u, 0x0440u, 0x0432u, 0x043Du, 0x044Fu, + 0x0434u, 0x0443u, 0x0442u, 0x0447u, 0x043Eu, 0x043Fu, 0x0437u, 0x044Bu, 0x0439u, 0x0433u, + 0x0431u, 0x0436u, 0x0430u, 0x0448u, 0x044Eu, 0x0445u, 0x044Cu, 0x044Du, 0x0032u, 0x0444u, + 0x0446u, 0x0031u, +}; + +static const uint16_t mcotxt_be_extension_symbols[11] = { + 0x0034u, 0x0451u, 0x0033u, 0x0036u, 0x0038u, 0x0035u, 0x0037u, 0x0030u, 0x0039u, 0x0456u, + 0x045Eu, +}; + +static const uint8_t mcotxt_be_start_top4[4] = { + 0u, 12u, 8u, 5u, +}; + +static const uint8_t mcotxt_be_punct_start_top4[4] = { + 0u, 12u, 28u, 38u, +}; + +static const uint8_t mcotxt_be_top4[172] = { + 8u, 15u, 7u, 4u, 22u, 14u, 0u, 2u, 14u, 26u, 0u, 22u, 14u, 0u, 5u, 22u, 12u, 0u, 5u, 14u, 0u, + 12u, 8u, 6u, 14u, 22u, 5u, 11u, 0u, 22u, 5u, 14u, 22u, 14u, 5u, 11u, 0u, 12u, 16u, 8u, 5u, 14u, + 22u, 11u, 0u, 12u, 21u, 4u, 14u, 22u, 0u, 26u, 5u, 22u, 12u, 8u, 0u, 10u, 20u, 12u, 14u, 6u, 22u, + 5u, 22u, 0u, 8u, 14u, 0u, 18u, 2u, 12u, 0u, 12u, 1u, 4u, 14u, 6u, 10u, 22u, 14u, 17u, 11u, 6u, + 5u, 8u, 11u, 22u, 0u, 2u, 4u, 12u, 5u, 26u, 22u, 12u, 13u, 0u, 12u, 10u, 14u, 0u, 22u, 16u, 0u, + 8u, 23u, 12u, 12u, 1u, 29u, 7u, 0u, 28u, 39u, 32u, 14u, 22u, 29u, 11u, 5u, 22u, 7u, 17u, 39u, + 37u, 0u, 31u, 0u, 37u, 36u, 39u, 0u, 12u, 3u, 14u, 0u, 39u, 28u, 32u, 0u, 36u, 38u, 40u, 35u, 0u, + 37u, 14u, 0u, 39u, 35u, 4u, 0u, 35u, 28u, 37u, 39u, 0u, 28u, 37u, 39u, 0u, 3u, 34u, 0u, 14u, 5u, + 22u, 0u, 14u, 5u, 22u, +}; + +static const mcotxt_uppercase_pair_t mcotxt_be_uppercase_map[32] = { + { 0x0401u, 33u }, /* Ё -> ё */ + { 0x0406u, 41u }, /* І -> і */ + { 0x040Eu, 42u }, /* Ў -> ў */ + { 0x0410u, 22u }, /* А -> а */ + { 0x0411u, 20u }, /* Б -> б */ + { 0x0412u, 7u }, /* В -> в */ + { 0x0413u, 19u }, /* Г -> г */ + { 0x0414u, 10u }, /* Д -> д */ + { 0x0415u, 5u }, /* Е -> е */ + { 0x0416u, 21u }, /* Ж -> ж */ + { 0x0417u, 16u }, /* З -> з */ + { 0x0419u, 18u }, /* Й -> й */ + { 0x041Au, 1u }, /* К -> к */ + { 0x041Bu, 2u }, /* Л -> л */ + { 0x041Cu, 3u }, /* М -> м */ + { 0x041Du, 8u }, /* Н -> н */ + { 0x041Eu, 14u }, /* О -> о */ + { 0x041Fu, 15u }, /* П -> п */ + { 0x0420u, 6u }, /* Р -> р */ + { 0x0421u, 4u }, /* С -> с */ + { 0x0422u, 12u }, /* Т -> т */ + { 0x0423u, 11u }, /* У -> у */ + { 0x0424u, 29u }, /* Ф -> ф */ + { 0x0425u, 25u }, /* Х -> х */ + { 0x0426u, 30u }, /* Ц -> ц */ + { 0x0427u, 13u }, /* Ч -> ч */ + { 0x0428u, 23u }, /* Ш -> ш */ + { 0x042Bu, 17u }, /* Ы -> ы */ + { 0x042Cu, 26u }, /* Ь -> ь */ + { 0x042Du, 27u }, /* Э -> э */ + { 0x042Eu, 24u }, /* Ю -> ю */ + { 0x042Fu, 9u }, /* Я -> я */ +}; + +#endif /* MCOTXT_MODEL_BE_H */ diff --git a/src/helpers/mcotxt/generated/model_de.h b/src/helpers/mcotxt/generated/model_de.h new file mode 100644 index 00000000000..eec4006e057 --- /dev/null +++ b/src/helpers/mcotxt/generated/model_de.h @@ -0,0 +1,92 @@ +/* GENERATED FILE - DO NOT EDIT BY HAND. */ +/* MCOtxt v1 static TOP-4 model (START + AFTER_PUNCT): DE (wire id 3). */ +#ifndef MCOTXT_MODEL_DE_H +#define MCOTXT_MODEL_DE_H + +#include + +#ifndef MCOTXT_UPPERCASE_PAIR_T_DEFINED +#define MCOTXT_UPPERCASE_PAIR_T_DEFINED +#if defined(__GNUC__) || defined(__clang__) +#define MCOTXT_PACKED __attribute__((packed)) +#else +#define MCOTXT_PACKED +#endif +typedef struct MCOTXT_PACKED { + uint16_t uppercase_codepoint; + uint8_t lowercase_symbol_index; +} mcotxt_uppercase_pair_t; +#endif + +static const uint8_t mcotxt_de_available = 1u; +static const char mcotxt_de_wire_hash[] = "00b971288fee56f37535ce03b4ac0d8ebddfc12ecbd5ef282ab4b0f4a659f451"; +static const uint8_t mcotxt_de_language_id = 3u; +static const uint8_t mcotxt_de_primary_count = 32u; +static const uint8_t mcotxt_de_extension_count = 9u; +static const uint8_t mcotxt_de_symbol_count = 41u; +static const uint8_t mcotxt_de_uppercase_count = 30u; + +static const uint16_t mcotxt_de_primary_symbols[32] = { + 0x0020u, 0x0073u, 0x0074u, 0x0062u, 0x0068u, 0x006Fu, 0x0067u, 0x006Cu, 0x0069u, 0x006Du, + 0x006Eu, 0x0072u, 0x0075u, 0x0077u, 0x006Bu, 0x0066u, 0x0065u, 0x0063u, 0x0061u, 0x0064u, + 0x007Au, 0x0070u, 0x00FCu, 0x0076u, 0x006Au, 0x00F6u, 0x00E4u, 0x0031u, 0x0033u, 0x0032u, + 0x0079u, 0x0035u, +}; + +static const uint16_t mcotxt_de_extension_symbols[9] = { + 0x0039u, 0x0034u, 0x0036u, 0x0037u, 0x0038u, 0x00DFu, 0x0078u, 0x0030u, 0x0071u, +}; + +static const uint8_t mcotxt_de_start_top4[4] = { + 6u, 9u, 4u, 19u, +}; + +static const uint8_t mcotxt_de_punct_start_top4[4] = { + 0u, 3u, 19u, 18u, +}; + +static const uint8_t mcotxt_de_top4[164] = { + 18u, 19u, 9u, 8u, 0u, 2u, 16u, 17u, 0u, 16u, 18u, 2u, 16u, 8u, 18u, 0u, 0u, 18u, 16u, 2u, 11u, + 8u, 10u, 21u, 16u, 12u, 11u, 0u, 7u, 16u, 8u, 0u, 10u, 17u, 16u, 2u, 5u, 16u, 8u, 18u, 0u, 19u, + 16u, 18u, 0u, 16u, 6u, 18u, 1u, 2u, 10u, 11u, 16u, 8u, 18u, 5u, 16u, 5u, 18u, 0u, 16u, 18u, 0u, + 11u, 10u, 11u, 0u, 8u, 4u, 14u, 5u, 0u, 12u, 10u, 7u, 17u, 16u, 18u, 0u, 8u, 12u, 16u, 2u, 0u, + 1u, 16u, 21u, 15u, 37u, 11u, 3u, 17u, 5u, 16u, 8u, 12u, 16u, 18u, 5u, 12u, 10u, 11u, 7u, 6u, 2u, + 26u, 12u, 10u, 0u, 39u, 27u, 29u, 0u, 27u, 39u, 36u, 0u, 39u, 35u, 28u, 2u, 0u, 16u, 5u, 0u, 39u, + 4u, 35u, 0u, 4u, 39u, 36u, 0u, 35u, 39u, 34u, 0u, 27u, 28u, 34u, 0u, 28u, 27u, 33u, 0u, 27u, 28u, + 34u, 16u, 0u, 2u, 8u, 0u, 8u, 2u, 16u, 0u, 39u, 34u, 35u, 12u, 0u, 2u, 1u, +}; + +static const mcotxt_uppercase_pair_t mcotxt_de_uppercase_map[30] = { + { 0x0041u, 18u }, /* A -> a */ + { 0x0042u, 3u }, /* B -> b */ + { 0x0043u, 17u }, /* C -> c */ + { 0x0044u, 19u }, /* D -> d */ + { 0x0045u, 16u }, /* E -> e */ + { 0x0046u, 15u }, /* F -> f */ + { 0x0047u, 6u }, /* G -> g */ + { 0x0048u, 4u }, /* H -> h */ + { 0x0049u, 8u }, /* I -> i */ + { 0x004Au, 24u }, /* J -> j */ + { 0x004Bu, 14u }, /* K -> k */ + { 0x004Cu, 7u }, /* L -> l */ + { 0x004Du, 9u }, /* M -> m */ + { 0x004Eu, 10u }, /* N -> n */ + { 0x004Fu, 5u }, /* O -> o */ + { 0x0050u, 21u }, /* P -> p */ + { 0x0051u, 40u }, /* Q -> q */ + { 0x0052u, 11u }, /* R -> r */ + { 0x0053u, 1u }, /* S -> s */ + { 0x0054u, 2u }, /* T -> t */ + { 0x0055u, 12u }, /* U -> u */ + { 0x0056u, 23u }, /* V -> v */ + { 0x0057u, 13u }, /* W -> w */ + { 0x0058u, 38u }, /* X -> x */ + { 0x0059u, 30u }, /* Y -> y */ + { 0x005Au, 20u }, /* Z -> z */ + { 0x00C4u, 26u }, /* Ä -> ä */ + { 0x00D6u, 25u }, /* Ö -> ö */ + { 0x00DCu, 22u }, /* Ü -> ü */ + { 0x1E9Eu, 37u }, /* ẞ -> ß */ +}; + +#endif /* MCOTXT_MODEL_DE_H */ diff --git a/src/helpers/mcotxt/generated/model_en.h b/src/helpers/mcotxt/generated/model_en.h new file mode 100644 index 00000000000..f97047c7e68 --- /dev/null +++ b/src/helpers/mcotxt/generated/model_en.h @@ -0,0 +1,87 @@ +/* GENERATED FILE - DO NOT EDIT BY HAND. */ +/* MCOtxt v1 static TOP-4 model (START + AFTER_PUNCT): EN (wire id 0). */ +#ifndef MCOTXT_MODEL_EN_H +#define MCOTXT_MODEL_EN_H + +#include + +#ifndef MCOTXT_UPPERCASE_PAIR_T_DEFINED +#define MCOTXT_UPPERCASE_PAIR_T_DEFINED +#if defined(__GNUC__) || defined(__clang__) +#define MCOTXT_PACKED __attribute__((packed)) +#else +#define MCOTXT_PACKED +#endif +typedef struct MCOTXT_PACKED { + uint16_t uppercase_codepoint; + uint8_t lowercase_symbol_index; +} mcotxt_uppercase_pair_t; +#endif + +static const uint8_t mcotxt_en_available = 1u; +static const char mcotxt_en_wire_hash[] = "55988b3bb2a000adf6e768a8541df5a25fec1628b4ec661a10b577e3af8b3770"; +static const uint8_t mcotxt_en_language_id = 0u; +static const uint8_t mcotxt_en_primary_count = 32u; +static const uint8_t mcotxt_en_extension_count = 5u; +static const uint8_t mcotxt_en_symbol_count = 37u; +static const uint8_t mcotxt_en_uppercase_count = 26u; + +static const uint16_t mcotxt_en_primary_symbols[32] = { + 0x0020u, 0x006Fu, 0x0064u, 0x0061u, 0x0069u, 0x006Cu, 0x0072u, 0x006Du, 0x0079u, 0x0074u, + 0x0063u, 0x0070u, 0x0077u, 0x0073u, 0x0066u, 0x0067u, 0x0068u, 0x0062u, 0x006Eu, 0x0065u, + 0x006Bu, 0x0075u, 0x0076u, 0x0078u, 0x0031u, 0x006Au, 0x0033u, 0x0032u, 0x0038u, 0x0034u, + 0x007Au, 0x0035u, +}; + +static const uint16_t mcotxt_en_extension_symbols[5] = { + 0x0071u, 0x0036u, 0x0037u, 0x0039u, 0x0030u, +}; + +static const uint8_t mcotxt_en_start_top4[4] = { + 16u, 7u, 9u, 4u, +}; + +static const uint8_t mcotxt_en_punct_start_top4[4] = { + 0u, 13u, 9u, 7u, +}; + +static const uint8_t mcotxt_en_top4[148] = { + 9u, 3u, 4u, 13u, 18u, 6u, 0u, 21u, 0u, 1u, 19u, 3u, 18u, 9u, 5u, 0u, 18u, 9u, 13u, 0u, 5u, 1u, + 0u, 19u, 19u, 0u, 18u, 1u, 19u, 1u, 0u, 3u, 0u, 1u, 19u, 13u, 0u, 16u, 1u, 19u, 1u, 16u, 3u, 19u, + 19u, 13u, 0u, 1u, 19u, 4u, 3u, 0u, 0u, 9u, 19u, 16u, 0u, 1u, 6u, 4u, 0u, 1u, 16u, 19u, 19u, 3u, + 1u, 4u, 19u, 21u, 1u, 3u, 15u, 0u, 4u, 19u, 0u, 6u, 13u, 18u, 19u, 0u, 13u, 4u, 9u, 13u, 0u, 6u, + 19u, 4u, 29u, 3u, 0u, 9u, 11u, 4u, 36u, 0u, 31u, 27u, 21u, 1u, 3u, 19u, 0u, 36u, 33u, 2u, 0u, + 27u, 31u, 36u, 0u, 7u, 33u, 36u, 0u, 36u, 29u, 33u, 19u, 30u, 8u, 4u, 0u, 36u, 33u, 27u, 21u, 0u, + 7u, 9u, 0u, 36u, 35u, 28u, 0u, 5u, 10u, 6u, 0u, 33u, 2u, 35u, 0u, 36u, 7u, 35u, +}; + +static const mcotxt_uppercase_pair_t mcotxt_en_uppercase_map[26] = { + { 0x0041u, 3u }, /* A -> a */ + { 0x0042u, 17u }, /* B -> b */ + { 0x0043u, 10u }, /* C -> c */ + { 0x0044u, 2u }, /* D -> d */ + { 0x0045u, 19u }, /* E -> e */ + { 0x0046u, 14u }, /* F -> f */ + { 0x0047u, 15u }, /* G -> g */ + { 0x0048u, 16u }, /* H -> h */ + { 0x0049u, 4u }, /* I -> i */ + { 0x004Au, 25u }, /* J -> j */ + { 0x004Bu, 20u }, /* K -> k */ + { 0x004Cu, 5u }, /* L -> l */ + { 0x004Du, 7u }, /* M -> m */ + { 0x004Eu, 18u }, /* N -> n */ + { 0x004Fu, 1u }, /* O -> o */ + { 0x0050u, 11u }, /* P -> p */ + { 0x0051u, 32u }, /* Q -> q */ + { 0x0052u, 6u }, /* R -> r */ + { 0x0053u, 13u }, /* S -> s */ + { 0x0054u, 9u }, /* T -> t */ + { 0x0055u, 21u }, /* U -> u */ + { 0x0056u, 22u }, /* V -> v */ + { 0x0057u, 12u }, /* W -> w */ + { 0x0058u, 23u }, /* X -> x */ + { 0x0059u, 8u }, /* Y -> y */ + { 0x005Au, 30u }, /* Z -> z */ +}; + +#endif /* MCOTXT_MODEL_EN_H */ diff --git a/src/helpers/mcotxt/generated/model_fr.h b/src/helpers/mcotxt/generated/model_fr.h new file mode 100644 index 00000000000..5229ff8fd3c --- /dev/null +++ b/src/helpers/mcotxt/generated/model_fr.h @@ -0,0 +1,108 @@ +/* GENERATED FILE - DO NOT EDIT BY HAND. */ +/* MCOtxt v1 static TOP-4 model (START + AFTER_PUNCT): FR (wire id 2). */ +#ifndef MCOTXT_MODEL_FR_H +#define MCOTXT_MODEL_FR_H + +#include + +#ifndef MCOTXT_UPPERCASE_PAIR_T_DEFINED +#define MCOTXT_UPPERCASE_PAIR_T_DEFINED +#if defined(__GNUC__) || defined(__clang__) +#define MCOTXT_PACKED __attribute__((packed)) +#else +#define MCOTXT_PACKED +#endif +typedef struct MCOTXT_PACKED { + uint16_t uppercase_codepoint; + uint8_t lowercase_symbol_index; +} mcotxt_uppercase_pair_t; +#endif + +static const uint8_t mcotxt_fr_available = 1u; +static const char mcotxt_fr_wire_hash[] = "910c5821a202a9b35313a9b939355f09d1c4293fe2ed246fed0123639d379ccb"; +static const uint8_t mcotxt_fr_language_id = 2u; +static const uint8_t mcotxt_fr_primary_count = 32u; +static const uint8_t mcotxt_fr_extension_count = 21u; +static const uint8_t mcotxt_fr_symbol_count = 53u; +static const uint8_t mcotxt_fr_uppercase_count = 42u; + +static const uint16_t mcotxt_fr_primary_symbols[32] = { + 0x0020u, 0x0075u, 0x0074u, 0x0063u, 0x006Du, 0x0072u, 0x006Cu, 0x006Fu, 0x0069u, 0x0065u, + 0x0061u, 0x0073u, 0x006Eu, 0x0076u, 0x0070u, 0x00E9u, 0x006Au, 0x0062u, 0x0064u, 0x0067u, + 0x0068u, 0x0066u, 0x0071u, 0x0079u, 0x00E7u, 0x00E0u, 0x006Bu, 0x0032u, 0x0078u, 0x0031u, + 0x0035u, 0x007Au, +}; + +static const uint16_t mcotxt_fr_extension_symbols[21] = { + 0x0034u, 0x0077u, 0x0033u, 0x0037u, 0x0036u, 0x00E8u, 0x0030u, 0x0038u, 0x00EAu, 0x0039u, + 0x00F4u, 0x00EEu, 0x00F9u, 0x00FBu, 0x00FCu, 0x00EBu, 0x0153u, 0x00E2u, 0x00E6u, 0x00EFu, + 0x00FFu, +}; + +static const uint8_t mcotxt_fr_start_top4[4] = { + 17u, 11u, 2u, 3u, +}; + +static const uint8_t mcotxt_fr_punct_start_top4[4] = { + 0u, 10u, 9u, 12u, +}; + +static const uint8_t mcotxt_fr_top4[212] = { + 18u, 14u, 6u, 11u, 5u, 0u, 2u, 8u, 0u, 9u, 8u, 7u, 7u, 20u, 9u, 10u, 9u, 10u, 7u, 8u, 0u, 9u, + 10u, 7u, 9u, 10u, 6u, 0u, 12u, 1u, 8u, 5u, 0u, 11u, 12u, 5u, 0u, 11u, 12u, 5u, 8u, 0u, 12u, 11u, + 0u, 2u, 9u, 10u, 0u, 2u, 9u, 11u, 9u, 7u, 8u, 10u, 10u, 7u, 9u, 5u, 2u, 0u, 9u, 14u, 9u, 7u, 1u, + 10u, 7u, 8u, 6u, 9u, 9u, 8u, 0u, 1u, 9u, 5u, 10u, 12u, 9u, 7u, 10u, 0u, 10u, 5u, 8u, 7u, 1u, 0u, + 22u, 5u, 0u, 9u, 7u, 2u, 10u, 1u, 7u, 0u, 0u, 9u, 11u, 10u, 0u, 9u, 10u, 8u, 0u, 38u, 29u, 41u, + 0u, 9u, 2u, 7u, 0u, 34u, 29u, 35u, 0u, 38u, 41u, 27u, 0u, 10u, 9u, 8u, 0u, 41u, 32u, 34u, 8u, 9u, + 10u, 7u, 0u, 34u, 32u, 39u, 34u, 29u, 38u, 0u, 0u, 39u, 27u, 32u, 11u, 5u, 2u, 4u, 0u, 38u, 36u, + 35u, 0u, 36u, 4u, 21u, 2u, 4u, 13u, 0u, 0u, 10u, 21u, 9u, 2u, 6u, 14u, 0u, 12u, 2u, 6u, 3u, 0u, + 9u, 11u, 10u, 5u, 0u, 9u, 11u, 5u, 0u, 9u, 11u, 6u, 0u, 9u, 11u, 8u, 0u, 9u, 11u, 0u, 9u, 11u, + 10u, 0u, 9u, 11u, 10u, 0u, 9u, 11u, 10u, 0u, 9u, 11u, 10u, +}; + +static const mcotxt_uppercase_pair_t mcotxt_fr_uppercase_map[42] = { + { 0x0041u, 10u }, /* A -> a */ + { 0x0042u, 17u }, /* B -> b */ + { 0x0043u, 3u }, /* C -> c */ + { 0x0044u, 18u }, /* D -> d */ + { 0x0045u, 9u }, /* E -> e */ + { 0x0046u, 21u }, /* F -> f */ + { 0x0047u, 19u }, /* G -> g */ + { 0x0048u, 20u }, /* H -> h */ + { 0x0049u, 8u }, /* I -> i */ + { 0x004Au, 16u }, /* J -> j */ + { 0x004Bu, 26u }, /* K -> k */ + { 0x004Cu, 6u }, /* L -> l */ + { 0x004Du, 4u }, /* M -> m */ + { 0x004Eu, 12u }, /* N -> n */ + { 0x004Fu, 7u }, /* O -> o */ + { 0x0050u, 14u }, /* P -> p */ + { 0x0051u, 22u }, /* Q -> q */ + { 0x0052u, 5u }, /* R -> r */ + { 0x0053u, 11u }, /* S -> s */ + { 0x0054u, 2u }, /* T -> t */ + { 0x0055u, 1u }, /* U -> u */ + { 0x0056u, 13u }, /* V -> v */ + { 0x0057u, 33u }, /* W -> w */ + { 0x0058u, 28u }, /* X -> x */ + { 0x0059u, 23u }, /* Y -> y */ + { 0x005Au, 31u }, /* Z -> z */ + { 0x00C0u, 25u }, /* À -> à */ + { 0x00C2u, 49u }, /*  -> â */ + { 0x00C6u, 50u }, /* Æ -> æ */ + { 0x00C7u, 24u }, /* Ç -> ç */ + { 0x00C8u, 37u }, /* È -> è */ + { 0x00C9u, 15u }, /* É -> é */ + { 0x00CAu, 40u }, /* Ê -> ê */ + { 0x00CBu, 47u }, /* Ë -> ë */ + { 0x00CEu, 43u }, /* Î -> î */ + { 0x00CFu, 51u }, /* Ï -> ï */ + { 0x00D4u, 42u }, /* Ô -> ô */ + { 0x00D9u, 44u }, /* Ù -> ù */ + { 0x00DBu, 45u }, /* Û -> û */ + { 0x00DCu, 46u }, /* Ü -> ü */ + { 0x0152u, 48u }, /* Œ -> œ */ + { 0x0178u, 52u }, /* Ÿ -> ÿ */ +}; + +#endif /* MCOTXT_MODEL_FR_H */ diff --git a/src/helpers/mcotxt/generated/model_it.h b/src/helpers/mcotxt/generated/model_it.h new file mode 100644 index 00000000000..47ce7612ded --- /dev/null +++ b/src/helpers/mcotxt/generated/model_it.h @@ -0,0 +1,99 @@ +/* GENERATED FILE - DO NOT EDIT BY HAND. */ +/* MCOtxt v1 static TOP-4 model (START + AFTER_PUNCT): IT (wire id 4). */ +#ifndef MCOTXT_MODEL_IT_H +#define MCOTXT_MODEL_IT_H + +#include + +#ifndef MCOTXT_UPPERCASE_PAIR_T_DEFINED +#define MCOTXT_UPPERCASE_PAIR_T_DEFINED +#if defined(__GNUC__) || defined(__clang__) +#define MCOTXT_PACKED __attribute__((packed)) +#else +#define MCOTXT_PACKED +#endif +typedef struct MCOTXT_PACKED { + uint16_t uppercase_codepoint; + uint8_t lowercase_symbol_index; +} mcotxt_uppercase_pair_t; +#endif + +static const uint8_t mcotxt_it_available = 1u; +static const char mcotxt_it_wire_hash[] = "f6d073a1d6ae6344eaac445b86fb8a454b276a740fd40651956b1ff28f5af270"; +static const uint8_t mcotxt_it_language_id = 4u; +static const uint8_t mcotxt_it_primary_count = 32u; +static const uint8_t mcotxt_it_extension_count = 14u; +static const uint8_t mcotxt_it_symbol_count = 46u; +static const uint8_t mcotxt_it_uppercase_count = 35u; + +static const uint16_t mcotxt_it_primary_symbols[32] = { + 0x0020u, 0x0074u, 0x006Cu, 0x006Du, 0x0070u, 0x0075u, 0x0069u, 0x0072u, 0x0063u, 0x0073u, + 0x0067u, 0x0076u, 0x0065u, 0x006Fu, 0x0061u, 0x006Eu, 0x0064u, 0x0062u, 0x0068u, 0x007Au, + 0x0066u, 0x0071u, 0x00E8u, 0x0031u, 0x0033u, 0x0032u, 0x0034u, 0x0077u, 0x006Bu, 0x0037u, + 0x0036u, 0x0038u, +}; + +static const uint16_t mcotxt_it_extension_symbols[14] = { + 0x0035u, 0x0079u, 0x00E0u, 0x00ECu, 0x0039u, 0x00F2u, 0x0030u, 0x00F9u, 0x006Au, 0x00E9u, + 0x0078u, 0x00EDu, 0x00F3u, 0x00FAu, +}; + +static const uint8_t mcotxt_it_start_top4[4] = { + 17u, 8u, 9u, 6u, +}; + +static const uint8_t mcotxt_it_punct_start_top4[4] = { + 0u, 6u, 14u, 7u, +}; + +static const uint8_t mcotxt_it_top4[184] = { + 14u, 9u, 16u, 8u, 12u, 6u, 13u, 1u, 14u, 0u, 12u, 6u, 12u, 6u, 14u, 13u, 12u, 13u, 9u, 7u, 13u, + 15u, 1u, 14u, 0u, 13u, 14u, 15u, 12u, 14u, 6u, 13u, 13u, 6u, 18u, 12u, 12u, 14u, 1u, 6u, 6u, 10u, + 7u, 14u, 12u, 14u, 13u, 6u, 0u, 7u, 15u, 9u, 0u, 15u, 7u, 4u, 0u, 15u, 2u, 7u, 13u, 0u, 14u, 1u, + 14u, 6u, 13u, 12u, 5u, 12u, 6u, 14u, 13u, 12u, 6u, 14u, 6u, 14u, 19u, 13u, 14u, 6u, 12u, 13u, 5u, + 1u, 0u, 7u, 0u, 13u, 14u, 6u, 0u, 38u, 25u, 14u, 0u, 17u, 24u, 18u, 0u, 38u, 31u, 36u, 0u, 36u, + 32u, 18u, 6u, 14u, 0u, 12u, 0u, 6u, 7u, 5u, 24u, 0u, 20u, 32u, 0u, 23u, 31u, 32u, 0u, 30u, 17u, + 20u, 0u, 38u, 23u, 30u, 1u, 0u, 10u, 9u, 0u, 13u, 14u, 6u, 0u, 14u, 15u, 13u, 0u, 13u, 23u, 25u, + 0u, 12u, 13u, 14u, 0u, 38u, 25u, 30u, 0u, 13u, 14u, 6u, 15u, 13u, 0u, 14u, 0u, 13u, 14u, 6u, 0u, + 8u, 5u, 16u, 0u, 13u, 14u, 6u, 0u, 13u, 14u, 6u, 0u, 13u, 14u, 6u, +}; + +static const mcotxt_uppercase_pair_t mcotxt_it_uppercase_map[35] = { + { 0x0041u, 14u }, /* A -> a */ + { 0x0042u, 17u }, /* B -> b */ + { 0x0043u, 8u }, /* C -> c */ + { 0x0044u, 16u }, /* D -> d */ + { 0x0045u, 12u }, /* E -> e */ + { 0x0046u, 20u }, /* F -> f */ + { 0x0047u, 10u }, /* G -> g */ + { 0x0048u, 18u }, /* H -> h */ + { 0x0049u, 6u }, /* I -> i */ + { 0x004Au, 40u }, /* J -> j */ + { 0x004Bu, 28u }, /* K -> k */ + { 0x004Cu, 2u }, /* L -> l */ + { 0x004Du, 3u }, /* M -> m */ + { 0x004Eu, 15u }, /* N -> n */ + { 0x004Fu, 13u }, /* O -> o */ + { 0x0050u, 4u }, /* P -> p */ + { 0x0051u, 21u }, /* Q -> q */ + { 0x0052u, 7u }, /* R -> r */ + { 0x0053u, 9u }, /* S -> s */ + { 0x0054u, 1u }, /* T -> t */ + { 0x0055u, 5u }, /* U -> u */ + { 0x0056u, 11u }, /* V -> v */ + { 0x0057u, 27u }, /* W -> w */ + { 0x0058u, 42u }, /* X -> x */ + { 0x0059u, 33u }, /* Y -> y */ + { 0x005Au, 19u }, /* Z -> z */ + { 0x00C0u, 34u }, /* À -> à */ + { 0x00C8u, 22u }, /* È -> è */ + { 0x00C9u, 41u }, /* É -> é */ + { 0x00CCu, 35u }, /* Ì -> ì */ + { 0x00CDu, 43u }, /* Í -> í */ + { 0x00D2u, 37u }, /* Ò -> ò */ + { 0x00D3u, 44u }, /* Ó -> ó */ + { 0x00D9u, 39u }, /* Ù -> ù */ + { 0x00DAu, 45u }, /* Ú -> ú */ +}; + +#endif /* MCOTXT_MODEL_IT_H */ diff --git a/src/helpers/mcotxt/generated/model_ru.h b/src/helpers/mcotxt/generated/model_ru.h new file mode 100644 index 00000000000..2a387760a9e --- /dev/null +++ b/src/helpers/mcotxt/generated/model_ru.h @@ -0,0 +1,97 @@ +/* GENERATED FILE - DO NOT EDIT BY HAND. */ +/* MCOtxt v1 static TOP-4 model (START + AFTER_PUNCT): RU (wire id 1). */ +#ifndef MCOTXT_MODEL_RU_H +#define MCOTXT_MODEL_RU_H + +#include + +#ifndef MCOTXT_UPPERCASE_PAIR_T_DEFINED +#define MCOTXT_UPPERCASE_PAIR_T_DEFINED +#if defined(__GNUC__) || defined(__clang__) +#define MCOTXT_PACKED __attribute__((packed)) +#else +#define MCOTXT_PACKED +#endif +typedef struct MCOTXT_PACKED { + uint16_t uppercase_codepoint; + uint8_t lowercase_symbol_index; +} mcotxt_uppercase_pair_t; +#endif + +static const uint8_t mcotxt_ru_available = 1u; +static const char mcotxt_ru_wire_hash[] = "d123c4978a635bf1b26fe37261b7e96a0206b3ba15b2a80542e560f00d1eb193"; +static const uint8_t mcotxt_ru_language_id = 1u; +static const uint8_t mcotxt_ru_primary_count = 32u; +static const uint8_t mcotxt_ru_extension_count = 12u; +static const uint8_t mcotxt_ru_symbol_count = 44u; +static const uint8_t mcotxt_ru_uppercase_count = 33u; + +static const uint16_t mcotxt_ru_primary_symbols[32] = { + 0x0020u, 0x0441u, 0x043Cu, 0x0443u, 0x0440u, 0x043Au, 0x0435u, 0x043Bu, 0x043Du, 0x0438u, + 0x044Fu, 0x0442u, 0x0431u, 0x0434u, 0x0432u, 0x0433u, 0x0447u, 0x0430u, 0x0437u, 0x043Eu, + 0x044Bu, 0x043Fu, 0x0439u, 0x0445u, 0x0436u, 0x0448u, 0x044Eu, 0x044Cu, 0x044Du, 0x0449u, + 0x0446u, 0x0444u, +}; + +static const uint16_t mcotxt_ru_extension_symbols[12] = { + 0x0031u, 0x0032u, 0x0033u, 0x0451u, 0x0034u, 0x0035u, 0x0038u, 0x0037u, 0x0036u, 0x0039u, + 0x0030u, 0x044Au, +}; + +static const uint8_t mcotxt_ru_start_top4[4] = { + 21u, 8u, 0u, 13u, +}; + +static const uint8_t mcotxt_ru_punct_start_top4[4] = { + 0u, 11u, 32u, 5u, +}; + +static const uint8_t mcotxt_ru_top4[176] = { + 8u, 21u, 1u, 14u, 11u, 6u, 0u, 19u, 0u, 6u, 19u, 17u, 0u, 11u, 13u, 24u, 19u, 17u, 6u, 9u, 19u, + 17u, 0u, 9u, 0u, 11u, 8u, 4u, 9u, 27u, 19u, 6u, 17u, 19u, 6u, 9u, 0u, 11u, 8u, 7u, 0u, 11u, 18u, + 7u, 19u, 17u, 0u, 27u, 19u, 4u, 20u, 6u, 17u, 19u, 6u, 9u, 0u, 6u, 17u, 19u, 19u, 13u, 17u, 9u, + 6u, 11u, 17u, 9u, 0u, 7u, 5u, 11u, 17u, 0u, 8u, 9u, 0u, 11u, 13u, 14u, 0u, 22u, 25u, 7u, 19u, 4u, + 9u, 17u, 0u, 11u, 16u, 5u, 19u, 0u, 17u, 3u, 6u, 8u, 13u, 9u, 6u, 9u, 17u, 5u, 0u, 11u, 16u, 1u, + 0u, 8u, 5u, 1u, 11u, 5u, 8u, 7u, 6u, 17u, 35u, 9u, 9u, 6u, 17u, 0u, 9u, 19u, 7u, 6u, 0u, 42u, + 32u, 36u, 0u, 42u, 37u, 33u, 0u, 42u, 34u, 33u, 0u, 11u, 2u, 7u, 0u, 42u, 34u, 37u, 0u, 42u, 37u, + 40u, 0u, 40u, 42u, 33u, 0u, 39u, 42u, 37u, 0u, 37u, 38u, 42u, 0u, 41u, 42u, 37u, 0u, 42u, 2u, 5u, + 6u, 10u, 35u, 9u, +}; + +static const mcotxt_uppercase_pair_t mcotxt_ru_uppercase_map[33] = { + { 0x0401u, 35u }, /* Ё -> ё */ + { 0x0410u, 17u }, /* А -> а */ + { 0x0411u, 12u }, /* Б -> б */ + { 0x0412u, 14u }, /* В -> в */ + { 0x0413u, 15u }, /* Г -> г */ + { 0x0414u, 13u }, /* Д -> д */ + { 0x0415u, 6u }, /* Е -> е */ + { 0x0416u, 24u }, /* Ж -> ж */ + { 0x0417u, 18u }, /* З -> з */ + { 0x0418u, 9u }, /* И -> и */ + { 0x0419u, 22u }, /* Й -> й */ + { 0x041Au, 5u }, /* К -> к */ + { 0x041Bu, 7u }, /* Л -> л */ + { 0x041Cu, 2u }, /* М -> м */ + { 0x041Du, 8u }, /* Н -> н */ + { 0x041Eu, 19u }, /* О -> о */ + { 0x041Fu, 21u }, /* П -> п */ + { 0x0420u, 4u }, /* Р -> р */ + { 0x0421u, 1u }, /* С -> с */ + { 0x0422u, 11u }, /* Т -> т */ + { 0x0423u, 3u }, /* У -> у */ + { 0x0424u, 31u }, /* Ф -> ф */ + { 0x0425u, 23u }, /* Х -> х */ + { 0x0426u, 30u }, /* Ц -> ц */ + { 0x0427u, 16u }, /* Ч -> ч */ + { 0x0428u, 25u }, /* Ш -> ш */ + { 0x0429u, 29u }, /* Щ -> щ */ + { 0x042Au, 43u }, /* Ъ -> ъ */ + { 0x042Bu, 20u }, /* Ы -> ы */ + { 0x042Cu, 27u }, /* Ь -> ь */ + { 0x042Du, 28u }, /* Э -> э */ + { 0x042Eu, 26u }, /* Ю -> ю */ + { 0x042Fu, 10u }, /* Я -> я */ +}; + +#endif /* MCOTXT_MODEL_RU_H */ diff --git a/src/helpers/mcotxt/generated/model_uk.h b/src/helpers/mcotxt/generated/model_uk.h new file mode 100644 index 00000000000..dac78da82dc --- /dev/null +++ b/src/helpers/mcotxt/generated/model_uk.h @@ -0,0 +1,97 @@ +/* GENERATED FILE - DO NOT EDIT BY HAND. */ +/* MCOtxt v1 static TOP-4 model (START + AFTER_PUNCT): UK (wire id 5). */ +#ifndef MCOTXT_MODEL_UK_H +#define MCOTXT_MODEL_UK_H + +#include + +#ifndef MCOTXT_UPPERCASE_PAIR_T_DEFINED +#define MCOTXT_UPPERCASE_PAIR_T_DEFINED +#if defined(__GNUC__) || defined(__clang__) +#define MCOTXT_PACKED __attribute__((packed)) +#else +#define MCOTXT_PACKED +#endif +typedef struct MCOTXT_PACKED { + uint16_t uppercase_codepoint; + uint8_t lowercase_symbol_index; +} mcotxt_uppercase_pair_t; +#endif + +static const uint8_t mcotxt_uk_available = 1u; +static const char mcotxt_uk_wire_hash[] = "949992c57156526843cdb9634c5c14bee23218878f116ef90be2a8497274c131"; +static const uint8_t mcotxt_uk_language_id = 5u; +static const uint8_t mcotxt_uk_primary_count = 32u; +static const uint8_t mcotxt_uk_extension_count = 12u; +static const uint8_t mcotxt_uk_symbol_count = 44u; +static const uint8_t mcotxt_uk_uppercase_count = 33u; + +static const uint16_t mcotxt_uk_primary_symbols[32] = { + 0x0020u, 0x043Au, 0x0441u, 0x0440u, 0x0456u, 0x043Cu, 0x0443u, 0x043Du, 0x0437u, 0x0431u, + 0x043Bu, 0x0434u, 0x043Eu, 0x0438u, 0x044Fu, 0x0430u, 0x0435u, 0x0442u, 0x0432u, 0x0448u, + 0x0447u, 0x0439u, 0x0433u, 0x043Fu, 0x0436u, 0x044Eu, 0x0446u, 0x0445u, 0x0454u, 0x0449u, + 0x0444u, 0x0457u, +}; + +static const uint16_t mcotxt_uk_extension_symbols[12] = { + 0x0032u, 0x0033u, 0x044Cu, 0x0031u, 0x0034u, 0x0035u, 0x0038u, 0x0036u, 0x0037u, 0x0030u, + 0x0039u, 0x0491u, +}; + +static const uint8_t mcotxt_uk_start_top4[4] = { + 0u, 17u, 7u, 11u, +}; + +static const uint8_t mcotxt_uk_punct_start_top4[4] = { + 0u, 37u, 41u, 35u, +}; + +static const uint8_t mcotxt_uk_top4[176] = { + 23u, 18u, 7u, 17u, 12u, 15u, 13u, 6u, 17u, 34u, 14u, 23u, 12u, 15u, 16u, 13u, 0u, 11u, 7u, 10u, + 12u, 15u, 0u, 16u, 0u, 18u, 17u, 10u, 15u, 12u, 16u, 4u, 15u, 0u, 7u, 4u, 6u, 15u, 12u, 4u, 16u, + 34u, 13u, 14u, 12u, 15u, 16u, 6u, 0u, 5u, 18u, 11u, 0u, 18u, 17u, 2u, 0u, 1u, 17u, 18u, 0u, 7u, + 17u, 10u, 0u, 7u, 3u, 5u, 15u, 13u, 12u, 34u, 0u, 15u, 13u, 4u, 13u, 12u, 16u, 6u, 13u, 15u, 16u, + 12u, 0u, 7u, 17u, 12u, 12u, 15u, 4u, 3u, 3u, 12u, 4u, 16u, 16u, 7u, 0u, 13u, 0u, 28u, 17u, 18u, + 16u, 4u, 25u, 14u, 0u, 12u, 13u, 17u, 0u, 17u, 5u, 11u, 12u, 16u, 15u, 6u, 4u, 12u, 15u, 17u, 0u, + 27u, 31u, 7u, 0u, 32u, 39u, 41u, 0u, 32u, 33u, 41u, 0u, 7u, 1u, 2u, 32u, 41u, 0u, 37u, 0u, 33u, + 40u, 41u, 0u, 18u, 41u, 32u, 41u, 0u, 39u, 32u, 38u, 0u, 32u, 33u, 0u, 41u, 1u, 5u, 0u, 41u, 1u, + 5u, 39u, 0u, 42u, 22u, 0u, 12u, 15u, 7u, +}; + +static const mcotxt_uppercase_pair_t mcotxt_uk_uppercase_map[33] = { + { 0x0404u, 28u }, /* Є -> є */ + { 0x0406u, 4u }, /* І -> і */ + { 0x0407u, 31u }, /* Ї -> ї */ + { 0x0410u, 15u }, /* А -> а */ + { 0x0411u, 9u }, /* Б -> б */ + { 0x0412u, 18u }, /* В -> в */ + { 0x0413u, 22u }, /* Г -> г */ + { 0x0414u, 11u }, /* Д -> д */ + { 0x0415u, 16u }, /* Е -> е */ + { 0x0416u, 24u }, /* Ж -> ж */ + { 0x0417u, 8u }, /* З -> з */ + { 0x0418u, 13u }, /* И -> и */ + { 0x0419u, 21u }, /* Й -> й */ + { 0x041Au, 1u }, /* К -> к */ + { 0x041Bu, 10u }, /* Л -> л */ + { 0x041Cu, 5u }, /* М -> м */ + { 0x041Du, 7u }, /* Н -> н */ + { 0x041Eu, 12u }, /* О -> о */ + { 0x041Fu, 23u }, /* П -> п */ + { 0x0420u, 3u }, /* Р -> р */ + { 0x0421u, 2u }, /* С -> с */ + { 0x0422u, 17u }, /* Т -> т */ + { 0x0423u, 6u }, /* У -> у */ + { 0x0424u, 30u }, /* Ф -> ф */ + { 0x0425u, 27u }, /* Х -> х */ + { 0x0426u, 26u }, /* Ц -> ц */ + { 0x0427u, 20u }, /* Ч -> ч */ + { 0x0428u, 19u }, /* Ш -> ш */ + { 0x0429u, 29u }, /* Щ -> щ */ + { 0x042Cu, 34u }, /* Ь -> ь */ + { 0x042Eu, 25u }, /* Ю -> ю */ + { 0x042Fu, 14u }, /* Я -> я */ + { 0x0490u, 43u }, /* Ґ -> ґ */ +}; + +#endif /* MCOTXT_MODEL_UK_H */ diff --git a/test/test_mcotxt/test_mcotxt.cpp b/test/test_mcotxt/test_mcotxt.cpp new file mode 100644 index 00000000000..6670d8215a4 --- /dev/null +++ b/test/test_mcotxt/test_mcotxt.cpp @@ -0,0 +1,730 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +// Most-significant bit first, the stream's own order. +class StreamWriter { + public: + void bits(uint32_t value, int count) { + for (int i = count - 1; i >= 0; i--) { + if (_bits % 8 == 0) _bytes.push_back(0); + _bytes.back() |= (uint8_t)(((value >> i) & 1u) << (7 - (_bits % 8))); + _bits++; + } + } + void headerField(uint32_t value) { + if (value < 7) { + bits(value, 3); + } else { + bits(7, 3); + bits(value - 7, 8); + } + } + // version 1, generation [generation], inline languages. + void header(uint8_t language_a, uint8_t language_b = 7, uint32_t generation = 0) { + headerField(1); + headerField(generation); + bits(language_a, 3); + bits(language_b, 3); + } + void top4(int rank) { + switch (rank) { + case 0: bits(0, 2); break; + case 1: bits(2, 3); break; + case 2: bits(6, 4); break; + default: bits(7, 4); break; + } + } + void primary(uint8_t id) { bits(2, 2); bits(id, 5); } + void punctuation(uint8_t id) { bits(6, 3); bits(id, 5); } + void extension(uint8_t id) { bits(14, 4); bits(id, 5); } + void shift() { bits(30, 5); } + void toggleLanguage() { bits(62, 6); } + void toggleCase() { bits(63, 6); bits(3, 3); } + void utf8Run(const std::vector& run) { + bits(63, 6); + bits(2, 3); + bits((uint32_t)(run.size() - 1), 5); + for (uint8_t b : run) bits(b, 8); + } + void byte(uint8_t value) { bits(value, 8); } + + const uint8_t* data() const { return _bytes.data(); } + size_t size() const { return _bytes.size(); } + uint32_t bitLength() const { return _bits; } + + private: + std::vector _bytes; + uint32_t _bits = 0; +}; + +std::string decode(const StreamWriter& w, mcotxt::DecodeStatus expected = mcotxt::DecodeStatus::Ok, + mcotxt::DecodeResult* result_out = nullptr) { + char out[512]; + size_t length = 0; + const mcotxt::DecodeResult result = + mcotxt::decodeStream(w.data(), w.size(), w.bitLength(), out, sizeof(out), length); + EXPECT_EQ((int)result.status, (int)expected); + if (result_out) *result_out = result; + return std::string(out, length); +} + +std::string utf8(uint32_t codepoint) { + uint8_t tmp[4]; + const size_t n = mcotxt::utf8Encode(codepoint, tmp); + return std::string((const char*)tmp, n); +} + +} // namespace + +// ---------------------------------------------------------------- models + +TEST(MCOtxtModels, TablesMatchTheManifest) { + const mcotxt::Model* en = mcotxt::modelFor(0); + ASSERT_NE(en, nullptr); + EXPECT_EQ(en->primary_count, 32); + EXPECT_EQ(en->extension_count, 5); + EXPECT_EQ(en->uppercase_count, 26); + EXPECT_STREQ(en->wire_hash, "55988b3bb2a000adf6e768a8541df5a25fec1628b4ec661a10b577e3af8b3770"); + const mcotxt::Model* ru = mcotxt::modelFor(1); + ASSERT_NE(ru, nullptr); + EXPECT_EQ(ru->extension_count, 12); + EXPECT_STREQ(ru->wire_hash, "d123c4978a635bf1b26fe37261b7e96a0206b3ba15b2a80542e560f00d1eb193"); + for (uint8_t id = 0; id < mcotxt::kLanguageCount; id++) { + const mcotxt::Model* model = mcotxt::modelFor(id); + ASSERT_NE(model, nullptr) << (int)id; + EXPECT_EQ(model->language_id, id); + EXPECT_EQ(model->symbolAt(0), 0x20u) << "SPACE is always primary[0]"; + for (uint8_t rank = 0; rank < 4; rank++) { + EXPECT_LT(model->start_top4[rank], model->symbolCount()); + EXPECT_LT(model->punct_start_top4[rank], model->symbolCount()); + } + } + EXPECT_EQ(mcotxt::modelFor(7), nullptr); + EXPECT_EQ(mcotxt::modelFor(255), nullptr); +} + +TEST(MCOtxtModels, LookupsAgree) { + const mcotxt::Model* en = mcotxt::modelFor(0); + const int a = en->indexOf('a'); + ASSERT_GE(a, 0); + EXPECT_EQ(en->indexOfUppercase('A'), a); + EXPECT_EQ(en->uppercaseOf((uint8_t)a), 'A'); + EXPECT_EQ(en->uppercaseOf(0), 0u) << "SPACE has no uppercase form"; + EXPECT_EQ(en->indexOf(0x0439), -1) << "Cyrillic is not English"; + EXPECT_EQ(mcotxt::languageIdForCode("ru"), 1); + EXPECT_EQ(mcotxt::languageIdForCode("BE"), 6); + EXPECT_EQ(mcotxt::languageIdForCode("xx"), -1); + EXPECT_EQ(mcotxt::languageIdForCode("eng"), -1); + EXPECT_STREQ(mcotxt::languageCode(2), "fr"); + EXPECT_EQ(mcotxt::punctuationIdOf('.'), 1); + EXPECT_EQ(mcotxt::punctuationIdOf(0x0A), 31); + EXPECT_EQ(mcotxt::punctuationAt(8), 0x2014u); + EXPECT_EQ(mcotxt::punctuationIdOf('1'), -1) << "digits are language symbols"; +} + +// --------------------------------------------------------------- decoder + +TEST(MCOtxtDecoder, RawUtf8Mode) { + StreamWriter w; + w.headerField(1); + w.headerField(0); + w.bits(7, 3); + w.bits(1, 3); + w.bits(0, 4); + EXPECT_EQ(w.bitLength(), 16u); + for (char c : std::string("hi \xD0\xBF")) w.byte((uint8_t)c); + mcotxt::DecodeResult result; + EXPECT_EQ(decode(w, mcotxt::DecodeStatus::Ok, &result), "hi \xD0\xBF"); + EXPECT_TRUE(result.raw_utf8); +} + +TEST(MCOtxtDecoder, RawUtf8RejectsPaddingAlignmentAndBadBytes) { + { + StreamWriter w; + w.headerField(1); w.headerField(0); w.bits(7, 3); w.bits(1, 3); w.bits(1, 4); + w.byte('a'); + decode(w, mcotxt::DecodeStatus::InvalidUtf8); + } + { + StreamWriter w; + w.headerField(1); w.headerField(0); w.bits(7, 3); w.bits(1, 3); w.bits(0, 4); + w.byte('a'); w.bits(1, 3); + decode(w, mcotxt::DecodeStatus::InvalidUtf8); + } + { + StreamWriter w; + w.headerField(1); w.headerField(0); w.bits(7, 3); w.bits(1, 3); w.bits(0, 4); + w.byte(0xC0); w.byte(0x80); // overlong NUL + decode(w, mcotxt::DecodeStatus::InvalidUtf8); + } +} + +TEST(MCOtxtDecoder, RawUtf8IgnoresTheGeneration) { + StreamWriter w; + w.headerField(1); w.headerField(9); w.bits(7, 3); w.bits(1, 3); w.bits(0, 4); + w.byte('x'); + EXPECT_EQ(decode(w), "x"); +} + +TEST(MCOtxtDecoder, HeaderRejections) { + { + StreamWriter w; + w.header(0); + w.bits(0, 0); + // version 2 + StreamWriter v; + v.headerField(2); v.headerField(0); v.bits(0, 3); v.bits(7, 3); + decode(v, mcotxt::DecodeStatus::UnknownVersion); + } + { + StreamWriter w; + w.header(0, 7, 1); + decode(w, mcotxt::DecodeStatus::UnsupportedGeneration); + } + { + StreamWriter w; + w.headerField(1); w.headerField(0); w.bits(7, 3); w.bits(2, 3); + decode(w, mcotxt::DecodeStatus::UnsupportedHeader); + } + { + StreamWriter w; + w.header(0, 0); // A and B must differ + decode(w, mcotxt::DecodeStatus::UnknownLanguage); + } + { + StreamWriter w; + w.headerField(1); w.headerField(0); w.bits(7, 3); // cut inside the header + decode(w, mcotxt::DecodeStatus::UnexpectedEnd); + } +} + +TEST(MCOtxtDecoder, RejectsBitLengthLargerThanTheBufferWithoutOverflow) { + const uint8_t data[] = {0}; + char out[8]; + size_t length = 0; + const mcotxt::DecodeResult result = + mcotxt::decodeStream(data, sizeof(data), UINT32_MAX, out, sizeof(out), length); + EXPECT_EQ((int)result.status, (int)mcotxt::DecodeStatus::Malformed); + EXPECT_EQ(length, 0u); +} + +TEST(MCOtxtDecoder, ExtendedPairHeader) { + StreamWriter w; + w.headerField(1); w.headerField(0); w.bits(7, 3); w.bits(0, 3); + w.bits(1, 8); w.bits(255, 8); + w.primary(1); + mcotxt::DecodeResult result; + EXPECT_EQ(decode(w, mcotxt::DecodeStatus::Ok, &result), utf8(mcotxt::modelFor(1)->symbolAt(1))); + EXPECT_EQ(result.language_a, 1); + EXPECT_EQ(result.language_b, mcotxt::kLanguageNone); + + StreamWriter bad; + bad.headerField(1); bad.headerField(0); bad.bits(7, 3); bad.bits(0, 3); + bad.bits(255, 8); bad.bits(0, 8); + decode(bad, mcotxt::DecodeStatus::UnknownLanguage); +} + +TEST(MCOtxtDecoder, Top4FollowsTheContextRows) { + for (uint8_t id = 0; id < mcotxt::kLanguageCount; id++) { + const mcotxt::Model* model = mcotxt::modelFor(id); + for (int rank = 0; rank < 4; rank++) { + StreamWriter w; + w.header(id); + w.top4(rank); + w.top4(0); + const uint8_t first = model->start_top4[rank]; + const uint8_t second = model->rowFor(first)[0]; + EXPECT_EQ(decode(w), utf8(model->symbolAt(first)) + utf8(model->symbolAt(second))) + << "language " << (int)id << " rank " << rank; + } + } +} + +TEST(MCOtxtDecoder, LiteralsAndPunctuationContexts) { + const mcotxt::Model* en = mcotxt::modelFor(0); + StreamWriter w; + w.header(0); + w.primary(3); + w.extension(0); + w.punctuation(1); // '.', then AFTER_PUNCT + w.top4(0); + w.punctuation(0); // SPACE keeps the SYMBOL context + w.top4(1); + w.punctuation(31); // LF gives START + w.top4(2); + const uint8_t after_punct = en->punct_start_top4[0]; + const uint8_t after_space = en->rowFor(after_punct)[1]; + const uint8_t after_lf = en->start_top4[2]; + EXPECT_EQ(decode(w), utf8(en->symbolAt(3)) + utf8(en->symbolAt(en->primary_count)) + "." + + utf8(en->symbolAt(after_punct)) + " " + utf8(en->symbolAt(after_space)) + + "\n" + utf8(en->symbolAt(after_lf))); + + // PRIMARY has a five-bit ID and every v1 model has all 32 entries, so an + // out-of-range PRIMARY cannot be represented on the wire. EXTENSION IDs + // still have unused values and exercise the decoder's range check. + StreamWriter bad_extension; + bad_extension.header(0); + bad_extension.extension(5); // EN has five extension symbols + decode(bad_extension, mcotxt::DecodeStatus::InvalidToken); +} + +TEST(MCOtxtDecoder, ShiftAndCapsMode) { + const mcotxt::Model* en = mcotxt::modelFor(0); + const uint8_t a = (uint8_t)en->indexOf('a'); + const uint8_t one = (uint8_t)en->indexOf('1'); + { + StreamWriter w; + w.header(0); + w.shift(); w.primary(a); w.primary(a); + w.toggleCase(); w.primary(a); w.shift(); w.primary(a); w.primary(a); + w.toggleCase(); w.primary(a); + EXPECT_EQ(decode(w), "AaAaAa"); + } + { + StreamWriter w; + w.header(0); + w.toggleCase(); w.primary(one); // digits ignore caps mode + EXPECT_EQ(decode(w), "1"); + } + { + StreamWriter w; + w.header(0); + w.shift(); w.primary(one); + decode(w, mcotxt::DecodeStatus::InvalidShift); + } + { + StreamWriter w; + w.header(0); + w.shift(); w.shift(); w.primary(a); + decode(w, mcotxt::DecodeStatus::InvalidShift); + } + { + StreamWriter w; + w.header(0); + w.shift(); w.punctuation(1); + decode(w, mcotxt::DecodeStatus::InvalidShift); + } + { + StreamWriter w; + w.header(0); + w.primary(a); w.shift(); + decode(w, mcotxt::DecodeStatus::InvalidShift); + } +} + +TEST(MCOtxtDecoder, ToggleLanguage) { + const mcotxt::Model* ru = mcotxt::modelFor(1); + StreamWriter w; + w.header(0, 1); + w.toggleLanguage(); w.primary(1); + w.toggleLanguage(); w.primary(3); + EXPECT_EQ(decode(w), utf8(ru->symbolAt(1)) + "a"); + + StreamWriter without_b; + without_b.header(0); + without_b.toggleLanguage(); + decode(without_b, mcotxt::DecodeStatus::InvalidToken); +} + +TEST(MCOtxtDecoder, SwitchOtherLanguageAndResetContext) { + const mcotxt::Model* fr = mcotxt::modelFor(2); + StreamWriter w; + w.header(0); + w.bits(63, 6); w.bits(0, 3); w.bits(2, 8); // SWITCH_OTHER_LANGUAGE fr + w.top4(0); + w.bits(63, 6); w.bits(1, 3); // RESET_CONTEXT + w.top4(1); + EXPECT_EQ(decode(w), utf8(fr->symbolAt(fr->start_top4[0])) + utf8(fr->symbolAt(fr->start_top4[1]))); + + StreamWriter unknown; + unknown.header(0); + unknown.bits(63, 6); unknown.bits(0, 3); unknown.bits(255, 8); + decode(unknown, mcotxt::DecodeStatus::InvalidToken); + + StreamWriter reserved; + reserved.header(0); + reserved.bits(63, 6); reserved.bits(5, 3); + decode(reserved, mcotxt::DecodeStatus::InvalidToken); +} + +TEST(MCOtxtDecoder, Utf8Run) { + StreamWriter w; + w.header(0); + w.utf8Run({0xF0, 0x9F, 0x98, 0x80}); + w.top4(0); + const mcotxt::Model* en = mcotxt::modelFor(0); + EXPECT_EQ(decode(w), std::string("\xF0\x9F\x98\x80") + utf8(en->symbolAt(en->start_top4[0]))); + + StreamWriter bad; + bad.header(0); + bad.utf8Run({0xF0, 0x9F, 0x98}); // cut sequence + decode(bad, mcotxt::DecodeStatus::InvalidUtf8); + + StreamWriter cut; + cut.header(0); + cut.bits(63, 6); cut.bits(2, 3); cut.bits(3, 5); cut.byte('a'); + decode(cut, mcotxt::DecodeStatus::UnexpectedEnd); +} + +TEST(MCOtxtDecoder, PaddingIsNotSelfTerminating) { + StreamWriter w; + w.header(0); + w.punctuation(1); // 20 bits total; four zero padding bits are two TOP4 tokens + char out[64]; + size_t length; + // Reading the padded byte count instead of the bit count turns the padding + // into TOP4 tokens: the caller must pass the exact bit length. + mcotxt::decodeStream(w.data(), w.size(), (uint32_t)w.size() * 8, out, sizeof(out), length); + EXPECT_GT(length, 1u); + mcotxt::decodeStream(w.data(), w.size(), w.bitLength(), out, sizeof(out), length); + EXPECT_EQ(std::string(out, length), "."); +} + +TEST(MCOtxtDecoder, OutputCapacityIsHonoured) { + StreamWriter w; + w.header(0); + for (int i = 0; i < 10; i++) w.primary(3); + char out[6]; + size_t length; + const mcotxt::DecodeResult result = + mcotxt::decodeStream(w.data(), w.size(), w.bitLength(), out, sizeof(out), length); + EXPECT_EQ((int)result.status, (int)mcotxt::DecodeStatus::OutputTooSmall); + // The start that fit stays in the buffer, terminated. + const mcotxt::Model* en = mcotxt::modelFor(0); + ASSERT_NE(en, nullptr); + std::string expected; + while (expected.size() + utf8(en->symbolAt(3)).size() + 1 <= sizeof(out)) { + expected += utf8(en->symbolAt(3)); + } + EXPECT_EQ(std::string(out), expected); + EXPECT_EQ(length, expected.size()); + + // RAW_UTF8 cuts on a character boundary too. + StreamWriter raw; + raw.headerField(1); + raw.headerField(0); + raw.bits(7, 3); + raw.bits(1, 3); + raw.bits(0, 4); + for (char c : std::string("h\xC3\xA9llo")) raw.byte((uint8_t)c); + char small[4]; + const mcotxt::DecodeResult cut = + mcotxt::decodeStream(raw.data(), raw.size(), raw.bitLength(), small, sizeof(small), length); + EXPECT_EQ((int)cut.status, (int)mcotxt::DecodeStatus::OutputTooSmall); + EXPECT_TRUE(cut.raw_utf8); + EXPECT_EQ(std::string(small), "h\xC3\xA9"); + EXPECT_EQ(length, 3u); + + // And a UTF8_RUN keeps the characters that fit. + StreamWriter run; + run.header(0); + run.utf8Run({'x', 'y', 'z'}); + char two[3]; + const mcotxt::DecodeResult part = + mcotxt::decodeStream(run.data(), run.size(), run.bitLength(), two, sizeof(two), length); + EXPECT_EQ((int)part.status, (int)mcotxt::DecodeStatus::OutputTooSmall); + EXPECT_EQ(std::string(two), "xy"); +} + +// ----------------------------------------------------------------- parts + +namespace { + +std::string partText(const char* prefix, const std::string& text, const mcotxt::TextPart* parts, + size_t index, size_t count) { + char out[256]; + const size_t length = mcotxt::formatPart(prefix, strlen(prefix), text.data(), parts[index], + index, count, out, sizeof(out)); + return std::string(out, length); +} + +} // namespace + +TEST(MCOtxtParts, OneMessageWhenItFits) { + const std::string text = "hello world"; + mcotxt::TextPart parts[mcotxt::kMaxParts]; + ASSERT_EQ(mcotxt::splitForApp(text.data(), text.size(), 40, 5, parts, mcotxt::kMaxParts), 1u); + EXPECT_EQ(partText("Bob: ", text, parts, 0, 1), "Bob: hello world"); + // Exactly the budget still goes as one; one byte more does not. + const std::string full(35, 'x'); + EXPECT_EQ(mcotxt::splitForApp(full.data(), full.size(), 40, 5, parts, mcotxt::kMaxParts), 1u); + const std::string over(36, 'x'); + EXPECT_GT(mcotxt::splitForApp(over.data(), over.size(), 40, 5, parts, mcotxt::kMaxParts), 1u); + EXPECT_EQ(mcotxt::splitForApp(text.data(), 0, 40, 5, parts, mcotxt::kMaxParts), 0u); +} + +TEST(MCOtxtParts, BreaksAtSpacesAndNumbersTheParts) { + std::string text; + for (int i = 1; i <= 30; i++) { + if (!text.empty()) text += ' '; + char word[12]; + snprintf(word, sizeof(word), "word%02d", i); + text += word; + } + mcotxt::TextPart parts[mcotxt::kMaxParts]; + const size_t count = mcotxt::splitForApp(text.data(), text.size(), 40, 5, parts, mcotxt::kMaxParts); + ASSERT_GT(count, 1u); + std::string joined; + for (size_t i = 0; i < count; i++) { + const std::string part = partText("Bob: ", text, parts, i, count); + EXPECT_LE(part.size(), 40u) << part; + const std::string head = i == 0 ? "Bob: " : "Bob: ..."; + EXPECT_EQ(part.compare(0, head.size(), head), 0) << part; + char mark[16]; + snprintf(mark, sizeof(mark), "%s [%u/%u]", i + 1 < count ? "..." : "", + (unsigned)(i + 1), (unsigned)count); + ASSERT_GT(part.size(), strlen(mark)); + EXPECT_EQ(part.compare(part.size() - strlen(mark), strlen(mark), mark), 0) << part; + const std::string chunk = text.substr(parts[i].offset, parts[i].length); + EXPECT_EQ(part, head + chunk + mark); + EXPECT_NE(chunk.front(), ' '); + EXPECT_NE(chunk.back(), ' '); + if (!joined.empty()) joined += ' '; + joined += chunk; + } + EXPECT_EQ(joined, text) << "chunks only break at spaces, so the words come back whole"; +} + +TEST(MCOtxtParts, FirstAndMiddlePartsEndWithEllipsesButLastDoesNot) { + const std::string text = "abcdefghijklmnopqrstuvwx"; + mcotxt::TextPart parts[mcotxt::kMaxParts]; + const size_t count = mcotxt::splitForApp(text.data(), text.size(), 24, 5, parts, mcotxt::kMaxParts); + ASSERT_EQ(count, 3u); + EXPECT_EQ(partText("Bob: ", text, parts, 0, count), "Bob: abcdefghij... [1/3]"); + EXPECT_EQ(partText("Bob: ", text, parts, 1, count), "Bob: ...klmnopq... [2/3]"); + EXPECT_EQ(partText("Bob: ", text, parts, 2, count), "Bob: ...rstuvwx [3/3]"); +} + +TEST(MCOtxtParts, LastPartUsesFullBudgetWithoutReservingTrailingEllipses) { + const std::string text = "abcdefghijklmnopqrst"; + mcotxt::TextPart parts[mcotxt::kMaxParts]; + const size_t count = mcotxt::splitForApp(text.data(), text.size(), 24, 5, parts, mcotxt::kMaxParts); + ASSERT_EQ(count, 2u); + EXPECT_EQ(partText("Bob: ", text, parts, 0, count), "Bob: abcdefghij... [1/2]"); + EXPECT_EQ(partText("Bob: ", text, parts, 1, count), "Bob: ...klmnopqrst [2/2]"); + EXPECT_EQ(partText("Bob: ", text, parts, 0, count).size(), 24u); + EXPECT_EQ(partText("Bob: ", text, parts, 1, count).size(), 24u); +} + +TEST(MCOtxtParts, CutsOnCharacterBoundariesWithoutSpaces) { + std::string text; + for (int i = 0; i < 60; i++) text += "\xD0\xB0"; // 120 bytes, no breaks + mcotxt::TextPart parts[mcotxt::kMaxParts]; + const size_t count = mcotxt::splitForApp(text.data(), text.size(), 32, 0, parts, mcotxt::kMaxParts); + ASSERT_GT(count, 1u); + std::string joined; + for (size_t i = 0; i < count; i++) { + EXPECT_EQ(parts[i].length % 2, 0u) << "a two-byte letter is never split"; + EXPECT_LE(partText("", text, parts, i, count).size(), 32u); + joined += text.substr(parts[i].offset, parts[i].length); + } + EXPECT_EQ(joined, text); +} + +TEST(MCOtxtParts, MarksGrowWithTheCountAndTooManyPartsFail) { + const std::string text(400, 'a'); // no breaks: every part fills its budget + mcotxt::TextPart parts[mcotxt::kMaxParts]; + const size_t count = mcotxt::splitForApp(text.data(), text.size(), 40, 0, parts, mcotxt::kMaxParts); + ASSERT_GE(count, 10u) << "two-digit marks were planned for"; + std::string joined; + for (size_t i = 0; i < count; i++) { + EXPECT_LE(partText("", text, parts, i, count).size(), 40u); + joined += text.substr(parts[i].offset, parts[i].length); + } + EXPECT_EQ(joined, text); + const std::string last = partText("", text, parts, count - 1, count); + const std::string mark = " [" + std::to_string(count) + "/" + std::to_string(count) + "]"; + EXPECT_EQ(last.substr(last.size() - mark.size()), mark); + EXPECT_EQ(mcotxt::splitForApp(text.data(), text.size(), 40, 0, parts, 4), 0u); + // A prefix that leaves no room fails too. + EXPECT_EQ(mcotxt::splitForApp(text.data(), text.size(), 40, 40, parts, mcotxt::kMaxParts), 0u); +} + +TEST(MCOtxtParts, SplitPartsAlwaysFormatInsideFrameTextBudget) { + const std::string prefix = "LongestVisibleSenderName: "; + std::string text; + for (int i = 0; i < 80; i++) { + if (!text.empty()) text += ' '; + text += "message"; + } + constexpr size_t kFrameTextBudget = 167; // MAX_FRAME_SIZE - CHANNEL_MSG_RECV_V3 header + mcotxt::TextPart parts[mcotxt::kMaxParts]; + const size_t count = mcotxt::splitForApp(text.data(), text.size(), kFrameTextBudget, + prefix.size(), parts, mcotxt::kMaxParts); + ASSERT_GT(count, 1u); + char out[256]; + std::string joined; + for (size_t i = 0; i < count; i++) { + const size_t length = mcotxt::formatPart(prefix.c_str(), prefix.size(), text.data(), + parts[i], i, count, out, sizeof(out)); + ASSERT_GT(length, 0u); + EXPECT_LE(length, kFrameTextBudget); + const std::string chunk = text.substr(parts[i].offset, parts[i].length); + if (!joined.empty()) joined += ' '; + joined += chunk; + } + EXPECT_EQ(joined, text); +} + +// ------------------------------------------------------------- transport + +TEST(MCOtxtTransport, TextPayloadRecognition) { + EXPECT_TRUE(mcotxt::isTextPayload("mct:abc")); + EXPECT_TRUE(mcotxt::isTextPayload(" mct:abc")); + EXPECT_FALSE(mcotxt::isTextPayload("mct:")); + EXPECT_FALSE(mcotxt::isTextPayload("mcmp3:abc")); + EXPECT_FALSE(mcotxt::isTextPayload("hello")); + EXPECT_FALSE(mcotxt::isTextPayload(nullptr)); +} + +TEST(MCOtxtTransport, BinaryEnvelopeDecodes) { + // senderNameLength 0 | 0x31 | flags 0x02 | timestamp | name (UTF-8 string) | + // text (UTF-8 string). + std::vector envelope = {0x00, 0x31, 0x02, 0x78, 0x56, 0x34, 0x12, + 0x01, 0x03, 'B', 'o', 'b', + 0x01, 0x02, 'h', 'i'}; + char out[64]; + mcotxt::DecodedMessage message; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, envelope.data(), envelope.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::Ok); + EXPECT_STREQ(out, "hi"); + EXPECT_TRUE(message.has_sender); + EXPECT_STREQ(message.sender, "Bob"); + EXPECT_TRUE(message.has_timestamp); + EXPECT_EQ(message.timestamp, 0x12345678u); + EXPECT_FALSE(message.has_reply); + + // The envelope name is the fallback when the container carries none. + std::vector named = {0x03, 'A', 'n', 'n', 0x31, 0x04, 0x01, 0x02, 'h', 'i'}; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, named.data(), named.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::Ok); + EXPECT_STREQ(message.sender, "Ann"); + EXPECT_FALSE(message.has_timestamp); + + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, envelope.data(), envelope.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::Ok); + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0xAE1C, envelope.data(), envelope.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::NotMCOtxt); + std::vector image = {0x00, 0x13, 0x00}; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, image.data(), image.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::NotMCOtxt); + std::vector newer = {0x00, 0x32, 0x04, 0x01, 0x02, 'h', 'i'}; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, newer.data(), newer.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::UnsupportedVersion); + EXPECT_EQ(message.declared_version, 2); + std::vector trailing = {0x00, 0x31, 0x04, 0x01, 0x02, 'h', 'i', 0x00}; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, trailing.data(), trailing.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::Malformed); + std::vector flags = {0x00, 0x31, 0x84, 0x01, 0x02, 'h', 'i'}; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, flags.data(), flags.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::Malformed); +} + +TEST(MCOtxtTransport, BinaryEnvelopeRecognition) { + std::vector ok = {0x00, 0x31, 0x04, 0x01, 0x02, 'h', 'i'}; + EXPECT_TRUE(mcotxt::isBinaryEnvelope(0x0120, ok.data(), ok.size())); + std::vector named = {0x03, 'B', 'o', 'b', 0x32}; // any revision counts + EXPECT_TRUE(mcotxt::isBinaryEnvelope(0x0120, named.data(), named.size())); + std::vector other = {0x00, 0x41}; // another subtype + EXPECT_FALSE(mcotxt::isBinaryEnvelope(0x0120, other.data(), other.size())); + EXPECT_FALSE(mcotxt::isBinaryEnvelope(0x0121, ok.data(), ok.size())); + std::vector cut = {0x05, 'B'}; // name longer than the payload + EXPECT_FALSE(mcotxt::isBinaryEnvelope(0x0120, cut.data(), cut.size())); + const std::vector huge_name = {0xFF, 0xFF, 0xFF, 0xFF, 0x0F, 0x31}; + EXPECT_FALSE(mcotxt::isBinaryEnvelope(0x0120, huge_name.data(), huge_name.size())); + char out[8]; + mcotxt::DecodedMessage message; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, huge_name.data(), huge_name.size(), + out, sizeof(out), message), + (int)mcotxt::MessageStatus::NotMCOtxt); + EXPECT_FALSE(mcotxt::isBinaryEnvelope(0x0120, nullptr, 0)); +} + +TEST(MCOtxtTransport, RejectsOverflowingMCOtxtBitLength) { + // outer name 0 | subtype v1 | inherited timestamp | MCOtxt mode | + // bit_length UINT32_MAX. No stream bytes follow. + const std::vector envelope = { + 0x00, 0x31, 0x04, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F}; + char out[8]; + mcotxt::DecodedMessage message; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, envelope.data(), envelope.size(), + out, sizeof(out), message), + (int)mcotxt::MessageStatus::Malformed); +} + +TEST(MCOtxtTransport, TooLongKeepsTheStart) { + // A UTF-8 mode text into a buffer for three bytes. + std::vector plain = {0x00, 0x31, 0x04, 0x01, 0x05, 'h', 'e', 'l', 'l', 'o'}; + char out[4]; + mcotxt::DecodedMessage message; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, plain.data(), plain.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::TooLong); + EXPECT_STREQ(out, "hel"); + // A failed text decode does not claim a successfully parsed envelope sender. + std::vector named = {0x03, 'B', 'o', 'b', 0x31, 0x04, 0x01, 0x05, + 'h', 'e', 'l', 'l', 'o'}; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, named.data(), named.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::TooLong); + EXPECT_FALSE(message.has_sender); +} + +TEST(MCOtxtTransport, ReplyAnchorNeedsBothHalves) { + std::vector reply = {0x00, 0x31, 0x05, 0x01, 0x03, 'B', 'o', 'b', + 0x10, 0x00, 0x00, 0x00, 0x01, 0x02, 'o', 'k'}; + char out[64]; + mcotxt::DecodedMessage message; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, reply.data(), reply.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::Ok); + EXPECT_TRUE(message.has_reply); + EXPECT_STREQ(message.reply_author, "Bob"); + EXPECT_EQ(message.reply_timestamp, 0x10u); + EXPECT_STREQ(out, "ok"); + + bool truncated = false; + EXPECT_TRUE(mcotxt::ensureReplyMentionPrefix(message, out, sizeof(out), truncated)); + EXPECT_FALSE(truncated); + EXPECT_STREQ(out, "@[Bob] ok"); + EXPECT_TRUE(mcotxt::ensureReplyMentionPrefix(message, out, sizeof(out), truncated)); + EXPECT_FALSE(truncated); + EXPECT_STREQ(out, "@[Bob] ok"); + + std::vector cut = {0x00, 0x31, 0x05, 0x01, 0x03, 'B', 'o', 'b', 0x10, 0x00}; + EXPECT_EQ((int)mcotxt::decodeBinaryEnvelope(0x0120, cut.data(), cut.size(), out, + sizeof(out), message), + (int)mcotxt::MessageStatus::Malformed); +} + +TEST(MCOtxtTransport, TextPayloadRejectsGarbage) { + char out[64]; + mcotxt::DecodedMessage message; + EXPECT_EQ((int)mcotxt::decodeTextPayload("mct:!!!\x01", out, sizeof(out), message), + (int)mcotxt::MessageStatus::Malformed); + EXPECT_EQ((int)mcotxt::decodeTextPayload("plain text", out, sizeof(out), message), + (int)mcotxt::MessageStatus::NotMCOtxt); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/test_mcotxt_delivery/fixtures.h b/test/test_mcotxt_delivery/fixtures.h new file mode 100644 index 00000000000..30880aedc7a --- /dev/null +++ b/test/test_mcotxt_delivery/fixtures.h @@ -0,0 +1,185 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "../../examples/companion_radio/MCOtxtDelivery.h" +#include "golden.h" +#include +#include +#include +#include +#include + +namespace { + +using Bytes = std::vector; + +Bytes fromHex(const char* hex) { + Bytes result; + for (size_t i = 0; hex[i]; i += 2) { + const char byte[] = {hex[i], hex[i + 1], 0}; + result.push_back((uint8_t)strtoul(byte, nullptr, 16)); + } + return result; +} + +void integer(Bytes& out, uint32_t value) { + for (int i = 0; i < 4; ++i) out.push_back((uint8_t)(value >> (8 * i))); +} + +uint32_t timestamp(const Bytes& frame) { + const size_t p = frame[0] == 17 ? 7 : 4; + return (uint32_t)frame[p] | ((uint32_t)frame[p + 1] << 8) | + ((uint32_t)frame[p + 2] << 16) | ((uint32_t)frame[p + 3] << 24); +} + +void varuint(Bytes& out, size_t value) { + do { + out.push_back((uint8_t)((value & 0x7f) | (value > 0x7f ? 0x80 : 0))); + value >>= 7; + } while (value); +} + +void stringField(Bytes& out, const std::string& value) { + out.push_back(1); // counted UTF-8 mode + varuint(out, value.size()); + out.insert(out.end(), value.begin(), value.end()); +} + +Bytes container(const Golden& golden, const std::string& sender = "", const std::string& reply = "", + bool inherited = true) { + Bytes out = {0x31, (uint8_t)((inherited ? 4 : 0) | (sender.empty() ? 0 : 2) | (reply.empty() ? 0 : 1))}; + if (!inherited) integer(out, 0x12345678); + if (!sender.empty()) stringField(out, sender); + if (!reply.empty()) { stringField(out, reply); integer(out, 0x10203040); } + out.push_back(0); // MCOtxt string mode + Dart's counted frame + const Bytes stream = fromHex(golden.frame_hex); + out.insert(out.end(), stream.begin(), stream.end()); + return out; +} + +std::string textToken(const Bytes& body) { + char out[1024]; + size_t length; + EXPECT_TRUE(mesh::base91::encode(body.data(), body.size(), out, sizeof(out), length)); + return std::string("mct:") + out; +} + +Bytes channel(const std::string& text, bool v3 = true) { + Bytes frame = v3 ? Bytes{17, 12, 0, 0} : Bytes{8}; + frame.insert(frame.end(), {2, 0x82, 0}); // channel index, packed path, plain + integer(frame, 0x11223344); + frame.insert(frame.end(), text.begin(), text.end()); + return frame; +} + +Bytes direct(const std::string& text, uint8_t type = 0, bool v3 = true) { + Bytes frame = v3 ? Bytes{16, 12, 0, 0} : Bytes{7}; + frame.insert(frame.end(), {1, 2, 3, 4, 5, 6, 0x81, type}); + integer(frame, 0x11223344); + if (type == 2) frame.insert(frame.end(), {0xAA, 0xBB, 0xCC, 0xDD}); + frame.insert(frame.end(), text.begin(), text.end()); + return frame; +} + +Bytes binary(const Bytes& body, const std::string& outer_name = "") { + Bytes payload; + varuint(payload, outer_name.size()); + payload.insert(payload.end(), outer_name.begin(), outer_name.end()); + payload.insert(payload.end(), body.begin(), body.end()); + Bytes frame = {27, 12, 0, 0, 2, 0x82, 0x20, 0x01, (uint8_t)payload.size()}; + frame.insert(frame.end(), payload.begin(), payload.end()); + return frame; +} + +size_t headerSize(const Bytes& f) { + switch (f[0]) { + case 7: return 13 + (f[8] == 2 ? 4 : 0); + case 8: return 8; + case 16: return 16 + (f[11] == 2 ? 4 : 0); + case 17: return 11; + default: return 9; + } +} + +std::string bodyText(const Bytes& f) { + const size_t offset = headerSize(f); + return std::string(f.begin() + offset, f.end()); +} + +void start(MCOtxtDelivery& delivery, const std::string& info) { + delivery.appStart((const uint8_t*)info.data(), info.size()); +} + +Bytes next(MCOtxtDelivery& delivery, const Bytes& source, bool& consume, + uint8_t version = 4, uint32_t now = 100) { + uint8_t guarded[MCOtxtDelivery::kFrameSize + 3]; + memset(guarded, 0xCD, sizeof(guarded)); + const int length = delivery.next(source.data(), source.size(), version, now, guarded + 1, consume); + EXPECT_GE(length, 0); + EXPECT_LE(length, (int)MCOtxtDelivery::kFrameSize); + EXPECT_EQ(guarded[0], 0xCD); + EXPECT_EQ(guarded[sizeof(guarded) - 1], 0xCD); + return Bytes(guarded + 1, guarded + 1 + length); +} + +std::vector drain(MCOtxtDelivery& delivery, const Bytes& source, uint8_t version = 4) { + EXPECT_LE(source.size(), (size_t)MCOtxtDelivery::kFrameSize) << "fixture must fit the original offline queue"; + std::vector result; + bool consume = false; + for (int i = 0; i < 25 && !consume; ++i) { + const Bytes before = source; + const Bytes f = next(delivery, source, consume, version, 100 + i); + EXPECT_EQ(source, before); // raw queue bytes never change + if (!f.empty()) result.push_back(f); + } + EXPECT_TRUE(consume); + return result; +} + +bool validUtf8(const std::string& text) { + size_t pos = 0; + uint32_t cp; + while (pos < text.size()) { + if (!mcotxt::utf8Decode((const uint8_t*)text.data(), text.size(), pos, cp)) return false; + } + return true; +} + +// Reassemble without split decorations to detect dropped/repeated text. +std::string reassemble(const std::vector& frames, const std::string& prefix) { + std::string result; + for (size_t i = 0; i < frames.size(); ++i) { + std::string text = bodyText(frames[i]); + EXPECT_TRUE(validUtf8(text)); + EXPECT_LE(text.size(), (size_t)MCOtxtDelivery::kTextSize); + EXPECT_EQ(text.substr(0, prefix.size()), prefix); + text.erase(0, prefix.size()); + if (frames.size() > 1) { + const std::string mark = " [" + std::to_string(i + 1) + "/" + std::to_string(frames.size()) + "]"; + EXPECT_EQ(text.substr(text.size() - mark.size()), mark); + text.erase(text.size() - mark.size()); + if (i + 1 < frames.size()) { + EXPECT_EQ(text.substr(text.size() - 3), "..."); + text.erase(text.size() - 3); + } + if (i > 0) { EXPECT_EQ(text.substr(0, 3), "..."); text.erase(0, 3); } + } + result += text; + } + return result; +} + +std::string withoutSpaces(std::string s) { + s.erase(std::remove(s.begin(), s.end(), ' '), s.end()); + return s; +} + +} // namespace + diff --git a/test/test_mcotxt_delivery/golden.h b/test/test_mcotxt_delivery/golden.h new file mode 100644 index 00000000000..c75d701b33b --- /dev/null +++ b/test/test_mcotxt_delivery/golden.h @@ -0,0 +1,11 @@ +#pragma once + +// Fixed MCOtxtFrame output from the independent Dart encoder in MCOa. +// Model generation 0; language pair RU/EN. Do not regenerate from the C++ decoder. +struct Golden { const char* text; const char* frame_hex; }; + +static const Golden kEnglish = {"Hello world!", "52208fbc1144d320a8b0b0c0"}; +static const Golden kRussian = {"Привет мир!", "48208f09e724d0a4c4c3"}; +static const Golden kReply = {"о, а вот тебе ответ о, а вот тебе ответ о, а вот тебе ответ о, а вот тебе ответ о, а вот тебе ответ о, а вот тебе ответ о, а вот тебе ответ о, а вот тебе ответ о, а вот тебе ответ", "d706208a784511dd34b8d31ca6a7258061144774d2e34c729a9c960184511dd34b8d31ca6a7258061144774d2e34c729a9c960184511dd34b8d31ca6a7258061144774d2e34c729a9c960184511dd34b8d31ca6a7258061144774d2e34c729a9c960184511dd34b8d31ca6a724"}; +static const Golden kLong = {"аааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааааа", "e413208a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d1a3468d10"}; +static const Golden kUnicode = {"😀 ёжик идёт домой — привет!", "c201208fd0fc27e6201b8761e2b499bc34d35426ac64093ce4b0c0"}; diff --git a/test/test_mcotxt_delivery/test_mcotxt_delivery.cpp b/test/test_mcotxt_delivery/test_mcotxt_delivery.cpp new file mode 100644 index 00000000000..4746e25f81c --- /dev/null +++ b/test/test_mcotxt_delivery/test_mcotxt_delivery.cpp @@ -0,0 +1,528 @@ +#include "fixtures.h" + +TEST(MCOtxtInterop, DartFramesDecodeExactly) { + for (const Golden* golden : {&kEnglish, &kRussian, &kReply, &kLong, &kUnicode}) { + Bytes envelope = {0}; + const Bytes body = container(*golden); + envelope.insert(envelope.end(), body.begin(), body.end()); + char output[2048]; + mcotxt::DecodedMessage metadata; + EXPECT_EQ(mcotxt::decodeBinaryEnvelope(0x0120, envelope.data(), envelope.size(), output, + sizeof(output), metadata), mcotxt::MessageStatus::Ok); + EXPECT_STREQ(output, golden->text); + } +} + +TEST(MCOtxtDelivery, CapabilityIsAnExactCaseSensitiveItemNotAnAppName) { + const Bytes raw = channel("Bob: " + textToken(container(kRussian))); + for (const std::string info : {"cap=mctxt", "Any;cap=mctxt", "Other;cap=mctxt,other", + "Other;cap=other,mctxt,another", "Other;cap=other,mctxt", + "a;cap=other;cap=mctxt", "a;cap=;cap=mctxt", + "a;cap=,other,,mctxt,;version=1"}) { + SCOPED_TRACE(info); + MCOtxtDelivery d; start(d, info); + EXPECT_EQ(drain(d, raw), std::vector{raw}); + } + for (const std::string info : {"", "MeshCoreOpen", "cap=", "a;cap=;version=1", "a;cap=,,,", + "a;cap=other,another", "a;cap=mctxt2", "a;cap=MCTXT", + "a;mctxt", "a;cap=xmctxt", "a;othercap=mctxt"}) { + SCOPED_TRACE(info); + MCOtxtDelivery d; start(d, info); + auto frames = drain(d, raw); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]), "Bob: Привет мир!"); + } +} + +TEST(MCOtxtDelivery, CapabilityParsingStopsAtFieldLengthAndNul) { + const Bytes raw = channel("Bob: " + textToken(container(kRussian))); + const std::string info = "Any;cap=other,mctxt"; + MCOtxtDelivery d; + // The buffer contains mctxt, but it is outside the supplied APP_START length. + d.appStart((const uint8_t*)info.data(), info.size() - 1); + auto frames = drain(d, raw); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]), "Bob: Привет мир!"); + + const uint8_t terminated[] = "Any;cap=\0mctxt"; + d.appStart(terminated, sizeof(terminated) - 1); + frames = drain(d, raw); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]), "Bob: Привет мир!"); + + d.appStart(nullptr, 0); + frames = drain(d, raw); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]), "Bob: Привет мир!"); +} + +TEST(MCOtxtDelivery, PlainMessagesAndCliAreUntouched) { + MCOtxtDelivery d; + for (const Bytes raw : {channel("Bob: ordinary text"), direct(textToken(container(kRussian)), 1)}) { + EXPECT_EQ(drain(d, raw), std::vector{raw}); + d.resetMessage(); + } +} + +TEST(MCOtxtDelivery, ReplyMentionOnlyInFirstNumberedPartAndOriginalTimestampOnEveryPart) { + const Bytes raw = channel("Bob: " + textToken(container(kReply, "", "Ann"))); + MCOtxtDelivery d; + auto frames = drain(d, raw); + ASSERT_GT(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]).find("Bob: @[Ann] "), 0u); + for (size_t i = 0; i < frames.size(); ++i) { + EXPECT_EQ(timestamp(frames[i]), 0x11223344u); + EXPECT_EQ(frames[i][4], 2); EXPECT_EQ(frames[i][5], 0x82); + if (i > 0) EXPECT_EQ(bodyText(frames[i]).find("@[Ann]"), std::string::npos); + } + EXPECT_EQ(withoutSpaces(reassemble(frames, "Bob: ")), withoutSpaces(std::string("@[Ann] ") + kReply.text)); +} + +TEST(MCOtxtDelivery, EmbeddedContainersPreserveProsePunctuationAndMultipleTokens) { + const std::string token = textToken(container(kRussian)); + MCOtxtDelivery d; + const auto frames = drain(d, channel("Bob: цитата («" + token + "»), then " + textToken(container(kEnglish)) + ".")); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]), "Bob: цитата («Привет мир!»), then Hello world!."); +} + +TEST(MCOtxtDelivery, InlinePunctuationCannotChangeTheLastDecodedCharacter) { + // Regression: the trailing '.' used to be accepted as a noncanonical Base91 + // tail, changing the final compressed '!' to SPACE and swallowing the dot. + const std::string input = "quote: mct:3O2W>2lB8FIA/BPZ{Y)kxL>uwPw."; + const std::string expected = "@[N] quote: Hello world!."; + MCOtxtDelivery delivery; + const auto frames = drain(delivery, channel("Bob: " + input)); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]), "Bob: " + expected); + EXPECT_EQ(std::string(mcotxt::displayText(("Bob: " + input).c_str(), true)), "Bob: " + expected); +} + +TEST(MCOtxtDelivery, AdjacentBase91PunctuationIsPreservedAcrossTokenEndings) { + for (const Golden* golden : {&kEnglish, &kRussian, &kUnicode}) { + for (size_t n = 0; n <= 31; ++n) { + const std::string name(n, 'N'); + for (bool reply : {false, true}) { + const std::string token = textToken(reply ? container(*golden, "", name) + : container(*golden, name)); + for (const std::string punctuation : {".", ",", ")", "]", "!", "?", ";", ":", "\"", ")...", "]!"}) { + SCOPED_TRACE(token + punctuation); + const std::string input = "quote: (" + token + punctuation; + const std::string expected = (reply && !name.empty() ? "@[" + name + "] " : "") + + "quote: (" + golden->text + punctuation; + char output[mcotxt::kScratchBytes]; + ASSERT_TRUE(mcotxt::decodeText(input.c_str(), output, sizeof(output))); + ASSERT_EQ(std::string(output), expected); + } + } + } + } +} + +TEST(MCOtxtDelivery, NoncanonicalBase91AndNonzeroStreamPaddingAreRejected) { + char output[mcotxt::kScratchBytes]; + mcotxt::DecodedMessage metadata; + const std::string token = textToken(container(kEnglish, "NNNNN")); + EXPECT_EQ(mcotxt::decodeTextPayload((token + ".").c_str(), output, sizeof(output), metadata), + mcotxt::MessageStatus::Malformed); + // The independent English stream has 82 significant bits; these low bits + // are padding even though Base91 can canonically encode them as nonzero. + Bytes body = container(kEnglish); + body.back() |= 1; + EXPECT_EQ(mcotxt::decodeTextPayload(textToken(body).c_str(), output, sizeof(output), metadata), + mcotxt::MessageStatus::Malformed); + MCOtxtDelivery delivery; + const Bytes raw = binary(body); + EXPECT_EQ(drain(delivery, raw), std::vector{raw}); +} + +TEST(MCOtxtNames, Full31ByteNamesAndReplyRemainUnchangedAcrossParts) { + const std::string name(31, 'N'); + const std::string prefix = name + ": "; + MCOtxtDelivery delivery; + auto frames = drain(delivery, channel(prefix + textToken(container(kEnglish, "", name)))); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]), prefix + "@[" + name + "] Hello world!"); + delivery.resetMessage(); + frames = drain(delivery, binary(container(kReply, name))); + ASSERT_GT(frames.size(), 2u); + frames.erase(frames.begin()); + EXPECT_EQ(withoutSpaces(reassemble(frames, prefix)), withoutSpaces(kReply.text)); +} + +TEST(MCOtxtNames, InvalidUtf8BeyondTheClippedContainerNameIsStillRejected) { + const Bytes raw = binary(container(kEnglish, std::string(32, 'N') + "\xc0\x80")); + MCOtxtDelivery delivery; + EXPECT_EQ(drain(delivery, raw), std::vector{raw}); +} + +TEST(MCOtxtDelivery, EmbeddedReplyIsMovedToBeginningBeforeSplitting) { + MCOtxtDelivery d; + const auto frames = drain(d, channel("Bob: цитата: " + textToken(container(kRussian, "", "Ann")))); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]), "Bob: @[Ann] цитата: Привет мир!"); +} + +TEST(MCOtxtDelivery, TokenMustStartAtAWordBoundary) { + const std::string token = textToken(container(kRussian)); + for (const std::string prefix : {"word", "слово", "_", "9"}) { + MCOtxtDelivery d; + const Bytes raw = channel("Bob: " + prefix + token); + EXPECT_EQ(drain(d, raw), std::vector{raw}); + } +} + +TEST(MCOtxtDelivery, RawBinaryFirstThenDecodedTextWithContainerSenderAndReply) { + const Bytes raw = binary(container(kReply, "Bob", "Ann", false)); + MCOtxtDelivery d; + auto frames = drain(d, raw); + ASSERT_GT(frames.size(), 2u); + EXPECT_EQ(frames[0], raw); + frames.erase(frames.begin()); + for (const Bytes& frame : frames) EXPECT_EQ(timestamp(frame), 0x12345678u); + EXPECT_EQ(withoutSpaces(reassemble(frames, "Bob: ")), withoutSpaces(std::string("@[Ann] ") + kReply.text)); +} + +TEST(MCOtxtDelivery, CapableClientGetsOnlyOriginalBinary) { + MCOtxtDelivery d; start(d, "ArbitraryClient;cap=mctxt"); + const Bytes raw = binary(container(kRussian, "Bob")); + EXPECT_EQ(drain(d, raw), std::vector{raw}); +} + +TEST(MCOtxtDelivery, OuterBinaryNameFallbackAndStableInheritedTimestamp) { + MCOtxtDelivery d; + auto frames = drain(d, binary(container(kReply), "Bob")); + ASSERT_GT(frames.size(), 2u); + EXPECT_EQ(frames[1][0], 17); + for (size_t i = 1; i < frames.size(); ++i) EXPECT_EQ(timestamp(frames[i]), 100u); + frames.erase(frames.begin()); + EXPECT_EQ(withoutSpaces(reassemble(frames, "Bob: ")), withoutSpaces(kReply.text)); +} + +TEST(MCOtxtDelivery, GeneratedFramesRespectLegacyAndV3LayoutsAndSignedRoomPrefix) { + for (bool source_v3 : {false, true}) for (uint8_t target : Bytes{0, 4}) { + for (uint8_t type : Bytes{0, 2}) { + MCOtxtDelivery d; + const auto frames = drain(d, direct(textToken(container(kRussian)), type, source_v3), target); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(frames[0][0], target >= 3 ? 16 : 7); + EXPECT_EQ(bodyText(frames[0]), "Привет мир!"); + if (type == 2) EXPECT_EQ(frames[0][headerSize(frames[0]) - 4], 0xAA); + } + MCOtxtDelivery d; + auto frames = drain(d, binary(container(kRussian, "Bob")), target); + ASSERT_EQ(frames.size(), 2u); + EXPECT_EQ(frames[1][0], target >= 3 ? 17 : 8); + EXPECT_EQ(bodyText(frames[1]), "Bob: Привет мир!"); + } +} + +TEST(MCOtxtDelivery, NewAppReevaluatesRetainedRawHeadWithoutLeakingOldCursorOrCapability) { + const Bytes raw = channel("Bob: " + textToken(container(kReply))); + MCOtxtDelivery d; + bool consumed; + const Bytes first = next(d, raw, consumed); + EXPECT_FALSE(consumed); + start(d, "decoder;cap=mctxt"); + EXPECT_EQ(next(d, raw, consumed), raw); EXPECT_TRUE(consumed); + start(d, "legacy"); + EXPECT_EQ(next(d, raw, consumed), first); EXPECT_FALSE(consumed); + d.resetMessage(); // the real queue calls this when overflow evicts its head + EXPECT_EQ(bodyText(drain(d, channel("Bob: " + textToken(container(kRussian))))[0]), "Bob: Привет мир!"); +} + +TEST(MCOtxtDelivery, UnknownMalformedAndOtherBinaryRemainAvailableRaw) { + for (Bytes body : {Bytes{0x32, 4, 1, 2, 'h', 'i'}, Bytes{0x31, 0xFF}, Bytes{0x13, 0}}) { + MCOtxtDelivery d; + const Bytes raw = binary(body); + EXPECT_EQ(drain(d, raw), std::vector{raw}); + } + MCOtxtDelivery d; + const Bytes raw = channel("Bob: mct:!!!"); + EXPECT_EQ(drain(d, raw), std::vector{raw}); +} + +TEST(MCOtxtDelivery, InsufficientDecodeWorkspaceFallsBackToRawWithoutClipping) { + char out[32]; + EXPECT_FALSE(mcotxt::decodeText(textToken(container(kReply)).c_str(), out, sizeof(out))); +} + +TEST(MCOtxtDelivery, ConflictingEmbeddedRepliesStayRawRatherThanLosingAnAnchor) { + MCOtxtDelivery d; + const Bytes raw = channel("Bob: " + textToken(container(kRussian, "", "Ann")) + + " " + textToken(container(kEnglish, "", "Eve"))); + EXPECT_EQ(drain(d, raw), std::vector{raw}); +} + +TEST(MCOtxtDelivery, EmbeddedNulInDecodedTextNeverSilentlyTruncatesTheMessage) { + // UTF-8 mode and RAW_UTF8 stream mode can represent NUL, but a companion + // C-string cannot. Both must fall back to their original source bytes. + for (const Bytes& body : {Bytes{0x31, 4, 1, 3, 'a', 0, 'b'}, + Bytes{0x31, 4, 0, 40, 0x23, 0x90, 'a', 0, 'b'}}) { + MCOtxtDelivery d; + const Bytes raw = channel("Bob: " + textToken(body)); + EXPECT_EQ(drain(d, raw), std::vector{raw}); + d.resetMessage(); + const Bytes binary_raw = binary(body); + EXPECT_EQ(drain(d, binary_raw), std::vector{binary_raw}); + } +} + +TEST(MCOtxtDelivery, EmptyDecodedTextAndLateCapabilityDoNotNeedSpecialQueueRecords) { + const Bytes empty = {0x31, 4, 1, 0}; + MCOtxtDelivery d; + const auto frames = drain(d, direct(textToken(empty))); + ASSERT_EQ(frames.size(), 1u); + EXPECT_TRUE(bodyText(frames[0]).empty()); + start(d, std::string(80, 'x') + ";cap=mctxt"); + const Bytes raw = channel("Bob: " + textToken(container(kRussian))); + EXPECT_EQ(drain(d, raw), std::vector{raw}); +} + +TEST(MCOtxtDelivery, EveryTruncationAndBitMutationOfAValidContainerIsBounded) { + const Bytes good = container(kReply, "Bob", "Ann"); + for (size_t cut = 0; cut <= good.size(); ++cut) { + MCOtxtDelivery d; + drain(d, binary(Bytes(good.begin(), good.begin() + cut))); + } + for (size_t pos = 0; pos < good.size(); ++pos) for (uint8_t bit = 0; bit < 8; ++bit) { + Bytes body = good; + body[pos] ^= (uint8_t)(1u << bit); + MCOtxtDelivery d; + drain(d, binary(body)); + } +} + +TEST(MCOtxtDelivery, BoundedRandomFramesAndTruncatedContainersNeverOverrunOutput) { + std::mt19937 random(0x4D4354); + MCOtxtDelivery d; + for (size_t trial = 0; trial < 4000; ++trial) { + Bytes raw(random() % (MCOtxtDelivery::kFrameSize + 1)); + for (auto& byte : raw) byte = (uint8_t)random(); + if (raw.size() >= 9 && trial % 2 == 0) { raw[0] = 27; raw[6] = 0x20; raw[7] = 1; raw[8] = (uint8_t)(raw.size() - 9); } + bool consumed; + next(d, raw, consumed, (uint8_t)(trial % 5)); + d.resetMessage(); + } +} + +TEST(MCOtxtNames, SenderAndReplyAreClippedToFirmwareNameLimit) { + for (const std::string sender : {std::string(96, 'A'), std::string("Боб")}) { + const std::string author(40, 'R'); + const std::string expected_sender = sender == "Боб" ? sender : std::string(31, 'A'); + const std::string expected = expected_sender + ": @[" + std::string(31, 'R') + "] Hello world!"; + const Bytes raw = binary(container(kEnglish, sender, author)); + MCOtxtDelivery delivery; + const auto frames = drain(delivery, raw); + ASSERT_EQ(frames.size(), 2u); + EXPECT_EQ(bodyText(frames[1]), expected); + EXPECT_STREQ(mcotxt::displayBinary(0x0120, raw.data() + 9, raw[8]), + expected.c_str()); + } +} + +TEST(MCOtxtNames, Utf8ClippingKeepsWholeCharactersAndTheSameInlineReply) { + std::string author; + for (int i = 0; i < 17; ++i) author += "я"; + const std::string clipped = author.substr(0, 30); // the next two-byte character cannot fit + const std::string token = textToken(container(kEnglish, "", author)); + const std::string input = "Bob: quote: " + token + " then " + token + "."; + MCOtxtDelivery delivery; + const auto frames = drain(delivery, channel(input)); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]), "Bob: @[" + clipped + "] quote: Hello world! then Hello world!."); +} + +TEST(MCOtxtNames, ReplyMetadataDoesNotConsumeTextWorkspace) { + const std::string input = "quote: " + textToken(container(kEnglish, "", "Ann")); + char output[40]; + ASSERT_TRUE(mcotxt::decodeText(input.c_str(), output, sizeof(output))); + EXPECT_STREQ(output, "@[Ann] quote: Hello world!"); +} + +TEST(MCOtxtNames, DistinctAuthorsWithinTheFirmwareLimitStayDistinct) { + const std::string common(30, 'A'); + const std::string input = "Bob: " + textToken(container(kEnglish, "", common + "X")) + + " " + textToken(container(kEnglish, "", common + "Y")); + const Bytes raw = channel(input); + MCOtxtDelivery delivery; + EXPECT_EQ(drain(delivery, raw), std::vector{raw}); +} + +TEST(MCOtxtNames, EnvelopeSenderIsClippedAndItsWholeUtf8FieldIsValidated) { + const std::string sender(96, 'A'); + const Bytes raw = binary(container(kEnglish), sender); + MCOtxtDelivery delivery; + const auto frames = drain(delivery, raw); + ASSERT_EQ(frames.size(), 2u); + EXPECT_EQ(bodyText(frames[1]), std::string(31, 'A') + ": Hello world!"); + const Bytes invalid = binary(container(kEnglish), std::string(32, 'A') + "\xc0\x80"); + delivery.resetMessage(); + EXPECT_EQ(drain(delivery, invalid), std::vector{invalid}); + + const std::string unicode_name = "яяяяяяяяяяяяяяяяя"; // 34 bytes + const std::string expected_name = "яяяяяяяяяяяяяяя"; // 30 complete bytes + for (bool inner : {false, true}) { + delivery.resetMessage(); + const Bytes unicode_raw = inner ? binary(container(kEnglish, unicode_name)) + : binary(container(kEnglish), unicode_name); + const auto unicode_frames = drain(delivery, unicode_raw); + ASSERT_EQ(unicode_frames.size(), 2u); + EXPECT_EQ(bodyText(unicode_frames[1]), expected_name + ": Hello world!"); + } +} + +TEST(MCOtxtNames, CompressedSenderIsClippedLikeAnUncompressedNodeName) { + // A name is a counted MCOtxt string too; its decoded length is not its wire size. + Bytes body = {0x31, 6, 0}; // inherited timestamp, compressed sender + const Bytes name = fromHex(kReply.frame_hex); + body.insert(body.end(), name.begin(), name.end()); + body.push_back(0); + const Bytes text = fromHex(kEnglish.frame_hex); + body.insert(body.end(), text.begin(), text.end()); + const Bytes raw = binary(body); + MCOtxtDelivery delivery; + auto frames = drain(delivery, raw); + ASSERT_EQ(frames.size(), 2u); + EXPECT_EQ(frames[0], raw); + frames.erase(frames.begin()); + EXPECT_EQ(bodyText(frames[0]), "о, а вот тебе отве: Hello world!"); + EXPECT_STREQ(mcotxt::displayBinary(0x0120, raw.data() + 9, raw[8]), + "о, а вот тебе отве: Hello world!"); +} + +TEST(MCOtxtNames, CompressedReplyAuthorUsesTheSameBoundedNameField) { + Bytes body = {0x31, 7}; + stringField(body, "Bob"); + body.push_back(0); // compressed reply author, longer than a whole text frame + const Bytes author = fromHex(kReply.frame_hex); + body.insert(body.end(), author.begin(), author.end()); + integer(body, 0x10203040); + body.push_back(0); + const Bytes text = fromHex(kEnglish.frame_hex); + body.insert(body.end(), text.begin(), text.end()); + const Bytes raw = binary(body); + MCOtxtDelivery delivery; + const auto frames = drain(delivery, raw); + ASSERT_EQ(frames.size(), 2u); + EXPECT_EQ(bodyText(frames[1]), "Bob: @[о, а вот тебе отве] Hello world!"); +} + +TEST(MCOtxtNames, MetadataOwnsItsNamesAfterTheOutputBufferIsReused) { + const std::string author(40, 'R'); + const std::string token = textToken(container(kEnglish, "Bob", author)); + char output[128]; + mcotxt::DecodedMessage metadata; + ASSERT_EQ(mcotxt::decodeTextPayload(token.c_str(), output, sizeof(output), metadata), + mcotxt::MessageStatus::Ok); + EXPECT_EQ(sizeof(metadata.sender), 32u); + EXPECT_EQ(sizeof(metadata.reply_author), 32u); + EXPECT_STREQ(metadata.sender, "Bob"); + EXPECT_STREQ(metadata.reply_author, std::string(31, 'R').c_str()); + EXPECT_STREQ(output, "Hello world!"); + memset(output, 'X', sizeof(output)); + EXPECT_STREQ(metadata.sender, "Bob"); + EXPECT_STREQ(metadata.reply_author, std::string(31, 'R').c_str()); +} + +TEST(MCOtxtNames, InlineTokenBudgetTracksTransportFrameBudget) { + Bytes raw; + std::string expected; + size_t token_size = 0; + for (size_t n = 1; n + 5 <= MCOtxtDelivery::kTextSize; ++n) { + const std::string text(n, 'A'); + Bytes body = {0x31, 4}; + stringField(body, text); + const std::string token = textToken(body); + const Bytes candidate = channel("Bob: " + token); + if (candidate.size() > MCOtxtDelivery::kFrameSize) continue; + raw = candidate; + expected = "Bob: " + text; + token_size = token.size(); + } + ASSERT_FALSE(raw.empty()); + RecordProperty("input_frame_bytes", (int)raw.size()); + RecordProperty("encoded_token_bytes", (int)token_size); + MCOtxtDelivery delivery; + const auto frames = drain(delivery, raw); + ASSERT_EQ(frames.size(), 1u); + EXPECT_EQ(bodyText(frames[0]), expected); +} + +TEST(MCOtxtDisplay, PlainAndMalformedTextKeepTheirOriginalRepresentation) { + for (const std::string input : {"Bob: ordinary text", "Bob: mct:!!!", "mct:"}) { + EXPECT_EQ(mcotxt::displayText(input.c_str(), true), input.c_str()); + } + EXPECT_EQ(mcotxt::displayText(nullptr, false), nullptr); +} + +TEST(MCOtxtDisplay, ChannelPrefixPrecedesReplyAndEmbeddedDecodedText) { + const std::string input = "Bob: quote: " + textToken(container(kRussian, "", "Ann")); + const std::string before = input; + EXPECT_STREQ(mcotxt::displayText(input.c_str(), true), "Bob: @[Ann] quote: Привет мир!"); + EXPECT_EQ(input, before); +} + +TEST(MCOtxtDisplay, DirectTextIsDecodedOnceWithoutBlePartDecorations) { + const std::string input = textToken(container(kReply, "", "Ann")); + const std::string expected = std::string("@[Ann] ") + kReply.text; + EXPECT_STREQ(mcotxt::displayText(input.c_str(), false), expected.c_str()); +} + +TEST(MCOtxtDisplay, BinaryPreviewPreservesSenderReplyAndOriginalBytes) { + const Bytes raw = binary(container(kRussian, "Bob", "Ann")); + const Bytes before = raw; + EXPECT_STREQ(mcotxt::displayBinary(0x0120, raw.data() + 9, raw[8]), + "Bob: @[Ann] Привет мир!"); + EXPECT_EQ(raw, before); +} + +TEST(MCOtxtDisplay, BinarySenderFallsBackToEnvelopeOrUnknown) { + for (const std::string name : {"Bob", ""}) { + const Bytes raw = binary(container(kEnglish), name); + const std::string expected = (name.empty() ? "?" : name) + std::string(": Hello world!"); + EXPECT_STREQ(mcotxt::displayBinary(0x0120, raw.data() + 9, raw[8]), expected.c_str()); + } +} + +TEST(MCOtxtDisplay, UnknownAndMalformedBinaryDoNotCreateTextPreviews) { + for (const Bytes body : {Bytes{0x32, 4, 1, 2, 'h', 'i'}, Bytes{0x31, 0xff}, Bytes{0x13, 0}}) { + const Bytes raw = binary(body); + EXPECT_EQ(mcotxt::displayBinary(0x0120, raw.data() + 9, raw[8]), nullptr); + } + const Bytes raw = binary(container(kEnglish)); + EXPECT_EQ(mcotxt::displayBinary(0x9999, raw.data() + 9, raw[8]), nullptr); +} + +TEST(MCOtxtDisplay, PreviewCopiesFitExistingUiBuffersOnUtf8Boundaries) { + const std::string input = "Bob: я😀!"; + for (size_t capacity = 0; capacity <= input.size() + 1; ++capacity) { + char guarded[64]; + memset(guarded, 0x5a, sizeof(guarded)); + mcotxt::copyDisplayPreview(guarded + 1, capacity, input.c_str()); + EXPECT_EQ(guarded[0], 0x5a); + EXPECT_EQ(guarded[capacity + 1], 0x5a); + if (!capacity) continue; + const std::string copied(guarded + 1); + EXPECT_LT(copied.size(), capacity); + EXPECT_TRUE(validUtf8(copied)); + EXPECT_EQ(input.substr(0, copied.size()), copied); + } + char preview[12]; + mcotxt::copyDisplayPreview(preview, sizeof(preview), input.c_str()); + EXPECT_STREQ(preview, "Bob: я😀"); +} + +TEST(MCOtxtDisplay, CopiedPreviewSurvivesSubsequentUseOfSharedWorkspace) { + const std::string input = "Bob: " + textToken(container(kRussian)); + char preview[80]; + mcotxt::copyDisplayPreview(preview, sizeof(preview), mcotxt::displayText(input.c_str(), true)); + const Bytes other = binary(container(kEnglish, "Ann")); + ASSERT_NE(mcotxt::displayBinary(0x0120, other.data() + 9, other[8]), nullptr); + EXPECT_STREQ(preview, "Bob: Привет мир!"); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tools/test_mcotxt_queue.cpp.in b/tools/test_mcotxt_queue.cpp.in new file mode 100644 index 00000000000..49b223d848a --- /dev/null +++ b/tools/test_mcotxt_queue.cpp.in @@ -0,0 +1,140 @@ +// Filled by test_mcotxt_queue.py. Only hardware-independent doubles live here; +// Queue and SYNC placeholders are replaced with current production method bodies. +#include "@FIXTURES@" + +class MyMesh { + public: + enum { OFFLINE_QUEUE_SIZE = 16, RESP_CODE_CHANNEL_MSG_RECV = 8, + RESP_CODE_CHANNEL_MSG_RECV_V3 = 17, RESP_CODE_CHANNEL_DATA_RECV = 27, + RESP_CODE_NO_MORE_MESSAGES = 3 }; + struct Frame { + uint8_t len; + uint8_t buf[MAX_FRAME_SIZE]; + bool isChannelMsg() const; + }; + Frame offline_queue[OFFLINE_QUEUE_SIZE]; + int offline_queue_len = 0; + MCOtxtDelivery mcotxt_delivery; + uint8_t app_target_ver = 4; + uint8_t out_frame[MAX_FRAME_SIZE + 1]; + struct Clock { uint32_t getCurrentTime() { return 100; } } clock; + Clock* getRTCClock() { return &clock; } + struct Serial { + std::vector frames; + void writeFrame(uint8_t* frame, int length) { frames.emplace_back(frame, frame + length); } + } serial; + Serial* _serial = &serial; + struct Listener { + int count = 0; + void onQueueSizeChanged(int value) { count = value; } + } listener; + Listener* _listener = &listener; + void addToOfflineQueue(const uint8_t frame[], int len); + int getFromOfflineQueue(uint8_t frame[]); + void sync() { @SYNC@ } +}; + +@QUEUE@ + +void enqueue(MyMesh& mesh, const Bytes& frame) { + ASSERT_LE(frame.size(), (size_t)MAX_FRAME_SIZE); + mesh.addToOfflineQueue(frame.data(), (int)frame.size()); +} + +void fillDirect(MyMesh& mesh) { + const Bytes filler = direct("ordinary"); + while (mesh.offline_queue_len < MyMesh::OFFLINE_QUEUE_SIZE) enqueue(mesh, filler); +} + +TEST(MCOtxtQueue, EvictingPartialChannelHeadRestartsNextSource) { + MyMesh mesh; + enqueue(mesh, channel("Bob: " + textToken(container(kReply)))); + enqueue(mesh, channel("Ann: " + textToken(container(kReply)))); + fillDirect(mesh); + mesh.sync(); + EXPECT_NE(bodyText(mesh.serial.frames.back()).find(" [1/"), std::string::npos); + enqueue(mesh, channel("New: ordinary")); + EXPECT_EQ(mesh.offline_queue_len, MyMesh::OFFLINE_QUEUE_SIZE); + mesh.sync(); + EXPECT_EQ(bodyText(mesh.serial.frames.back()).find("Ann: "), 0u); + EXPECT_NE(bodyText(mesh.serial.frames.back()).find(" [1/"), std::string::npos); +} + +TEST(MCOtxtQueue, EvictingLaterChannelDoesNotRestartDirectHead) { + MyMesh mesh; + const Bytes first = direct(textToken(container(kReply))); + enqueue(mesh, first); + enqueue(mesh, channel("Bob: ordinary")); + fillDirect(mesh); + mesh.sync(); + enqueue(mesh, direct("new")); + EXPECT_EQ(mesh.offline_queue_len, MyMesh::OFFLINE_QUEUE_SIZE); + mesh.sync(); + EXPECT_NE(bodyText(mesh.serial.frames.back()).find(" [2/"), std::string::npos); + EXPECT_EQ(Bytes(mesh.offline_queue[0].buf, + mesh.offline_queue[0].buf + mesh.offline_queue[0].len), first); +} + +TEST(MCOtxtQueue, FullDirectQueueDropsIncomingWithoutLosingCursor) { + MyMesh mesh; + enqueue(mesh, direct(textToken(container(kReply)))); + fillDirect(mesh); + mesh.sync(); + enqueue(mesh, channel("New: dropped")); + mesh.sync(); + EXPECT_NE(bodyText(mesh.serial.frames.back()).find(" [2/"), std::string::npos); + const auto& last = mesh.offline_queue[MyMesh::OFFLINE_QUEUE_SIZE - 1]; + EXPECT_EQ(Bytes(last.buf, last.buf + last.len), direct("ordinary")); +} + +TEST(MCOtxtQueue, ExhaustedBinaryDoesNotHideNextEntry) { + MyMesh mesh; + const Bytes bad = binary(Bytes{0x31, 0xff}); + const Bytes good = channel("Bob: next"); + enqueue(mesh, bad); + enqueue(mesh, good); + mesh.sync(); + EXPECT_EQ(mesh.serial.frames.back(), bad); + mesh.sync(); + EXPECT_EQ(mesh.serial.frames.back(), good); + EXPECT_EQ(mesh.offline_queue_len, 0); + EXPECT_EQ(mesh.listener.count, 0); +} + +TEST(MCOtxtQueue, RemovingLastMalformedBinaryClearsListenerCount) { + MyMesh mesh; + enqueue(mesh, binary(Bytes{0x31, 0xff})); + mesh.sync(); + ASSERT_EQ(mesh.listener.count, 1); + mesh.sync(); + EXPECT_EQ(mesh.offline_queue_len, 0); + EXPECT_EQ(mesh.listener.count, 0); + ASSERT_EQ(mesh.serial.frames.back().size(), 1u); + EXPECT_EQ(mesh.serial.frames.back()[0], MyMesh::RESP_CODE_NO_MORE_MESSAGES); +} + +TEST(MCOtxtQueue, NewAppReceivesRetainedHeadAsOriginalFrame) { + MyMesh mesh; + const Bytes raw = channel("Bob: " + textToken(container(kReply))); + enqueue(mesh, raw); + mesh.sync(); + EXPECT_EQ(mesh.offline_queue_len, 1); + start(mesh.mcotxt_delivery, "Capable;cap=mctxt"); + mesh.sync(); + EXPECT_EQ(mesh.serial.frames.back(), raw); + EXPECT_EQ(mesh.offline_queue_len, 0); +} + +TEST(MCOtxtQueue, PollingAnAlreadyEmptyQueueDoesNotResetUiState) { + MyMesh mesh; + mesh.listener.count = -1; // sentinel: no queue callback has been requested + mesh.sync(); + EXPECT_EQ(mesh.listener.count, -1); + ASSERT_EQ(mesh.serial.frames.back().size(), 1u); + EXPECT_EQ(mesh.serial.frames.back()[0], MyMesh::RESP_CODE_NO_MORE_MESSAGES); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tools/test_mcotxt_queue.py b/tools/test_mcotxt_queue.py new file mode 100644 index 00000000000..2e584a0a978 --- /dev/null +++ b/tools/test_mcotxt_queue.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Exercise actual MyMesh queue methods with native doubles, without Arduino. + +Requires an installed C++ compiler and Google Test sources; never installs them. +The small harness substitutes unrelated radio/filesystem/UI dependencies, while +the enqueue/drain/SYNC method bodies come verbatim from the current MyMesh.cpp. +This complements the board build; it is not a replacement for hardware tests. +""" +import argparse +from pathlib import Path +import shutil +import subprocess +import tempfile + + +def main(): + repo = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--compiler', help='g++, clang++, or cl.exe in a developer shell') + parser.add_argument('--gtest', type=Path, + default=repo / '.pio/libdeps/native/googletest') + parser.add_argument('--asan', action='store_true') + args = parser.parse_args() + compiler = args.compiler or shutil.which('g++') or shutil.which('clang++') or shutil.which('cl') + if not compiler: + parser.error('No compiler found; supply --compiler (no installation is attempted).') + gtest = args.gtest.resolve() + if not (gtest / 'src/gtest-all.cc').is_file(): + gtest /= 'googletest' + if not (gtest / 'src/gtest-all.cc').is_file(): + parser.error('Google Test sources not found; supply --gtest.') + + mesh = (repo / 'examples/companion_radio/MyMesh.cpp').read_text(encoding='utf-8') + template = Path(__file__).with_suffix('.cpp.in').read_text(encoding='utf-8') + # Fail clearly when these source landmarks change instead of silently testing + # an obsolete hand-copied implementation of the queue. + queue = mesh[mesh.index('bool MyMesh::Frame::isChannelMsg('): + mesh.index('float MyMesh::getAirtimeBudgetFactor(')] + sync = mesh.split('} else if (cmd_frame[0] == CMD_SYNC_NEXT_MESSAGE) {', 1)[1] + sync = sync.split('} else if (cmd_frame[0] == CMD_SET_RADIO_PARAMS)', 1)[0] + fixtures = repo / 'test/test_mcotxt_delivery/fixtures.h' + source = template.replace('@FIXTURES@', fixtures.as_posix()) + source = source.replace('@QUEUE@', queue).replace('@SYNC@', sync) + sources = [repo / 'src/helpers/Base91.cpp', + *sorted((repo / 'src/helpers/mcotxt').glob('*.cpp')), + repo / 'examples/companion_radio/MCOtxtDelivery.cpp', + gtest / 'src/gtest-all.cc'] + with tempfile.TemporaryDirectory(prefix='mcotxt-queue-') as temporary: + work = Path(temporary) + generated = work / 'queue.cpp' + generated.write_text(source, encoding='utf-8') + msvc = Path(compiler).name.lower() in ('cl', 'cl.exe') + executable = work / ('queue.exe' if msvc else 'queue') + includes = [repo / 'src', gtest, gtest / 'include'] + if msvc: + command = [compiler, '/nologo', '/std:c++17', '/EHsc', '/utf-8', '/Od', '/Zi'] + command += ['/I' + str(path) for path in includes] + if args.asan: + command += ['/fsanitize=address'] + command += [*map(str, sources), str(generated), '/Fe:' + str(executable)] + else: + command = [compiler, '-std=c++17', '-pthread', '-O1', '-g'] + command += ['-I' + str(path) for path in includes] + if args.asan: + command += ['-fsanitize=address', '-fno-omit-frame-pointer'] + command += [*map(str, sources), str(generated), '-o', str(executable)] + subprocess.run(command, cwd=work, check=True) + subprocess.run([str(executable), '--gtest_color=no'], cwd=work, check=True) + + +if __name__ == '__main__': + main()