diff --git a/src/agent_passport/canonical.py b/src/agent_passport/canonical.py index 8bc5961..49db8e8 100644 --- a/src/agent_passport/canonical.py +++ b/src/agent_passport/canonical.py @@ -13,6 +13,45 @@ import re +class JCSCanonicalizationError(ValueError): + """A value cannot be canonicalized under RFC 8785. + + Subclasses ValueError so existing fail-closed handlers that catch ValueError + around canonicalization keep working (they fail closed rather than crash). + + Carries a stable machine-readable ``category`` and ``reason`` matching the + TypeScript and Go SDKs, so cross-language callers can branch on the failure + without parsing the message. + """ + + #: Stable machine-readable category, shared across the APS SDKs. + category = "invalid_unicode" + + def __init__(self, message: str, reason: str = "lone_surrogate") -> None: + super().__init__(message) + #: Specific failure within the category, e.g. "lone_surrogate". + self.reason = reason + + +def _assert_no_lone_surrogate(s: str) -> None: + """Reject a string containing an unpaired UTF-16 surrogate. + + A lone surrogate (U+D800..U+DFFF) is not a valid Unicode scalar and has no + UTF-8 encoding, so RFC 8785 requires rejecting the input rather than escaping + it or replacing it with U+FFFD. Python's JSON parser resolves valid surrogate + PAIRS to their single non-BMP code point, so any character that remains in the + surrogate range is unpaired. + """ + for ch in s: + o = ord(ch) + if 0xD800 <= o <= 0xDFFF: + raise JCSCanonicalizationError( + "canonicalize_jcs: string contains an unpaired UTF-16 surrogate " + f"(U+{o:04X}); a lone surrogate has no valid UTF-8 encoding and " + "RFC 8785 requires rejection" + ) + + def has_non_finite(obj) -> bool: """Return True if obj contains a NaN or Infinity float anywhere within it. @@ -149,11 +188,17 @@ def canonicalize_jcs(obj) -> str: raise ValueError(f"Cannot canonicalize {obj}") return _es_number(obj) if isinstance(obj, str): + _assert_no_lone_surrogate(obj) return json.dumps(obj, ensure_ascii=False) if isinstance(obj, list): return "[" + ",".join(canonicalize_jcs(item) for item in obj) + "]" if isinstance(obj, dict): # Preserve null values in objects (unlike legacy canonicalize) + # Validate keys before sorting: _canonical_keys encodes each key as + # utf-16-be, which itself raises on a lone surrogate. Check first so the + # failure is the clean typed error, not a raw UnicodeEncodeError. + for key in obj.keys(): + _assert_no_lone_surrogate(key) pairs = [] for key in _canonical_keys(obj.keys()): pairs.append( diff --git a/tests/test_jcs_lone_surrogate.py b/tests/test_jcs_lone_surrogate.py new file mode 100644 index 0000000..26983c1 --- /dev/null +++ b/tests/test_jcs_lone_surrogate.py @@ -0,0 +1,145 @@ +# Copyright 2026 Tymofii Pidlisnyi. Apache-2.0 license. See LICENSE. +"""RFC 8785: canonicalize_jcs must reject lone/unpaired UTF-16 surrogates. + +A lone surrogate is not a valid Unicode scalar and has no UTF-8 encoding, so the +input is invalid and must be rejected, not escaped or replaced. A valid surrogate +pair (a non-BMP character) must still canonicalize to its raw UTF-8 bytes. +""" + +import pytest + +from agent_passport.canonical import JCSCanonicalizationError, canonicalize_jcs + +HIGH = chr(0xD800) # lone high surrogate +LOW = chr(0xDFFF) # lone low surrogate +EMOJI = chr(0x1F600) # valid non-BMP scalar (U+1F600) + + +def test_reject_lone_high_surrogate(): + with pytest.raises(JCSCanonicalizationError): + canonicalize_jcs({"v": HIGH}) + + +def test_reject_lone_low_surrogate(): + with pytest.raises(JCSCanonicalizationError): + canonicalize_jcs({"v": LOW}) + + +def test_reject_lone_surrogate_after_valid_pair(): + # A valid non-BMP scalar followed by a lone surrogate: detection must not be + # fooled by the earlier valid character. + with pytest.raises(JCSCanonicalizationError): + canonicalize_jcs({"v": EMOJI + HIGH}) + + +def test_reject_lone_surrogate_in_key(): + with pytest.raises(JCSCanonicalizationError): + canonicalize_jcs({HIGH: "x"}) + + +def test_reject_bare_lone_surrogate_string(): + with pytest.raises(JCSCanonicalizationError): + canonicalize_jcs(HIGH) + + +def test_valid_non_bmp_unchanged(): + # A valid non-BMP character canonicalizes to raw UTF-8, unchanged by this fix. + assert canonicalize_jcs(EMOJI) == '"' + EMOJI + '"' + assert canonicalize_jcs({"v": EMOJI}) == '{"v":"' + EMOJI + '"}' + # The canonical bytes are the raw 4-byte UTF-8 of U+1F600. + assert canonicalize_jcs({"v": EMOJI}).encode("utf-8") == b'{"v":"\xf0\x9f\x98\x80"}' + + +def test_error_is_valueerror_subclass(): + # Existing fail-closed handlers that catch ValueError keep working. + assert issubclass(JCSCanonicalizationError, ValueError) + + +# --- property-name and structural coverage --- + + +def test_reject_lone_surrogate_nested_value(): + with pytest.raises(JCSCanonicalizationError): + canonicalize_jcs({"a": {"b": HIGH}}) + + +def test_reject_lone_surrogate_array_element(): + with pytest.raises(JCSCanonicalizationError): + canonicalize_jcs({"a": [HIGH]}) + + +def test_reject_lone_surrogate_nested_key(): + with pytest.raises(JCSCanonicalizationError): + canonicalize_jcs({"a": {LOW: "x"}}) + + +def test_reject_valid_pair_followed_by_lone_low(): + # Off-by-one guard: a valid non-BMP scalar then a lone low surrogate rejects. + with pytest.raises(JCSCanonicalizationError): + canonicalize_jcs({"v": EMOJI + LOW}) + + +# --- error contract --- + + +def test_error_contract_category_reason_and_no_leak(): + # Caught by the documented type (ValueError), carries the stable category and + # reason, and does not leak the offending string into the message. + with pytest.raises(ValueError) as exc: + canonicalize_jcs({"v": HIGH}) + err = exc.value + assert isinstance(err, JCSCanonicalizationError) + assert err.category == "invalid_unicode" + assert err.reason == "lone_surrogate" + assert HIGH not in str(err) + + +# --- signing boundary --- + + +def test_sign_preimage_api_fails_closed_on_lone_surrogate(): + # A real content-id API (compute_action_ref) routes through canonicalize_jcs; + # a lone surrogate in the signed preimage must fail closed with the typed + # error and produce no output, with no fallback to the legacy canonicalizer. + from agent_passport.action_ref import compute_action_ref + + with pytest.raises(JCSCanonicalizationError): + compute_action_ref("agent-1", "read", [HIGH], "2026-07-13T00:00:00Z") + + +# --- adversarial raw payloads reach the same terminal state as Go --- +# The same raw JSON text pinned in the Go scanner tests. Python preserves a lone +# surrogate through json.loads and rejects it at canonicalize_jcs, so it reaches +# the same accept/reject terminal state as the Go raw-JSON path. + +import json # noqa: E402 + +_REJECT = { + "space-separated-non-adjacent": r'{"v":"\uD800 \uDC00"}', + "newline-separated-non-adjacent": r'{"v":"\uD800\n\uDC00"}', + "lone-low-first": r'{"v":"\uDC00"}', + "low-then-high": r'{"v":"\uDC00\uD800"}', + "high-then-literal-low": r'{"v":"\uD800\\uDC00"}', + "lone-in-key": r'{"\uD800":"x"}', + "lowercase-hex": r'{"v":"\ud800"}', + "literal-backslash-then-lone": r'{"v":"\\\uD800"}', +} + +_ACCEPT = { + "valid-adjacent-pair": r'{"v":"😀"}', + "escaped-backslash-literal": r'{"v":"\\uD800"}', + "double-backslash-literal": r'{"v":"\\\\uD800"}', + "genuine-replacement-char": r'{"v":"�"}', +} + + +@pytest.mark.parametrize("raw", list(_REJECT.values()), ids=list(_REJECT.keys())) +def test_adversarial_raw_reject(raw): + with pytest.raises(JCSCanonicalizationError): + canonicalize_jcs(json.loads(raw)) + + +@pytest.mark.parametrize("raw", list(_ACCEPT.values()), ids=list(_ACCEPT.keys())) +def test_adversarial_raw_accept(raw): + # Must not raise. + canonicalize_jcs(json.loads(raw))