diff --git a/.github/workflows/sandbox-log-redaction-quality-ci.yml b/.github/workflows/sandbox-log-redaction-quality-ci.yml index 3ca2f33e0..ac3df2e82 100644 --- a/.github/workflows/sandbox-log-redaction-quality-ci.yml +++ b/.github/workflows/sandbox-log-redaction-quality-ci.yml @@ -10,6 +10,7 @@ on: - "scripts/ci/redact_sensitive_log.py" - "scripts/ci/sandboxed_verify.py" - "scripts/ci/sandboxed_web_e2e.py" + - "tests/test_atomic_json_redaction.py" - "tests/test_command_wrapper_redaction.py" - "tests/test_opencode_security_boundaries.py" - "tests/test_sandboxed_verify.py" @@ -67,6 +68,7 @@ jobs: run: | test "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha || github.sha }}" python -m coverage run --branch -m pytest \ + tests/test_atomic_json_redaction.py \ tests/test_command_wrapper_redaction.py \ tests/test_opencode_security_boundaries.py \ tests/test_sandboxed_verify.py \ @@ -100,6 +102,7 @@ jobs: scripts/ci/redact_sensitive_log.py \ scripts/ci/sandboxed_verify.py \ scripts/ci/sandboxed_web_e2e.py \ + tests/test_atomic_json_redaction.py \ tests/test_command_wrapper_redaction.py \ tests/test_opencode_security_boundaries.py \ tests/test_sandboxed_verify.py \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 08f3ed4d7..d83b7ebe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Redacted complete multiline JSON and bounded JSON spans before line splitting with an iterative, resource-limited token/span rewriter that preserves untouched layout, escape spelling, duplicate-member order/count, scalar categories, container shape, stable markers, and mixed diagnostic records; malformed sensitive candidates and exhausted byte/depth/token/string/replacement/work limits fail closed without parser diagnostics. - Redacted opaque Docker/Podman login credentials inside bounded GNU `env` split-string and supported shell `-c` wrapper operands with one shared literal matcher, input/token/work budget, and four-level depth boundary; malformed or compound grammar fails closed while valid `--password-stdin` registries, env unset/chdir operands, Docker publish ports, SSH ports, and unrelated `login` arguments remain visible. - Redacted credential-shaped stdout, stderr, timeout evidence, and bounded backend/frontend service-log tails emitted by sandboxed verification and web-E2E subprocesses before those values become CI or review evidence. The boundary now canonicalizes safe ANSI styling and fails closed on single- or multiline rendering controls, scans JSON keys and opaque string values without corrupting scalar types or result schemas, preserves benign metadata and diagnostic domains, removes authorization headers, URL userinfo, and private-key blocks, protects raw and escaped explicitly allowed values before setup and tail selection, handles separated credential options, preserves markers and line boundaries, falls back safely on excessive JSON nesting, and parses high-volume diagnostics within bounded time. Normal executed argv, child exit, timeout, readiness, and network behavior remains unchanged; ambiguous short or fixed-evidence-colliding allowed values and setup/launch exceptions return redacted code `126` evidence before a raw traceback can escape. - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. diff --git a/docs/doctoring/sandbox-log-redaction.md b/docs/doctoring/sandbox-log-redaction.md index ee8a51c93..531604318 100644 --- a/docs/doctoring/sandbox-log-redaction.md +++ b/docs/doctoring/sandbox-log-redaction.md @@ -41,7 +41,11 @@ A consumer-path review then found additional integration hazards: post-processin Issue #907 then exposed a distinct argv-evidence boundary: direct Docker/Podman login options were protected, but opaque credentials inside supported `env` split-string and shell `-c` operands were still treated as ordinary data. The repair recognizes only exact `env -S`, `env --split-string`, `env --split-string=...`, and exact `sh`/`bash`/`dash`/`ksh`/`zsh` basenames with an exact or combined `-c` selector. One root-owned context compiles caller literals once and shares limits of 65,536 UTF-8 input bytes, 4,096 parsed tokens, 262,144 cumulative scan bytes, and four wrapper levels. Unsupported quoting, expansion, comments, escapes, compound-shell syntax, ambiguous trailing argv, or exhausted root budgets fail closed; the depth boundary replaces the remaining nested operand rather than scanning it again. The command is never executed or rewritten at its execution boundary. -The same repair fixes option-context false positives: only an exact Docker/Podman `login` subcommand gives `-p` password meaning; Docker publish ports, SSH ports, unrelated `login` arguments, GNU env unset/chdir operands, and the registry after valid `--password-stdin` remain visible. The invalid `--password-stdin=...` spelling remains conservatively redacted. Atomic multiline/duplicate-key raw JSON preservation remains separate Issue #908 and output/service-file memory quotas remain Issue #766. +The same repair fixes option-context false positives: only an exact Docker/Podman `login` subcommand gives `-p` password meaning; Docker publish ports, SSH ports, unrelated `login` arguments, GNU env unset/chdir operands, and the registry after valid `--password-stdin` remain visible. The invalid `--password-stdin=...` spelling remains conservatively redacted. Output/service-file memory quotas remain separate Issue #766. + +Issue #908 then exposed an ordering and representation defect in the raw-text path. Splitting evidence into lines before structural handling loses the association between a multiline sensitive key, colon, and value. Parsing a complete document into a Python dictionary would avoid that split but collapse duplicate members and normalize whitespace, escapes, number spelling, and punctuation. The repair therefore recognizes complete JSON and JSON spans before line splitting with an iterative token/span state machine. It decodes only bounded key and string tokens for classification, retains every untouched source slice, and applies non-overlapping scalar/key span replacements afterward. Duplicate members retain order and count; sensitive arrays and objects retain shape while scalar leaves keep their string, integer, floating-point, boolean, or null category. + +The raw parser shares one root budget of 65,536 input bytes, depth 64, 8,192 tokens, 32,768 bytes per string token, 2,048 replacements, and 262,144 cumulative work units. Limit exhaustion and malformed structural evidence with a sensitive key fail closed to one stable marker without exception text. Command strings and argv arrays reuse the bounded wrapper redactor; when its fail-closed representation changes argv length, the JSON path preserves array shape by replacing every original element. Complete valid spans can coexist with prefixes, suffixes, and multiple records. This does not change `redact_json_value()` for already-materialized trusted objects and does not claim Issue #766's output-memory closure. ## Test-first evidence @@ -65,12 +69,16 @@ The follow-up fail-first contract independently reproduced eight initial failure The wrapper follow-up began with **31 focused failures** covering both GNU env split spellings, all five supported shells, combined `-c` selectors, env-to-shell nesting, malformed/compound operands, trailing positional ambiguity, resource limits, depth, and option false positives. The bounded implementation reports **142 passed** in the permanent focused quality selection with exact **100% statement and branch coverage** across the three owned modules (`784` statements and `302` branches), followed by **1,060 passed plus 16 subtests** and exact complete owned-production coverage (`7,395` statements and `2,960` branches) in the complete repository suite and a passing Strix quick gate. Fixtures construct opaque credentials at runtime. +The atomic JSON follow-up began on exact parent `18a6d125fead8cb95972fe3e1a97e4cc4163e9d2` with a test-only head that produced the intended local RED result: **5 failed, 1 passed**. The failures reproduced multiline separation, duplicate-key loss, scalar/container normalization, mixed-record handling, and malformed-candidate leakage before production changed. The GREEN boundary reports **155 focused tests passed** with exact **100% statement and branch coverage** across the three owned modules (`1,039` statements and `416` branches). The complete repository suite reports **1,073 passed plus 16 subtests** and exact complete owned-production coverage (`7,650` statements and `3,074` branches); public callable docstrings, compilation, diff integrity, and the central Strix quick gate also pass. Additional fixtures exercise escaped spelling, empty containers, iterative depth, token, byte, string and replacement exhaustion, malformed parser states, shape-preserving command fallback, benign oversized input, and trusted materialized-object collision handling without committing a fixed credential-shaped literal. + ## Security and privacy interpretation Redaction is defense in depth, not authorization. It does not make arbitrary sensitive material safe to publish and it does not authorize repositories to pass secrets into sandbox commands. The existing environment minimization remains the primary ingress control; deterministic output redaction limits accidental disclosure if a child process or service emits sensitive-looking evidence anyway. ANSI styling canonicalization preserves visible diagnostic text and line separators, while cursor movement, backspace, multiline/unterminated control payloads, and invisible Unicode format controls fail closed for every affected evidence line or value. Structured JSON retains benign failure, policy, count, status, expiry, type, and usage metadata, while credential-denoting keys, credential material used as a key, credential-shaped string values, authorization headers, URL userinfo, and multiline private-key blocks are replaced without changing boolean or numeric types. Result markers remain one-line valid JSON with stable trusted keys after redaction, and the collector's literal `api.deepseek.com` classification signal remains visible. Literal protection is limited to non-empty values of names explicitly passed through `--allow-env`; values shorter than eight characters or colliding with fixed evidence text are rejected before execution, while whitespace-bearing values remain supported. The ordinary safe environment allowlist is not treated as secret, avoiding blanket removal of paths, locale data, or other useful diagnostics. +Raw JSON layout preservation applies before line-oriented normalization. It does not interpret arbitrary prefixes as trusted JSON, publish parser errors, reconstruct a dictionary, or silently accept a partially parsed sensitive candidate. Existing escape spelling and line endings are therefore diagnostic evidence rather than data to canonicalize. A bounded benign non-JSON or malformed fragment still uses the general redactor; a fragment that establishes a sensitive structural key but cannot establish a safe value boundary fails closed. + The redactor operates on CI-facing text only. It does not mutate files in the copied repository, service log files on disk, subprocess input, or successful child-process status and lifetime. The wrappers separately apply the documented pre-execution and exception code `126` policy. Operators should therefore interpret `[REDACTED]` as evidence suppression, not as successful removal of sensitive data from the source system that produced it. ## Rollback diff --git a/scripts/ci/redact_sensitive_log.py b/scripts/ci/redact_sensitive_log.py index 12ff807d0..7f9543c04 100644 --- a/scripts/ci/redact_sensitive_log.py +++ b/scripts/ci/redact_sensitive_log.py @@ -137,6 +137,12 @@ MAX_COMMAND_INPUT_BYTES = 65_536 MAX_COMMAND_TOKENS = 4_096 MAX_COMMAND_WORK = 262_144 +MAX_RAW_JSON_INPUT_BYTES = 65_536 +MAX_RAW_JSON_DEPTH = 64 +MAX_RAW_JSON_TOKENS = 8_192 +MAX_RAW_JSON_STRING_BYTES = 32_768 +MAX_RAW_JSON_REPLACEMENTS = 2_048 +MAX_RAW_JSON_WORK = 262_144 PROVIDER_TOKEN_RES = ( re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b"), re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"), @@ -860,6 +866,414 @@ def _redact_jwt_candidate(match: re.Match[str]) -> str: return candidate +class _RawJsonError(Exception): + """Signal malformed or over-budget raw JSON structural evidence.""" + + +def _raw_json_spend(budget: dict[str, int], *, tokens: int = 0, work: int = 0) -> None: + """Charge one raw JSON token/span operation to its bounded root budget.""" + budget["tokens"] -= tokens + budget["work"] -= work + if budget["tokens"] < 0 or budget["work"] < 0: + raise _RawJsonError + + +def _raw_json_skip_space(text: str, cursor: int, budget: dict[str, int]) -> int: + """Advance over JSON whitespace while charging each inspected character.""" + start = cursor + while cursor < len(text) and text[cursor] in " \t\r\n": + cursor += 1 + _raw_json_spend(budget, work=cursor - start) + return cursor + + +def _raw_json_string(text: str, cursor: int, budget: dict[str, int]) -> int: + """Return the exclusive end of one validated, bounded JSON string token.""" + start = cursor + cursor += 1 + escaped = False + while cursor < len(text): + character = text[cursor] + cursor += 1 + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == '"': + token = text[start:cursor] + if len(token.encode("utf-8")) > MAX_RAW_JSON_STRING_BYTES: + raise _RawJsonError + try: + json.loads(token) + except (ValueError, RecursionError) as error: + raise _RawJsonError from error + _raw_json_spend(budget, tokens=1, work=cursor - start) + return cursor + elif ord(character) < 0x20: + raise _RawJsonError + raise _RawJsonError + + +def _raw_json_scalar(text: str, cursor: int, budget: dict[str, int]) -> tuple[int, str]: + """Return the exclusive end and category of one non-container JSON scalar.""" + if text[cursor] == '"': + return _raw_json_string(text, cursor, budget), "string" + for literal, category in (("true", "boolean"), ("false", "boolean"), ("null", "null")): + if text.startswith(literal, cursor): + _raw_json_spend(budget, tokens=1, work=len(literal)) + return cursor + len(literal), category + match = re.match(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?", text[cursor:]) + if match is None: + raise _RawJsonError + token = match.group(0) + _raw_json_spend(budget, tokens=1, work=len(token)) + return cursor + len(token), ("float" if any(mark in token for mark in ".eE") else "integer") + + +def _raw_json_start_value( + text: str, + cursor: int, + depth: int, + budget: dict[str, int], +) -> tuple[dict[str, Any], int, bool]: + """Start one bounded value and report whether it opened a container.""" + if depth > MAX_RAW_JSON_DEPTH: + raise _RawJsonError + cursor = _raw_json_skip_space(text, cursor, budget) + if cursor >= len(text): + raise _RawJsonError + start = cursor + opener = text[cursor] + if opener not in "{[": + end, category = _raw_json_scalar(text, cursor, budget) + return {"kind": category, "start": start, "end": end, "children": []}, end, False + _raw_json_spend(budget, tokens=1, work=1) + return ( + { + "kind": "object" if opener == "{" else "array", + "start": start, + "children": [], + }, + cursor + 1, + True, + ) + + +def _raw_json_attach_child( + frame: dict[str, Any], + child: dict[str, Any], +) -> None: + """Attach pending object-key metadata to one parsed child node.""" + key = frame.pop("key", None) + if key is not None: + child["key"] = key + child["key_start"] = frame.pop("key_start") + child["key_end"] = frame.pop("key_end") + child["sensitive"] = key is not None and ( + _key_identifies_credentials(key) + or (isinstance(key, str) and _contains_unsafe_render_controls(key)) + ) + child["command_evidence"] = key is not None and _key_identifies_command_evidence(key) + frame["node"]["children"].append(child) + + +def _raw_json_parse_value( + text: str, + cursor: int, + depth: int, + budget: dict[str, int], +) -> tuple[dict[str, Any], int]: + """Iteratively parse one bounded JSON value into source-span nodes.""" + root, cursor, _ = _raw_json_start_value(text, cursor, depth, budget) + stack: list[dict[str, Any]] = [ + { + "node": root, + "depth": depth, + "state": "key_or_end" if root["kind"] == "object" else "value_or_end", + } + ] + while stack: + frame = stack[-1] + node = frame["node"] + kind = node["kind"] + closer = "}" if kind == "object" else "]" + state = frame["state"] + cursor = _raw_json_skip_space(text, cursor, budget) + + if state in {"key_or_end", "value_or_end"} and ( + cursor < len(text) and text[cursor] == closer + ): + _raw_json_spend(budget, tokens=1, work=1) + cursor += 1 + node["end"] = cursor + stack.pop() + continue + + if state in {"key_or_end", "key"}: + if cursor >= len(text) or text[cursor] != '"': + raise _RawJsonError + key_start = cursor + key_end = _raw_json_string(text, cursor, budget) + key = json.loads(text[cursor:key_end]) + cursor = _raw_json_skip_space(text, key_end, budget) + if cursor >= len(text) or text[cursor] != ":": + raise _RawJsonError + _raw_json_spend(budget, tokens=1, work=1) + frame.update( + state="value", + key=key, + key_start=key_start, + key_end=key_end, + ) + cursor += 1 + continue + + if state in {"value_or_end", "value"}: + child, cursor, child_opened = _raw_json_start_value( + text, + cursor, + frame["depth"] + 1, + budget, + ) + _raw_json_attach_child(frame, child) + frame["state"] = "comma_or_end" + if child_opened: + stack.append( + { + "node": child, + "depth": frame["depth"] + 1, + "state": ( + "key_or_end" if child["kind"] == "object" else "value_or_end" + ), + } + ) + continue + + if state != "comma_or_end" or cursor >= len(text): + raise _RawJsonError + if text[cursor] == closer: + _raw_json_spend(budget, tokens=1, work=1) + cursor += 1 + node["end"] = cursor + stack.pop() + continue + if text[cursor] != ",": + raise _RawJsonError + _raw_json_spend(budget, tokens=1, work=1) + frame["state"] = "key" if kind == "object" else "value" + cursor += 1 + return root, cursor + + +def _raw_json_add_replacement( + replacements: list[tuple[int, int, str]], + start: int, + end: int, + replacement: str, +) -> None: + """Append one span replacement while enforcing the root replacement limit.""" + replacements.append((start, end, replacement)) + if len(replacements) > MAX_RAW_JSON_REPLACEMENTS: + raise _RawJsonError + + +def _raw_json_leaf_replacements( + node: dict[str, Any], + *, + text: str, + literal_pattern: re.Pattern[str] | None, + force: bool, + replacements: list[tuple[int, int, str]], +) -> None: + """Iteratively collect layout-preserving replacements from a span tree.""" + pending: list[tuple[dict[str, Any], bool]] = [(node, force)] + while pending: + current, inherited_force = pending.pop() + if "key_start" in current: + key = str(current["key"]) + cleaned_key = str(current.get("rendered_key", key)) + if cleaned_key != key: + _raw_json_add_replacement( + replacements, + current["key_start"], + current["key_end"], + json.dumps(cleaned_key, ensure_ascii=False), + ) + current_force = inherited_force or bool(current.get("sensitive")) + kind = current["kind"] + if current.get("command_evidence") and kind == "string": + decoded = json.loads(text[current["start"] : current["end"]]) + cleaned_command = _redact_command_text_with_pattern(decoded, literal_pattern) + if cleaned_command != decoded: + _raw_json_add_replacement( + replacements, + current["start"], + current["end"], + json.dumps(cleaned_command, ensure_ascii=False), + ) + continue + if current.get("command_evidence") and kind == "array" and all( + child["kind"] == "string" for child in current["children"] + ): + decoded_arguments = [ + json.loads(text[child["start"] : child["end"]]) + for child in current["children"] + ] + cleaned_arguments = _redact_command_argv_with_pattern( + decoded_arguments, + literal_pattern, + ) + if len(cleaned_arguments) != len(current["children"]): + cleaned_arguments = [REDACTED] * len(current["children"]) + for child, original, cleaned in zip( + current["children"], + decoded_arguments, + cleaned_arguments, + strict=True, + ): + if cleaned != original: + _raw_json_add_replacement( + replacements, + child["start"], + child["end"], + json.dumps(cleaned, ensure_ascii=False), + ) + continue + if kind == "object": + used_keys: set[str] = set() + used_sources: dict[str, set[str]] = {} + collision_counts: dict[str, int] = {} + for child in current["children"]: + key = str(child["key"]) + cleaned_key = _redact_unstructured(key, literal_pattern) + if cleaned_key in used_keys and key not in used_sources.get(cleaned_key, set()): + collision_index = collision_counts.get(cleaned_key, 2) + while f"{cleaned_key}#{collision_index}" in used_keys: + collision_index += 1 + collision_counts[cleaned_key] = collision_index + 1 + cleaned_key = f"{cleaned_key}#{collision_index}" + child["rendered_key"] = cleaned_key + used_keys.add(cleaned_key) + used_sources.setdefault(cleaned_key, set()).add(key) + if kind in {"object", "array"}: + pending.extend( + (child, current_force) for child in reversed(current["children"]) + ) + continue + if not current_force and kind == "string": + decoded = json.loads(text[current["start"] : current["end"]]) + cleaned = _redact_unstructured(decoded, literal_pattern) + if cleaned != decoded: + _raw_json_add_replacement( + replacements, + current["start"], + current["end"], + json.dumps(cleaned, ensure_ascii=False), + ) + if not current_force or kind == "null": + continue + replacement = { + "string": json.dumps(REDACTED), + "integer": "0", + "float": "0.0", + "boolean": "false", + }[kind] + _raw_json_add_replacement( + replacements, + current["start"], + current["end"], + replacement, + ) + + +def _looks_like_sensitive_json_candidate(text: str) -> bool: + """Return whether a malformed structural suffix names a sensitive JSON key.""" + for match in re.finditer(r'"(?:\\.|[^"\\])*"\s*:', text): + try: + key = json.loads(match.group(0).rsplit(":", 1)[0].rstrip()) + except (ValueError, RecursionError): + continue + if _key_identifies_credentials(key): + return True + return False + + +def _redact_plain_json_gap( + text: str, + literal_pattern: re.Pattern[str] | None, +) -> str: + """Redact non-JSON slices without invoking structural line normalization.""" + output: list[str] = [] + for raw_line in text.splitlines(keepends=True): + separator = LINE_SEPARATOR_END_RE.search(raw_line) + line_end = separator.start() if separator is not None else len(raw_line) + output.append(_redact_unstructured(raw_line[:line_end], literal_pattern)) + output.append(raw_line[line_end:]) + return "".join(output) + + +def _redact_raw_json_spans( + text: str, + literal_pattern: re.Pattern[str] | None, +) -> tuple[str, bool]: + """Rewrite complete bounded JSON spans before any line-oriented processing.""" + if len(text.encode("utf-8")) > MAX_RAW_JSON_INPUT_BYTES: + if _looks_like_sensitive_json_candidate(text): + return REDACTED, True + return text, False + replacements: list[tuple[int, int, str]] = [] + spans: list[tuple[int, int]] = [] + cursor = 0 + parsed = False + budget = {"tokens": MAX_RAW_JSON_TOKENS, "work": MAX_RAW_JSON_WORK} + while cursor < len(text): + if text[cursor] not in "{[": + cursor += 1 + continue + start = cursor + try: + node, end = _raw_json_parse_value(text, start, 0, budget) + except _RawJsonError: + if _looks_like_sensitive_json_candidate(text[start:]): + return REDACTED, True + cursor += 1 + continue + parsed = True + spans.append((start, end)) + try: + _raw_json_leaf_replacements( + node, + text=text, + literal_pattern=literal_pattern, + force=False, + replacements=replacements, + ) + except _RawJsonError: + return REDACTED, True + cursor = end + if not parsed: + return text, False + replacements.sort() + output: list[str] = [] + gap_start = 0 + replacement_index = 0 + for span_start, span_end in spans: + output.append(_redact_plain_json_gap(text[gap_start:span_start], literal_pattern)) + span_cursor = span_start + while replacement_index < len(replacements): + start, end, replacement = replacements[replacement_index] + if start >= span_end: + break + output.append(text[span_cursor:start]) + output.append(replacement) + span_cursor = end + replacement_index += 1 + output.append(text[span_cursor:span_end]) + gap_start = span_end + output.append(_redact_plain_json_gap(text[gap_start:], literal_pattern)) + return "".join(output), True + + def _redact_line( line: str, json_literal_pattern: re.Pattern[str] | None, @@ -897,6 +1311,12 @@ def redact_text(text: str, *, sensitive_values: Sequence[str] = ()) -> str: ) unstructured_literal_pattern = _compile_literal_pattern(single_line_values) text = _redact_literal_values(text, _compile_literal_pattern(multiline_values)) + text, raw_json_processed = _redact_raw_json_spans( + text, + unstructured_literal_pattern, + ) + if raw_json_processed: + return text output: list[str] = [] for raw_line in text.splitlines(keepends=True): separator = LINE_SEPARATOR_END_RE.search(raw_line) diff --git a/tests/test_atomic_json_redaction.py b/tests/test_atomic_json_redaction.py new file mode 100644 index 000000000..2f05ac1a8 --- /dev/null +++ b/tests/test_atomic_json_redaction.py @@ -0,0 +1,203 @@ +"""Fail-first contracts for layout-preserving raw JSON log redaction.""" + +from __future__ import annotations + +from scripts.ci import redact_sensitive_log as redactor + + +def _credential(suffix: str = "731") -> str: + """Construct opaque credential material only at runtime.""" + return "-".join(("marble", "river", "opaque", suffix)) + + +def test_multiline_sensitive_value_is_rewritten_before_line_splitting() -> None: + """Whitespace between key, colon, and value is preserved byte-for-byte.""" + credential = _credential() + source = '{\n "password"\n :\n "' + credential + '"\n}\n' + expected = source.replace(credential, redactor.REDACTED) + + assert redactor.redact_text(source) == expected + + +def test_duplicate_sensitive_keys_preserve_order_count_and_layout() -> None: + """Duplicate object members survive span rewriting without dict collapse.""" + first = _credential("first") + second = _credential("second") + source = ( + '{ "token" : "' + + first + + '", "status":"failed", "token" : "' + + second + + '" }' + ) + + cleaned = redactor.redact_text(source) + + assert first not in cleaned + assert second not in cleaned + assert cleaned.count('"token"') == 2 + assert cleaned.count(f'"{redactor.REDACTED}"') == 2 + assert cleaned.replace(f'"{redactor.REDACTED}"', '"VALUE"') == source.replace( + f'"{first}"', + '"VALUE"', + ).replace(f'"{second}"', '"VALUE"') + + +def test_sensitive_scalar_categories_and_container_shape_are_preserved() -> None: + """Sensitive JSON values keep type categories and recursive container shape.""" + credential = _credential() + source = ( + '{"password":"' + + credential + + '","token":12,"secret":1.25,"auth":true,' + '"credential":null,"private_key":["leaf",7,false,null,{"x":"y"}]}' + ) + expected = ( + '{"password":"[REDACTED]","token":0,"secret":0.0,"auth":false,' + '"credential":null,"private_key":["[REDACTED]",0,false,null,' + '{"x":"[REDACTED]"}]}' + ) + + assert redactor.redact_text(source) == expected + + +def test_prefixed_and_multiple_json_records_preserve_untouched_slices() -> None: + """Bounded structural spans can coexist with ordinary diagnostic text.""" + first = _credential("one") + second = _credential("two") + source = ( + "prefix diagnostic\n" + f'{{\n "password": "{first}"\n}}\n' + "middle diagnostic\n" + f'{{"token":"{second}","status":"failed"}}\n' + "suffix diagnostic\n" + ) + + cleaned = redactor.redact_text(source) + + assert first not in cleaned + assert second not in cleaned + assert cleaned.startswith("prefix diagnostic\n{\n") + assert "\n}\nmiddle diagnostic\n" in cleaned + assert cleaned.endswith("\nsuffix diagnostic\n") + assert '"status":"failed"' in cleaned + + +def test_malformed_multiline_sensitive_candidate_fails_closed() -> None: + """A structural candidate spanning lines cannot fall back and leak its tail.""" + credential = _credential() + source = '{\n "password"\n :\n "' + credential + + cleaned = redactor.redact_text(source) + + assert credential not in cleaned + assert redactor.REDACTED in cleaned + + +def test_raw_json_limits_are_explicit_and_fail_closed() -> None: + """Parser byte, depth, token, string, replacement, and work limits are fixed.""" + assert redactor.MAX_RAW_JSON_INPUT_BYTES == 65_536 + assert redactor.MAX_RAW_JSON_DEPTH == 64 + assert redactor.MAX_RAW_JSON_TOKENS == 8_192 + assert redactor.MAX_RAW_JSON_STRING_BYTES == 32_768 + assert redactor.MAX_RAW_JSON_REPLACEMENTS == 2_048 + assert redactor.MAX_RAW_JSON_WORK == 262_144 + + oversized = '{"password":"' + "x" * redactor.MAX_RAW_JSON_INPUT_BYTES + '"}' + assert redactor.redact_text(oversized) == redactor.REDACTED + + +def test_escaped_keys_empty_containers_and_layout_remain_unchanged() -> None: + """Classification decodes tokens without normalizing untouched source slices.""" + credential = _credential("escaped") + source = ( + '{"pa\\u0073sword" : "' + + credential + + '","note":"line\\nkept","empty_array":[],"empty_object":{}}' + ) + expected = source.replace(credential, redactor.REDACTED) + + assert redactor.redact_text(source) == expected + + +def test_iterative_depth_and_replacement_limits_fail_closed() -> None: + """Deep trees and excessive rewrite counts return one bounded marker.""" + deep = ( + '{"password":' + + "[" * (redactor.MAX_RAW_JSON_DEPTH + 1) + + '"leaf"' + + "]" * (redactor.MAX_RAW_JSON_DEPTH + 1) + + "}" + ) + replacements = ( + '{"password":[' + + ",".join("1" for _ in range(redactor.MAX_RAW_JSON_REPLACEMENTS + 1)) + + "]}" + ) + + assert redactor.redact_text(deep) == redactor.REDACTED + assert redactor.redact_text(replacements) == redactor.REDACTED + + +def test_token_limit_is_bounded_for_benign_high_volume_json() -> None: + """Excessive benign token volume returns through bounded plain-text handling.""" + source = "[" + ",".join("0" for _ in range(5_000)) + "]" + + assert redactor.redact_text(source) == source + + +def test_oversized_benign_input_does_not_become_a_false_secret_finding() -> None: + """The byte limit fails closed only when credential context is present.""" + source = '{"status":"' + "x" * redactor.MAX_RAW_JSON_INPUT_BYTES + '"}' + + assert redactor.redact_text(source) == source + + +def test_malformed_sensitive_json_states_fail_closed_without_diagnostics() -> None: + """Every unsafe parser boundary emits only the stable redaction marker.""" + credential = _credential("malformed") + malformed = ( + '{"password":', + '{"password" "' + credential + '","token":}', + '{"password":"' + credential + '", bad}', + '{"password":1', + '{"password":"\\q"}', + '{"password":"' + credential + '\n"}', + '{"password":"' + "x" * redactor.MAX_RAW_JSON_STRING_BYTES + '"}', + '{"\\q":0,"password":', + ) + + assert all(redactor.redact_text(source) == redactor.REDACTED for source in malformed) + assert redactor.redact_text('{"status":') == '{"status":' + + +def test_command_fields_preserve_json_shape_when_wrapper_fails_closed() -> None: + """Command evidence keeps array shape even when its wrapper collapses evidence.""" + source = '{"command":"echo ok","argv":["sh","-c"," "]}' + + assert redactor.redact_text(source) == ( + '{"command":"echo ok","argv":' + '["[REDACTED]","[REDACTED]","[REDACTED]"]}' + ) + + +def test_materialized_json_redaction_keeps_collision_and_command_contracts() -> None: + """Trusted object redaction remains separate from raw layout preservation.""" + value = { + "[REDACTED]#2": "kept", + "alpha-secret": "first", + "beta-secret": "second", + "command": {"password": "nested"}, + } + + cleaned = redactor.redact_json_value( + value, + sensitive_values=("alpha-secret", "beta-secret"), + ) + + assert cleaned == { + "[REDACTED]#2": "kept", + "[REDACTED]": redactor.REDACTED, + "[REDACTED]#3": redactor.REDACTED, + "command": {"password": redactor.REDACTED}, + }