fix(pii): require corroboration before redacting a bare digit run as a credit card

src/memory/store/safety/pii.rs

@@ -99,10 +99,38 @@ static PHONE_NANP_RE: LazyLock<Regex> = LazyLock::new(|| {
 static SSN_RE: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r"\b\d{3}-\d{2}-\d{4}\b").expect("ssn"));
 
-// Credit card: 13-19 digits with optional spaces/dashes every 4. Luhn-gated.
+// Credit card: 13-19 digits with optional spaces/dashes every 4. Every match
+// is Luhn-gated; a match with no separators at all additionally needs
+// corroboration — a real network IIN at an issued length, or a card keyword
+// nearby — because Luhn alone passes ~10% of arbitrary digit runs, and bare
+// 13-digit epoch-millisecond timestamps were being redacted out of stored
+// JSON envelopes at exactly that rate (opencompany#1201). Same split as
+// Aadhaar below: formatted keeps the checksum-only gate, bare needs more.
 static CC_RE: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r"\b(?:\d[\s\-]?){13,19}\b").expect("credit card"));
 
+// Card keyword corroborating a bare digit run. Three tiers, matched
+// case-insensitively:
+//
+// * Standalone words, bounded by `[\W_]` rather than `\b` — the regex crate
+//   counts `_` as a word character, so `\bcard\b` never fires inside
+//   `card_number`, which is among the most common serialized key shapes a
+//   stored payload carries. The explicit class keeps `pan` from firing
+//   inside `japan` while still matching `card_number=` and `cc=`.
+// * Compound identifiers matched as substrings, because camelCase provides
+//   no boundary of any kind: `cardNumber`, `creditCard`, `ccNum`, `cardNo`,
+//   `panNumber` all lowercase into these.
+// * Native-script terms, per this module's multilingual mandate (the Aadhaar
+//   and My Number patterns already carry theirs). CJK terms match as
+//   substrings — CJK text does not put `[\W_]` between a word and the
+//   digits that follow it.
+static CC_KEYWORD_RE: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(
+        r"(?i)(?:^|[\W_])(?:card|credit|debit|visa|mastercard|amex|american\s?express|discover|jcb|diners|unionpay|maestro|hipercard|elo|rupay|cvv|cvc|cc|pan|tarjeta|cart[aã]o|carte|karte|карта|карты|карту|картой|карте|кредитка)(?:[\W_]|$)|(?i:cardnumber|creditcard|ccnum|cardno|pannumber|カード|信用卡|卡号|银行卡|카드)",
+    )
+    .expect("cc keyword")
+});
+
 // IBAN: 2 letter country code + 2 check digits + 11-30 alphanumeric.
 // Allow optional spaces every 4 chars (common human format).
 static IBAN_RE: LazyLock<Regex> =
@@ -203,8 +231,12 @@ pub fn redact_pii(text: &str) -> Sanitized<String> {
 /// JIDs like `12025551234-1543890267@g.us`, telegram numeric peer IDs,
 /// millisecond timestamps, padded counters) is too high to use as a hard
 /// rejection signal. Content scrubbing via [`redact_pii`] still applies
-/// those patterns — false positives are tolerable there because they only
-/// replace bytes inside a string, not reject the whole write.
+/// those patterns — a content false positive replaces bytes inside a string
+/// rather than rejecting the whole write, which is cheaper but *not* free:
+/// a redaction landing inside structured content corrupts it for whatever
+/// wrote it (opencompany#1201 — timestamps in stored JSON envelopes), which
+/// is why the credit-card pattern's bare form now demands corroboration
+/// beyond its checksum.
 pub fn has_likely_pii(value: &str) -> bool {
     let nview = NormalizedView::build(value);
     let cand = scan_candidates(&nview.normalized);
@@ -283,7 +315,7 @@ fn collect_redactions_inner(norm: &str, cand: &Candidates, include_bare_numeric:
     if include_bare_numeric {
         // Credit card before bare CPF/CNPJ to avoid catching a 13-19 digit run as CPF/CNPJ.
         if cand.cc {
-            push_checksum(&mut hits, norm, &CC_RE, PII_CC, valid_luhn);
+            push_credit_cards(&mut hits, norm);
         }
         if cand.cnpj_bare {
             push_checksum(&mut hits, norm, &CNPJ_BARE_RE, PII_CNPJ, |s| {
@@ -407,6 +439,67 @@ fn push_captured(
     }
 }
 
+/// Credit-card collection. Every match must pass Luhn; a bare match (no
+/// separators) additionally needs structural or contextual corroboration.
+///
+/// Luhn alone passes ~10% of arbitrary digit runs, and 13-19 contiguous
+/// digits is a common machine-identifier shape — 13-digit epoch-millisecond
+/// timestamps above all, which were being redacted out of stored JSON
+/// envelopes at that rate and corrupting them (opencompany#1201). The strict
+/// boundary set ([`collect_strict_redactions`]) already excludes credit card
+/// for exactly this reason; this brings the content path to the same
+/// judgement without giving up real cards: a separated run keeps the
+/// Luhn-only gate it always had, and a bare run still redacts when its
+/// prefix is a real network IIN at an issued length
+/// ([`plausible_card_number`]) or a card keyword sits within
+/// [`CC_KEYWORD_WINDOW`] bytes.
+fn push_credit_cards(hits: &mut Vec<Hit>, norm: &str) {
+    for m in CC_RE.find_iter(norm) {
+        let s = m.as_str();
+        if !valid_luhn(s) {
+            continue;
+        }
+        // Judge bare-vs-separated on the interior of the digit run: the
+        // regex's per-digit `[\s\-]?` can capture one trailing separator
+        // (`"…773 "`), which is not the human 4-4-4-4 grouping this
+        // distinction is after.
+        let interior = s.trim_matches(|c: char| !c.is_ascii_digit());
+        let bare = interior.bytes().all(|b| b.is_ascii_digit());
+        let corroborated =
+            !bare || plausible_card_number(&digits(s)) || cc_keyword_near(norm, m.start(), m.end());
+        if corroborated {
+            hits.push(Hit {
+                start: m.start(),
+                end: m.end(),
+                token: PII_CC,
+            });
+        }
+    }
+}
+
+/// Bytes of context searched either side of a bare digit run for a card
+/// keyword. 64 bytes rather than 32 because the window is counted in bytes
+/// while text is not: 32 bytes is only ~10 CJK characters or 8 emoji, so a
+/// run of either could evict an English keyword that a reader would call
+/// adjacent. 64 comfortably spans `{"payment_method":{"card":{"number":…`
+/// and a `カード`-prefixed line alike.
+const CC_KEYWORD_WINDOW: usize = 64;
+
+/// True when [`CC_KEYWORD_RE`] matches within the window around
+/// `start..end`, widened outward to char boundaries so the slice cannot
+/// split a multi-byte character.
+fn cc_keyword_near(norm: &str, start: usize, end: usize) -> bool {
+    let mut lo = start.saturating_sub(CC_KEYWORD_WINDOW);
+    while lo > 0 && !norm.is_char_boundary(lo) {
+        lo -= 1;
+    }
+    let mut hi = (end + CC_KEYWORD_WINDOW).min(norm.len());
+    while hi < norm.len() && !norm.is_char_boundary(hi) {
+        hi += 1;
+    }
+    CC_KEYWORD_RE.is_match(&norm[lo..hi])
+}
+
 // Sort by start asc, length desc. Then walk in order, dropping any hit whose
 // range overlaps a kept hit. Result: earlier + longer wins; no double-redact.
 fn dedupe_overlaps(hits: &mut Vec<Hit>) {

src/memory/store/safety/pii/checks.rs

@@ -79,6 +79,103 @@ pub(super) fn valid_luhn(s: &str) -> bool {
     sum.is_multiple_of(10)
 }
 
+/// True when a digit string has a plausible payment-card shape: a known
+/// major-network IIN prefix at a length that network actually issues.
+///
+/// The structural gate behind bare (separator-less) credit-card redaction.
+/// Luhn alone passes ~10% of arbitrary digit runs — the same raw
+/// false-positive rate that put bare Aadhaar behind a keyword — and 13-19
+/// contiguous digits is a common machine-identifier shape: 13-digit
+/// epoch-millisecond timestamps (`17…`/`18…` for decades either side of now)
+/// sit squarely in the window and were being redacted out of stored JSON at
+/// that rate (opencompany#1201). No card network issues from a `17`/`18`
+/// prefix, so requiring a real IIN removes that entire class while keeping
+/// every number a major network could actually have issued.
+///
+/// The table lists each supported network's published IIN ranges at the
+/// lengths that network issues: Visa, Mastercard (incl. the 2-series), Amex,
+/// Discover, JCB, Diners Club, UnionPay, Maestro, Mir, RuPay, and the
+/// Brazilian networks Elo and Hipercard (in scope by this module's own
+/// design: it already carries bare and formatted CPF/CNPJ). It is a
+/// *corroboration* tier, not an acquirer's validator: an exhaustive BIN
+/// registry is a licensed, continuously updated database, and a range missing
+/// here is not silently dropped from redaction — a bare PAN on an unlisted
+/// network still redacts whenever a card keyword appears within the keyword
+/// window (the third gate in `push_credit_cards`).
+///
+/// One dated caveat, so nobody inherits a stronger claim than the code makes:
+/// "timestamps can never corroborate" is prefix-and-length dependent, not
+/// absolute. 13-digit epoch-milliseconds stay out of every range until the
+/// year 2096 (`4…`, Visa's 13-digit arm); but 16-digit epoch-MICROsecond
+/// stamps enter Mir's `2200-2204` window in late 2039 and Mastercard's
+/// 2-series `2221-2720` from ~2040 to ~2056. If this code outlives that,
+/// those stamps redact at Luhn's ~10% again and this gate needs a rethink.
+pub(super) fn plausible_card_number(d: &[u32]) -> bool {
+    let len = d.len();
+    if !(13..=19).contains(&len) {
+        return false;
+    }
+    // len >= 13 makes the first four digits always present.
+    let p2 = d[0] * 10 + d[1];
+    let p3 = p2 * 10 + d[2];
+    let p4 = p3 * 10 + d[3];
+    match d[0] {
+        // Visa: 16 standard, 13 legacy, 19 extended. (Also where Elo's
+        // 4-prefixed ranges land, at the same 16.)
+        4 => matches!(len, 13 | 16 | 19),
+        5 => {
+            // Mastercard 51-55 (16 only).
+            ((51..=55).contains(&p2) && len == 16)
+                // Maestro 5018/5020/5038/5893 and 56-58. Maestro issues
+                // 12-19; the floor here is 13 because CC_RE requires 13
+                // digits, so the explicit bound below is the whole window
+                // this function can see — kept explicit so the "at an
+                // issued length" promise stays visibly true.
+                || ((matches!(p4, 5018 | 5020 | 5038 | 5893) || (56..=58).contains(&p2))
+                    && (13..=19).contains(&len))
+                // Elo 5041/5066/5067 (16), RuPay 508 (16).
+                || ((matches!(p4, 5041 | 5066 | 5067) || p3 == 508) && len == 16)
+        }
+        2 => {
+            // Mastercard 2-series 2221-2720 (16 only).
+            ((2221..=2720).contains(&p4) && len == 16)
+                // Mir 2200-2204 (16 only).
+                || ((2200..=2204).contains(&p4) && len == 16)
+        }
+        3 => {
+            // Amex 34/37 (15 only).
+            (matches!(p2, 34 | 37) && len == 15)
+                // JCB 3528-3589 (16-19).
+                || ((3528..=3589).contains(&p4) && (16..=19).contains(&len))
+                // Diners Club 36 / 300-305 / 3095 / 38-39, one scheme, one
+                // length rule: 14 (Diners International / Carte Blanche
+                // classic — 30569309025904, the canonical test PAN, is 14)
+                // through 19.
+                || ((p2 == 36
+                    || (300..=305).contains(&p3)
+                    || p4 == 3095
+                    || matches!(p2, 38 | 39))
+                    && (14..=19).contains(&len))
+        }
+        6 => {
+            // Discover 6011 / 644-649 / 65 (16 or 19).
+            ((p4 == 6011 || (644..=649).contains(&p3) || p2 == 65) && matches!(len, 16 | 19))
+                // UnionPay 62 (16-19), which also covers the
+                // Discover-processed 622126-622925 range.
+                || (p2 == 62 && (16..=19).contains(&len))
+                // Maestro 6304/6759/6761-6763, bounded as the 5-prefix
+                // Maestro arm above.
+                || (matches!(p4, 6304 | 6759 | 6761 | 6762 | 6763) && (13..=19).contains(&len))
+                // RuPay 60 (16) beyond the 65 range shared with Discover;
+                // Elo 6277/6362/6363 and Hipercard 6062 (16).
+                || ((p2 == 60 || matches!(p4, 6277 | 6362 | 6363 | 6062)) && len == 16)
+        }
+        // RuPay 81/82 (16).
+        8 => matches!(p2, 81 | 82) && len == 16,
+        _ => false,
+    }
+}
+
 // IBAN mod-97. Steps: strip spaces, move first 4 chars to end, expand letters
 // (A=10..Z=35), divide as a big-integer mod 97, require remainder == 1.
 pub(super) fn valid_iban(s: &str) -> bool {

src/memory/store/safety/pii/checks_tests.rs

@@ -41,3 +41,127 @@ fn identity_validators_cover_checksums_reserved_values_and_prefixes() {
     assert!(!valid_nino("DA123456A"));
     assert!(!valid_nino("AA12345A"));
 }
+
+#[test]
+fn plausible_card_number_requires_a_real_iin_at_an_issued_length() {
+    // No network's prefix: epoch-millisecond timestamps and other machine ids.
+    assert!(!plausible_card_number(&digits("1787178633773"))); // 13-digit epoch ms
+    assert!(!plausible_card_number(&digits("1700000000000")));
+    assert!(!plausible_card_number(&digits("9111111111111119")));
+    assert!(!plausible_card_number(&digits("2000000000000000"))); // year-2033 epoch-µs shape
+    assert!(!plausible_card_number(&digits("1900000000000"))); // 13-digit, no IIN starts 1
+
+    // Out of the card length window entirely.
+    assert!(!plausible_card_number(&digits("411111111111"))); // 12
+    assert!(!plausible_card_number(&digits("41111111111111111111"))); // 20
+}
+
+// The accept direction, per network, at its boundary lengths — with the
+// reject cases one step past each length and each range edge. Luhn is
+// irrelevant here: the function judges shape only, and the redaction path
+// tests Luhn separately.
+#[test]
+fn plausible_card_number_accepts_each_network_at_its_boundary_lengths() {
+    // Visa 13/16/19; nothing between or past.
+    assert!(plausible_card_number(&digits("4222222222222"))); // 13
+    assert!(plausible_card_number(&digits("4111111111111111"))); // 16
+    assert!(plausible_card_number(&digits("4111111111111111111"))); // 19
+    assert!(!plausible_card_number(&digits("41111111111111"))); // 14
+    assert!(!plausible_card_number(&digits("411111111111111"))); // 15
+    assert!(!plausible_card_number(&digits("41111111111111111"))); // 17
+
+    // Mastercard 51-55 and 2221-2720, 16 only; edges out both sides.
+    assert!(plausible_card_number(&digits("5100000000000000")));
+    assert!(plausible_card_number(&digits("5500005555555559")));
+    assert!(plausible_card_number(&digits("2221000000000009")));
+    assert!(plausible_card_number(&digits("2720000000000000")));
+    assert!(!plausible_card_number(&digits("550000555555555"))); // 15
+    assert!(!plausible_card_number(&digits("55000055555555590"))); // 17
+    assert!(!plausible_card_number(&digits("5000000000000000"))); // 50: not MC
+    assert!(!plausible_card_number(&digits("2220000000000000"))); // below 2221
+    assert!(!plausible_card_number(&digits("2721000000000000"))); // above 2720
+
+    // Mir 2200-2204, 16 only.
+    assert!(plausible_card_number(&digits("2200000000000004")));
+    assert!(plausible_card_number(&digits("2204000000000000")));
+    assert!(!plausible_card_number(&digits("2205000000000009"))); // past range
+    assert!(!plausible_card_number(&digits("220000000000000"))); // 15
+    assert!(!plausible_card_number(&digits("22000000000000004"))); // 17
+
+    // Amex 34/37, 15 only.
+    assert!(plausible_card_number(&digits("378282246310005")));
+    assert!(plausible_card_number(&digits("340000000000009")));
+    assert!(!plausible_card_number(&digits("37828224631000"))); // 14
+    assert!(!plausible_card_number(&digits("3782822463100051"))); // 16
+    assert!(!plausible_card_number(&digits("350000000000000"))); // 35 alone
+
+    // JCB 3528-3589, 16-19.
+    assert!(plausible_card_number(&digits("3530111333300000")));
+    assert!(plausible_card_number(&digits("3589000000000000000"))); // 19
+    assert!(!plausible_card_number(&digits("3527000000000000")));
+    assert!(!plausible_card_number(&digits("3590000000000000")));
+    assert!(!plausible_card_number(&digits("353011133330000"))); // 15
+
+    // Diners Club — one scheme, one rule: 36, 300-305, 3095, 38, 39 all at
+    // 14-19. 30569309025904 is the canonical Diners test PAN; regression
+    // for the review finding that 14-digit Diners had been split away.
+    assert!(plausible_card_number(&digits("30569309025904"))); // 300-305 @ 14
+    assert!(plausible_card_number(&digits("38520000023237"))); // 38 @ 14
+    assert!(plausible_card_number(&digits("36700102000000"))); // 36 @ 14
+    assert!(plausible_card_number(&digits("30950000000000"))); // 3095 @ 14
+    assert!(plausible_card_number(&digits("39000000000005"))); // 39 @ 14
+    assert!(plausible_card_number(&digits("3050000000000000002"))); // 305 @ 19
+    assert!(!plausible_card_number(&digits("3060000000000000"))); // 306
+    assert!(!plausible_card_number(&digits("3700010200000"))); // 37 @ 13: not Diners
+
+    // Discover 6011 / 644-649 / 65 at 16 or 19.
+    assert!(plausible_card_number(&digits("6011111111111117")));
+    assert!(plausible_card_number(&digits("6440000000000000")));
+    assert!(plausible_card_number(&digits("6500000000000000000"))); // 19
+    assert!(!plausible_card_number(&digits("60111111111111170"))); // 17
+    assert!(!plausible_card_number(&digits("6430000000000000"))); // 643
+
+    // UnionPay 62 at 16-19.
+    assert!(plausible_card_number(&digits("6200000000000005")));
+    assert!(plausible_card_number(&digits("6200000000000000005"))); // 19
+    assert!(!plausible_card_number(&digits("620000000000000"))); // 15
+
+    // Maestro (5018/5020/5038/5893, 56-58, 6304/6759/6761-6763) at 13-19.
+    assert!(plausible_card_number(&digits("5018000000000"))); // 13
+    assert!(plausible_card_number(&digits("5600000000002"))); // 56 @ 13
+    assert!(plausible_card_number(&digits("5800000000000000008"))); // 58 @ 19
+    assert!(plausible_card_number(&digits("6759000000000000000"))); // 19
+    assert!(plausible_card_number(&digits("6763000000000000")));
+    assert!(!plausible_card_number(&digits("5019000000000000"))); // 5019
+    assert!(!plausible_card_number(&digits("5900000000000000"))); // 59
+    assert!(!plausible_card_number(&digits("6760000000000000"))); // 6760
+
+    // RuPay 60 / 508 / 81 / 82 at 16.
+    assert!(plausible_card_number(&digits("6069850000000000")));
+    assert!(plausible_card_number(&digits("5080000000000002")));
+    assert!(plausible_card_number(&digits("8100000000000000")));
+    assert!(plausible_card_number(&digits("8200000000000000")));
+    assert!(!plausible_card_number(&digits("8300000000000000"))); // 83
+    assert!(!plausible_card_number(&digits("810000000000000"))); // 15
+    assert!(!plausible_card_number(&digits("5090000000000000"))); // 509
+
+    // Elo 5041/5066/5067/6277/6362/6363 and Hipercard 6062, 16.
+    assert!(plausible_card_number(&digits("5067310000000010")));
+    assert!(plausible_card_number(&digits("5041000000000000")));
+    assert!(plausible_card_number(&digits("6362000000000009")));
+    assert!(plausible_card_number(&digits("6277000000000000")));
+    // Hipercard 6062 sits inside RuPay's blanket 60 range as well — listed
+    // in the table in its own right, but note 60xx@16 is corroborable for
+    // any xx, which is what the published RuPay range says.
+    assert!(plausible_card_number(&digits("6062821234567890"))); // Hipercard
+    assert!(!plausible_card_number(&digits("5065000000000001"))); // 5065
+    assert!(!plausible_card_number(&digits("6364000000000007"))); // 6364
+    assert!(!plausible_card_number(&digits("506731000000001"))); // 15
+
+    // The documented expiry (see the function doc): 16-digit
+    // epoch-microsecond stamps enter Mir/Mastercard-2-series territory
+    // around 2039/2040. Asserted as truth, not as an endorsement — when
+    // this line starts mattering, the gate needs a rethink.
+    assert!(plausible_card_number(&digits("2221787178633773"))); // µs in ~2040
+    assert!(!plausible_card_number(&digits("1787178633773000"))); // µs today
+}