diff --git a/src/memory/store/safety/pii.rs b/src/memory/store/safety/pii.rs index a0a6cec..a552fde 100644 --- a/src/memory/store/safety/pii.rs +++ b/src/memory/store/safety/pii.rs @@ -99,10 +99,38 @@ static PHONE_NANP_RE: LazyLock = LazyLock::new(|| { static SSN_RE: LazyLock = LazyLock::new(|| Regex::new(r"\b\d{3}-\d{2}-\d{4}\b").expect("ssn")); -// Credit card: 13-19 digits with optional spaces/dashes every 4. Luhn-gated. +// Credit card: 13-19 digits with optional spaces/dashes every 4. Every match +// is Luhn-gated; a match with no separators at all additionally needs +// corroboration — a real network IIN at an issued length, or a card keyword +// nearby — because Luhn alone passes ~10% of arbitrary digit runs, and bare +// 13-digit epoch-millisecond timestamps were being redacted out of stored +// JSON envelopes at exactly that rate (opencompany#1201). Same split as +// Aadhaar below: formatted keeps the checksum-only gate, bare needs more. static CC_RE: LazyLock = LazyLock::new(|| Regex::new(r"\b(?:\d[\s\-]?){13,19}\b").expect("credit card")); +// Card keyword corroborating a bare digit run. Three tiers, matched +// case-insensitively: +// +// * Standalone words, bounded by `[\W_]` rather than `\b` — the regex crate +// counts `_` as a word character, so `\bcard\b` never fires inside +// `card_number`, which is among the most common serialized key shapes a +// stored payload carries. The explicit class keeps `pan` from firing +// inside `japan` while still matching `card_number=` and `cc=`. +// * Compound identifiers matched as substrings, because camelCase provides +// no boundary of any kind: `cardNumber`, `creditCard`, `ccNum`, `cardNo`, +// `panNumber` all lowercase into these. +// * Native-script terms, per this module's multilingual mandate (the Aadhaar +// and My Number patterns already carry theirs). CJK terms match as +// substrings — CJK text does not put `[\W_]` between a word and the +// digits that follow it. +static CC_KEYWORD_RE: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?i)(?:^|[\W_])(?:card|credit|debit|visa|mastercard|amex|american\s?express|discover|jcb|diners|unionpay|maestro|hipercard|elo|rupay|cvv|cvc|cc|pan|tarjeta|cart[aã]o|carte|karte|карта|карты|карту|картой|карте|кредитка)(?:[\W_]|$)|(?i:cardnumber|creditcard|ccnum|cardno|pannumber|カード|信用卡|卡号|银行卡|카드)", + ) + .expect("cc keyword") +}); + // IBAN: 2 letter country code + 2 check digits + 11-30 alphanumeric. // Allow optional spaces every 4 chars (common human format). static IBAN_RE: LazyLock = @@ -203,8 +231,12 @@ pub fn redact_pii(text: &str) -> Sanitized { /// JIDs like `12025551234-1543890267@g.us`, telegram numeric peer IDs, /// millisecond timestamps, padded counters) is too high to use as a hard /// rejection signal. Content scrubbing via [`redact_pii`] still applies -/// those patterns — false positives are tolerable there because they only -/// replace bytes inside a string, not reject the whole write. +/// those patterns — a content false positive replaces bytes inside a string +/// rather than rejecting the whole write, which is cheaper but *not* free: +/// a redaction landing inside structured content corrupts it for whatever +/// wrote it (opencompany#1201 — timestamps in stored JSON envelopes), which +/// is why the credit-card pattern's bare form now demands corroboration +/// beyond its checksum. pub fn has_likely_pii(value: &str) -> bool { let nview = NormalizedView::build(value); let cand = scan_candidates(&nview.normalized); @@ -283,7 +315,7 @@ fn collect_redactions_inner(norm: &str, cand: &Candidates, include_bare_numeric: if include_bare_numeric { // Credit card before bare CPF/CNPJ to avoid catching a 13-19 digit run as CPF/CNPJ. if cand.cc { - push_checksum(&mut hits, norm, &CC_RE, PII_CC, valid_luhn); + push_credit_cards(&mut hits, norm); } if cand.cnpj_bare { push_checksum(&mut hits, norm, &CNPJ_BARE_RE, PII_CNPJ, |s| { @@ -407,6 +439,67 @@ fn push_captured( } } +/// Credit-card collection. Every match must pass Luhn; a bare match (no +/// separators) additionally needs structural or contextual corroboration. +/// +/// Luhn alone passes ~10% of arbitrary digit runs, and 13-19 contiguous +/// digits is a common machine-identifier shape — 13-digit epoch-millisecond +/// timestamps above all, which were being redacted out of stored JSON +/// envelopes at that rate and corrupting them (opencompany#1201). The strict +/// boundary set ([`collect_strict_redactions`]) already excludes credit card +/// for exactly this reason; this brings the content path to the same +/// judgement without giving up real cards: a separated run keeps the +/// Luhn-only gate it always had, and a bare run still redacts when its +/// prefix is a real network IIN at an issued length +/// ([`plausible_card_number`]) or a card keyword sits within +/// [`CC_KEYWORD_WINDOW`] bytes. +fn push_credit_cards(hits: &mut Vec, norm: &str) { + for m in CC_RE.find_iter(norm) { + let s = m.as_str(); + if !valid_luhn(s) { + continue; + } + // Judge bare-vs-separated on the interior of the digit run: the + // regex's per-digit `[\s\-]?` can capture one trailing separator + // (`"…773 "`), which is not the human 4-4-4-4 grouping this + // distinction is after. + let interior = s.trim_matches(|c: char| !c.is_ascii_digit()); + let bare = interior.bytes().all(|b| b.is_ascii_digit()); + let corroborated = + !bare || plausible_card_number(&digits(s)) || cc_keyword_near(norm, m.start(), m.end()); + if corroborated { + hits.push(Hit { + start: m.start(), + end: m.end(), + token: PII_CC, + }); + } + } +} + +/// Bytes of context searched either side of a bare digit run for a card +/// keyword. 64 bytes rather than 32 because the window is counted in bytes +/// while text is not: 32 bytes is only ~10 CJK characters or 8 emoji, so a +/// run of either could evict an English keyword that a reader would call +/// adjacent. 64 comfortably spans `{"payment_method":{"card":{"number":…` +/// and a `カード`-prefixed line alike. +const CC_KEYWORD_WINDOW: usize = 64; + +/// True when [`CC_KEYWORD_RE`] matches within the window around +/// `start..end`, widened outward to char boundaries so the slice cannot +/// split a multi-byte character. +fn cc_keyword_near(norm: &str, start: usize, end: usize) -> bool { + let mut lo = start.saturating_sub(CC_KEYWORD_WINDOW); + while lo > 0 && !norm.is_char_boundary(lo) { + lo -= 1; + } + let mut hi = (end + CC_KEYWORD_WINDOW).min(norm.len()); + while hi < norm.len() && !norm.is_char_boundary(hi) { + hi += 1; + } + CC_KEYWORD_RE.is_match(&norm[lo..hi]) +} + // Sort by start asc, length desc. Then walk in order, dropping any hit whose // range overlaps a kept hit. Result: earlier + longer wins; no double-redact. fn dedupe_overlaps(hits: &mut Vec) { diff --git a/src/memory/store/safety/pii/checks.rs b/src/memory/store/safety/pii/checks.rs index 04fe09d..2769dd7 100644 --- a/src/memory/store/safety/pii/checks.rs +++ b/src/memory/store/safety/pii/checks.rs @@ -79,6 +79,103 @@ pub(super) fn valid_luhn(s: &str) -> bool { sum.is_multiple_of(10) } +/// True when a digit string has a plausible payment-card shape: a known +/// major-network IIN prefix at a length that network actually issues. +/// +/// The structural gate behind bare (separator-less) credit-card redaction. +/// Luhn alone passes ~10% of arbitrary digit runs — the same raw +/// false-positive rate that put bare Aadhaar behind a keyword — and 13-19 +/// contiguous digits is a common machine-identifier shape: 13-digit +/// epoch-millisecond timestamps (`17…`/`18…` for decades either side of now) +/// sit squarely in the window and were being redacted out of stored JSON at +/// that rate (opencompany#1201). No card network issues from a `17`/`18` +/// prefix, so requiring a real IIN removes that entire class while keeping +/// every number a major network could actually have issued. +/// +/// The table lists each supported network's published IIN ranges at the +/// lengths that network issues: Visa, Mastercard (incl. the 2-series), Amex, +/// Discover, JCB, Diners Club, UnionPay, Maestro, Mir, RuPay, and the +/// Brazilian networks Elo and Hipercard (in scope by this module's own +/// design: it already carries bare and formatted CPF/CNPJ). It is a +/// *corroboration* tier, not an acquirer's validator: an exhaustive BIN +/// registry is a licensed, continuously updated database, and a range missing +/// here is not silently dropped from redaction — a bare PAN on an unlisted +/// network still redacts whenever a card keyword appears within the keyword +/// window (the third gate in `push_credit_cards`). +/// +/// One dated caveat, so nobody inherits a stronger claim than the code makes: +/// "timestamps can never corroborate" is prefix-and-length dependent, not +/// absolute. 13-digit epoch-milliseconds stay out of every range until the +/// year 2096 (`4…`, Visa's 13-digit arm); but 16-digit epoch-MICROsecond +/// stamps enter Mir's `2200-2204` window in late 2039 and Mastercard's +/// 2-series `2221-2720` from ~2040 to ~2056. If this code outlives that, +/// those stamps redact at Luhn's ~10% again and this gate needs a rethink. +pub(super) fn plausible_card_number(d: &[u32]) -> bool { + let len = d.len(); + if !(13..=19).contains(&len) { + return false; + } + // len >= 13 makes the first four digits always present. + let p2 = d[0] * 10 + d[1]; + let p3 = p2 * 10 + d[2]; + let p4 = p3 * 10 + d[3]; + match d[0] { + // Visa: 16 standard, 13 legacy, 19 extended. (Also where Elo's + // 4-prefixed ranges land, at the same 16.) + 4 => matches!(len, 13 | 16 | 19), + 5 => { + // Mastercard 51-55 (16 only). + ((51..=55).contains(&p2) && len == 16) + // Maestro 5018/5020/5038/5893 and 56-58. Maestro issues + // 12-19; the floor here is 13 because CC_RE requires 13 + // digits, so the explicit bound below is the whole window + // this function can see — kept explicit so the "at an + // issued length" promise stays visibly true. + || ((matches!(p4, 5018 | 5020 | 5038 | 5893) || (56..=58).contains(&p2)) + && (13..=19).contains(&len)) + // Elo 5041/5066/5067 (16), RuPay 508 (16). + || ((matches!(p4, 5041 | 5066 | 5067) || p3 == 508) && len == 16) + } + 2 => { + // Mastercard 2-series 2221-2720 (16 only). + ((2221..=2720).contains(&p4) && len == 16) + // Mir 2200-2204 (16 only). + || ((2200..=2204).contains(&p4) && len == 16) + } + 3 => { + // Amex 34/37 (15 only). + (matches!(p2, 34 | 37) && len == 15) + // JCB 3528-3589 (16-19). + || ((3528..=3589).contains(&p4) && (16..=19).contains(&len)) + // Diners Club 36 / 300-305 / 3095 / 38-39, one scheme, one + // length rule: 14 (Diners International / Carte Blanche + // classic — 30569309025904, the canonical test PAN, is 14) + // through 19. + || ((p2 == 36 + || (300..=305).contains(&p3) + || p4 == 3095 + || matches!(p2, 38 | 39)) + && (14..=19).contains(&len)) + } + 6 => { + // Discover 6011 / 644-649 / 65 (16 or 19). + ((p4 == 6011 || (644..=649).contains(&p3) || p2 == 65) && matches!(len, 16 | 19)) + // UnionPay 62 (16-19), which also covers the + // Discover-processed 622126-622925 range. + || (p2 == 62 && (16..=19).contains(&len)) + // Maestro 6304/6759/6761-6763, bounded as the 5-prefix + // Maestro arm above. + || (matches!(p4, 6304 | 6759 | 6761 | 6762 | 6763) && (13..=19).contains(&len)) + // RuPay 60 (16) beyond the 65 range shared with Discover; + // Elo 6277/6362/6363 and Hipercard 6062 (16). + || ((p2 == 60 || matches!(p4, 6277 | 6362 | 6363 | 6062)) && len == 16) + } + // RuPay 81/82 (16). + 8 => matches!(p2, 81 | 82) && len == 16, + _ => false, + } +} + // IBAN mod-97. Steps: strip spaces, move first 4 chars to end, expand letters // (A=10..Z=35), divide as a big-integer mod 97, require remainder == 1. pub(super) fn valid_iban(s: &str) -> bool { diff --git a/src/memory/store/safety/pii/checks_tests.rs b/src/memory/store/safety/pii/checks_tests.rs index 77aeea5..e05291e 100644 --- a/src/memory/store/safety/pii/checks_tests.rs +++ b/src/memory/store/safety/pii/checks_tests.rs @@ -41,3 +41,127 @@ fn identity_validators_cover_checksums_reserved_values_and_prefixes() { assert!(!valid_nino("DA123456A")); assert!(!valid_nino("AA12345A")); } + +#[test] +fn plausible_card_number_requires_a_real_iin_at_an_issued_length() { + // No network's prefix: epoch-millisecond timestamps and other machine ids. + assert!(!plausible_card_number(&digits("1787178633773"))); // 13-digit epoch ms + assert!(!plausible_card_number(&digits("1700000000000"))); + assert!(!plausible_card_number(&digits("9111111111111119"))); + assert!(!plausible_card_number(&digits("2000000000000000"))); // year-2033 epoch-µs shape + assert!(!plausible_card_number(&digits("1900000000000"))); // 13-digit, no IIN starts 1 + + // Out of the card length window entirely. + assert!(!plausible_card_number(&digits("411111111111"))); // 12 + assert!(!plausible_card_number(&digits("41111111111111111111"))); // 20 +} + +// The accept direction, per network, at its boundary lengths — with the +// reject cases one step past each length and each range edge. Luhn is +// irrelevant here: the function judges shape only, and the redaction path +// tests Luhn separately. +#[test] +fn plausible_card_number_accepts_each_network_at_its_boundary_lengths() { + // Visa 13/16/19; nothing between or past. + assert!(plausible_card_number(&digits("4222222222222"))); // 13 + assert!(plausible_card_number(&digits("4111111111111111"))); // 16 + assert!(plausible_card_number(&digits("4111111111111111111"))); // 19 + assert!(!plausible_card_number(&digits("41111111111111"))); // 14 + assert!(!plausible_card_number(&digits("411111111111111"))); // 15 + assert!(!plausible_card_number(&digits("41111111111111111"))); // 17 + + // Mastercard 51-55 and 2221-2720, 16 only; edges out both sides. + assert!(plausible_card_number(&digits("5100000000000000"))); + assert!(plausible_card_number(&digits("5500005555555559"))); + assert!(plausible_card_number(&digits("2221000000000009"))); + assert!(plausible_card_number(&digits("2720000000000000"))); + assert!(!plausible_card_number(&digits("550000555555555"))); // 15 + assert!(!plausible_card_number(&digits("55000055555555590"))); // 17 + assert!(!plausible_card_number(&digits("5000000000000000"))); // 50: not MC + assert!(!plausible_card_number(&digits("2220000000000000"))); // below 2221 + assert!(!plausible_card_number(&digits("2721000000000000"))); // above 2720 + + // Mir 2200-2204, 16 only. + assert!(plausible_card_number(&digits("2200000000000004"))); + assert!(plausible_card_number(&digits("2204000000000000"))); + assert!(!plausible_card_number(&digits("2205000000000009"))); // past range + assert!(!plausible_card_number(&digits("220000000000000"))); // 15 + assert!(!plausible_card_number(&digits("22000000000000004"))); // 17 + + // Amex 34/37, 15 only. + assert!(plausible_card_number(&digits("378282246310005"))); + assert!(plausible_card_number(&digits("340000000000009"))); + assert!(!plausible_card_number(&digits("37828224631000"))); // 14 + assert!(!plausible_card_number(&digits("3782822463100051"))); // 16 + assert!(!plausible_card_number(&digits("350000000000000"))); // 35 alone + + // JCB 3528-3589, 16-19. + assert!(plausible_card_number(&digits("3530111333300000"))); + assert!(plausible_card_number(&digits("3589000000000000000"))); // 19 + assert!(!plausible_card_number(&digits("3527000000000000"))); + assert!(!plausible_card_number(&digits("3590000000000000"))); + assert!(!plausible_card_number(&digits("353011133330000"))); // 15 + + // Diners Club — one scheme, one rule: 36, 300-305, 3095, 38, 39 all at + // 14-19. 30569309025904 is the canonical Diners test PAN; regression + // for the review finding that 14-digit Diners had been split away. + assert!(plausible_card_number(&digits("30569309025904"))); // 300-305 @ 14 + assert!(plausible_card_number(&digits("38520000023237"))); // 38 @ 14 + assert!(plausible_card_number(&digits("36700102000000"))); // 36 @ 14 + assert!(plausible_card_number(&digits("30950000000000"))); // 3095 @ 14 + assert!(plausible_card_number(&digits("39000000000005"))); // 39 @ 14 + assert!(plausible_card_number(&digits("3050000000000000002"))); // 305 @ 19 + assert!(!plausible_card_number(&digits("3060000000000000"))); // 306 + assert!(!plausible_card_number(&digits("3700010200000"))); // 37 @ 13: not Diners + + // Discover 6011 / 644-649 / 65 at 16 or 19. + assert!(plausible_card_number(&digits("6011111111111117"))); + assert!(plausible_card_number(&digits("6440000000000000"))); + assert!(plausible_card_number(&digits("6500000000000000000"))); // 19 + assert!(!plausible_card_number(&digits("60111111111111170"))); // 17 + assert!(!plausible_card_number(&digits("6430000000000000"))); // 643 + + // UnionPay 62 at 16-19. + assert!(plausible_card_number(&digits("6200000000000005"))); + assert!(plausible_card_number(&digits("6200000000000000005"))); // 19 + assert!(!plausible_card_number(&digits("620000000000000"))); // 15 + + // Maestro (5018/5020/5038/5893, 56-58, 6304/6759/6761-6763) at 13-19. + assert!(plausible_card_number(&digits("5018000000000"))); // 13 + assert!(plausible_card_number(&digits("5600000000002"))); // 56 @ 13 + assert!(plausible_card_number(&digits("5800000000000000008"))); // 58 @ 19 + assert!(plausible_card_number(&digits("6759000000000000000"))); // 19 + assert!(plausible_card_number(&digits("6763000000000000"))); + assert!(!plausible_card_number(&digits("5019000000000000"))); // 5019 + assert!(!plausible_card_number(&digits("5900000000000000"))); // 59 + assert!(!plausible_card_number(&digits("6760000000000000"))); // 6760 + + // RuPay 60 / 508 / 81 / 82 at 16. + assert!(plausible_card_number(&digits("6069850000000000"))); + assert!(plausible_card_number(&digits("5080000000000002"))); + assert!(plausible_card_number(&digits("8100000000000000"))); + assert!(plausible_card_number(&digits("8200000000000000"))); + assert!(!plausible_card_number(&digits("8300000000000000"))); // 83 + assert!(!plausible_card_number(&digits("810000000000000"))); // 15 + assert!(!plausible_card_number(&digits("5090000000000000"))); // 509 + + // Elo 5041/5066/5067/6277/6362/6363 and Hipercard 6062, 16. + assert!(plausible_card_number(&digits("5067310000000010"))); + assert!(plausible_card_number(&digits("5041000000000000"))); + assert!(plausible_card_number(&digits("6362000000000009"))); + assert!(plausible_card_number(&digits("6277000000000000"))); + // Hipercard 6062 sits inside RuPay's blanket 60 range as well — listed + // in the table in its own right, but note 60xx@16 is corroborable for + // any xx, which is what the published RuPay range says. + assert!(plausible_card_number(&digits("6062821234567890"))); // Hipercard + assert!(!plausible_card_number(&digits("5065000000000001"))); // 5065 + assert!(!plausible_card_number(&digits("6364000000000007"))); // 6364 + assert!(!plausible_card_number(&digits("506731000000001"))); // 15 + + // The documented expiry (see the function doc): 16-digit + // epoch-microsecond stamps enter Mir/Mastercard-2-series territory + // around 2039/2040. Asserted as truth, not as an endorsement — when + // this line starts mattering, the gate needs a rethink. + assert!(plausible_card_number(&digits("2221787178633773"))); // µs in ~2040 + assert!(!plausible_card_number(&digits("1787178633773000"))); // µs today +} diff --git a/src/memory/store/safety/pii_tests.rs b/src/memory/store/safety/pii_tests.rs index 125638b..e6b4c48 100644 --- a/src/memory/store/safety/pii_tests.rs +++ b/src/memory/store/safety/pii_tests.rs @@ -139,6 +139,105 @@ fn credit_card_amex_redacted() { fn credit_card_invalid_luhn_kept() { unchanged("invoice 4111 1111 1111 1112"); } +#[test] +fn credit_card_bare_visa_redacted_without_keyword() { + // A real network IIN (Visa `4`) at an issued length: bare runs with an + // issued card shape still redact with no keyword anywhere near. + redacts("4111111111111111", PII_CC); +} +#[test] +fn credit_card_bare_amex_redacted_without_keyword() { + redacts("378282246310005", PII_CC); +} +#[test] +fn credit_card_keyword_corroborates_a_bare_non_iin_run() { + // Luhn-valid, but `17` is no network's IIN — the keyword is what makes + // this a card mention rather than a machine identifier. + redacts("card 1787178633773", PII_CC); +} +#[test] +fn bare_luhn_valid_timestamp_kept() { + // A 13-digit epoch-millisecond timestamp that happens to pass Luhn + // (~10% of them do). No IIN, no keyword: not a card. + unchanged("run 1787178633773 finished"); +} +#[test] +fn credit_card_keyword_matches_serialized_key_shapes() { + // The keyword net is what the IIN table's incompleteness leans on, so it + // has to fire for the key shapes serialized payloads actually use. The + // digit run is Luhn-valid with no network's IIN (`17…`), so only the + // keyword tier can be doing the work in each of these. + for text in [ + r#"{"card_number":"1787178633773"}"#, + r#"{"cardNumber":"1787178633773"}"#, + r#"{"credit_card":"1787178633773"}"#, + r#"{"creditCard":"1787178633773"}"#, + r#"{"ccNumber":"1787178633773"}"#, + r#"{"card_no":"1787178633773"}"#, + "CARD_NUMBER=1787178633773", + "cc=1787178633773", + ] { + let out = redact_pii(text); + assert!( + out.value.contains(PII_CC), + "expected the keyword tier to corroborate: {text:?} -> {out:?}" + ); + } +} + +#[test] +fn credit_card_keyword_speaks_more_than_english() { + // Multilingual mandate: native card words corroborate too, and the + // window is wide enough that non-ASCII text does not evict them. + for text in [ + "カード 1787178633773", + "信用卡 1787178633773", + "카드 1787178633773", + "карта 1787178633773", + "tarjeta 1787178633773", + "cartão 1787178633773", + "card 😀😀😀😀😀😀😀😀 1787178633773", + ] { + let out = redact_pii(text); + assert!( + out.value.contains(PII_CC), + "expected corroboration: {text:?} -> {out:?}" + ); + } +} + +#[test] +fn credit_card_keyword_still_respects_word_boundaries() { + // `pan` must not fire inside an unrelated word: Luhn-valid non-IIN run + // next to `japan` stays untouched. + unchanged("japan 1787178633773 spans"); +} + +#[test] +fn bare_brazilian_network_pans_redact_without_keyword() { + // Elo and Hipercard are in the IIN table (the module targets Brazilian + // PII by design — it carries CPF/CNPJ), so their bare PANs corroborate + // structurally, keyword or not. + redacts("5067310000000010", PII_CC); + redacts("6062821234567890", PII_CC); +} + +#[test] +fn bare_diners_14_digit_pan_redacts() { + // Regression for the review finding: the canonical 14-digit Diners test + // PAN redacted before the corroboration gate and must keep redacting. + redacts("30569309025904", PII_CC); +} + +#[test] +fn json_envelope_luhn_valid_timestamp_kept() { + // opencompany#1201: the exact corruption — a serialized record whose + // `at_millis` passed Luhn was redacted into unparseable JSON, and the + // read side then dropped the whole record as undecodable. + unchanged( + r#"{"v":1,"record":{"cycle_id":"c1","summary":"summary 1","at_millis":1787178633773}}"#, + ); +} // --- IBAN --- #[test]