From cfc821210b2ebb970e4fa800ede3cb677d79a4f6 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 20 Aug 2026 15:06:05 +0530 Subject: [PATCH 1/3] fix(pii): require corroboration before redacting a bare digit run as a credit card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Luhn alone passes ~10% of arbitrary digit runs, and 13-19 contiguous digits is a common machine-identifier shape — above all 13-digit epoch-millisecond timestamps, which were being redacted out of stored JSON envelopes at that rate and corrupting them into unparseable records (opencompany#1201: the embedded namespace driver's conformance suite went red on ~36% of runs because a scrubbed at_millis broke the envelope and the read side dropped the whole record). A separated run (the human 4-4-4-4 grouping) keeps the Luhn-only gate it always had. A bare run now additionally needs a real network IIN prefix at an issued length (plausible_card_number) or a card keyword within the keyword window (CC_KEYWORD_RE) — so bare card dumps still redact with no keyword anywhere near, while no unassigned prefix can corroborate. Same split the file already applies to Aadhaar (checksum when formatted, keyword when bare), and the same judgement has_likely_pii already makes by excluding credit card from the strict boundary set. --- src/memory/store/safety/pii.rs | 76 ++++++++++++++++++++- src/memory/store/safety/pii/checks.rs | 48 +++++++++++++ src/memory/store/safety/pii/checks_tests.rs | 28 ++++++++ src/memory/store/safety/pii_tests.rs | 31 +++++++++ 4 files changed, 181 insertions(+), 2 deletions(-) diff --git a/src/memory/store/safety/pii.rs b/src/memory/store/safety/pii.rs index a0a6cec..a719582 100644 --- a/src/memory/store/safety/pii.rs +++ b/src/memory/store/safety/pii.rs @@ -99,10 +99,25 @@ static PHONE_NANP_RE: LazyLock = LazyLock::new(|| { static SSN_RE: LazyLock = LazyLock::new(|| Regex::new(r"\b\d{3}-\d{2}-\d{4}\b").expect("ssn")); -// Credit card: 13-19 digits with optional spaces/dashes every 4. Luhn-gated. +// Credit card: 13-19 digits with optional spaces/dashes every 4. Every match +// is Luhn-gated; a match with no separators at all additionally needs +// corroboration — a real network IIN at an issued length, or a card keyword +// nearby — because Luhn alone passes ~10% of arbitrary digit runs, and bare +// 13-digit epoch-millisecond timestamps were being redacted out of stored +// JSON envelopes at exactly that rate (opencompany#1201). Same split as +// Aadhaar below: formatted keeps the checksum-only gate, bare needs more. static CC_RE: LazyLock = LazyLock::new(|| Regex::new(r"\b(?:\d[\s\-]?){13,19}\b").expect("credit card")); +// Card keyword corroborating a bare digit run. Word-bounded, so `pan` (the +// payment-industry term) cannot fire inside `japan` or `panel`. +static CC_KEYWORD_RE: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?i)\b(?:card|credit|debit|visa|mastercard|amex|american\s?express|discover|jcb|diners|unionpay|cvv|cvc|pan)\b", + ) + .expect("cc keyword") +}); + // IBAN: 2 letter country code + 2 check digits + 11-30 alphanumeric. // Allow optional spaces every 4 chars (common human format). static IBAN_RE: LazyLock = @@ -283,7 +298,7 @@ fn collect_redactions_inner(norm: &str, cand: &Candidates, include_bare_numeric: if include_bare_numeric { // Credit card before bare CPF/CNPJ to avoid catching a 13-19 digit run as CPF/CNPJ. if cand.cc { - push_checksum(&mut hits, norm, &CC_RE, PII_CC, valid_luhn); + push_credit_cards(&mut hits, norm); } if cand.cnpj_bare { push_checksum(&mut hits, norm, &CNPJ_BARE_RE, PII_CNPJ, |s| { @@ -407,6 +422,63 @@ fn push_captured( } } +/// Credit-card collection. Every match must pass Luhn; a bare match (no +/// separators) additionally needs structural or contextual corroboration. +/// +/// Luhn alone passes ~10% of arbitrary digit runs, and 13-19 contiguous +/// digits is a common machine-identifier shape — 13-digit epoch-millisecond +/// timestamps above all, which were being redacted out of stored JSON +/// envelopes at that rate and corrupting them (opencompany#1201). The strict +/// boundary set ([`collect_strict_redactions`]) already excludes credit card +/// for exactly this reason; this brings the content path to the same +/// judgement without giving up real cards: a separated run keeps the +/// Luhn-only gate it always had, and a bare run still redacts when its +/// prefix is a real network IIN at an issued length +/// ([`plausible_card_number`]) or a card keyword sits within +/// [`CC_KEYWORD_WINDOW`] bytes. +fn push_credit_cards(hits: &mut Vec, norm: &str) { + for m in CC_RE.find_iter(norm) { + let s = m.as_str(); + if !valid_luhn(s) { + continue; + } + // Judge bare-vs-separated on the interior of the digit run: the + // regex's per-digit `[\s\-]?` can capture one trailing separator + // (`"…773 "`), which is not the human 4-4-4-4 grouping this + // distinction is after. + let interior = s.trim_matches(|c: char| !c.is_ascii_digit()); + let bare = interior.bytes().all(|b| b.is_ascii_digit()); + let corroborated = + !bare || plausible_card_number(&digits(s)) || cc_keyword_near(norm, m.start(), m.end()); + if corroborated { + hits.push(Hit { + start: m.start(), + end: m.end(), + token: PII_CC, + }); + } + } +} + +/// Bytes of context searched either side of a bare digit run for a card +/// keyword. Wide enough for "credit card number is" plus punctuation. +const CC_KEYWORD_WINDOW: usize = 32; + +/// True when [`CC_KEYWORD_RE`] matches within the window around +/// `start..end`, widened outward to char boundaries so the slice cannot +/// split a multi-byte character. +fn cc_keyword_near(norm: &str, start: usize, end: usize) -> bool { + let mut lo = start.saturating_sub(CC_KEYWORD_WINDOW); + while lo > 0 && !norm.is_char_boundary(lo) { + lo -= 1; + } + let mut hi = (end + CC_KEYWORD_WINDOW).min(norm.len()); + while hi < norm.len() && !norm.is_char_boundary(hi) { + hi += 1; + } + CC_KEYWORD_RE.is_match(&norm[lo..hi]) +} + // Sort by start asc, length desc. Then walk in order, dropping any hit whose // range overlaps a kept hit. Result: earlier + longer wins; no double-redact. fn dedupe_overlaps(hits: &mut Vec) { diff --git a/src/memory/store/safety/pii/checks.rs b/src/memory/store/safety/pii/checks.rs index 04fe09d..3b22c07 100644 --- a/src/memory/store/safety/pii/checks.rs +++ b/src/memory/store/safety/pii/checks.rs @@ -79,6 +79,54 @@ pub(super) fn valid_luhn(s: &str) -> bool { sum.is_multiple_of(10) } +/// True when a digit string has a plausible payment-card shape: a known +/// major-network IIN prefix at a length that network actually issues. +/// +/// The structural gate behind bare (separator-less) credit-card redaction. +/// Luhn alone passes ~10% of arbitrary digit runs — the same raw +/// false-positive rate that put bare Aadhaar behind a keyword — and 13-19 +/// contiguous digits is a common machine-identifier shape: 13-digit +/// epoch-millisecond timestamps (`17…`/`18…` for decades either side of now) +/// sit squarely in the window and were being redacted out of stored JSON at +/// that rate (opencompany#1201). No card network issues from a `17`/`18` +/// prefix, so requiring a real IIN removes that entire class while keeping +/// every number a major network could actually have issued. +/// +/// Deliberately conservative on both axes: majors only (Visa, Mastercard +/// incl. the 2-series, Amex, Discover, JCB, Diners, UnionPay), each prefix +/// only at its network's issued lengths. +pub(super) fn plausible_card_number(d: &[u32]) -> bool { + let len = d.len(); + if !(13..=19).contains(&len) { + return false; + } + // len >= 13 makes the first four digits always present. + let p2 = d[0] * 10 + d[1]; + let p3 = p2 * 10 + d[2]; + let p4 = p3 * 10 + d[3]; + match d[0] { + // Visa; 13-digit cards are legacy but real. + 4 => matches!(len, 13 | 16 | 19), + // Mastercard 51-55. + 5 => (51..=55).contains(&p2) && len == 16, + // Mastercard 2-series. + 2 => (2221..=2720).contains(&p4) && len == 16, + 3 => { + // Amex 34/37, JCB 3528-3589, Diners 300-305/36/38. + (matches!(p2, 34 | 37) && len == 15) + || ((3528..=3589).contains(&p4) && (16..=19).contains(&len)) + || (((300..=305).contains(&p3) || matches!(p2, 36 | 38)) + && (14..=19).contains(&len)) + } + 6 => { + // Discover 6011 / 644-649 / 65, UnionPay 62. + ((p4 == 6011 || (644..=649).contains(&p3) || p2 == 65) && matches!(len, 16 | 19)) + || (p2 == 62 && (16..=19).contains(&len)) + } + _ => false, + } +} + // IBAN mod-97. Steps: strip spaces, move first 4 chars to end, expand letters // (A=10..Z=35), divide as a big-integer mod 97, require remainder == 1. pub(super) fn valid_iban(s: &str) -> bool { diff --git a/src/memory/store/safety/pii/checks_tests.rs b/src/memory/store/safety/pii/checks_tests.rs index 77aeea5..2d8a7b2 100644 --- a/src/memory/store/safety/pii/checks_tests.rs +++ b/src/memory/store/safety/pii/checks_tests.rs @@ -41,3 +41,31 @@ fn identity_validators_cover_checksums_reserved_values_and_prefixes() { assert!(!valid_nino("DA123456A")); assert!(!valid_nino("AA12345A")); } + +#[test] +fn plausible_card_number_requires_a_real_iin_at_an_issued_length() { + // Issued shapes on major networks. + assert!(plausible_card_number(&digits("4111111111111111"))); // Visa 16 + assert!(plausible_card_number(&digits("4222222222222"))); // Visa 13 (legacy) + assert!(plausible_card_number(&digits("5500005555555559"))); // Mastercard 55 + assert!(plausible_card_number(&digits("2221000000000009"))); // Mastercard 2-series + assert!(plausible_card_number(&digits("378282246310005"))); // Amex 37, 15 + assert!(plausible_card_number(&digits("6011111111111117"))); // Discover + assert!(plausible_card_number(&digits("3530111333300000"))); // JCB + assert!(plausible_card_number(&digits("36700102000000"))); // Diners 36, 14 + assert!(plausible_card_number(&digits("6200000000000005"))); // UnionPay + + // Right prefix at a length the network does not issue. + assert!(!plausible_card_number(&digits("41111111111111"))); // Visa at 14 + assert!(!plausible_card_number(&digits("37828224631000"))); // Amex at 14 + assert!(!plausible_card_number(&digits("55000055555555590"))); // MC at 17 + + // No network's prefix: epoch-millisecond timestamps and other machine ids. + assert!(!plausible_card_number(&digits("1787178633773"))); // 13-digit epoch ms + assert!(!plausible_card_number(&digits("1700000000000"))); + assert!(!plausible_card_number(&digits("9111111111111119"))); + + // Out of the card length window entirely. + assert!(!plausible_card_number(&digits("411111111111"))); // 12 + assert!(!plausible_card_number(&digits("41111111111111111111"))); // 20 +} diff --git a/src/memory/store/safety/pii_tests.rs b/src/memory/store/safety/pii_tests.rs index 125638b..05b6d4e 100644 --- a/src/memory/store/safety/pii_tests.rs +++ b/src/memory/store/safety/pii_tests.rs @@ -139,6 +139,37 @@ fn credit_card_amex_redacted() { fn credit_card_invalid_luhn_kept() { unchanged("invoice 4111 1111 1111 1112"); } +#[test] +fn credit_card_bare_visa_redacted_without_keyword() { + // A real network IIN (Visa `4`) at an issued length: bare runs with an + // issued card shape still redact with no keyword anywhere near. + redacts("4111111111111111", PII_CC); +} +#[test] +fn credit_card_bare_amex_redacted_without_keyword() { + redacts("378282246310005", PII_CC); +} +#[test] +fn credit_card_keyword_corroborates_a_bare_non_iin_run() { + // Luhn-valid, but `17` is no network's IIN — the keyword is what makes + // this a card mention rather than a machine identifier. + redacts("card 1787178633773", PII_CC); +} +#[test] +fn bare_luhn_valid_timestamp_kept() { + // A 13-digit epoch-millisecond timestamp that happens to pass Luhn + // (~10% of them do). No IIN, no keyword: not a card. + unchanged("run 1787178633773 finished"); +} +#[test] +fn json_envelope_luhn_valid_timestamp_kept() { + // opencompany#1201: the exact corruption — a serialized record whose + // `at_millis` passed Luhn was redacted into unparseable JSON, and the + // read side then dropped the whole record as undecodable. + unchanged( + r#"{"v":1,"record":{"cycle_id":"c1","summary":"summary 1","at_millis":1787178633773}}"#, + ); +} // --- IBAN --- #[test] From 2276271648b2d7822aff4adffecaf6f3deedc541 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 20 Aug 2026 15:06:05 +0530 Subject: [PATCH 2/3] fix(pii): exact IIN ranges and per-network lengths in plausible_card_number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up (#154, CodeRabbit): replace the first cut's blanket predicates with each network's published ranges at its issued lengths, adding Maestro (5018/5020/5038/5893, 6304/6759/6761-6763), Mir (2200-2204), RuPay (60/81/82 beyond the shared 65) and Diners 3095/38/39. The doc now states the table's role explicitly: corroboration, not an acquirer's validator — an unlisted range still redacts via the keyword gate. Errata, recorded rather than hidden: the edit that produced this commit also clobbered the per-network accept-direction tests it claimed to add (a script reused a stale buffer and overwrote its own insertion). The suite that actually pins this table lands in the next commit. --- src/memory/store/safety/pii/checks.rs | 50 ++++++++++++++++----- src/memory/store/safety/pii/checks_tests.rs | 18 +------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/memory/store/safety/pii/checks.rs b/src/memory/store/safety/pii/checks.rs index 3b22c07..1b09537 100644 --- a/src/memory/store/safety/pii/checks.rs +++ b/src/memory/store/safety/pii/checks.rs @@ -92,9 +92,14 @@ pub(super) fn valid_luhn(s: &str) -> bool { /// prefix, so requiring a real IIN removes that entire class while keeping /// every number a major network could actually have issued. /// -/// Deliberately conservative on both axes: majors only (Visa, Mastercard -/// incl. the 2-series, Amex, Discover, JCB, Diners, UnionPay), each prefix -/// only at its network's issued lengths. +/// The table lists each supported network's published IIN ranges at the +/// lengths that network issues: Visa, Mastercard (incl. the 2-series), Amex, +/// Discover, JCB, Diners Club, UnionPay, Maestro, Mir, and RuPay. It is a +/// *corroboration* tier, not an acquirer's validator: an exhaustive BIN +/// registry is a licensed, continuously updated database, and a range missing +/// here is not silently dropped from redaction — a bare PAN on an unlisted +/// network still redacts whenever a card keyword appears within the keyword +/// window (the third gate in `push_credit_cards`). pub(super) fn plausible_card_number(d: &[u32]) -> bool { let len = d.len(); if !(13..=19).contains(&len) { @@ -105,24 +110,45 @@ pub(super) fn plausible_card_number(d: &[u32]) -> bool { let p3 = p2 * 10 + d[2]; let p4 = p3 * 10 + d[3]; match d[0] { - // Visa; 13-digit cards are legacy but real. + // Visa: 16 standard, 13 legacy, 19 extended. 4 => matches!(len, 13 | 16 | 19), - // Mastercard 51-55. - 5 => (51..=55).contains(&p2) && len == 16, - // Mastercard 2-series. - 2 => (2221..=2720).contains(&p4) && len == 16, + 5 => { + // Mastercard 51-55 (16 only). + ((51..=55).contains(&p2) && len == 16) + // Maestro 5018/5020/5038/5893 (12-19 issued; 13 is this + // module's floor because CC_RE requires 13 digits). + || matches!(p4, 5018 | 5020 | 5038 | 5893) + } + 2 => { + // Mastercard 2-series 2221-2720 (16 only). + ((2221..=2720).contains(&p4) && len == 16) + // Mir 2200-2204 (16 only). + || ((2200..=2204).contains(&p4) && len == 16) + } 3 => { - // Amex 34/37, JCB 3528-3589, Diners 300-305/36/38. + // Amex 34/37 (15 only). (matches!(p2, 34 | 37) && len == 15) + // JCB 3528-3589 (16-19). || ((3528..=3589).contains(&p4) && (16..=19).contains(&len)) - || (((300..=305).contains(&p3) || matches!(p2, 36 | 38)) - && (14..=19).contains(&len)) + // Diners Club: 36 at the classic 14 up to 19; 300-305, 3095, + // 38-39 at 16-19. + || (p2 == 36 && (14..=19).contains(&len)) + || (((300..=305).contains(&p3) || p4 == 3095 || matches!(p2, 38 | 39)) + && (16..=19).contains(&len)) } 6 => { - // Discover 6011 / 644-649 / 65, UnionPay 62. + // Discover 6011 / 644-649 / 65 (16 or 19). ((p4 == 6011 || (644..=649).contains(&p3) || p2 == 65) && matches!(len, 16 | 19)) + // UnionPay 62 (16-19), which also covers the + // Discover-processed 622126-622925 range. || (p2 == 62 && (16..=19).contains(&len)) + // Maestro 6304/6759/6761-6763 (13-19, floor as above). + || matches!(p4, 6304 | 6759 | 6761 | 6762 | 6763) + // RuPay 60 (16), beyond the 65 range shared with Discover. + || (p2 == 60 && len == 16) } + // RuPay 81/82 (16). + 8 => matches!(p2, 81 | 82) && len == 16, _ => false, } } diff --git a/src/memory/store/safety/pii/checks_tests.rs b/src/memory/store/safety/pii/checks_tests.rs index 2d8a7b2..01c2ec6 100644 --- a/src/memory/store/safety/pii/checks_tests.rs +++ b/src/memory/store/safety/pii/checks_tests.rs @@ -44,26 +44,12 @@ fn identity_validators_cover_checksums_reserved_values_and_prefixes() { #[test] fn plausible_card_number_requires_a_real_iin_at_an_issued_length() { - // Issued shapes on major networks. - assert!(plausible_card_number(&digits("4111111111111111"))); // Visa 16 - assert!(plausible_card_number(&digits("4222222222222"))); // Visa 13 (legacy) - assert!(plausible_card_number(&digits("5500005555555559"))); // Mastercard 55 - assert!(plausible_card_number(&digits("2221000000000009"))); // Mastercard 2-series - assert!(plausible_card_number(&digits("378282246310005"))); // Amex 37, 15 - assert!(plausible_card_number(&digits("6011111111111117"))); // Discover - assert!(plausible_card_number(&digits("3530111333300000"))); // JCB - assert!(plausible_card_number(&digits("36700102000000"))); // Diners 36, 14 - assert!(plausible_card_number(&digits("6200000000000005"))); // UnionPay - - // Right prefix at a length the network does not issue. - assert!(!plausible_card_number(&digits("41111111111111"))); // Visa at 14 - assert!(!plausible_card_number(&digits("37828224631000"))); // Amex at 14 - assert!(!plausible_card_number(&digits("55000055555555590"))); // MC at 17 - // No network's prefix: epoch-millisecond timestamps and other machine ids. assert!(!plausible_card_number(&digits("1787178633773"))); // 13-digit epoch ms assert!(!plausible_card_number(&digits("1700000000000"))); assert!(!plausible_card_number(&digits("9111111111111119"))); + assert!(!plausible_card_number(&digits("2000000000000000"))); // year-2033 epoch-µs shape + assert!(!plausible_card_number(&digits("1900000000000"))); // 13-digit, no IIN starts 1 // Out of the card length window entirely. assert!(!plausible_card_number(&digits("411111111111"))); // 12 From ef1eb19bf6fa698c812e9c70667773e47f66507d Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 20 Aug 2026 15:06:05 +0530 Subject: [PATCH 3/3] fix(pii): Diners at 14, a keyword net that matches serialized keys, and the boundary suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up (#154, maintainer review). Three substantive fixes and the tests that were missing: - Diners Club is one scheme with one length rule: 36, 300-305, 3095 and 38-39 all accept 14-19. The previous split (36 at 14, the rest at 16+) regressed canonical 14-digit Diners PANs (30569309025904) that redacted before the corroboration gate existed. - The keyword net now matches the key shapes serialized payloads actually use. [\W_] boundaries instead of \b (the regex crate counts _ as a word character, so \bcard\b never fired inside card_number), a substring tier for camelCase compounds (cardNumber, creditCard, ccNum, cardNo, panNumber), and native-script terms per the module's multilingual mandate (カード, 信用卡, 卡号, 카드, карта…, tarjeta, cartão, carte, Karte). Window widened 32 -> 64 bytes so non-ASCII text cannot evict an adjacent keyword. - Brazilian networks join the table in their own right (Elo 5041/5066/5067/6277/6362/6363, Hipercard 6062, plus RuPay 508 and Maestro 56-58), consistent with a module that already carries bare and formatted CPF/CNPJ. The per-network boundary suite exists this time — accept direction at each network's edge lengths, reject one past each length and range edge — plus redaction-path tests for the serialized-key shapes, the multilingual keywords, and the 14-digit Diners regression. Docs pick up the Maestro length note, the corrected false-positive framing on has_likely_pii, and the dated caveat that 16-digit epoch-microsecond stamps enter the Mir/Mastercard-2-series windows around 2039-2056. --- src/memory/store/safety/pii.rs | 35 +++++-- src/memory/store/safety/pii/checks.rs | 51 ++++++--- src/memory/store/safety/pii/checks_tests.rs | 110 ++++++++++++++++++++ src/memory/store/safety/pii_tests.rs | 68 ++++++++++++ 4 files changed, 243 insertions(+), 21 deletions(-) diff --git a/src/memory/store/safety/pii.rs b/src/memory/store/safety/pii.rs index a719582..a552fde 100644 --- a/src/memory/store/safety/pii.rs +++ b/src/memory/store/safety/pii.rs @@ -109,11 +109,24 @@ static SSN_RE: LazyLock = static CC_RE: LazyLock = LazyLock::new(|| Regex::new(r"\b(?:\d[\s\-]?){13,19}\b").expect("credit card")); -// Card keyword corroborating a bare digit run. Word-bounded, so `pan` (the -// payment-industry term) cannot fire inside `japan` or `panel`. +// Card keyword corroborating a bare digit run. Three tiers, matched +// case-insensitively: +// +// * Standalone words, bounded by `[\W_]` rather than `\b` — the regex crate +// counts `_` as a word character, so `\bcard\b` never fires inside +// `card_number`, which is among the most common serialized key shapes a +// stored payload carries. The explicit class keeps `pan` from firing +// inside `japan` while still matching `card_number=` and `cc=`. +// * Compound identifiers matched as substrings, because camelCase provides +// no boundary of any kind: `cardNumber`, `creditCard`, `ccNum`, `cardNo`, +// `panNumber` all lowercase into these. +// * Native-script terms, per this module's multilingual mandate (the Aadhaar +// and My Number patterns already carry theirs). CJK terms match as +// substrings — CJK text does not put `[\W_]` between a word and the +// digits that follow it. static CC_KEYWORD_RE: LazyLock = LazyLock::new(|| { Regex::new( - r"(?i)\b(?:card|credit|debit|visa|mastercard|amex|american\s?express|discover|jcb|diners|unionpay|cvv|cvc|pan)\b", + r"(?i)(?:^|[\W_])(?:card|credit|debit|visa|mastercard|amex|american\s?express|discover|jcb|diners|unionpay|maestro|hipercard|elo|rupay|cvv|cvc|cc|pan|tarjeta|cart[aã]o|carte|karte|карта|карты|карту|картой|карте|кредитка)(?:[\W_]|$)|(?i:cardnumber|creditcard|ccnum|cardno|pannumber|カード|信用卡|卡号|银行卡|카드)", ) .expect("cc keyword") }); @@ -218,8 +231,12 @@ pub fn redact_pii(text: &str) -> Sanitized { /// JIDs like `12025551234-1543890267@g.us`, telegram numeric peer IDs, /// millisecond timestamps, padded counters) is too high to use as a hard /// rejection signal. Content scrubbing via [`redact_pii`] still applies -/// those patterns — false positives are tolerable there because they only -/// replace bytes inside a string, not reject the whole write. +/// those patterns — a content false positive replaces bytes inside a string +/// rather than rejecting the whole write, which is cheaper but *not* free: +/// a redaction landing inside structured content corrupts it for whatever +/// wrote it (opencompany#1201 — timestamps in stored JSON envelopes), which +/// is why the credit-card pattern's bare form now demands corroboration +/// beyond its checksum. pub fn has_likely_pii(value: &str) -> bool { let nview = NormalizedView::build(value); let cand = scan_candidates(&nview.normalized); @@ -461,8 +478,12 @@ fn push_credit_cards(hits: &mut Vec, norm: &str) { } /// Bytes of context searched either side of a bare digit run for a card -/// keyword. Wide enough for "credit card number is" plus punctuation. -const CC_KEYWORD_WINDOW: usize = 32; +/// keyword. 64 bytes rather than 32 because the window is counted in bytes +/// while text is not: 32 bytes is only ~10 CJK characters or 8 emoji, so a +/// run of either could evict an English keyword that a reader would call +/// adjacent. 64 comfortably spans `{"payment_method":{"card":{"number":…` +/// and a `カード`-prefixed line alike. +const CC_KEYWORD_WINDOW: usize = 64; /// True when [`CC_KEYWORD_RE`] matches within the window around /// `start..end`, widened outward to char boundaries so the slice cannot diff --git a/src/memory/store/safety/pii/checks.rs b/src/memory/store/safety/pii/checks.rs index 1b09537..2769dd7 100644 --- a/src/memory/store/safety/pii/checks.rs +++ b/src/memory/store/safety/pii/checks.rs @@ -94,12 +94,22 @@ pub(super) fn valid_luhn(s: &str) -> bool { /// /// The table lists each supported network's published IIN ranges at the /// lengths that network issues: Visa, Mastercard (incl. the 2-series), Amex, -/// Discover, JCB, Diners Club, UnionPay, Maestro, Mir, and RuPay. It is a +/// Discover, JCB, Diners Club, UnionPay, Maestro, Mir, RuPay, and the +/// Brazilian networks Elo and Hipercard (in scope by this module's own +/// design: it already carries bare and formatted CPF/CNPJ). It is a /// *corroboration* tier, not an acquirer's validator: an exhaustive BIN /// registry is a licensed, continuously updated database, and a range missing /// here is not silently dropped from redaction — a bare PAN on an unlisted /// network still redacts whenever a card keyword appears within the keyword /// window (the third gate in `push_credit_cards`). +/// +/// One dated caveat, so nobody inherits a stronger claim than the code makes: +/// "timestamps can never corroborate" is prefix-and-length dependent, not +/// absolute. 13-digit epoch-milliseconds stay out of every range until the +/// year 2096 (`4…`, Visa's 13-digit arm); but 16-digit epoch-MICROsecond +/// stamps enter Mir's `2200-2204` window in late 2039 and Mastercard's +/// 2-series `2221-2720` from ~2040 to ~2056. If this code outlives that, +/// those stamps redact at Luhn's ~10% again and this gate needs a rethink. pub(super) fn plausible_card_number(d: &[u32]) -> bool { let len = d.len(); if !(13..=19).contains(&len) { @@ -110,14 +120,21 @@ pub(super) fn plausible_card_number(d: &[u32]) -> bool { let p3 = p2 * 10 + d[2]; let p4 = p3 * 10 + d[3]; match d[0] { - // Visa: 16 standard, 13 legacy, 19 extended. + // Visa: 16 standard, 13 legacy, 19 extended. (Also where Elo's + // 4-prefixed ranges land, at the same 16.) 4 => matches!(len, 13 | 16 | 19), 5 => { // Mastercard 51-55 (16 only). ((51..=55).contains(&p2) && len == 16) - // Maestro 5018/5020/5038/5893 (12-19 issued; 13 is this - // module's floor because CC_RE requires 13 digits). - || matches!(p4, 5018 | 5020 | 5038 | 5893) + // Maestro 5018/5020/5038/5893 and 56-58. Maestro issues + // 12-19; the floor here is 13 because CC_RE requires 13 + // digits, so the explicit bound below is the whole window + // this function can see — kept explicit so the "at an + // issued length" promise stays visibly true. + || ((matches!(p4, 5018 | 5020 | 5038 | 5893) || (56..=58).contains(&p2)) + && (13..=19).contains(&len)) + // Elo 5041/5066/5067 (16), RuPay 508 (16). + || ((matches!(p4, 5041 | 5066 | 5067) || p3 == 508) && len == 16) } 2 => { // Mastercard 2-series 2221-2720 (16 only). @@ -130,11 +147,15 @@ pub(super) fn plausible_card_number(d: &[u32]) -> bool { (matches!(p2, 34 | 37) && len == 15) // JCB 3528-3589 (16-19). || ((3528..=3589).contains(&p4) && (16..=19).contains(&len)) - // Diners Club: 36 at the classic 14 up to 19; 300-305, 3095, - // 38-39 at 16-19. - || (p2 == 36 && (14..=19).contains(&len)) - || (((300..=305).contains(&p3) || p4 == 3095 || matches!(p2, 38 | 39)) - && (16..=19).contains(&len)) + // Diners Club 36 / 300-305 / 3095 / 38-39, one scheme, one + // length rule: 14 (Diners International / Carte Blanche + // classic — 30569309025904, the canonical test PAN, is 14) + // through 19. + || ((p2 == 36 + || (300..=305).contains(&p3) + || p4 == 3095 + || matches!(p2, 38 | 39)) + && (14..=19).contains(&len)) } 6 => { // Discover 6011 / 644-649 / 65 (16 or 19). @@ -142,10 +163,12 @@ pub(super) fn plausible_card_number(d: &[u32]) -> bool { // UnionPay 62 (16-19), which also covers the // Discover-processed 622126-622925 range. || (p2 == 62 && (16..=19).contains(&len)) - // Maestro 6304/6759/6761-6763 (13-19, floor as above). - || matches!(p4, 6304 | 6759 | 6761 | 6762 | 6763) - // RuPay 60 (16), beyond the 65 range shared with Discover. - || (p2 == 60 && len == 16) + // Maestro 6304/6759/6761-6763, bounded as the 5-prefix + // Maestro arm above. + || (matches!(p4, 6304 | 6759 | 6761 | 6762 | 6763) && (13..=19).contains(&len)) + // RuPay 60 (16) beyond the 65 range shared with Discover; + // Elo 6277/6362/6363 and Hipercard 6062 (16). + || ((p2 == 60 || matches!(p4, 6277 | 6362 | 6363 | 6062)) && len == 16) } // RuPay 81/82 (16). 8 => matches!(p2, 81 | 82) && len == 16, diff --git a/src/memory/store/safety/pii/checks_tests.rs b/src/memory/store/safety/pii/checks_tests.rs index 01c2ec6..e05291e 100644 --- a/src/memory/store/safety/pii/checks_tests.rs +++ b/src/memory/store/safety/pii/checks_tests.rs @@ -55,3 +55,113 @@ fn plausible_card_number_requires_a_real_iin_at_an_issued_length() { assert!(!plausible_card_number(&digits("411111111111"))); // 12 assert!(!plausible_card_number(&digits("41111111111111111111"))); // 20 } + +// The accept direction, per network, at its boundary lengths — with the +// reject cases one step past each length and each range edge. Luhn is +// irrelevant here: the function judges shape only, and the redaction path +// tests Luhn separately. +#[test] +fn plausible_card_number_accepts_each_network_at_its_boundary_lengths() { + // Visa 13/16/19; nothing between or past. + assert!(plausible_card_number(&digits("4222222222222"))); // 13 + assert!(plausible_card_number(&digits("4111111111111111"))); // 16 + assert!(plausible_card_number(&digits("4111111111111111111"))); // 19 + assert!(!plausible_card_number(&digits("41111111111111"))); // 14 + assert!(!plausible_card_number(&digits("411111111111111"))); // 15 + assert!(!plausible_card_number(&digits("41111111111111111"))); // 17 + + // Mastercard 51-55 and 2221-2720, 16 only; edges out both sides. + assert!(plausible_card_number(&digits("5100000000000000"))); + assert!(plausible_card_number(&digits("5500005555555559"))); + assert!(plausible_card_number(&digits("2221000000000009"))); + assert!(plausible_card_number(&digits("2720000000000000"))); + assert!(!plausible_card_number(&digits("550000555555555"))); // 15 + assert!(!plausible_card_number(&digits("55000055555555590"))); // 17 + assert!(!plausible_card_number(&digits("5000000000000000"))); // 50: not MC + assert!(!plausible_card_number(&digits("2220000000000000"))); // below 2221 + assert!(!plausible_card_number(&digits("2721000000000000"))); // above 2720 + + // Mir 2200-2204, 16 only. + assert!(plausible_card_number(&digits("2200000000000004"))); + assert!(plausible_card_number(&digits("2204000000000000"))); + assert!(!plausible_card_number(&digits("2205000000000009"))); // past range + assert!(!plausible_card_number(&digits("220000000000000"))); // 15 + assert!(!plausible_card_number(&digits("22000000000000004"))); // 17 + + // Amex 34/37, 15 only. + assert!(plausible_card_number(&digits("378282246310005"))); + assert!(plausible_card_number(&digits("340000000000009"))); + assert!(!plausible_card_number(&digits("37828224631000"))); // 14 + assert!(!plausible_card_number(&digits("3782822463100051"))); // 16 + assert!(!plausible_card_number(&digits("350000000000000"))); // 35 alone + + // JCB 3528-3589, 16-19. + assert!(plausible_card_number(&digits("3530111333300000"))); + assert!(plausible_card_number(&digits("3589000000000000000"))); // 19 + assert!(!plausible_card_number(&digits("3527000000000000"))); + assert!(!plausible_card_number(&digits("3590000000000000"))); + assert!(!plausible_card_number(&digits("353011133330000"))); // 15 + + // Diners Club — one scheme, one rule: 36, 300-305, 3095, 38, 39 all at + // 14-19. 30569309025904 is the canonical Diners test PAN; regression + // for the review finding that 14-digit Diners had been split away. + assert!(plausible_card_number(&digits("30569309025904"))); // 300-305 @ 14 + assert!(plausible_card_number(&digits("38520000023237"))); // 38 @ 14 + assert!(plausible_card_number(&digits("36700102000000"))); // 36 @ 14 + assert!(plausible_card_number(&digits("30950000000000"))); // 3095 @ 14 + assert!(plausible_card_number(&digits("39000000000005"))); // 39 @ 14 + assert!(plausible_card_number(&digits("3050000000000000002"))); // 305 @ 19 + assert!(!plausible_card_number(&digits("3060000000000000"))); // 306 + assert!(!plausible_card_number(&digits("3700010200000"))); // 37 @ 13: not Diners + + // Discover 6011 / 644-649 / 65 at 16 or 19. + assert!(plausible_card_number(&digits("6011111111111117"))); + assert!(plausible_card_number(&digits("6440000000000000"))); + assert!(plausible_card_number(&digits("6500000000000000000"))); // 19 + assert!(!plausible_card_number(&digits("60111111111111170"))); // 17 + assert!(!plausible_card_number(&digits("6430000000000000"))); // 643 + + // UnionPay 62 at 16-19. + assert!(plausible_card_number(&digits("6200000000000005"))); + assert!(plausible_card_number(&digits("6200000000000000005"))); // 19 + assert!(!plausible_card_number(&digits("620000000000000"))); // 15 + + // Maestro (5018/5020/5038/5893, 56-58, 6304/6759/6761-6763) at 13-19. + assert!(plausible_card_number(&digits("5018000000000"))); // 13 + assert!(plausible_card_number(&digits("5600000000002"))); // 56 @ 13 + assert!(plausible_card_number(&digits("5800000000000000008"))); // 58 @ 19 + assert!(plausible_card_number(&digits("6759000000000000000"))); // 19 + assert!(plausible_card_number(&digits("6763000000000000"))); + assert!(!plausible_card_number(&digits("5019000000000000"))); // 5019 + assert!(!plausible_card_number(&digits("5900000000000000"))); // 59 + assert!(!plausible_card_number(&digits("6760000000000000"))); // 6760 + + // RuPay 60 / 508 / 81 / 82 at 16. + assert!(plausible_card_number(&digits("6069850000000000"))); + assert!(plausible_card_number(&digits("5080000000000002"))); + assert!(plausible_card_number(&digits("8100000000000000"))); + assert!(plausible_card_number(&digits("8200000000000000"))); + assert!(!plausible_card_number(&digits("8300000000000000"))); // 83 + assert!(!plausible_card_number(&digits("810000000000000"))); // 15 + assert!(!plausible_card_number(&digits("5090000000000000"))); // 509 + + // Elo 5041/5066/5067/6277/6362/6363 and Hipercard 6062, 16. + assert!(plausible_card_number(&digits("5067310000000010"))); + assert!(plausible_card_number(&digits("5041000000000000"))); + assert!(plausible_card_number(&digits("6362000000000009"))); + assert!(plausible_card_number(&digits("6277000000000000"))); + // Hipercard 6062 sits inside RuPay's blanket 60 range as well — listed + // in the table in its own right, but note 60xx@16 is corroborable for + // any xx, which is what the published RuPay range says. + assert!(plausible_card_number(&digits("6062821234567890"))); // Hipercard + assert!(!plausible_card_number(&digits("5065000000000001"))); // 5065 + assert!(!plausible_card_number(&digits("6364000000000007"))); // 6364 + assert!(!plausible_card_number(&digits("506731000000001"))); // 15 + + // The documented expiry (see the function doc): 16-digit + // epoch-microsecond stamps enter Mir/Mastercard-2-series territory + // around 2039/2040. Asserted as truth, not as an endorsement — when + // this line starts mattering, the gate needs a rethink. + assert!(plausible_card_number(&digits("2221787178633773"))); // µs in ~2040 + assert!(!plausible_card_number(&digits("1787178633773000"))); // µs today +} diff --git a/src/memory/store/safety/pii_tests.rs b/src/memory/store/safety/pii_tests.rs index 05b6d4e..e6b4c48 100644 --- a/src/memory/store/safety/pii_tests.rs +++ b/src/memory/store/safety/pii_tests.rs @@ -161,6 +161,74 @@ fn bare_luhn_valid_timestamp_kept() { // (~10% of them do). No IIN, no keyword: not a card. unchanged("run 1787178633773 finished"); } +#[test] +fn credit_card_keyword_matches_serialized_key_shapes() { + // The keyword net is what the IIN table's incompleteness leans on, so it + // has to fire for the key shapes serialized payloads actually use. The + // digit run is Luhn-valid with no network's IIN (`17…`), so only the + // keyword tier can be doing the work in each of these. + for text in [ + r#"{"card_number":"1787178633773"}"#, + r#"{"cardNumber":"1787178633773"}"#, + r#"{"credit_card":"1787178633773"}"#, + r#"{"creditCard":"1787178633773"}"#, + r#"{"ccNumber":"1787178633773"}"#, + r#"{"card_no":"1787178633773"}"#, + "CARD_NUMBER=1787178633773", + "cc=1787178633773", + ] { + let out = redact_pii(text); + assert!( + out.value.contains(PII_CC), + "expected the keyword tier to corroborate: {text:?} -> {out:?}" + ); + } +} + +#[test] +fn credit_card_keyword_speaks_more_than_english() { + // Multilingual mandate: native card words corroborate too, and the + // window is wide enough that non-ASCII text does not evict them. + for text in [ + "カード 1787178633773", + "信用卡 1787178633773", + "카드 1787178633773", + "карта 1787178633773", + "tarjeta 1787178633773", + "cartão 1787178633773", + "card 😀😀😀😀😀😀😀😀 1787178633773", + ] { + let out = redact_pii(text); + assert!( + out.value.contains(PII_CC), + "expected corroboration: {text:?} -> {out:?}" + ); + } +} + +#[test] +fn credit_card_keyword_still_respects_word_boundaries() { + // `pan` must not fire inside an unrelated word: Luhn-valid non-IIN run + // next to `japan` stays untouched. + unchanged("japan 1787178633773 spans"); +} + +#[test] +fn bare_brazilian_network_pans_redact_without_keyword() { + // Elo and Hipercard are in the IIN table (the module targets Brazilian + // PII by design — it carries CPF/CNPJ), so their bare PANs corroborate + // structurally, keyword or not. + redacts("5067310000000010", PII_CC); + redacts("6062821234567890", PII_CC); +} + +#[test] +fn bare_diners_14_digit_pan_redacts() { + // Regression for the review finding: the canonical 14-digit Diners test + // PAN redacted before the corroboration gate and must keep redacting. + redacts("30569309025904", PII_CC); +} + #[test] fn json_envelope_luhn_valid_timestamp_kept() { // opencompany#1201: the exact corruption — a serialized record whose