From 0a0b4200903a40d92309496363c432155d684c1b Mon Sep 17 00:00:00 2001 From: David Crowe Date: Mon, 20 Apr 2026 03:24:57 -0700 Subject: [PATCH 1/4] transformabl-core: expand PiiType union + regulatory mapping Prepares for Presidio-parity and HIPAA-relevant recognizers by widening the PiiType union and adding regulatory-category mappings for the new types. Patterns themselves land in the next commit. New types: - US identifiers: us_bank_number, us_itin, us_passport, us_drivers_license - International banking: iban - Healthcare (HIPAA PHI): icd_10, icd_9, npi - Financial: crypto_wallet Regulatory mapping adds PCI (bank accounts, IBAN) and HIPAA (all three healthcare types) attribution so classify.ts can report compliance categories in ContentMetadata. --- packages/transformabl-core/src/classify.ts | 9 +++++++++ packages/transformabl-core/src/types.ts | 16 +++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/transformabl-core/src/classify.ts b/packages/transformabl-core/src/classify.ts index 89f2780..db3f858 100644 --- a/packages/transformabl-core/src/classify.ts +++ b/packages/transformabl-core/src/classify.ts @@ -58,6 +58,15 @@ const PII_TO_REGULATORY: Record = { phone: ["gdpr"], date_of_birth: ["gdpr", "coppa", "hipaa"], ip_address: ["gdpr"], + us_bank_number: ["pci", "gdpr"], + us_itin: ["pci", "gdpr"], + us_passport: ["gdpr"], + us_drivers_license: ["gdpr"], + iban: ["pci", "gdpr"], + icd_10: ["hipaa"], + icd_9: ["hipaa"], + npi: ["hipaa"], + crypto_wallet: ["gdpr"], }; /** diff --git a/packages/transformabl-core/src/types.ts b/packages/transformabl-core/src/types.ts index 171c9e7..e382c90 100644 --- a/packages/transformabl-core/src/types.ts +++ b/packages/transformabl-core/src/types.ts @@ -2,12 +2,26 @@ /** Categories of PII that can be detected. */ export type PiiType = + // Universal | "email" | "phone" | "ssn" | "credit_card" | "ip_address" - | "date_of_birth"; + | "date_of_birth" + // US identifiers (Presidio parity) + | "us_bank_number" + | "us_itin" + | "us_passport" + | "us_drivers_license" + // International banking + | "iban" + // Healthcare (HIPAA PHI) + | "icd_10" + | "icd_9" + | "npi" + // Financial + | "crypto_wallet"; /** A single PII detection match. */ export interface PiiMatch { From ce40b7985051aafe87963be162e5ad6b187c5ec3 Mon Sep 17 00:00:00 2001 From: David Crowe Date: Mon, 20 Apr 2026 03:25:08 -0700 Subject: [PATCH 2/4] transformabl-core: add Presidio-parity + healthcare PII recognizers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends detectPii with recognizers that Microsoft Presidio ships by default (US-focused) plus healthcare-specific additions that are HIPAA-standard but not covered by Presidio's core list. Added: - us_bank_number: 8-17 digits (loose by design) - us_itin: 9XX-[78]X-XXXX — more specific than SSN pattern - us_passport: 9 digits, or 1 letter + 8 digits - us_drivers_license: letter + 7-8 digits, or prefixed form - iban: ISO 13616 (2 letters + 2 digits + 11-30 alphanum) - icd_10: letter + 2 chars + optional .Xn (up to 4 decimals) - icd_9: 3 digits + optional .XX, or V/E codes - npi: 10-digit starting with 1 or 2 - crypto_wallet: BTC (P2PKH + bech32) + ETH (0x + 40 hex) Extended: - phone: now matches E.164 international (+) - ip_address: now matches IPv6 full and compressed forms Motivation: ACP's content governance layer silently passed through bank account numbers and ICD diagnosis codes (surfaced by an external multi-agent privacy benchmark, AgentLeak). These are standard PII categories in financial and healthcare contexts — a governance layer that misses them can't credibly claim PII coverage. Coverage surfaces via existing detectPii() API; no consumer changes required beyond bumping the dep pin. 26 new unit tests; all 68 tests pass. --- .../__tests__/detect.test.ts | 156 ++++++++++++++++++ packages/transformabl-core/src/detect.ts | 81 ++++++++- 2 files changed, 229 insertions(+), 8 deletions(-) diff --git a/packages/transformabl-core/__tests__/detect.test.ts b/packages/transformabl-core/__tests__/detect.test.ts index 7986fff..f770b74 100644 --- a/packages/transformabl-core/__tests__/detect.test.ts +++ b/packages/transformabl-core/__tests__/detect.test.ts @@ -145,6 +145,162 @@ describe("detectPii", () => { }); }); + // ───────────────────────────────────────────────────────────────────── + // Presidio-parity recognizers (v0.4.0) + // ───────────────────────────────────────────────────────────────────── + + describe("us_bank_number", () => { + it("detects 10-digit bank account", () => { + const matches = detectPii("account 9845302145 for deposit"); + expect(matches.some((m) => m.type === "us_bank_number")).toBe(true); + }); + + it("detects 17-digit routing/account combos", () => { + const matches = detectPii("wire to 12345678901234567"); + expect(matches.some((m) => m.type === "us_bank_number")).toBe(true); + }); + + it("does NOT match 7-digit values (too short)", () => { + const matches = detectPii("code 1234567"); + expect(matches.some((m) => m.type === "us_bank_number")).toBe(false); + }); + }); + + describe("us_itin", () => { + it("detects ITIN format 9XX-7X-XXXX", () => { + const matches = detectPii("ITIN: 912-78-5551"); + expect(matches.some((m) => m.type === "us_itin")).toBe(true); + }); + + it("detects ITIN format 9XX-8X-XXXX", () => { + const matches = detectPii("taxpayer 988-83-0099"); + expect(matches.some((m) => m.type === "us_itin")).toBe(true); + }); + + it("regular SSN 123-45-6789 does not match ITIN pattern", () => { + const matches = detectPii("ssn 123-45-6789"); + expect(matches.some((m) => m.type === "us_itin")).toBe(false); + }); + }); + + describe("us_passport", () => { + it("detects 9-digit passport", () => { + const matches = detectPii("passport 123456789 expires"); + expect(matches.some((m) => m.type === "us_passport")).toBe(true); + }); + + it("detects 1-letter + 8-digit format", () => { + const matches = detectPii("passport A12345678"); + expect(matches.some((m) => m.type === "us_passport")).toBe(true); + }); + }); + + describe("us_drivers_license", () => { + it("detects 1-letter + 7-digit DL", () => { + const matches = detectPii("DL D1234567 state CA"); + expect(matches.some((m) => m.type === "us_drivers_license")).toBe(true); + }); + + it("detects DL: prefix form", () => { + const matches = detectPii("license DL:12345678"); + expect(matches.some((m) => m.type === "us_drivers_license")).toBe(true); + }); + }); + + describe("iban", () => { + it("detects German IBAN", () => { + const matches = detectPii("wire to DE89370400440532013000"); + expect(matches.some((m) => m.type === "iban")).toBe(true); + }); + + it("detects UK IBAN", () => { + const matches = detectPii("acct GB82WEST12345698765432"); + expect(matches.some((m) => m.type === "iban")).toBe(true); + }); + + it("does NOT match arbitrary long alphanum", () => { + const matches = detectPii("token ABCDEFGHIJKLMNOP12345"); + expect(matches.some((m) => m.type === "iban")).toBe(false); + }); + }); + + describe("phone (international)", () => { + it("detects +44 UK number", () => { + const matches = detectPii("call +44 20 7946 0958"); + expect(matches.some((m) => m.type === "phone")).toBe(true); + }); + + it("detects +81 Japan number", () => { + const matches = detectPii("support +81-3-1234-5678"); + expect(matches.some((m) => m.type === "phone")).toBe(true); + }); + }); + + describe("ip_address (IPv6)", () => { + it("detects full-form IPv6", () => { + const matches = detectPii("addr 2001:0db8:85a3:0000:0000:8a2e:0370:7334"); + expect(matches.some((m) => m.type === "ip_address")).toBe(true); + }); + + it("detects compressed IPv6", () => { + const matches = detectPii("addr 2001:db8::8a2e:370:7334"); + expect(matches.some((m) => m.type === "ip_address")).toBe(true); + }); + }); + + describe("icd_10", () => { + it("detects F32.9 (major depressive disorder)", () => { + const matches = detectPii("diagnosis code F32.9 confirmed"); + expect(matches.some((m) => m.type === "icd_10")).toBe(true); + }); + + it("detects E11.65 (type 2 diabetes)", () => { + const matches = detectPii("pt has E11.65 on chart"); + expect(matches.some((m) => m.type === "icd_10")).toBe(true); + }); + + it("detects code without decimals", () => { + const matches = detectPii("dx: Z00 routine"); + expect(matches.some((m) => m.type === "icd_10")).toBe(true); + }); + }); + + describe("icd_9", () => { + it("detects numeric ICD-9 with decimal", () => { + const matches = detectPii("legacy code 250.01 on record"); + expect(matches.some((m) => m.type === "icd_9")).toBe(true); + }); + + it("detects V-code", () => { + const matches = detectPii("status V70.0"); + expect(matches.some((m) => m.type === "icd_9")).toBe(true); + }); + }); + + describe("npi", () => { + it("detects 10-digit NPI starting with 1", () => { + const matches = detectPii("provider NPI 1234567893 treating"); + expect(matches.some((m) => m.type === "npi")).toBe(true); + }); + }); + + describe("crypto_wallet", () => { + it("detects BTC legacy address", () => { + const matches = detectPii("send to 1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2"); + expect(matches.some((m) => m.type === "crypto_wallet")).toBe(true); + }); + + it("detects BTC bech32 address", () => { + const matches = detectPii("payout bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4"); + expect(matches.some((m) => m.type === "crypto_wallet")).toBe(true); + }); + + it("detects ETH address", () => { + const matches = detectPii("eth 0x742d35Cc6634C0532925a3b844Bc9e7595f0bEb7"); + expect(matches.some((m) => m.type === "crypto_wallet")).toBe(true); + }); + }); + describe("zero-width character bypass", () => { it("detects email with zero-width space inside", () => { const text = "contact john\u200B@example.com for info"; diff --git a/packages/transformabl-core/src/detect.ts b/packages/transformabl-core/src/detect.ts index 31885a5..8a6a358 100644 --- a/packages/transformabl-core/src/detect.ts +++ b/packages/transformabl-core/src/detect.ts @@ -1,13 +1,14 @@ // packages/transformabl-core/src/detect.ts // -// Regex-based PII detection. +// Regex-based PII detection. Recognizer set tracks Microsoft Presidio's +// default recognizer list (US-focused) plus healthcare-specific additions +// that are standard under HIPAA but not covered by Presidio's core list. // // FUTURE WORK: // - ML-based NER detection (plug in spaCy, Presidio, or cloud NER APIs) // - Address detection (street addresses, zip codes) // - Name detection (requires NER - too many false positives with regex) -// - International phone number formats -// - Passport numbers, driver's license numbers +// - Country-specific identifiers (UK NHS, AU ABN, IN PAN, etc.) import type { PiiType, PiiMatch } from "./types.js"; import { stripInvisible } from "./normalize.js"; @@ -28,31 +29,95 @@ const BUILTIN_PATTERNS: PiiPattern[] = [ }, { type: "phone", - // US phone numbers (10+ digits): (xxx) xxx-xxxx, xxx-xxx-xxxx, +1xxxxxxxxxx - // Requires area code (3 digits) + 7-digit number to avoid matching short numbers - pattern: /(?:\+1[-.\s]?)\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b|\b\(?\d{3}\)?[-.\s]\d{3}[-.\s]?\d{4}\b/g, + // US 10-digit formats AND international E.164 (+<7-14 digits>). + pattern: + /(?:\+1[-.\s]?)\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b|\b\(?\d{3}\)?[-.\s]\d{3}[-.\s]?\d{4}\b|\+(?!1[-.\s]?\d)\d{1,3}[-.\s]?\d{1,4}[-.\s]?\d{1,4}[-.\s]?\d{1,9}\b/g, }, { type: "ssn", // US Social Security Numbers: xxx-xx-xxxx pattern: /\b\d{3}-\d{2}-\d{4}\b/g, }, + { + type: "us_itin", + // US Individual Taxpayer Identification Number: 9XX-(7X|8X)-XXXX where + // the middle group starts with 7, 8, or 9. Must be checked BEFORE the + // generic SSN pattern since ITIN overlaps structurally with SSN — ITIN + // is more specific, so order matters when dedup'ing overlapping hits. + pattern: /\b9\d{2}-[78]\d-\d{4}\b/g, + }, { type: "credit_card", // Major card formats: Visa (13-19), MC (16), Amex (15), Discover (16), Diners (14) // Matches 13-19 digit sequences with optional separators (space or dash) pattern: /\b(?:4\d{3}|5[1-5]\d{2}|3[47]\d{2}|6(?:011|5\d{2})|3(?:0[0-5]|[68]\d)\d)[-\s]?\d{4,6}[-\s]?\d{4,5}(?:[-\s]?\d{1,4})?\b/g, }, + { + type: "us_bank_number", + // US bank account numbers: 8-17 digits. Loose by design — banks don't + // publish a format. Tightened with word boundaries + minimum length to + // avoid matching short IDs. Will false-positive on other long digit + // strings; acceptable for a governance layer that prefers over- + // redaction on financial context. + pattern: /\b\d{8,17}\b/g, + }, + { + type: "us_passport", + // US passport: 9 digits (older) or 1 letter + 8 digits (newer). + pattern: /\b(?:[A-Z]\d{8}|\d{9})\b/g, + }, + { + type: "us_drivers_license", + // US driver's license formats are state-specific. Approximation: + // 1 letter + 7-8 digits, or 7-9 all-digit sequences with "DL:" prefix. + // False-positive rate is non-trivial; callers can disable this type. + pattern: /\b(?:DL[:\s]?)?(?:[A-Z]\d{7,8}|\d{7,9})\b/g, + }, + { + type: "iban", + // ISO 13616 IBAN: 2 letters (country) + 2 check digits + 11-30 alphanum. + // Optional spaces every 4 chars are stripped by normalize.ts before scan. + pattern: /\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b/g, + }, { type: "ip_address", - // IPv4 addresses - pattern: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, + // IPv4 OR IPv6 (full + compressed forms). Three alternatives: + // (1) IPv4 dotted-quad + // (2) IPv6 full form — 8 hex groups separated by `:` + // (3) IPv6 compressed — contains `::`, with hex groups on either side + pattern: + /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b|\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b|(? Date: Mon, 20 Apr 2026 03:25:17 -0700 Subject: [PATCH 3/4] transformabl-core: bump to 0.4.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Minor version because the PiiType union is additive (no breaking changes for existing consumers) but the set of types returned by detectPii() has grown — callers that hard-code a closed set of expected types should review. --- package-lock.json | 10 ++++++++-- packages/transformabl-core/package.json | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0bf0456..840f710 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6601,7 +6601,7 @@ }, "packages/explicabl": { "name": "@gatewaystack/explicabl", - "version": "0.0.7", + "version": "0.0.8", "license": "MIT", "dependencies": { "express": "^4.22.0", @@ -6716,12 +6716,18 @@ }, "packages/transformabl-core": { "name": "@gatewaystack/transformabl-core", - "version": "0.3.0", + "version": "0.4.0", "license": "MIT", "devDependencies": { "typescript": "^5.6.3" } }, + "packages/transformabl/node_modules/@gatewaystack/transformabl-core": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@gatewaystack/transformabl-core/-/transformabl-core-0.3.0.tgz", + "integrity": "sha512-NWo2inIcHPGFSxSfzNI9VBZcZO9yxS4NVUsq/CvOe3RoOMGq0xt58C1Ex2Oa1uNZzQ3C/mPWzCRLeFbkd6ozIw==", + "license": "MIT" + }, "packages/validatabl": { "name": "@gatewaystack/validatabl", "version": "0.2.0", diff --git a/packages/transformabl-core/package.json b/packages/transformabl-core/package.json index 5cae6eb..137780b 100644 --- a/packages/transformabl-core/package.json +++ b/packages/transformabl-core/package.json @@ -1,6 +1,6 @@ { "name": "@gatewaystack/transformabl-core", - "version": "0.3.0", + "version": "0.4.0", "private": false, "license": "MIT", "type": "module", From ff4e1551908728e145ef398e6caed534a76ac757 Mon Sep 17 00:00:00 2001 From: David Crowe Date: Thu, 2 Jul 2026 10:15:54 -0700 Subject: [PATCH 4/4] fix(transformabl-core): ReDoS in email PII regex + scan-length cap (0.4.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unbounded quantifiers around a single mandatory @ made the email pattern O(n^2): a long alphanumeric run (no @ needed) forced scan-to-end-then- backtrack from every start position — ~1MB blocked the event loop minutes. Bound to RFC 5321 limits (local<=64, domain<=255, tld<=24) + a 512KB detectPii scan cap. Verified: 300k-char attack 4413ms -> 39ms, emails still detected. Security review 2026-07-01. --- packages/transformabl-core/package.json | 2 +- packages/transformabl-core/src/detect.ts | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/transformabl-core/package.json b/packages/transformabl-core/package.json index 137780b..c3cbf1b 100644 --- a/packages/transformabl-core/package.json +++ b/packages/transformabl-core/package.json @@ -1,6 +1,6 @@ { "name": "@gatewaystack/transformabl-core", - "version": "0.4.0", + "version": "0.4.1", "private": false, "license": "MIT", "type": "module", diff --git a/packages/transformabl-core/src/detect.ts b/packages/transformabl-core/src/detect.ts index 8a6a358..5628d01 100644 --- a/packages/transformabl-core/src/detect.ts +++ b/packages/transformabl-core/src/detect.ts @@ -25,7 +25,13 @@ interface PiiPattern { const BUILTIN_PATTERNS: PiiPattern[] = [ { type: "email", - pattern: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, + // Quantifiers are bounded to RFC 5321 limits (local ≤64, domain ≤255, + // TLD ≤24). The previous unbounded `+…+` form around a single mandatory + // `@` was quadratic (O(n²)) — a long alphanumeric run with no `@` made + // the engine scan-to-end-then-backtrack from every start position, so a + // ~1 MB string of `a`s could block the event loop for minutes (ReDoS). + // Bounding the work per start position makes it effectively linear. + pattern: /[a-zA-Z0-9._%+-]{1,64}@[a-zA-Z0-9.-]{1,255}\.[a-zA-Z]{2,24}/g, }, { type: "phone", @@ -135,6 +141,15 @@ export function detectPii( // obfuscated span — including the invisible chars themselves. const { text: scanText, map } = stripInvisible(text); + // Defense-in-depth against pathological inputs: cap the length we scan. + // Even with bounded patterns, running every pattern over an unbounded body + // is a DoS lever (this is an agent/LLM gateway — large tool-call payloads + // are normal). PII beyond this cap is not scanned; callers handling very + // large bodies should chunk. 512 KB comfortably covers real tool I/O. + const MAX_SCAN_LENGTH = 512 * 1024; + const boundedScanText = + scanText.length > MAX_SCAN_LENGTH ? scanText.slice(0, MAX_SCAN_LENGTH) : scanText; + const matches: PiiMatch[] = []; const allPatterns: Array<{ type: string; pattern: RegExp }> = [ ...BUILTIN_PATTERNS, @@ -146,7 +161,7 @@ export function detectPii( const regex = new RegExp(pattern.source, pattern.flags); let match: RegExpExecArray | null; - while ((match = regex.exec(scanText)) !== null) { + while ((match = regex.exec(boundedScanText)) !== null) { const s = match.index; const e = match.index + match[0].length;