From e6541e3a7b758e94c6c0a7c7426683556104b268 Mon Sep 17 00:00:00 2001 From: David Crowe Date: Thu, 23 Jul 2026 13:33:00 -0700 Subject: [PATCH] transformabl-core 0.4.1: redactPii overlap fix (H4) + scan-truncation flag (M2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds on the ReDoS fix in #30 (this branch's base). Adds the two remaining 0.4.1 items from #40 and bumps the facade so consumers can receive it. H4 — redactPii overlap corruption (src/redact.ts): Matches were substituted by descending start with each match's own stale `end`. When an inner overlapping match was replaced first, it shifted the text so an outer match's `end` sliced the wrong point, leaving trailing PII digits behind in placeholder/remove modes. #30's new recognizers make overlaps routine (us_bank_number's \d{8,17} covers the same run as npi / us_drivers_license). Fix: filter to the requested types, merge overlapping matches into maximal intervals, then substitute the disjoint spans right-to-left so no substitution can invalidate another's offsets. The merged span is masked/removed/labelled as a unit (placeholder uses the widest contributing type). match.value always equals text.slice(start,end), so non-overlapping behavior is unchanged. M2 — silent PII-scan truncation (src/detect.ts): #30 capped scanning at 512 KB with no signal, so PII past the cap flowed through unredacted and unlogged — fail-open that wasn't loud. Add detectPiiDetailed() returning { matches, scanTruncated, scannedLength, totalLength } and a scanTruncated flag on TransformResult; detectPii stays a thin back-compatible wrapper. Exported MAX_PII_SCAN_LENGTH. Fixed the now-false normalize.ts comment (detection no longer scans the full input). Facade: bump @gatewaystack/transformabl-core range ^0.3.0 -> ^0.4.1 (^0.3.0 can never resolve 0.4.x) and the facade to 0.3.0 (re-exports detectPiiDetailed + MAX_PII_SCAN_LENGTH). Republish needed after publishing core 0.4.1. Tests: redact.overlap.test.ts (partial/identical/contained overlaps, types filter, no-surviving-substring acceptance) and scanTruncation.test.ts (flag on/off, PII before vs past the cap, transform surfaces it). Full suite 198 green. --- package-lock.json | 12 +-- .../__tests__/redact.overlap.test.ts | 87 +++++++++++++++++++ .../__tests__/scanTruncation.test.ts | 60 +++++++++++++ packages/transformabl-core/src/detect.ts | 63 ++++++++++++-- packages/transformabl-core/src/normalize.ts | 6 +- packages/transformabl-core/src/redact.ts | 68 ++++++++++++--- packages/transformabl-core/src/transform.ts | 6 +- packages/transformabl-core/src/types.ts | 7 ++ packages/transformabl/package.json | 4 +- packages/transformabl/src/index.ts | 2 + 10 files changed, 280 insertions(+), 35 deletions(-) create mode 100644 packages/transformabl-core/__tests__/redact.overlap.test.ts create mode 100644 packages/transformabl-core/__tests__/scanTruncation.test.ts diff --git a/package-lock.json b/package-lock.json index 840f710..b136574 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6702,10 +6702,10 @@ }, "packages/transformabl": { "name": "@gatewaystack/transformabl", - "version": "0.2.0", + "version": "0.3.0", "license": "MIT", "dependencies": { - "@gatewaystack/transformabl-core": "^0.3.0" + "@gatewaystack/transformabl-core": "^0.4.1" }, "devDependencies": { "typescript": "^5.6.3" @@ -6716,18 +6716,12 @@ }, "packages/transformabl-core": { "name": "@gatewaystack/transformabl-core", - "version": "0.4.0", + "version": "0.4.1", "license": "MIT", "devDependencies": { "typescript": "^5.6.3" } }, - "packages/transformabl/node_modules/@gatewaystack/transformabl-core": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@gatewaystack/transformabl-core/-/transformabl-core-0.3.0.tgz", - "integrity": "sha512-NWo2inIcHPGFSxSfzNI9VBZcZO9yxS4NVUsq/CvOe3RoOMGq0xt58C1Ex2Oa1uNZzQ3C/mPWzCRLeFbkd6ozIw==", - "license": "MIT" - }, "packages/validatabl": { "name": "@gatewaystack/validatabl", "version": "0.2.0", diff --git a/packages/transformabl-core/__tests__/redact.overlap.test.ts b/packages/transformabl-core/__tests__/redact.overlap.test.ts new file mode 100644 index 0000000..119a307 --- /dev/null +++ b/packages/transformabl-core/__tests__/redact.overlap.test.ts @@ -0,0 +1,87 @@ +// Overlapping-match redaction (H4 / #40). +// +// Detection routinely emits overlapping spans — e.g. `us_bank_number` +// (`\d{8,17}`) covers the same digit run as `npi` / `us_drivers_license`. The +// old descending-start substitution let an inner match's replacement shift the +// text so an outer match's stale `end` sliced the wrong point, leaving trailing +// PII digits behind in placeholder/remove modes. These lock in the +// merge-into-maximal-intervals fix. + +import { describe, it, expect } from "vitest"; +import { redactPii } from "../src/redact.js"; +import type { PiiMatch } from "../src/types.js"; + +describe("redactPii — overlapping matches merge into maximal intervals", () => { + // A partial overlap that, under the old code, left the digits "23" behind: + // the inner npi span (higher start) was substituted first, shifting the text + // so the outer us_bank_number span's stale `end` sliced past its real end. + const text = "id 1234567890123 end MORE"; + const overlapping: PiiMatch[] = [ + { type: "us_bank_number", value: "1234567890123", start: 3, end: 16 }, // outer + { type: "npi", value: "234", start: 4, end: 7 }, // inner, higher start + ]; + + it("placeholder mode: the outer span is replaced once and no digits survive", () => { + const result = redactPii(text, overlapping, { mode: "placeholder" }); + expect(result).toBe("id [US_BANK_NUMBER] end MORE"); + expect(result).not.toMatch(/\d/); + }); + + it("remove mode: the outer span is removed cleanly and no digits survive", () => { + const result = redactPii(text, overlapping, { mode: "remove" }); + expect(result).toBe("id end MORE"); + expect(result).not.toMatch(/\d/); + }); + + it("acceptance (#40): output contains no substring of any original match value", () => { + for (const mode of ["placeholder", "remove"] as const) { + const result = redactPii(text, overlapping, { mode }); + for (const m of overlapping) { + expect(result).not.toContain(m.value); + } + expect(result).not.toMatch(/\d/); + } + }); + + it("labels the merged span by its widest contributing type", () => { + const result = redactPii(text, overlapping, { mode: "placeholder" }); + expect(result).toContain("[US_BANK_NUMBER]"); + expect(result).not.toContain("[NPI]"); + }); + + it("identical spans (same start and end) collapse to one replacement", () => { + // us_bank_number and us_drivers_license both match an 8-digit run. + const t = "acct 12345678 ok"; + const same: PiiMatch[] = [ + { type: "us_bank_number", value: "12345678", start: 5, end: 13 }, + { type: "us_drivers_license", value: "12345678", start: 5, end: 13 }, + ]; + const result = redactPii(t, same, { mode: "placeholder" }); + expect(result).toBe("acct [US_BANK_NUMBER] ok"); + expect(result).not.toMatch(/\d/); + }); + + it("respects `types`: only the requested type's span is redacted", () => { + // Only redact npi. Its [4,7] span ("234") is replaced; the surrounding + // bank digits are the caller's explicit choice to keep, and the merge must + // not sweep the excluded us_bank_number span in. + const result = redactPii(text, overlapping, { mode: "placeholder", types: ["npi"] }); + expect(result).toBe("id 1[NPI]567890123 end MORE"); + }); + + it("mask mode masks the whole merged span (keeps only edge chars)", () => { + const result = redactPii(text, overlapping, { mode: "mask" }); + // Full 13-char span masked keeping 2 at each edge: "12" + 9*'*' + "23". + expect(result).toBe("id 12*********23 end MORE"); + }); + + it("non-overlapping matches are unaffected by the merge", () => { + const t = "mail x@y.com then 123-45-6789 z"; + const matches: PiiMatch[] = [ + { type: "email", value: "x@y.com", start: 5, end: 12 }, + { type: "ssn", value: "123-45-6789", start: 18, end: 29 }, + ]; + const result = redactPii(t, matches, { mode: "placeholder" }); + expect(result).toBe("mail [EMAIL] then [SSN] z"); + }); +}); diff --git a/packages/transformabl-core/__tests__/scanTruncation.test.ts b/packages/transformabl-core/__tests__/scanTruncation.test.ts new file mode 100644 index 0000000..307297c --- /dev/null +++ b/packages/transformabl-core/__tests__/scanTruncation.test.ts @@ -0,0 +1,60 @@ +// PII-scan truncation loudness (M2 / #40). +// +// detectPii caps scanning at MAX_PII_SCAN_LENGTH (512 KB). Before this fix the +// cap was silent, so PII past it flowed through unredacted AND unlogged — +// a fail-open that wasn't loud. detectPiiDetailed / TransformResult now carry a +// `scanTruncated` flag so a consumer can tell "scanned clean" from "not fully +// scanned." + +import { describe, it, expect } from "vitest"; +import { + detectPii, + detectPiiDetailed, + MAX_PII_SCAN_LENGTH, +} from "../src/detect.js"; +import { transformContent } from "../src/transform.js"; + +describe("scan-cap truncation is reported (not silent)", () => { + it("does not flag truncation for normal-sized input", () => { + const r = detectPiiDetailed("contact john@example.com please"); + expect(r.scanTruncated).toBe(false); + expect(r.totalLength).toBe("contact john@example.com please".length); + expect(r.scannedLength).toBe(r.totalLength); + expect(r.matches.some((m) => m.type === "email")).toBe(true); + }); + + it("flags truncation and does NOT scan PII past the cap", () => { + const filler = "a".repeat(MAX_PII_SCAN_LENGTH + 100); + const input = `${filler} leak@example.com`; + const r = detectPiiDetailed(input); + + expect(r.scanTruncated).toBe(true); + expect(r.scannedLength).toBe(MAX_PII_SCAN_LENGTH); + expect(r.totalLength).toBe(input.length); + // The email is beyond the cap, so it is (knowingly) not found — the flag + // is what makes that honest rather than a silent miss. + expect(r.matches.some((m) => m.type === "email")).toBe(false); + }); + + it("still finds PII that falls before the cap in an over-cap input", () => { + const input = `early@example.com ${"a".repeat(MAX_PII_SCAN_LENGTH)} late@example.com`; + const r = detectPiiDetailed(input); + expect(r.scanTruncated).toBe(true); + const emails = r.matches.filter((m) => m.type === "email"); + expect(emails).toHaveLength(1); + expect(emails[0].value).toBe("early@example.com"); + }); + + it("transformContent surfaces scanTruncated on the result", () => { + const small = transformContent("john@example.com"); + expect(small.scanTruncated).toBe(false); + + const big = transformContent("a".repeat(MAX_PII_SCAN_LENGTH + 1)); + expect(big.scanTruncated).toBe(true); + }); + + it("detectPii (array wrapper) stays behaviourally identical to detectPiiDetailed().matches", () => { + const input = "ssn 123-45-6789 and mail a@b.co"; + expect(detectPii(input)).toEqual(detectPiiDetailed(input).matches); + }); +}); diff --git a/packages/transformabl-core/src/detect.ts b/packages/transformabl-core/src/detect.ts index 5628d01..992aad7 100644 --- a/packages/transformabl-core/src/detect.ts +++ b/packages/transformabl-core/src/detect.ts @@ -126,14 +126,57 @@ const BUILTIN_PATTERNS: PiiPattern[] = [ }, ]; +/** + * Defense-in-depth against pathological inputs: cap the length we scan. + * Even with bounded patterns, running every pattern over an unbounded body + * is a DoS lever (this is an agent/LLM gateway — large tool-call payloads + * are normal). PII beyond this cap is not scanned; callers handling very + * large bodies should chunk. 512 KB comfortably covers real tool I/O. + */ +export const MAX_PII_SCAN_LENGTH = 512 * 1024; + +/** Result of a PII scan, including whether the scan cap truncated the input. */ +export interface DetectionResult { + /** All matches found, in original-text coordinates, sorted by start. */ + matches: PiiMatch[]; + /** + * True when the (invisible-stripped) input was longer than + * {@link MAX_PII_SCAN_LENGTH} and PII past the cap was NOT scanned. When + * true, absence of matches beyond the cap is not evidence of absence — the + * caller must treat this as a loud fail-open signal, not silently pass the + * unscanned tail through as clean. + */ + scanTruncated: boolean; + /** Number of characters actually scanned. */ + scannedLength: number; + /** Full length of the invisible-stripped input. */ + totalLength: number; +} + /** * Detect PII in text content. * Returns all matches with their positions. + * + * Back-compatible thin wrapper over {@link detectPiiDetailed}. Callers that + * need to know whether the scan cap truncated the input (M2 / #40) should use + * {@link detectPiiDetailed} and inspect `scanTruncated`. */ export function detectPii( text: string, customPatterns?: Array<{ type: string; pattern: RegExp }> ): PiiMatch[] { + return detectPiiDetailed(text, customPatterns).matches; +} + +/** + * Detect PII and report scan-cap truncation. + * Returns matches plus a `scanTruncated` flag so a truncated scan is never a + * silent fail-open. + */ +export function detectPiiDetailed( + text: string, + customPatterns?: Array<{ type: string; pattern: RegExp }> +): DetectionResult { // Strip zero-width / invisible characters before scanning so an attacker // cannot defeat the regex by inserting them inside PII values (e.g. // `john\u200B@example.com`). Matches found in the stripped form are @@ -141,14 +184,10 @@ export function detectPii( // obfuscated span — including the invisible chars themselves. const { text: scanText, map } = stripInvisible(text); - // Defense-in-depth against pathological inputs: cap the length we scan. - // Even with bounded patterns, running every pattern over an unbounded body - // is a DoS lever (this is an agent/LLM gateway — large tool-call payloads - // are normal). PII beyond this cap is not scanned; callers handling very - // large bodies should chunk. 512 KB comfortably covers real tool I/O. - const MAX_SCAN_LENGTH = 512 * 1024; - const boundedScanText = - scanText.length > MAX_SCAN_LENGTH ? scanText.slice(0, MAX_SCAN_LENGTH) : scanText; + const scanTruncated = scanText.length > MAX_PII_SCAN_LENGTH; + const boundedScanText = scanTruncated + ? scanText.slice(0, MAX_PII_SCAN_LENGTH) + : scanText; const matches: PiiMatch[] = []; const allPatterns: Array<{ type: string; pattern: RegExp }> = [ @@ -187,5 +226,11 @@ export function detectPii( // Sort by position matches.sort((a, b) => a.start - b.start); - return matches; + + return { + matches, + scanTruncated, + scannedLength: boundedScanText.length, + totalLength: scanText.length, + }; } diff --git a/packages/transformabl-core/src/normalize.ts b/packages/transformabl-core/src/normalize.ts index 362dfec..9727093 100644 --- a/packages/transformabl-core/src/normalize.ts +++ b/packages/transformabl-core/src/normalize.ts @@ -37,8 +37,10 @@ const INVISIBLE_CHAR_RE = /[\u200B-\u200D\u2060\uFEFF]/; // Defensive upper bound on the normalization pass. Callers should enforce // their own payload-size limits upstream; this cap exists so an oversized // input cannot itself turn the O(n) character walk into a DoS vector. -// PII detection still runs on the full original input — the oversized- -// input path just skips ZW-normalization rather than iterating. +// Over this length ZW-normalization is skipped (the input is returned +// unchanged). Note the detection scan has its own, smaller cap +// (MAX_PII_SCAN_LENGTH, 512 KB) past which PII is not scanned — detectPii +// truncates and detectPiiDetailed reports it via `scanTruncated`. const MAX_NORMALIZE_LENGTH = 1_000_000; export function stripInvisible(input: string): StrippedText { diff --git a/packages/transformabl-core/src/redact.ts b/packages/transformabl-core/src/redact.ts index c3e12f2..8508508 100644 --- a/packages/transformabl-core/src/redact.ts +++ b/packages/transformabl-core/src/redact.ts @@ -2,7 +2,16 @@ // // PII redaction: mask, remove, or replace detected PII. -import type { PiiMatch, RedactionConfig, RedactionMode } from "./types.js"; +import type { PiiMatch, PiiType, RedactionConfig, RedactionMode } from "./types.js"; + +/** A maximal interval formed by merging overlapping matches. */ +interface MergedSpan { + start: number; + end: number; + /** Type of the widest contributing match — used for the placeholder label. */ + repType: PiiType; + repWidth: number; +} /** * Redact PII matches from text. @@ -11,6 +20,15 @@ import type { PiiMatch, RedactionConfig, RedactionMode } from "./types.js"; * - "mask": Replace middle characters with mask char (e.g., "jo**@example.com") * - "remove": Remove the PII entirely * - "placeholder": Replace with type label (e.g., "[EMAIL]") + * + * Overlapping matches are merged into maximal intervals before substitution. + * Detection routinely produces overlaps — e.g. `us_bank_number` (`\d{8,17}`) + * spans the same digit run as `npi` / `us_drivers_license`. Substituting them + * independently (the old descending-start loop) let an inner match's + * replacement shift the text so an outer match's now-stale `end` sliced the + * wrong point, leaving trailing PII digits behind in placeholder/remove modes + * (H4). Merging first guarantees the spans handed to substitution are + * disjoint, so no substitution can invalidate another's offsets. */ export function redactPii( text: string, @@ -26,30 +44,58 @@ export function redactPii( ? new Set(config.types) : null; // null = redact all - // Process matches in reverse order to preserve positions - const sorted = [...matches].sort((a, b) => b.start - a.start); - let result = text; + // Only redact matches the caller asked for. Filtering BEFORE the merge is + // what keeps an excluded type from being swept into a redacted span just + // because it overlapped an included one. Drop zero-width/inverted spans too. + const selected = matches.filter( + (m) => (!typesToRedact || typesToRedact.has(m.type)) && m.end > m.start + ); + if (selected.length === 0) return text; - for (const match of sorted) { - if (typesToRedact && !typesToRedact.has(match.type)) continue; + // Merge overlapping spans into maximal intervals. Sort ascending by start + // (then end) and extend the current interval whenever the next match starts + // before the current one ends. Touching spans (start === prev end) stay + // separate — they are already disjoint and safe to substitute independently. + const asc = [...selected].sort((a, b) => a.start - b.start || a.end - b.end); + const merged: MergedSpan[] = []; + for (const m of asc) { + const width = m.end - m.start; + const last = merged[merged.length - 1]; + if (last && m.start < last.end) { + if (m.end > last.end) last.end = m.end; + if (width > last.repWidth) { + last.repType = m.type; + last.repWidth = width; + } + } else { + merged.push({ start: m.start, end: m.end, repType: m.type, repWidth: width }); + } + } - let replacement: string; + // Substitute right-to-left so earlier offsets stay valid. Spans are disjoint, + // so the substring being replaced is exactly the original PII span — no other + // span's coordinates can have shifted underneath it. + let result = text; + for (let i = merged.length - 1; i >= 0; i--) { + const span = merged[i]; + const original = text.slice(span.start, span.end); + let replacement: string; switch (mode) { case "mask": - replacement = maskValue(match.value, maskChar, maskKeep); + replacement = maskValue(original, maskChar, maskKeep); break; case "remove": replacement = ""; break; case "placeholder": replacement = config?.placeholder - ? config.placeholder.replace("{TYPE}", match.type.toUpperCase()) - : `[${match.type.toUpperCase()}]`; + ? config.placeholder.replace("{TYPE}", span.repType.toUpperCase()) + : `[${span.repType.toUpperCase()}]`; break; } - result = result.slice(0, match.start) + replacement + result.slice(match.end); + result = result.slice(0, span.start) + replacement + result.slice(span.end); } return result; diff --git a/packages/transformabl-core/src/transform.ts b/packages/transformabl-core/src/transform.ts index d644754..065e2f2 100644 --- a/packages/transformabl-core/src/transform.ts +++ b/packages/transformabl-core/src/transform.ts @@ -3,7 +3,7 @@ // Full transformation pipeline: detect → classify → redact → extract metadata. import type { TransformConfig, TransformResult } from "./types.js"; -import { detectPii } from "./detect.js"; +import { detectPiiDetailed } from "./detect.js"; import { redactPii } from "./redact.js"; import { classifyContent } from "./classify.js"; import { extractMetadata } from "./metadata.js"; @@ -23,7 +23,8 @@ export function transformContent( config?: TransformConfig ): TransformResult { // 1. Detect PII - const piiMatches = detectPii(content, config?.customPatterns); + const detection = detectPiiDetailed(content, config?.customPatterns); + const piiMatches = detection.matches; // 2. Classify const classification = @@ -54,5 +55,6 @@ export function transformContent( classification, metadata, transformed: redacted !== content, + scanTruncated: detection.scanTruncated, }; } diff --git a/packages/transformabl-core/src/types.ts b/packages/transformabl-core/src/types.ts index e382c90..4fbb3c2 100644 --- a/packages/transformabl-core/src/types.ts +++ b/packages/transformabl-core/src/types.ts @@ -102,6 +102,13 @@ export interface TransformResult { metadata: ContentMetadata; /** Whether any transformation was applied. */ transformed: boolean; + /** + * True when the input exceeded the PII scan cap (512 KB of invisible- + * stripped text) and content past the cap was NOT scanned for PII. A + * consumer must surface this rather than treat the result as a clean pass — + * fail-open must be loud (M2 / #40). + */ + scanTruncated: boolean; } /** Configuration for the full transformation pipeline. */ diff --git a/packages/transformabl/package.json b/packages/transformabl/package.json index e32affa..0518084 100644 --- a/packages/transformabl/package.json +++ b/packages/transformabl/package.json @@ -1,6 +1,6 @@ { "name": "@gatewaystack/transformabl", - "version": "0.2.0", + "version": "0.3.0", "private": false, "license": "MIT", "type": "module", @@ -19,7 +19,7 @@ "prepublishOnly": "npm run build" }, "dependencies": { - "@gatewaystack/transformabl-core": "^0.3.0" + "@gatewaystack/transformabl-core": "^0.4.1" }, "peerDependencies": { "express": "^4.0.0 || ^5.0.0" diff --git a/packages/transformabl/src/index.ts b/packages/transformabl/src/index.ts index 66a7dbd..94a04da 100644 --- a/packages/transformabl/src/index.ts +++ b/packages/transformabl/src/index.ts @@ -80,6 +80,8 @@ export function transformabl(config?: TransformablMiddlewareConfig): RequestHand export { transformContent, detectPii, + detectPiiDetailed, + MAX_PII_SCAN_LENGTH, redactPii, classifyContent, extractMetadata,