transformabl-core 0.4.1: redactPii overlap fix (H4) + scan-truncation flag (M2)

packages/transformabl-core/__tests__/redact.overlap.test.ts

@@ -0,0 +1,87 @@
+// Overlapping-match redaction (H4 / #40).
+//
+// Detection routinely emits overlapping spans — e.g. `us_bank_number`
+// (`\d{8,17}`) covers the same digit run as `npi` / `us_drivers_license`. The
+// old descending-start substitution let an inner match's replacement shift the
+// text so an outer match's stale `end` sliced the wrong point, leaving trailing
+// PII digits behind in placeholder/remove modes. These lock in the
+// merge-into-maximal-intervals fix.
+
+import { describe, it, expect } from "vitest";
+import { redactPii } from "../src/redact.js";
+import type { PiiMatch } from "../src/types.js";
+
+describe("redactPii — overlapping matches merge into maximal intervals", () => {
+  // A partial overlap that, under the old code, left the digits "23" behind:
+  // the inner npi span (higher start) was substituted first, shifting the text
+  // so the outer us_bank_number span's stale `end` sliced past its real end.
+  const text = "id 1234567890123 end MORE";
+  const overlapping: PiiMatch[] = [
+    { type: "us_bank_number", value: "1234567890123", start: 3, end: 16 }, // outer
+    { type: "npi", value: "234", start: 4, end: 7 }, // inner, higher start
+  ];
+
+  it("placeholder mode: the outer span is replaced once and no digits survive", () => {
+    const result = redactPii(text, overlapping, { mode: "placeholder" });
+    expect(result).toBe("id [US_BANK_NUMBER] end MORE");
+    expect(result).not.toMatch(/\d/);
+  });
+
+  it("remove mode: the outer span is removed cleanly and no digits survive", () => {
+    const result = redactPii(text, overlapping, { mode: "remove" });
+    expect(result).toBe("id  end MORE");
+    expect(result).not.toMatch(/\d/);
+  });
+
+  it("acceptance (#40): output contains no substring of any original match value", () => {
+    for (const mode of ["placeholder", "remove"] as const) {
+      const result = redactPii(text, overlapping, { mode });
+      for (const m of overlapping) {
+        expect(result).not.toContain(m.value);
+      }
+      expect(result).not.toMatch(/\d/);
+    }
+  });
+
+  it("labels the merged span by its widest contributing type", () => {
+    const result = redactPii(text, overlapping, { mode: "placeholder" });
+    expect(result).toContain("[US_BANK_NUMBER]");
+    expect(result).not.toContain("[NPI]");
+  });
+
+  it("identical spans (same start and end) collapse to one replacement", () => {
+    // us_bank_number and us_drivers_license both match an 8-digit run.
+    const t = "acct 12345678 ok";
+    const same: PiiMatch[] = [
+      { type: "us_bank_number", value: "12345678", start: 5, end: 13 },
+      { type: "us_drivers_license", value: "12345678", start: 5, end: 13 },
+    ];
+    const result = redactPii(t, same, { mode: "placeholder" });
+    expect(result).toBe("acct [US_BANK_NUMBER] ok");
+    expect(result).not.toMatch(/\d/);
+  });
+
+  it("respects `types`: only the requested type's span is redacted", () => {
+    // Only redact npi. Its [4,7] span ("234") is replaced; the surrounding
+    // bank digits are the caller's explicit choice to keep, and the merge must
+    // not sweep the excluded us_bank_number span in.
+    const result = redactPii(text, overlapping, { mode: "placeholder", types: ["npi"] });
+    expect(result).toBe("id 1[NPI]567890123 end MORE");
+  });
+
+  it("mask mode masks the whole merged span (keeps only edge chars)", () => {
+    const result = redactPii(text, overlapping, { mode: "mask" });
+    // Full 13-char span masked keeping 2 at each edge: "12" + 9*'*' + "23".
+    expect(result).toBe("id 12*********23 end MORE");
+  });
+
+  it("non-overlapping matches are unaffected by the merge", () => {
+    const t = "mail [email protected] then 123-45-6789 z";
+    const matches: PiiMatch[] = [
+      { type: "email", value: "[email protected]", start: 5, end: 12 },
+      { type: "ssn", value: "123-45-6789", start: 18, end: 29 },
+    ];
+    const result = redactPii(t, matches, { mode: "placeholder" });
+    expect(result).toBe("mail [EMAIL] then [SSN] z");
+  });
+});

packages/transformabl-core/__tests__/scanTruncation.test.ts

@@ -0,0 +1,60 @@
+// PII-scan truncation loudness (M2 / #40).
+//
+// detectPii caps scanning at MAX_PII_SCAN_LENGTH (512 KB). Before this fix the
+// cap was silent, so PII past it flowed through unredacted AND unlogged —
+// a fail-open that wasn't loud. detectPiiDetailed / TransformResult now carry a
+// `scanTruncated` flag so a consumer can tell "scanned clean" from "not fully
+// scanned."
+
+import { describe, it, expect } from "vitest";
+import {
+  detectPii,
+  detectPiiDetailed,
+  MAX_PII_SCAN_LENGTH,
+} from "../src/detect.js";
+import { transformContent } from "../src/transform.js";
+
+describe("scan-cap truncation is reported (not silent)", () => {
+  it("does not flag truncation for normal-sized input", () => {
+    const r = detectPiiDetailed("contact [email protected] please");
+    expect(r.scanTruncated).toBe(false);
+    expect(r.totalLength).toBe("contact [email protected] please".length);
+    expect(r.scannedLength).toBe(r.totalLength);
+    expect(r.matches.some((m) => m.type === "email")).toBe(true);
+  });
+
+  it("flags truncation and does NOT scan PII past the cap", () => {
+    const filler = "a".repeat(MAX_PII_SCAN_LENGTH + 100);
+    const input = `${filler} [email protected]`;
+    const r = detectPiiDetailed(input);
+
+    expect(r.scanTruncated).toBe(true);
+    expect(r.scannedLength).toBe(MAX_PII_SCAN_LENGTH);
+    expect(r.totalLength).toBe(input.length);
+    // The email is beyond the cap, so it is (knowingly) not found — the flag
+    // is what makes that honest rather than a silent miss.
+    expect(r.matches.some((m) => m.type === "email")).toBe(false);
+  });
+
+  it("still finds PII that falls before the cap in an over-cap input", () => {
+    const input = `[email protected] ${"a".repeat(MAX_PII_SCAN_LENGTH)} [email protected]`;
+    const r = detectPiiDetailed(input);
+    expect(r.scanTruncated).toBe(true);
+    const emails = r.matches.filter((m) => m.type === "email");
+    expect(emails).toHaveLength(1);
+    expect(emails[0].value).toBe("[email protected]");
+  });
+
+  it("transformContent surfaces scanTruncated on the result", () => {
+    const small = transformContent("[email protected]");
+    expect(small.scanTruncated).toBe(false);
+
+    const big = transformContent("a".repeat(MAX_PII_SCAN_LENGTH + 1));
+    expect(big.scanTruncated).toBe(true);
+  });
+
+  it("detectPii (array wrapper) stays behaviourally identical to detectPiiDetailed().matches", () => {
+    const input = "ssn 123-45-6789 and mail [email protected]";
+    expect(detectPii(input)).toEqual(detectPiiDetailed(input).matches);
+  });
+});

packages/transformabl-core/src/detect.ts

@@ -126,29 +126,68 @@ const BUILTIN_PATTERNS: PiiPattern[] = [
   },
 ];
 
+/**
+ * Defense-in-depth against pathological inputs: cap the length we scan.
+ * Even with bounded patterns, running every pattern over an unbounded body
+ * is a DoS lever (this is an agent/LLM gateway — large tool-call payloads
+ * are normal). PII beyond this cap is not scanned; callers handling very
+ * large bodies should chunk. 512 KB comfortably covers real tool I/O.
+ */
+export const MAX_PII_SCAN_LENGTH = 512 * 1024;
+
+/** Result of a PII scan, including whether the scan cap truncated the input. */
+export interface DetectionResult {
+  /** All matches found, in original-text coordinates, sorted by start. */
+  matches: PiiMatch[];
+  /**
+   * True when the (invisible-stripped) input was longer than
+   * {@link MAX_PII_SCAN_LENGTH} and PII past the cap was NOT scanned. When
+   * true, absence of matches beyond the cap is not evidence of absence — the
+   * caller must treat this as a loud fail-open signal, not silently pass the
+   * unscanned tail through as clean.
+   */
+  scanTruncated: boolean;
+  /** Number of characters actually scanned. */
+  scannedLength: number;
+  /** Full length of the invisible-stripped input. */
+  totalLength: number;
+}
+
 /**
  * Detect PII in text content.
  * Returns all matches with their positions.
+ *
+ * Back-compatible thin wrapper over {@link detectPiiDetailed}. Callers that
+ * need to know whether the scan cap truncated the input (M2 / #40) should use
+ * {@link detectPiiDetailed} and inspect `scanTruncated`.
  */
 export function detectPii(
   text: string,
   customPatterns?: Array<{ type: string; pattern: RegExp }>
 ): PiiMatch[] {
+  return detectPiiDetailed(text, customPatterns).matches;
+}
+
+/**
+ * Detect PII and report scan-cap truncation.
+ * Returns matches plus a `scanTruncated` flag so a truncated scan is never a
+ * silent fail-open.
+ */
+export function detectPiiDetailed(
+  text: string,
+  customPatterns?: Array<{ type: string; pattern: RegExp }>
+): DetectionResult {
   // Strip zero-width / invisible characters before scanning so an attacker
   // cannot defeat the regex by inserting them inside PII values (e.g.
   // `john\[email protected]`). Matches found in the stripped form are
   // reported in ORIGINAL-text coordinates so redaction covers the whole
   // obfuscated span — including the invisible chars themselves.
   const { text: scanText, map } = stripInvisible(text);
 
-  // Defense-in-depth against pathological inputs: cap the length we scan.
-  // Even with bounded patterns, running every pattern over an unbounded body
-  // is a DoS lever (this is an agent/LLM gateway — large tool-call payloads
-  // are normal). PII beyond this cap is not scanned; callers handling very
-  // large bodies should chunk. 512 KB comfortably covers real tool I/O.
-  const MAX_SCAN_LENGTH = 512 * 1024;
-  const boundedScanText =
-    scanText.length > MAX_SCAN_LENGTH ? scanText.slice(0, MAX_SCAN_LENGTH) : scanText;
+  const scanTruncated = scanText.length > MAX_PII_SCAN_LENGTH;
+  const boundedScanText = scanTruncated
+    ? scanText.slice(0, MAX_PII_SCAN_LENGTH)
+    : scanText;
 
   const matches: PiiMatch[] = [];
   const allPatterns: Array<{ type: string; pattern: RegExp }> = [
@@ -187,5 +226,11 @@ export function detectPii(
 
   // Sort by position
   matches.sort((a, b) => a.start - b.start);
-  return matches;
+
+  return {
+    matches,
+    scanTruncated,
+    scannedLength: boundedScanText.length,
+    totalLength: scanText.length,
+  };
 }

packages/transformabl-core/src/normalize.ts

@@ -37,8 +37,10 @@ const INVISIBLE_CHAR_RE = /[\u200B-\u200D\u2060\uFEFF]/;
 // Defensive upper bound on the normalization pass. Callers should enforce
 // their own payload-size limits upstream; this cap exists so an oversized
 // input cannot itself turn the O(n) character walk into a DoS vector.
-// PII detection still runs on the full original input — the oversized-
-// input path just skips ZW-normalization rather than iterating.
+// Over this length ZW-normalization is skipped (the input is returned
+// unchanged). Note the detection scan has its own, smaller cap
+// (MAX_PII_SCAN_LENGTH, 512 KB) past which PII is not scanned — detectPii
+// truncates and detectPiiDetailed reports it via `scanTruncated`.
 const MAX_NORMALIZE_LENGTH = 1_000_000;
 
 export function stripInvisible(input: string): StrippedText {

packages/transformabl-core/src/redact.ts

@@ -2,7 +2,16 @@
 //
 // PII redaction: mask, remove, or replace detected PII.
 
-import type { PiiMatch, RedactionConfig, RedactionMode } from "./types.js";
+import type { PiiMatch, PiiType, RedactionConfig, RedactionMode } from "./types.js";
+
+/** A maximal interval formed by merging overlapping matches. */
+interface MergedSpan {
+  start: number;
+  end: number;
+  /** Type of the widest contributing match — used for the placeholder label. */
+  repType: PiiType;
+  repWidth: number;
+}
 
 /**
  * Redact PII matches from text.
@@ -11,6 +20,15 @@ import type { PiiMatch, RedactionConfig, RedactionMode } from "./types.js";
  * - "mask": Replace middle characters with mask char (e.g., "jo**@example.com")
  * - "remove": Remove the PII entirely
  * - "placeholder": Replace with type label (e.g., "[EMAIL]")
+ *
+ * Overlapping matches are merged into maximal intervals before substitution.
+ * Detection routinely produces overlaps — e.g. `us_bank_number` (`\d{8,17}`)
+ * spans the same digit run as `npi` / `us_drivers_license`. Substituting them
+ * independently (the old descending-start loop) let an inner match's
+ * replacement shift the text so an outer match's now-stale `end` sliced the
+ * wrong point, leaving trailing PII digits behind in placeholder/remove modes
+ * (H4). Merging first guarantees the spans handed to substitution are
+ * disjoint, so no substitution can invalidate another's offsets.
  */
 export function redactPii(
   text: string,
@@ -26,30 +44,58 @@ export function redactPii(
     ? new Set(config.types)
     : null; // null = redact all
 
-  // Process matches in reverse order to preserve positions
-  const sorted = [...matches].sort((a, b) => b.start - a.start);
-  let result = text;
+  // Only redact matches the caller asked for. Filtering BEFORE the merge is
+  // what keeps an excluded type from being swept into a redacted span just
+  // because it overlapped an included one. Drop zero-width/inverted spans too.
+  const selected = matches.filter(
+    (m) => (!typesToRedact || typesToRedact.has(m.type)) && m.end > m.start
+  );
+  if (selected.length === 0) return text;
 
-  for (const match of sorted) {
-    if (typesToRedact && !typesToRedact.has(match.type)) continue;
+  // Merge overlapping spans into maximal intervals. Sort ascending by start
+  // (then end) and extend the current interval whenever the next match starts
+  // before the current one ends. Touching spans (start === prev end) stay
+  // separate — they are already disjoint and safe to substitute independently.
+  const asc = [...selected].sort((a, b) => a.start - b.start || a.end - b.end);
+  const merged: MergedSpan[] = [];
+  for (const m of asc) {
+    const width = m.end - m.start;
+    const last = merged[merged.length - 1];
+    if (last && m.start < last.end) {
+      if (m.end > last.end) last.end = m.end;
+      if (width > last.repWidth) {
+        last.repType = m.type;
+        last.repWidth = width;
+      }
+    } else {
+      merged.push({ start: m.start, end: m.end, repType: m.type, repWidth: width });
+    }
+  }
 
-    let replacement: string;
+  // Substitute right-to-left so earlier offsets stay valid. Spans are disjoint,
+  // so the substring being replaced is exactly the original PII span — no other
+  // span's coordinates can have shifted underneath it.
+  let result = text;
+  for (let i = merged.length - 1; i >= 0; i--) {
+    const span = merged[i];
+    const original = text.slice(span.start, span.end);
 
+    let replacement: string;
     switch (mode) {
       case "mask":
-        replacement = maskValue(match.value, maskChar, maskKeep);
+        replacement = maskValue(original, maskChar, maskKeep);
         break;
       case "remove":
         replacement = "";
         break;
       case "placeholder":
         replacement = config?.placeholder
-          ? config.placeholder.replace("{TYPE}", match.type.toUpperCase())
-          : `[${match.type.toUpperCase()}]`;
+          ? config.placeholder.replace("{TYPE}", span.repType.toUpperCase())
+          : `[${span.repType.toUpperCase()}]`;
         break;
     }
 
-    result = result.slice(0, match.start) + replacement + result.slice(match.end);
+    result = result.slice(0, span.start) + replacement + result.slice(span.end);
   }
 
   return result;