From 7d5cbc9ebf495819edcefe58c21df820d74f5d29 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 08:34:49 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=AB=EF=B8=8F=20feat:=20Add=20Bounded?= =?UTF-8?q?=20PII=20Text=20Transformation=20(#16295)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌫️ feat: Add Bounded PII Text Transformation * ⚡ perf: Reuse Bounded Placeholder Scan * fix: Respect PII Transformation Session Boundaries --------- Co-authored-by: Lia --- librechat.example.yaml | 6 +- .../src/protection/detectors/pattern.spec.ts | 26 ++ .../api/src/protection/detectors/pattern.ts | 114 ++++++++- packages/api/src/protection/index.ts | 1 + packages/api/src/protection/transform.spec.ts | 227 ++++++++++++++++++ packages/api/src/protection/transform.ts | 184 ++++++++++++++ packages/data-provider/src/filters.spec.ts | 38 ++- packages/data-provider/src/filters.ts | 9 +- 8 files changed, 589 insertions(+), 16 deletions(-) create mode 100644 packages/api/src/protection/transform.spec.ts create mode 100644 packages/api/src/protection/transform.ts diff --git a/librechat.example.yaml b/librechat.example.yaml index e407c300dd..3f25483564 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -1390,7 +1390,11 @@ endpoints: # messages: # unattributedAssistantContent: model_output # `model_output` (default) or `inspect` # pii: -# action: audit # `block` (default) or `audit` +# action: audit # `block` (default), `audit`, or `redact` (reserved for transformation-aware ingress; current routes still block) +# # `redact` requires the later encrypted owner-view and canonical-message integration. +# # maxCharacters: 65536 # bounded total inspected text per transformation session +# # maxMatches: 256 # bounded matches per transformation session +# # Custom patterns may set category: email, phone, name, credential, or custom. # fields: [name, text, summary, quote, answer, decision_response, decision_reason, content_part, attachment_reference, assembled_context] # starterPatterns: [sk_prefix, bearer_header, api_key_header] # customPatterns: diff --git a/packages/api/src/protection/detectors/pattern.spec.ts b/packages/api/src/protection/detectors/pattern.spec.ts index 992eb6e593..cc84a1a6ad 100644 --- a/packages/api/src/protection/detectors/pattern.spec.ts +++ b/packages/api/src/protection/detectors/pattern.spec.ts @@ -72,6 +72,32 @@ describe('pattern content inspector', () => { ).toMatchObject({ ruleId: 'bearer_header' }); }); + it.each([ + { config: { starterPatterns: ['bearer_header'] }, text: 'Bearer contract-token' }, + { + config: { + starterPatterns: [], + customPatterns: [{ id: 'custom', label: 'Custom', regex: 'ORG-[A-Z]+' }], + }, + text: 'ORG-SECRET', + }, + ])('enforces a zero remaining allowance when locating $text', ({ config, text }) => { + const inspector = createPatternContentInspector(config, { linearTime: true }); + + expect(inspector.locate('clean text', 0)).toEqual([]); + expect(inspector.locate('', 0)).toEqual([]); + expect(() => inspector.locate(text, 0)).toThrow(PatternConfigurationError); + expect(inspector.locate(text, 1)).toHaveLength(1); + }); + + it.each([-1, 0.5, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1])( + 'rejects invalid remaining match allowance %s before inspecting', + (allowance) => { + const inspector = createPatternContentInspector({}, { linearTime: true }); + expect(() => inspector.locate('clean text', allowance)).toThrow(PatternConfigurationError); + }, + ); + it('memoizes a compiled set by config identity and memory limit', () => { const compile = jest.spyOn(RE2Set.prototype, 'compile'); const config: MessageFilterPiiConfig = { diff --git a/packages/api/src/protection/detectors/pattern.ts b/packages/api/src/protection/detectors/pattern.ts index 373028a3a2..6456b931be 100644 --- a/packages/api/src/protection/detectors/pattern.ts +++ b/packages/api/src/protection/detectors/pattern.ts @@ -1,4 +1,4 @@ -import { RE2Set } from 're2js'; +import { RE2JS, RE2Set } from 're2js'; import { logger } from '@librechat/data-schemas'; import { MAX_PII_CUSTOM_REGEX_CHARACTERS, @@ -9,23 +9,25 @@ import { MAX_PII_PATTERNS_PER_SOURCE, getPiiRegexProgramSize, } from 'librechat-data-provider'; -import type { MessageFilterPiiConfig, FilterPiiCustomPatternConfig } from 'librechat-data-provider'; +import type { + FilterPiiCategory, + MessageFilterPiiConfig, + FilterPiiCustomPatternConfig, +} from 'librechat-data-provider'; import type { ProtectionFinding, TextContentFragment } from '../types'; -interface TestablePattern { - test(input: string): boolean; -} - interface CompiledPattern { readonly id: string; readonly label: string; - readonly pattern: TestablePattern; + readonly pattern: RegExp; + readonly category: FilterPiiCategory; } interface PreparedCustomPattern { readonly id: string; readonly label: string; readonly regex: string; + readonly category: FilterPiiCategory; } interface SnapshotPatternContentInspectorConfig { @@ -67,10 +69,18 @@ export interface PatternContentInspectorPreflightCost { readonly regexes: readonly string[]; } +export interface PatternTextMatch { + readonly start: number; + readonly end: number; + readonly category: FilterPiiCategory; +} + export interface PatternContentInspector { readonly active: boolean; inspectFragment(fragment: TextContentFragment): ProtectionFinding | null; inspect(fragments: Iterable): ProtectionFinding | null; + /** Internal match locations, never safe to return as public finding metadata. */ + locate(text: string, maxMatches: number): readonly PatternTextMatch[]; } export interface PatternContentInspectorConfig { @@ -86,9 +96,24 @@ export interface PatternContentInspectorOptions { } const STARTER_PATTERNS: readonly CompiledPattern[] = [ - { id: 'sk_prefix', label: 'sk- prefix token', pattern: /\b(sk-)[a-zA-Z0-9_-]+/ }, - { id: 'bearer_header', label: 'Bearer token', pattern: /\b(Bearer )[^\s"']+/i }, - { id: 'api_key_header', label: 'api-key header', pattern: /\b(api-key:?\s+)[^\s"']+/i }, + { + id: 'sk_prefix', + label: 'sk- prefix token', + pattern: /\b(sk-)[a-zA-Z0-9_-]+/, + category: 'credential', + }, + { + id: 'bearer_header', + label: 'Bearer token', + pattern: /\b(Bearer )[^\s"']+/i, + category: 'credential', + }, + { + id: 'api_key_header', + label: 'api-key header', + pattern: /\b(api-key:?\s+)[^\s"']+/i, + category: 'credential', + }, ]; const STARTER_BY_ID = new Map(STARTER_PATTERNS.map((pattern) => [pattern.id, pattern])); @@ -201,7 +226,25 @@ function readCustomPattern(candidate: unknown, index: number): FilterPiiCustomPa if (typeof regex !== 'string' || regex.length === 0 || regex.length > MAX_PII_PATTERN_LENGTH) { throw configurationError(`customPatterns[${index}].regex is invalid`); } - return { id, label, regex }; + let category: unknown; + try { + category = (candidate as FilterPiiCustomPatternConfig).category; + } catch { + throw configurationError(`customPatterns[${index}] could not be read safely`); + } + if ( + category != null && + (typeof category !== 'string' || + !['email', 'phone', 'name', 'credential', 'custom'].includes(category)) + ) { + throw configurationError(`customPatterns[${index}].category is invalid`); + } + return { + id, + label, + regex, + ...(category == null ? {} : { category: category as FilterPiiCategory }), + }; } function snapshotConfig( @@ -240,7 +283,7 @@ function snapshotConfig( throw configurationError('customPatterns could not be read safely'); } const pattern = readCustomPattern(patternCandidate, index); - custom.push(pattern); + custom.push({ ...pattern, category: pattern.category ?? 'custom' }); regexes.push(pattern.regex); regexCharacters += pattern.regex.length; if (regexCharacters > MAX_PII_CUSTOM_REGEX_CHARACTERS) { @@ -357,10 +400,17 @@ function createSequentialInspector(patterns: readonly CompiledPattern[]): Patter function createInspector( active: boolean, inspectFragment: (fragment: TextContentFragment) => ProtectionFinding | null, + locate?: (text: string, maxMatches: number) => readonly PatternTextMatch[], ): PatternContentInspector { return { active, inspectFragment, + locate(text, maxMatches) { + if (locate == null) { + throw configurationError('redaction requires a linear-time pattern inspector'); + } + return locate(text, maxMatches); + }, inspect(fragments) { for (const fragment of fragments) { const finding = inspectFragment(fragment); @@ -416,7 +466,44 @@ function createLinearInspector( return pattern == null ? null : findingFor(pattern, fragment); }; - return createInspector(prepared.cost.active, inspectFragment); + const compiled = new Map(); + const locate = (text: string, maxMatches: number): readonly PatternTextMatch[] => { + if (!Number.isSafeInteger(maxMatches) || maxMatches < 0) { + throw configurationError('redaction match limit must be a non-negative safe integer'); + } + const matches: PatternTextMatch[] = []; + const selected = customSet == null ? [] : customSet.match(text); + const candidates: Array = prepared.starter.map( + (pattern) => [ + `${pattern.pattern.ignoreCase ? '(?i)' : ''}${pattern.pattern.source}`, + pattern.category, + ], + ); + for (const index of selected) { + const pattern = prepared.custom[index]; + if (pattern != null) { + candidates.push([pattern.regex, pattern.category]); + } + } + for (const [expression, category] of candidates) { + let regex = compiled.get(expression); + if (regex == null) { + regex = RE2JS.compile(expression); + compiled.set(expression, regex); + } + const matcher = regex.matcher(text); + while (matcher.find()) { + const start = matcher.start(); + const end = matcher.end(); + if (end <= start || matches.length >= maxMatches) { + throw configurationError('redaction match limit or empty match encountered'); + } + matches.push({ start, end, category }); + } + } + return matches; + }; + return createInspector(prepared.cost.active, inspectFragment, locate); } function cacheLinearInspector( @@ -480,6 +567,7 @@ export function createPatternContentInspector( id: pattern.id, label: pattern.label, pattern: new RegExp(pattern.regex), + category: pattern.category, })); const inspector = createSequentialInspector([...prepared.starter, ...custom]); if (options.cacheResult !== false) { diff --git a/packages/api/src/protection/index.ts b/packages/api/src/protection/index.ts index 18e325633a..264f49b381 100644 --- a/packages/api/src/protection/index.ts +++ b/packages/api/src/protection/index.ts @@ -1,5 +1,6 @@ export * from './types'; export * from './runtime'; +export * from './transform'; export * from './title'; export * from './legacy'; export * from './provenance'; diff --git a/packages/api/src/protection/transform.spec.ts b/packages/api/src/protection/transform.spec.ts new file mode 100644 index 0000000000..88022c2981 --- /dev/null +++ b/packages/api/src/protection/transform.spec.ts @@ -0,0 +1,227 @@ +import type { FiltersConfig } from 'librechat-data-provider'; +import type { TextContentFragment } from './types'; +import { createPiiTextTransformer, PiiTransformationError } from './transform'; +import { inspectContent } from './runtime'; + +jest.mock('@librechat/data-schemas', () => ({ + logger: { warn: jest.fn(), error: jest.fn(), info: jest.fn(), debug: jest.fn() }, +})); + +const pii: NonNullable['pii']> = { + action: 'redact' as const, + starterPatterns: [], + fields: ['text'], + customPatterns: [ + { + id: 'email', + label: 'Email address', + regex: '(?i)[a-z]+@[a-z]+\\.[a-z]+', + category: 'email' as const, + }, + ], +}; + +function fragment(text: string, extras: Partial = {}): TextContentFragment { + return { + id: 'chat.text', + path: '/text', + text, + source: 'message', + field: 'text', + format: 'plain', + treatment: 'replaceable', + provenance: 'user', + ...extras, + } as TextContentFragment; +} + +describe('opt-in PII text transformation', () => { + it('leaves the existing inspection path blocking until a caller integrates transformation', () => { + const filters: FiltersConfig = { messages: { pii } }; + expect(inspectContent([fragment('Alice@Example.com')], { filters })).toMatchObject({ + detectorId: 'pii-pattern', + ruleId: 'email', + }); + }); + + it('uses typed placeholders and returns no matched values in metadata', () => { + const session = createPiiTextTransformer(pii).createSession(); + const first = session.transform(fragment('😀 Alice@Example.com, Alice@Example.com')); + const second = session.transform(fragment('Next: Bob@Example.com and Alice@Example.com')); + + expect(first).toEqual({ + version: 1, + content: '😀 [EMAIL_1], [EMAIL_1]', + replacements: 2, + categories: [{ category: 'EMAIL', count: 2 }], + }); + expect(second.content).toBe('Next: [EMAIL_2] and [EMAIL_1]'); + expect(JSON.stringify(first.categories)).not.toContain('Alice'); + expect(JSON.stringify(second.categories)).not.toContain('Bob'); + const newTurn = createPiiTextTransformer(pii).createSession(); + expect(newTurn.transform(fragment('Bob@Example.com')).content).toBe('[EMAIL_1]'); + }); + + it('never aliases a user-supplied placeholder to another private value', () => { + const session = createPiiTextTransformer(pii).createSession(); + expect(session.transform(fragment('[EMAIL_1] Alice@Example.com')).content).toBe( + '[EMAIL_1] [EMAIL_2]', + ); + expect(() => session.transform(fragment('A forged [EMAIL_2] marker'))).toThrow( + PiiTransformationError, + ); + }); + + it('reserves literal markers across selected fragments without sharing session state', () => { + const transformer = createPiiTextTransformer(pii); + const session = transformer.createSession(); + const independent = transformer.createSession(); + + expect(session.transform(fragment('[EMAIL_1] [EMAIL_3]'))).toEqual({ + version: 1, + content: '[EMAIL_1] [EMAIL_3]', + replacements: 0, + categories: [], + }); + expect(session.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_2]'); + expect(independent.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_1]'); + expect(session.transform(fragment('Bob@Example.com Alice@Example.com')).content).toBe( + '[EMAIL_4] [EMAIL_2]', + ); + expect(session.transform(fragment('[EMAIL_1]'))).toMatchObject({ replacements: 0 }); + expect(() => session.transform(fragment('[EMAIL_4]'))).toThrow(PiiTransformationError); + }); + + it('still charges selected literal markers against the character budget', () => { + const session = createPiiTextTransformer({ ...pii, maxCharacters: 18 }).createSession(); + expect(session.transform(fragment('[EMAIL_1]')).replacements).toBe(0); + expect(session.transform(fragment('[EMAIL_2]')).replacements).toBe(0); + expect(() => session.transform(fragment('[EMAIL_3]'))).toThrow(PiiTransformationError); + }); + + it('does not charge an excluded field against the selected character budget', () => { + const text = 'Alice@Example.com'; + const session = createPiiTextTransformer({ + ...pii, + maxCharacters: text.length, + }).createSession(); + const excluded = fragment(text.repeat(100), { field: 'summary' }); + + expect(session.transform(excluded)).toEqual({ + version: 1, + content: excluded.text, + replacements: 0, + categories: [], + }); + expect(session.transform(fragment(text)).content).toBe('[EMAIL_1]'); + }); + + it('neither validates nor reserves excluded markers before or after selected fragments', () => { + const session = createPiiTextTransformer(pii).createSession(); + const excluded = fragment('[EMAIL_1] [EMAIL_2]', { field: 'summary' }); + + expect(session.transform(excluded).content).toBe(excluded.text); + expect(session.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_1]'); + expect(session.transform(excluded).content).toBe(excluded.text); + expect(session.transform(fragment('Bob@Example.com')).content).toBe('[EMAIL_2]'); + }); + + it('accepts clean and empty fragments at the match limit but rejects one more match', () => { + const session = createPiiTextTransformer({ ...pii, maxMatches: 1 }).createSession(); + + expect(session.transform(fragment('Alice@Example.com')).replacements).toBe(1); + for (const text of ['No private details here.', '']) { + expect(session.transform(fragment(text))).toEqual({ + version: 1, + content: text, + replacements: 0, + categories: [], + }); + } + expect(() => session.transform(fragment('Alice@Example.com'))).toThrow(PiiTransformationError); + }); + + it('redacts selected built-in credential headers without leaking the value', () => { + const session = createPiiTextTransformer({ + action: 'redact', + starterPatterns: ['bearer_header'], + }).createSession(); + const result = session.transform(fragment('Authorization: bEaReR contract-token')); + expect(result.content).toBe('Authorization: [CREDENTIAL_1]'); + expect(JSON.stringify(result.categories)).not.toContain('contract-token'); + }); + + it('merges overlapping patterns so partial sensitive values never survive', () => { + const config = { + action: 'redact' as const, + starterPatterns: [], + customPatterns: [ + { id: 'early', label: 'early', regex: 'ORG-SECRET', category: 'credential' as const }, + { id: 'later', label: 'later', regex: 'SECRET-DATA', category: 'custom' as const }, + ], + }; + const result = createPiiTextTransformer(config) + .createSession() + .transform(fragment('prefix ORG-SECRET-DATA suffix')); + expect(result.content).toBe('prefix [CREDENTIAL_1] suffix'); + expect(result.replacements).toBe(1); + expect(result.content).not.toContain('SECRET'); + }); + + it('respects configured fields without attempting to replace other fields', () => { + const session = createPiiTextTransformer(pii).createSession(); + expect(session.transform(fragment('Alice@Example.com', { field: 'summary' })).content).toBe( + 'Alice@Example.com', + ); + expect(session.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_1]'); + }); + + it('fails closed for non-replaceable matches and structured or URI content', () => { + for (const extras of [ + { treatment: 'inspect_only' as const }, + { format: 'json' as const }, + { format: 'uri' as const }, + ]) { + expect(() => + createPiiTextTransformer(pii) + .createSession() + .transform(fragment('Alice@Example.com', extras)), + ).toThrow(PiiTransformationError); + } + }); + + it('bounds characters and matches across all fragments in one session', () => { + const config = { + ...pii, + maxCharacters: 50, + maxMatches: 2, + }; + const session = createPiiTextTransformer(config).createSession(); + expect(session.transform(fragment('A@b.com A@b.com')).replacements).toBe(2); + expect(() => session.transform(fragment('C@d.com'))).toThrow(PiiTransformationError); + expect(() => + createPiiTextTransformer({ ...config, maxCharacters: 4 }) + .createSession() + .transform(fragment('Alice@Example.com')), + ).toThrow(PiiTransformationError); + }); + + it('rejects empty regex matches, invalid limits and missing redact action without echoing input', () => { + expect(() => createPiiTextTransformer({ ...pii, maxMatches: -1 })).toThrow(); + expect(() => createPiiTextTransformer({ ...pii, maxMatches: 0 })).toThrow(); + expect(() => createPiiTextTransformer({ ...pii, action: 'block' })).toThrow(); + const session = createPiiTextTransformer({ + action: 'redact', + starterPatterns: [], + customPatterns: [{ id: 'empty', label: 'empty', regex: 'a*' }], + }).createSession(); + let failure: Error | undefined; + try { + session.transform(fragment('secret value')); + } catch (error) { + failure = error as Error; + } + expect(failure).toBeInstanceOf(PiiTransformationError); + expect(failure?.message).not.toContain('secret value'); + }); +}); diff --git a/packages/api/src/protection/transform.ts b/packages/api/src/protection/transform.ts new file mode 100644 index 0000000000..ddfd9695ae --- /dev/null +++ b/packages/api/src/protection/transform.ts @@ -0,0 +1,184 @@ +import { + DEFAULT_PII_REDACTION_MAX_CHARACTERS, + DEFAULT_PII_REDACTION_MAX_MATCHES, + MAX_PII_PATTERNS_PER_SOURCE, +} from 'librechat-data-provider'; +import type { FilterPiiAction, FilterPiiCategory } from 'librechat-data-provider'; +import type { PatternContentInspectorConfig, PatternTextMatch } from './detectors/pattern'; +import type { TextContentFragment } from './types'; +import { createPatternContentInspector, PatternConfigurationError } from './detectors/pattern'; + +export type PiiPlaceholderType = Uppercase; + +export interface PiiRedactionConfig extends PatternContentInspectorConfig { + readonly action?: FilterPiiAction; + readonly fields?: readonly string[]; + readonly maxCharacters?: number; + readonly maxMatches?: number; +} + +export interface PiiRedactionMetadata { + readonly category: PiiPlaceholderType; + readonly count: number; +} + +export interface PiiTransformationResult { + readonly version: 1; + readonly content: string; + readonly replacements: number; + readonly categories: readonly PiiRedactionMetadata[]; +} + +export class PiiTransformationError extends Error { + constructor(reason: 'limit' | 'unreplaceable' | 'inspection') { + super(`PII transformation failed: ${reason}`); + this.name = 'PiiTransformationError'; + } +} + +export interface PiiTransformationSession { + /** Applies this rule to selected text fields; the caller must still enforce all other policies. */ + transform(fragment: TextContentFragment): PiiTransformationResult; +} + +function readLimit(value: number | undefined, fallback: number, maximum: number): number { + if (value == null) { + return fallback; + } + if (!Number.isSafeInteger(value) || value <= 0 || value > maximum) { + throw new PatternConfigurationError('Invalid PII transformation limit'); + } + return value; +} + +function mergedMatches(matches: readonly PatternTextMatch[]): PatternTextMatch[] { + const sorted = [...matches].sort( + (left, right) => + left.start - right.start || + right.end - left.end || + left.category.localeCompare(right.category), + ); + const merged: PatternTextMatch[] = []; + for (const match of sorted) { + const previous = merged[merged.length - 1]; + if (previous == null || match.start >= previous.end) { + merged.push(match); + } else if (match.end > previous.end) { + merged[merged.length - 1] = { ...previous, end: match.end }; + } + } + return merged; +} + +/** + * Explicit, opt-in transformation. Legacy inspectors still reject `redact` findings + * until the authenticated submission and storage pipeline consumes this API. + * A session owns only ephemeral, bounded mappings; never log or persist it. + */ +export function createPiiTextTransformer(config: PiiRedactionConfig): { + createSession(): PiiTransformationSession; +} { + if (config.action !== 'redact') { + throw new PatternConfigurationError('PII transformation requires the redact action'); + } + const maxCharacters = readLimit( + config.maxCharacters, + DEFAULT_PII_REDACTION_MAX_CHARACTERS, + 262_144, + ); + const maxMatches = readLimit(config.maxMatches, DEFAULT_PII_REDACTION_MAX_MATCHES, 4_096); + let fields: ReadonlySet | undefined; + if (config.fields != null) { + try { + if (!Array.isArray(config.fields) || config.fields.length > MAX_PII_PATTERNS_PER_SOURCE) { + throw new PatternConfigurationError('Invalid PII transformation fields'); + } + fields = new Set(config.fields); + if (fields.size === 0 || [...fields].some((field) => typeof field !== 'string')) { + throw new PatternConfigurationError('Invalid PII transformation fields'); + } + } catch { + throw new PatternConfigurationError('Invalid PII transformation fields'); + } + } + const inspector = createPatternContentInspector(config, { linearTime: true }); + + return { + createSession() { + const placeholders = new Map(); + const issued = new Set(); + const reserved = new Set(); + const nextByCategory = new Map(); + let remainingCharacters = maxCharacters; + let remainingMatches = maxMatches; + return { + transform(fragment) { + let matches: readonly PatternTextMatch[]; + try { + if (fields != null && !fields.has(fragment.field)) { + return { version: 1, content: fragment.text, replacements: 0, categories: [] }; + } + if (typeof fragment.text !== 'string' || fragment.text.length > remainingCharacters) { + throw new PiiTransformationError('limit'); + } + remainingCharacters -= fragment.text.length; + for (const marker of fragment.text.matchAll( + /\[(?:EMAIL|PHONE|NAME|CREDENTIAL|CUSTOM)_[1-9]\d*\]/g, + )) { + reserved.add(marker[0]); + if (issued.has(marker[0])) { + throw new PiiTransformationError('inspection'); + } + } + matches = inspector.locate(fragment.text, remainingMatches); + } catch (error) { + if (error instanceof PiiTransformationError) { + throw error; + } + throw new PiiTransformationError('inspection'); + } + if (matches.length === 0) { + return { version: 1, content: fragment.text, replacements: 0, categories: [] }; + } + if ( + fragment.treatment !== 'replaceable' || + !['plain', 'markdown'].includes(fragment.format) + ) { + throw new PiiTransformationError('unreplaceable'); + } + const merged = mergedMatches(matches); + remainingMatches -= matches.length; + const categories = new Map(); + const parts: string[] = []; + let offset = 0; + for (const { start, end, category } of merged) { + const type = category.toUpperCase() as PiiPlaceholderType; + const value = fragment.text.slice(start, end); + const key = JSON.stringify([type, value]); + let placeholder = placeholders.get(key); + if (placeholder == null) { + let next = nextByCategory.get(type) ?? 0; + do { + next++; + placeholder = `[${type}_${next}]`; + } while (reserved.has(placeholder) || issued.has(placeholder)); + nextByCategory.set(type, next); + placeholders.set(key, placeholder); + issued.add(placeholder); + } + categories.set(type, (categories.get(type) ?? 0) + 1); + parts.push(fragment.text.slice(offset, start), placeholder); + offset = end; + } + parts.push(fragment.text.slice(offset)); + return { + version: 1, + content: parts.join(''), + replacements: merged.length, + categories: [...categories].map(([category, count]) => ({ category, count })), + }; + }, + }; + }, + }; +} diff --git a/packages/data-provider/src/filters.spec.ts b/packages/data-provider/src/filters.spec.ts index c92df1ea8a..eaae6af784 100644 --- a/packages/data-provider/src/filters.spec.ts +++ b/packages/data-provider/src/filters.spec.ts @@ -57,7 +57,7 @@ describe('filtersConfigSchema', () => { ).toBe(false); }); - it('accepts omitted, audit, and block actions while rejecting unsupported actions', () => { + it('accepts opt-in redact, audit, and block actions while rejecting unsupported actions', () => { expect(filtersConfigSchema.parse({ messages: { pii: {} } })).toEqual({ messages: { pii: {} }, }); @@ -71,8 +71,44 @@ describe('filtersConfigSchema', () => { prompts: { pii: { action: 'block' } }, }); expect(filtersConfigSchema.safeParse({ messages: { pii: { action: 'redact' } } }).success).toBe( + true, + ); + expect( + filtersConfigSchema.safeParse({ messages: { pii: { action: 'forward' } } }).success, + ).toBe(false); + }); + + it('bounds opt-in transformation controls and validates placeholder categories', () => { + expect(filtersConfigSchema.parse({ messages: { pii: { action: 'redact' } } })).toEqual({ + messages: { pii: { action: 'redact' } }, + }); + expect(filtersConfigSchema.safeParse({ messages: { pii: { maxCharacters: 0 } } }).success).toBe( false, ); + expect( + filtersConfigSchema.safeParse({ messages: { pii: { maxMatches: 4_097 } } }).success, + ).toBe(false); + expect( + filtersConfigSchema.safeParse({ + messages: { + pii: { + action: 'redact', + customPatterns: [{ id: 'email', label: 'Email', regex: 'a@b', category: 'email' }], + }, + }, + }).success, + ).toBe(true); + expect( + filtersConfigSchema.safeParse({ + messages: { + pii: { + customPatterns: [ + { id: 'email', label: 'Email', regex: 'a@b', category: 'secret-value' }, + ], + }, + }, + }).success, + ).toBe(false); }); it('combines active-pattern and selected-field checks', () => { diff --git a/packages/data-provider/src/filters.ts b/packages/data-provider/src/filters.ts index 9d010c727e..4368777d19 100644 --- a/packages/data-provider/src/filters.ts +++ b/packages/data-provider/src/filters.ts @@ -162,7 +162,10 @@ export const fileFilterFieldSchema = z.enum(FILE_FILTER_FIELDS); export const toolArgumentFilterFieldSchema = z.enum(TOOL_ARGUMENT_FILTER_FIELDS); export const modelParameterFilterFieldSchema = z.enum(MODEL_PARAMETER_FILTER_FIELDS); export const filterPiiStarterPatternSchema = z.enum(FILTER_PII_STARTER_PATTERNS); -export const filterPiiActionSchema = z.enum(['block', 'audit']); +export const filterPiiActionSchema = z.enum(['block', 'audit', 'redact']); +export const filterPiiCategorySchema = z.enum(['email', 'phone', 'name', 'credential', 'custom']); +export const DEFAULT_PII_REDACTION_MAX_CHARACTERS = 65_536; +export const DEFAULT_PII_REDACTION_MAX_MATCHES = 256; export const actionMetadataFilterFieldSchema = z.enum(ACTION_METADATA_FILTER_FIELDS); export const unattributedAssistantContentSchema = z.enum(['model_output', 'inspect']); export type UnattributedAssistantContent = z.infer; @@ -180,6 +183,7 @@ export type ToolArgumentFilterField = z.infer; export type ActionMetadataFilterField = z.infer; export type FilterPiiAction = z.infer; +export type FilterPiiCategory = z.infer; export const userSubmittedMessageFieldPathSchema = z .object({ @@ -279,6 +283,7 @@ export const filterPiiCustomPatternSchema = z id: z.string().min(1).max(MAX_PII_PATTERN_ID_LENGTH), label: z.string().min(1).max(MAX_PII_PATTERN_LABEL_LENGTH), regex: filterPiiRegexSchema, + category: filterPiiCategorySchema.optional(), }) .strict(); @@ -288,6 +293,8 @@ function createPiiFilterSchema(fieldSchema: Field) { return z .object({ action: filterPiiActionSchema.optional(), + maxCharacters: z.number().int().positive().max(262_144).optional(), + maxMatches: z.number().int().positive().max(4_096).optional(), fields: z.array(fieldSchema).min(1).max(MAX_PII_PATTERNS_PER_SOURCE).optional(), starterPatterns: z .array(filterPiiStarterPatternSchema)