🌫️ feat: Add Bounded PII Text Transformation (#16295)

* 🌫️ feat: Add Bounded PII Text Transformation

* ⚡ perf: Reuse Bounded Placeholder Scan

* fix: Respect PII Transformation Session Boundaries

---------

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
lia-by-librechat[bot] 2026-09-24 08:34:49 -04:00 • committed by GitHub
parent d0850e7389
commit 7d5cbc9ebf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 589 additions and 16 deletions

View file

@ -1390,7 +1390,11 @@ endpoints:
# messages:
# unattributedAssistantContent: model_output # `model_output` (default) or `inspect`
# pii:
# action: audit # `block` (default) or `audit`
# action: audit # `block` (default), `audit`, or `redact` (reserved for transformation-aware ingress; current routes still block)
# # `redact` requires the later encrypted owner-view and canonical-message integration.
# # maxCharacters: 65536 # bounded total inspected text per transformation session
# # maxMatches: 256 # bounded matches per transformation session
# # Custom patterns may set category: email, phone, name, credential, or custom.
# fields: [name, text, summary, quote, answer, decision_response, decision_reason, content_part, attachment_reference, assembled_context]
# starterPatterns: [sk_prefix, bearer_header, api_key_header]
# customPatterns:

View file

@ -72,6 +72,32 @@ describe('pattern content inspector', () => {
).toMatchObject({ ruleId: 'bearer_header' });
});
it.each([
{ config: { starterPatterns: ['bearer_header'] }, text: 'Bearer contract-token' },
{
config: {
starterPatterns: [],
customPatterns: [{ id: 'custom', label: 'Custom', regex: 'ORG-[A-Z]+' }],
},
text: 'ORG-SECRET',
},
])('enforces a zero remaining allowance when locating $text', ({ config, text }) => {
const inspector = createPatternContentInspector(config, { linearTime: true });
expect(inspector.locate('clean text', 0)).toEqual([]);
expect(inspector.locate('', 0)).toEqual([]);
expect(() => inspector.locate(text, 0)).toThrow(PatternConfigurationError);
expect(inspector.locate(text, 1)).toHaveLength(1);
});
it.each([-1, 0.5, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1])(
'rejects invalid remaining match allowance %s before inspecting',
(allowance) => {
const inspector = createPatternContentInspector({}, { linearTime: true });
expect(() => inspector.locate('clean text', allowance)).toThrow(PatternConfigurationError);
},
);
it('memoizes a compiled set by config identity and memory limit', () => {
const compile = jest.spyOn(RE2Set.prototype, 'compile');
const config: MessageFilterPiiConfig = {

View file

@ -1,4 +1,4 @@
import { RE2Set } from 're2js';
import { RE2JS, RE2Set } from 're2js';
import { logger } from '@librechat/data-schemas';
import {
MAX_PII_CUSTOM_REGEX_CHARACTERS,
@ -9,23 +9,25 @@ import {
MAX_PII_PATTERNS_PER_SOURCE,
getPiiRegexProgramSize,
} from 'librechat-data-provider';
import type { MessageFilterPiiConfig, FilterPiiCustomPatternConfig } from 'librechat-data-provider';
import type {
FilterPiiCategory,
MessageFilterPiiConfig,
FilterPiiCustomPatternConfig,
} from 'librechat-data-provider';
import type { ProtectionFinding, TextContentFragment } from '../types';
interface TestablePattern {
test(input: string): boolean;
}
interface CompiledPattern {
readonly id: string;
readonly label: string;
readonly pattern: TestablePattern;
readonly pattern: RegExp;
readonly category: FilterPiiCategory;
}
interface PreparedCustomPattern {
readonly id: string;
readonly label: string;
readonly regex: string;
readonly category: FilterPiiCategory;
}
interface SnapshotPatternContentInspectorConfig {
@ -67,10 +69,18 @@ export interface PatternContentInspectorPreflightCost {
readonly regexes: readonly string[];
}
export interface PatternTextMatch {
readonly start: number;
readonly end: number;
readonly category: FilterPiiCategory;
}
export interface PatternContentInspector {
readonly active: boolean;
inspectFragment(fragment: TextContentFragment): ProtectionFinding | null;
inspect(fragments: Iterable<TextContentFragment>): ProtectionFinding | null;
/** Internal match locations, never safe to return as public finding metadata. */
locate(text: string, maxMatches: number): readonly PatternTextMatch[];
}
export interface PatternContentInspectorConfig {
@ -86,9 +96,24 @@ export interface PatternContentInspectorOptions {
}
const STARTER_PATTERNS: readonly CompiledPattern[] = [
{ id: 'sk_prefix', label: 'sk- prefix token', pattern: /\b(sk-)[a-zA-Z0-9_-]+/ },
{ id: 'bearer_header', label: 'Bearer token', pattern: /\b(Bearer )[^\s"']+/i },
{ id: 'api_key_header', label: 'api-key header', pattern: /\b(api-key:?\s+)[^\s"']+/i },
{
id: 'sk_prefix',
label: 'sk- prefix token',
pattern: /\b(sk-)[a-zA-Z0-9_-]+/,
category: 'credential',
},
{
id: 'bearer_header',
label: 'Bearer token',
pattern: /\b(Bearer )[^\s"']+/i,
category: 'credential',
},
{
id: 'api_key_header',
label: 'api-key header',
pattern: /\b(api-key:?\s+)[^\s"']+/i,
category: 'credential',
},
];
const STARTER_BY_ID = new Map(STARTER_PATTERNS.map((pattern) => [pattern.id, pattern]));
@ -201,7 +226,25 @@ function readCustomPattern(candidate: unknown, index: number): FilterPiiCustomPa
if (typeof regex !== 'string' || regex.length === 0 || regex.length > MAX_PII_PATTERN_LENGTH) {
throw configurationError(`customPatterns[${index}].regex is invalid`);
}
return { id, label, regex };
let category: unknown;
try {
category = (candidate as FilterPiiCustomPatternConfig).category;
} catch {
throw configurationError(`customPatterns[${index}] could not be read safely`);
}
if (
category != null &&
(typeof category !== 'string' ||
!['email', 'phone', 'name', 'credential', 'custom'].includes(category))
) {
throw configurationError(`customPatterns[${index}].category is invalid`);
}
return {
id,
label,
regex,
...(category == null ? {} : { category: category as FilterPiiCategory }),
};
}
function snapshotConfig(
@ -240,7 +283,7 @@ function snapshotConfig(
throw configurationError('customPatterns could not be read safely');
}
const pattern = readCustomPattern(patternCandidate, index);
custom.push(pattern);
custom.push({ ...pattern, category: pattern.category ?? 'custom' });
regexes.push(pattern.regex);
regexCharacters += pattern.regex.length;
if (regexCharacters > MAX_PII_CUSTOM_REGEX_CHARACTERS) {
@ -357,10 +400,17 @@ function createSequentialInspector(patterns: readonly CompiledPattern[]): Patter
function createInspector(
active: boolean,
inspectFragment: (fragment: TextContentFragment) => ProtectionFinding | null,
locate?: (text: string, maxMatches: number) => readonly PatternTextMatch[],
): PatternContentInspector {
return {
active,
inspectFragment,
locate(text, maxMatches) {
if (locate == null) {
throw configurationError('redaction requires a linear-time pattern inspector');
}
return locate(text, maxMatches);
},
inspect(fragments) {
for (const fragment of fragments) {
const finding = inspectFragment(fragment);
@ -416,7 +466,44 @@ function createLinearInspector(
return pattern == null ? null : findingFor(pattern, fragment);
};
return createInspector(prepared.cost.active, inspectFragment);
const compiled = new Map<string, RE2JS>();
const locate = (text: string, maxMatches: number): readonly PatternTextMatch[] => {
if (!Number.isSafeInteger(maxMatches) || maxMatches < 0) {
throw configurationError('redaction match limit must be a non-negative safe integer');
}
const matches: PatternTextMatch[] = [];
const selected = customSet == null ? [] : customSet.match(text);
const candidates: Array<readonly [string, FilterPiiCategory]> = prepared.starter.map(
(pattern) => [
`${pattern.pattern.ignoreCase ? '(?i)' : ''}${pattern.pattern.source}`,
pattern.category,
],
);
for (const index of selected) {
const pattern = prepared.custom[index];
if (pattern != null) {
candidates.push([pattern.regex, pattern.category]);
}
}
for (const [expression, category] of candidates) {
let regex = compiled.get(expression);
if (regex == null) {
regex = RE2JS.compile(expression);
compiled.set(expression, regex);
}
const matcher = regex.matcher(text);
while (matcher.find()) {
const start = matcher.start();
const end = matcher.end();
if (end <= start || matches.length >= maxMatches) {
throw configurationError('redaction match limit or empty match encountered');
}
matches.push({ start, end, category });
}
}
return matches;
};
return createInspector(prepared.cost.active, inspectFragment, locate);
}
function cacheLinearInspector(
@ -480,6 +567,7 @@ export function createPatternContentInspector(
id: pattern.id,
label: pattern.label,
pattern: new RegExp(pattern.regex),
category: pattern.category,
}));
const inspector = createSequentialInspector([...prepared.starter, ...custom]);
if (options.cacheResult !== false) {

View file

@ -1,5 +1,6 @@
export * from './types';
export * from './runtime';
export * from './transform';
export * from './title';
export * from './legacy';
export * from './provenance';

View file

@ -0,0 +1,227 @@
import type { FiltersConfig } from 'librechat-data-provider';
import type { TextContentFragment } from './types';
import { createPiiTextTransformer, PiiTransformationError } from './transform';
import { inspectContent } from './runtime';
jest.mock('@librechat/data-schemas', () => ({
logger: { warn: jest.fn(), error: jest.fn(), info: jest.fn(), debug: jest.fn() },
}));
const pii: NonNullable<NonNullable<FiltersConfig['messages']>['pii']> = {
action: 'redact' as const,
starterPatterns: [],
fields: ['text'],
customPatterns: [
{
id: 'email',
label: 'Email address',
regex: '(?i)[a-z]+@[a-z]+\\.[a-z]+',
category: 'email' as const,
},
],
};
function fragment(text: string, extras: Partial<TextContentFragment> = {}): TextContentFragment {
return {
id: 'chat.text',
path: '/text',
text,
source: 'message',
field: 'text',
format: 'plain',
treatment: 'replaceable',
provenance: 'user',
...extras,
} as TextContentFragment;
}
describe('opt-in PII text transformation', () => {
it('leaves the existing inspection path blocking until a caller integrates transformation', () => {
const filters: FiltersConfig = { messages: { pii } };
expect(inspectContent([fragment('Alice@Example.com')], { filters })).toMatchObject({
detectorId: 'pii-pattern',
ruleId: 'email',
});
});
it('uses typed placeholders and returns no matched values in metadata', () => {
const session = createPiiTextTransformer(pii).createSession();
const first = session.transform(fragment('😀 Alice@Example.com, Alice@Example.com'));
const second = session.transform(fragment('Next: Bob@Example.com and Alice@Example.com'));
expect(first).toEqual({
version: 1,
content: '😀 [EMAIL_1], [EMAIL_1]',
replacements: 2,
categories: [{ category: 'EMAIL', count: 2 }],
});
expect(second.content).toBe('Next: [EMAIL_2] and [EMAIL_1]');
expect(JSON.stringify(first.categories)).not.toContain('Alice');
expect(JSON.stringify(second.categories)).not.toContain('Bob');
const newTurn = createPiiTextTransformer(pii).createSession();
expect(newTurn.transform(fragment('Bob@Example.com')).content).toBe('[EMAIL_1]');
});
it('never aliases a user-supplied placeholder to another private value', () => {
const session = createPiiTextTransformer(pii).createSession();
expect(session.transform(fragment('[EMAIL_1] Alice@Example.com')).content).toBe(
'[EMAIL_1] [EMAIL_2]',
);
expect(() => session.transform(fragment('A forged [EMAIL_2] marker'))).toThrow(
PiiTransformationError,
);
});
it('reserves literal markers across selected fragments without sharing session state', () => {
const transformer = createPiiTextTransformer(pii);
const session = transformer.createSession();
const independent = transformer.createSession();
expect(session.transform(fragment('[EMAIL_1] [EMAIL_3]'))).toEqual({
version: 1,
content: '[EMAIL_1] [EMAIL_3]',
replacements: 0,
categories: [],
});
expect(session.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_2]');
expect(independent.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_1]');
expect(session.transform(fragment('Bob@Example.com Alice@Example.com')).content).toBe(
'[EMAIL_4] [EMAIL_2]',
);
expect(session.transform(fragment('[EMAIL_1]'))).toMatchObject({ replacements: 0 });
expect(() => session.transform(fragment('[EMAIL_4]'))).toThrow(PiiTransformationError);
});
it('still charges selected literal markers against the character budget', () => {
const session = createPiiTextTransformer({ ...pii, maxCharacters: 18 }).createSession();
expect(session.transform(fragment('[EMAIL_1]')).replacements).toBe(0);
expect(session.transform(fragment('[EMAIL_2]')).replacements).toBe(0);
expect(() => session.transform(fragment('[EMAIL_3]'))).toThrow(PiiTransformationError);
});
it('does not charge an excluded field against the selected character budget', () => {
const text = 'Alice@Example.com';
const session = createPiiTextTransformer({
...pii,
maxCharacters: text.length,
}).createSession();
const excluded = fragment(text.repeat(100), { field: 'summary' });
expect(session.transform(excluded)).toEqual({
version: 1,
content: excluded.text,
replacements: 0,
categories: [],
});
expect(session.transform(fragment(text)).content).toBe('[EMAIL_1]');
});
it('neither validates nor reserves excluded markers before or after selected fragments', () => {
const session = createPiiTextTransformer(pii).createSession();
const excluded = fragment('[EMAIL_1] [EMAIL_2]', { field: 'summary' });
expect(session.transform(excluded).content).toBe(excluded.text);
expect(session.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_1]');
expect(session.transform(excluded).content).toBe(excluded.text);
expect(session.transform(fragment('Bob@Example.com')).content).toBe('[EMAIL_2]');
});
it('accepts clean and empty fragments at the match limit but rejects one more match', () => {
const session = createPiiTextTransformer({ ...pii, maxMatches: 1 }).createSession();
expect(session.transform(fragment('Alice@Example.com')).replacements).toBe(1);
for (const text of ['No private details here.', '']) {
expect(session.transform(fragment(text))).toEqual({
version: 1,
content: text,
replacements: 0,
categories: [],
});
}
expect(() => session.transform(fragment('Alice@Example.com'))).toThrow(PiiTransformationError);
});
it('redacts selected built-in credential headers without leaking the value', () => {
const session = createPiiTextTransformer({
action: 'redact',
starterPatterns: ['bearer_header'],
}).createSession();
const result = session.transform(fragment('Authorization: bEaReR contract-token'));
expect(result.content).toBe('Authorization: [CREDENTIAL_1]');
expect(JSON.stringify(result.categories)).not.toContain('contract-token');
});
it('merges overlapping patterns so partial sensitive values never survive', () => {
const config = {
action: 'redact' as const,
starterPatterns: [],
customPatterns: [
{ id: 'early', label: 'early', regex: 'ORG-SECRET', category: 'credential' as const },
{ id: 'later', label: 'later', regex: 'SECRET-DATA', category: 'custom' as const },
],
};
const result = createPiiTextTransformer(config)
.createSession()
.transform(fragment('prefix ORG-SECRET-DATA suffix'));
expect(result.content).toBe('prefix [CREDENTIAL_1] suffix');
expect(result.replacements).toBe(1);
expect(result.content).not.toContain('SECRET');
});
it('respects configured fields without attempting to replace other fields', () => {
const session = createPiiTextTransformer(pii).createSession();
expect(session.transform(fragment('Alice@Example.com', { field: 'summary' })).content).toBe(
'Alice@Example.com',
);
expect(session.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_1]');
});
it('fails closed for non-replaceable matches and structured or URI content', () => {
for (const extras of [
{ treatment: 'inspect_only' as const },
{ format: 'json' as const },
{ format: 'uri' as const },
]) {
expect(() =>
createPiiTextTransformer(pii)
.createSession()
.transform(fragment('Alice@Example.com', extras)),
).toThrow(PiiTransformationError);
}
});
it('bounds characters and matches across all fragments in one session', () => {
const config = {
...pii,
maxCharacters: 50,
maxMatches: 2,
};
const session = createPiiTextTransformer(config).createSession();
expect(session.transform(fragment('A@b.com A@b.com')).replacements).toBe(2);
expect(() => session.transform(fragment('C@d.com'))).toThrow(PiiTransformationError);
expect(() =>
createPiiTextTransformer({ ...config, maxCharacters: 4 })
.createSession()
.transform(fragment('Alice@Example.com')),
).toThrow(PiiTransformationError);
});
it('rejects empty regex matches, invalid limits and missing redact action without echoing input', () => {
expect(() => createPiiTextTransformer({ ...pii, maxMatches: -1 })).toThrow();
expect(() => createPiiTextTransformer({ ...pii, maxMatches: 0 })).toThrow();
expect(() => createPiiTextTransformer({ ...pii, action: 'block' })).toThrow();
const session = createPiiTextTransformer({
action: 'redact',
starterPatterns: [],
customPatterns: [{ id: 'empty', label: 'empty', regex: 'a*' }],
}).createSession();
let failure: Error | undefined;
try {
session.transform(fragment('secret value'));
} catch (error) {
failure = error as Error;
}
expect(failure).toBeInstanceOf(PiiTransformationError);
expect(failure?.message).not.toContain('secret value');
});
});

View file

@ -0,0 +1,184 @@
import {
DEFAULT_PII_REDACTION_MAX_CHARACTERS,
DEFAULT_PII_REDACTION_MAX_MATCHES,
MAX_PII_PATTERNS_PER_SOURCE,
} from 'librechat-data-provider';
import type { FilterPiiAction, FilterPiiCategory } from 'librechat-data-provider';
import type { PatternContentInspectorConfig, PatternTextMatch } from './detectors/pattern';
import type { TextContentFragment } from './types';
import { createPatternContentInspector, PatternConfigurationError } from './detectors/pattern';
export type PiiPlaceholderType = Uppercase<FilterPiiCategory>;
export interface PiiRedactionConfig extends PatternContentInspectorConfig {
readonly action?: FilterPiiAction;
readonly fields?: readonly string[];
readonly maxCharacters?: number;
readonly maxMatches?: number;
}
export interface PiiRedactionMetadata {
readonly category: PiiPlaceholderType;
readonly count: number;
}
export interface PiiTransformationResult {
readonly version: 1;
readonly content: string;
readonly replacements: number;
readonly categories: readonly PiiRedactionMetadata[];
}
export class PiiTransformationError extends Error {
constructor(reason: 'limit' | 'unreplaceable' | 'inspection') {
super(`PII transformation failed: ${reason}`);
this.name = 'PiiTransformationError';
}
}
export interface PiiTransformationSession {
/** Applies this rule to selected text fields; the caller must still enforce all other policies. */
transform(fragment: TextContentFragment): PiiTransformationResult;
}
function readLimit(value: number | undefined, fallback: number, maximum: number): number {
if (value == null) {
return fallback;
}
if (!Number.isSafeInteger(value) || value <= 0 || value > maximum) {
throw new PatternConfigurationError('Invalid PII transformation limit');
}
return value;
}
function mergedMatches(matches: readonly PatternTextMatch[]): PatternTextMatch[] {
const sorted = [...matches].sort(
(left, right) =>
left.start - right.start ||
right.end - left.end ||
left.category.localeCompare(right.category),
);
const merged: PatternTextMatch[] = [];
for (const match of sorted) {
const previous = merged[merged.length - 1];
if (previous == null || match.start >= previous.end) {
merged.push(match);
} else if (match.end > previous.end) {
merged[merged.length - 1] = { ...previous, end: match.end };
}
}
return merged;
}
/**
* Explicit, opt-in transformation. Legacy inspectors still reject `redact` findings
* until the authenticated submission and storage pipeline consumes this API.
* A session owns only ephemeral, bounded mappings; never log or persist it.
*/
export function createPiiTextTransformer(config: PiiRedactionConfig): {
createSession(): PiiTransformationSession;
} {
if (config.action !== 'redact') {
throw new PatternConfigurationError('PII transformation requires the redact action');
}
const maxCharacters = readLimit(
config.maxCharacters,
DEFAULT_PII_REDACTION_MAX_CHARACTERS,
262_144,
);
const maxMatches = readLimit(config.maxMatches, DEFAULT_PII_REDACTION_MAX_MATCHES, 4_096);
let fields: ReadonlySet<string> | undefined;
if (config.fields != null) {
try {
if (!Array.isArray(config.fields) || config.fields.length > MAX_PII_PATTERNS_PER_SOURCE) {
throw new PatternConfigurationError('Invalid PII transformation fields');
}
fields = new Set(config.fields);
if (fields.size === 0 || [...fields].some((field) => typeof field !== 'string')) {
throw new PatternConfigurationError('Invalid PII transformation fields');
}
} catch {
throw new PatternConfigurationError('Invalid PII transformation fields');
}
}
const inspector = createPatternContentInspector(config, { linearTime: true });
return {
createSession() {
const placeholders = new Map<string, string>();
const issued = new Set<string>();
const reserved = new Set<string>();
const nextByCategory = new Map<PiiPlaceholderType, number>();
let remainingCharacters = maxCharacters;
let remainingMatches = maxMatches;
return {
transform(fragment) {
let matches: readonly PatternTextMatch[];
try {
if (fields != null && !fields.has(fragment.field)) {
return { version: 1, content: fragment.text, replacements: 0, categories: [] };
}
if (typeof fragment.text !== 'string' || fragment.text.length > remainingCharacters) {
throw new PiiTransformationError('limit');
}
remainingCharacters -= fragment.text.length;
for (const marker of fragment.text.matchAll(
/\[(?:EMAIL|PHONE|NAME|CREDENTIAL|CUSTOM)_[1-9]\d*\]/g,
)) {
reserved.add(marker[0]);
if (issued.has(marker[0])) {
throw new PiiTransformationError('inspection');
}
}
matches = inspector.locate(fragment.text, remainingMatches);
} catch (error) {
if (error instanceof PiiTransformationError) {
throw error;
}
throw new PiiTransformationError('inspection');
}
if (matches.length === 0) {
return { version: 1, content: fragment.text, replacements: 0, categories: [] };
}
if (
fragment.treatment !== 'replaceable' ||
!['plain', 'markdown'].includes(fragment.format)
) {
throw new PiiTransformationError('unreplaceable');
}
const merged = mergedMatches(matches);
remainingMatches -= matches.length;
const categories = new Map<PiiPlaceholderType, number>();
const parts: string[] = [];
let offset = 0;
for (const { start, end, category } of merged) {
const type = category.toUpperCase() as PiiPlaceholderType;
const value = fragment.text.slice(start, end);
const key = JSON.stringify([type, value]);
let placeholder = placeholders.get(key);
if (placeholder == null) {
let next = nextByCategory.get(type) ?? 0;
do {
next++;
placeholder = `[${type}_${next}]`;
} while (reserved.has(placeholder) || issued.has(placeholder));
nextByCategory.set(type, next);
placeholders.set(key, placeholder);
issued.add(placeholder);
}
categories.set(type, (categories.get(type) ?? 0) + 1);
parts.push(fragment.text.slice(offset, start), placeholder);
offset = end;
}
parts.push(fragment.text.slice(offset));
return {
version: 1,
content: parts.join(''),
replacements: merged.length,
categories: [...categories].map(([category, count]) => ({ category, count })),
};
},
};
},
};
}

View file

@ -57,7 +57,7 @@ describe('filtersConfigSchema', () => {
).toBe(false);
});
it('accepts omitted, audit, and block actions while rejecting unsupported actions', () => {
it('accepts opt-in redact, audit, and block actions while rejecting unsupported actions', () => {
expect(filtersConfigSchema.parse({ messages: { pii: {} } })).toEqual({
messages: { pii: {} },
});
@ -71,8 +71,44 @@ describe('filtersConfigSchema', () => {
prompts: { pii: { action: 'block' } },
});
expect(filtersConfigSchema.safeParse({ messages: { pii: { action: 'redact' } } }).success).toBe(
true,
);
expect(
filtersConfigSchema.safeParse({ messages: { pii: { action: 'forward' } } }).success,
).toBe(false);
});
it('bounds opt-in transformation controls and validates placeholder categories', () => {
expect(filtersConfigSchema.parse({ messages: { pii: { action: 'redact' } } })).toEqual({
messages: { pii: { action: 'redact' } },
});
expect(filtersConfigSchema.safeParse({ messages: { pii: { maxCharacters: 0 } } }).success).toBe(
false,
);
expect(
filtersConfigSchema.safeParse({ messages: { pii: { maxMatches: 4_097 } } }).success,
).toBe(false);
expect(
filtersConfigSchema.safeParse({
messages: {
pii: {
action: 'redact',
customPatterns: [{ id: 'email', label: 'Email', regex: 'a@b', category: 'email' }],
},
},
}).success,
).toBe(true);
expect(
filtersConfigSchema.safeParse({
messages: {
pii: {
customPatterns: [
{ id: 'email', label: 'Email', regex: 'a@b', category: 'secret-value' },
],
},
},
}).success,
).toBe(false);
});
it('combines active-pattern and selected-field checks', () => {

View file

@ -162,7 +162,10 @@ export const fileFilterFieldSchema = z.enum(FILE_FILTER_FIELDS);
export const toolArgumentFilterFieldSchema = z.enum(TOOL_ARGUMENT_FILTER_FIELDS);
export const modelParameterFilterFieldSchema = z.enum(MODEL_PARAMETER_FILTER_FIELDS);
export const filterPiiStarterPatternSchema = z.enum(FILTER_PII_STARTER_PATTERNS);
export const filterPiiActionSchema = z.enum(['block', 'audit']);
export const filterPiiActionSchema = z.enum(['block', 'audit', 'redact']);
export const filterPiiCategorySchema = z.enum(['email', 'phone', 'name', 'credential', 'custom']);
export const DEFAULT_PII_REDACTION_MAX_CHARACTERS = 65_536;
export const DEFAULT_PII_REDACTION_MAX_MATCHES = 256;
export const actionMetadataFilterFieldSchema = z.enum(ACTION_METADATA_FILTER_FIELDS);
export const unattributedAssistantContentSchema = z.enum(['model_output', 'inspect']);
export type UnattributedAssistantContent = z.infer<typeof unattributedAssistantContentSchema>;
@ -180,6 +183,7 @@ export type ToolArgumentFilterField = z.infer<typeof toolArgumentFilterFieldSche
export type ModelParameterFilterField = z.infer<typeof modelParameterFilterFieldSchema>;
export type ActionMetadataFilterField = z.infer<typeof actionMetadataFilterFieldSchema>;
export type FilterPiiAction = z.infer<typeof filterPiiActionSchema>;
export type FilterPiiCategory = z.infer<typeof filterPiiCategorySchema>;
export const userSubmittedMessageFieldPathSchema = z
.object({
@ -279,6 +283,7 @@ export const filterPiiCustomPatternSchema = z
id: z.string().min(1).max(MAX_PII_PATTERN_ID_LENGTH),
label: z.string().min(1).max(MAX_PII_PATTERN_LABEL_LENGTH),
regex: filterPiiRegexSchema,
category: filterPiiCategorySchema.optional(),
})
.strict();
@ -288,6 +293,8 @@ function createPiiFilterSchema<Field extends z.ZodTypeAny>(fieldSchema: Field) {
return z
.object({
action: filterPiiActionSchema.optional(),
maxCharacters: z.number().int().positive().max(262_144).optional(),
maxMatches: z.number().int().positive().max(4_096).optional(),
fields: z.array(fieldSchema).min(1).max(MAX_PII_PATTERNS_PER_SOURCE).optional(),
starterPatterns: z
.array(filterPiiStarterPatternSchema)