mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-21 07:23:11 +00:00
🌫️ feat: Add Bounded PII Text Transformation (#16295)
* 🌫️ feat: Add Bounded PII Text Transformation * ⚡ perf: Reuse Bounded Placeholder Scan * fix: Respect PII Transformation Session Boundaries --------- Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
d0850e7389
commit
7d5cbc9ebf
8 changed files with 589 additions and 16 deletions
|
|
@ -1390,7 +1390,11 @@ endpoints:
|
|||
# messages:
|
||||
# unattributedAssistantContent: model_output # `model_output` (default) or `inspect`
|
||||
# pii:
|
||||
# action: audit # `block` (default) or `audit`
|
||||
# action: audit # `block` (default), `audit`, or `redact` (reserved for transformation-aware ingress; current routes still block)
|
||||
# # `redact` requires the later encrypted owner-view and canonical-message integration.
|
||||
# # maxCharacters: 65536 # bounded total inspected text per transformation session
|
||||
# # maxMatches: 256 # bounded matches per transformation session
|
||||
# # Custom patterns may set category: email, phone, name, credential, or custom.
|
||||
# fields: [name, text, summary, quote, answer, decision_response, decision_reason, content_part, attachment_reference, assembled_context]
|
||||
# starterPatterns: [sk_prefix, bearer_header, api_key_header]
|
||||
# customPatterns:
|
||||
|
|
|
|||
|
|
@ -72,6 +72,32 @@ describe('pattern content inspector', () => {
|
|||
).toMatchObject({ ruleId: 'bearer_header' });
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ config: { starterPatterns: ['bearer_header'] }, text: 'Bearer contract-token' },
|
||||
{
|
||||
config: {
|
||||
starterPatterns: [],
|
||||
customPatterns: [{ id: 'custom', label: 'Custom', regex: 'ORG-[A-Z]+' }],
|
||||
},
|
||||
text: 'ORG-SECRET',
|
||||
},
|
||||
])('enforces a zero remaining allowance when locating $text', ({ config, text }) => {
|
||||
const inspector = createPatternContentInspector(config, { linearTime: true });
|
||||
|
||||
expect(inspector.locate('clean text', 0)).toEqual([]);
|
||||
expect(inspector.locate('', 0)).toEqual([]);
|
||||
expect(() => inspector.locate(text, 0)).toThrow(PatternConfigurationError);
|
||||
expect(inspector.locate(text, 1)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([-1, 0.5, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1])(
|
||||
'rejects invalid remaining match allowance %s before inspecting',
|
||||
(allowance) => {
|
||||
const inspector = createPatternContentInspector({}, { linearTime: true });
|
||||
expect(() => inspector.locate('clean text', allowance)).toThrow(PatternConfigurationError);
|
||||
},
|
||||
);
|
||||
|
||||
it('memoizes a compiled set by config identity and memory limit', () => {
|
||||
const compile = jest.spyOn(RE2Set.prototype, 'compile');
|
||||
const config: MessageFilterPiiConfig = {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { RE2Set } from 're2js';
|
||||
import { RE2JS, RE2Set } from 're2js';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import {
|
||||
MAX_PII_CUSTOM_REGEX_CHARACTERS,
|
||||
|
|
@ -9,23 +9,25 @@ import {
|
|||
MAX_PII_PATTERNS_PER_SOURCE,
|
||||
getPiiRegexProgramSize,
|
||||
} from 'librechat-data-provider';
|
||||
import type { MessageFilterPiiConfig, FilterPiiCustomPatternConfig } from 'librechat-data-provider';
|
||||
import type {
|
||||
FilterPiiCategory,
|
||||
MessageFilterPiiConfig,
|
||||
FilterPiiCustomPatternConfig,
|
||||
} from 'librechat-data-provider';
|
||||
import type { ProtectionFinding, TextContentFragment } from '../types';
|
||||
|
||||
interface TestablePattern {
|
||||
test(input: string): boolean;
|
||||
}
|
||||
|
||||
interface CompiledPattern {
|
||||
readonly id: string;
|
||||
readonly label: string;
|
||||
readonly pattern: TestablePattern;
|
||||
readonly pattern: RegExp;
|
||||
readonly category: FilterPiiCategory;
|
||||
}
|
||||
|
||||
interface PreparedCustomPattern {
|
||||
readonly id: string;
|
||||
readonly label: string;
|
||||
readonly regex: string;
|
||||
readonly category: FilterPiiCategory;
|
||||
}
|
||||
|
||||
interface SnapshotPatternContentInspectorConfig {
|
||||
|
|
@ -67,10 +69,18 @@ export interface PatternContentInspectorPreflightCost {
|
|||
readonly regexes: readonly string[];
|
||||
}
|
||||
|
||||
export interface PatternTextMatch {
|
||||
readonly start: number;
|
||||
readonly end: number;
|
||||
readonly category: FilterPiiCategory;
|
||||
}
|
||||
|
||||
export interface PatternContentInspector {
|
||||
readonly active: boolean;
|
||||
inspectFragment(fragment: TextContentFragment): ProtectionFinding | null;
|
||||
inspect(fragments: Iterable<TextContentFragment>): ProtectionFinding | null;
|
||||
/** Internal match locations, never safe to return as public finding metadata. */
|
||||
locate(text: string, maxMatches: number): readonly PatternTextMatch[];
|
||||
}
|
||||
|
||||
export interface PatternContentInspectorConfig {
|
||||
|
|
@ -86,9 +96,24 @@ export interface PatternContentInspectorOptions {
|
|||
}
|
||||
|
||||
const STARTER_PATTERNS: readonly CompiledPattern[] = [
|
||||
{ id: 'sk_prefix', label: 'sk- prefix token', pattern: /\b(sk-)[a-zA-Z0-9_-]+/ },
|
||||
{ id: 'bearer_header', label: 'Bearer token', pattern: /\b(Bearer )[^\s"']+/i },
|
||||
{ id: 'api_key_header', label: 'api-key header', pattern: /\b(api-key:?\s+)[^\s"']+/i },
|
||||
{
|
||||
id: 'sk_prefix',
|
||||
label: 'sk- prefix token',
|
||||
pattern: /\b(sk-)[a-zA-Z0-9_-]+/,
|
||||
category: 'credential',
|
||||
},
|
||||
{
|
||||
id: 'bearer_header',
|
||||
label: 'Bearer token',
|
||||
pattern: /\b(Bearer )[^\s"']+/i,
|
||||
category: 'credential',
|
||||
},
|
||||
{
|
||||
id: 'api_key_header',
|
||||
label: 'api-key header',
|
||||
pattern: /\b(api-key:?\s+)[^\s"']+/i,
|
||||
category: 'credential',
|
||||
},
|
||||
];
|
||||
|
||||
const STARTER_BY_ID = new Map(STARTER_PATTERNS.map((pattern) => [pattern.id, pattern]));
|
||||
|
|
@ -201,7 +226,25 @@ function readCustomPattern(candidate: unknown, index: number): FilterPiiCustomPa
|
|||
if (typeof regex !== 'string' || regex.length === 0 || regex.length > MAX_PII_PATTERN_LENGTH) {
|
||||
throw configurationError(`customPatterns[${index}].regex is invalid`);
|
||||
}
|
||||
return { id, label, regex };
|
||||
let category: unknown;
|
||||
try {
|
||||
category = (candidate as FilterPiiCustomPatternConfig).category;
|
||||
} catch {
|
||||
throw configurationError(`customPatterns[${index}] could not be read safely`);
|
||||
}
|
||||
if (
|
||||
category != null &&
|
||||
(typeof category !== 'string' ||
|
||||
!['email', 'phone', 'name', 'credential', 'custom'].includes(category))
|
||||
) {
|
||||
throw configurationError(`customPatterns[${index}].category is invalid`);
|
||||
}
|
||||
return {
|
||||
id,
|
||||
label,
|
||||
regex,
|
||||
...(category == null ? {} : { category: category as FilterPiiCategory }),
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotConfig(
|
||||
|
|
@ -240,7 +283,7 @@ function snapshotConfig(
|
|||
throw configurationError('customPatterns could not be read safely');
|
||||
}
|
||||
const pattern = readCustomPattern(patternCandidate, index);
|
||||
custom.push(pattern);
|
||||
custom.push({ ...pattern, category: pattern.category ?? 'custom' });
|
||||
regexes.push(pattern.regex);
|
||||
regexCharacters += pattern.regex.length;
|
||||
if (regexCharacters > MAX_PII_CUSTOM_REGEX_CHARACTERS) {
|
||||
|
|
@ -357,10 +400,17 @@ function createSequentialInspector(patterns: readonly CompiledPattern[]): Patter
|
|||
function createInspector(
|
||||
active: boolean,
|
||||
inspectFragment: (fragment: TextContentFragment) => ProtectionFinding | null,
|
||||
locate?: (text: string, maxMatches: number) => readonly PatternTextMatch[],
|
||||
): PatternContentInspector {
|
||||
return {
|
||||
active,
|
||||
inspectFragment,
|
||||
locate(text, maxMatches) {
|
||||
if (locate == null) {
|
||||
throw configurationError('redaction requires a linear-time pattern inspector');
|
||||
}
|
||||
return locate(text, maxMatches);
|
||||
},
|
||||
inspect(fragments) {
|
||||
for (const fragment of fragments) {
|
||||
const finding = inspectFragment(fragment);
|
||||
|
|
@ -416,7 +466,44 @@ function createLinearInspector(
|
|||
return pattern == null ? null : findingFor(pattern, fragment);
|
||||
};
|
||||
|
||||
return createInspector(prepared.cost.active, inspectFragment);
|
||||
const compiled = new Map<string, RE2JS>();
|
||||
const locate = (text: string, maxMatches: number): readonly PatternTextMatch[] => {
|
||||
if (!Number.isSafeInteger(maxMatches) || maxMatches < 0) {
|
||||
throw configurationError('redaction match limit must be a non-negative safe integer');
|
||||
}
|
||||
const matches: PatternTextMatch[] = [];
|
||||
const selected = customSet == null ? [] : customSet.match(text);
|
||||
const candidates: Array<readonly [string, FilterPiiCategory]> = prepared.starter.map(
|
||||
(pattern) => [
|
||||
`${pattern.pattern.ignoreCase ? '(?i)' : ''}${pattern.pattern.source}`,
|
||||
pattern.category,
|
||||
],
|
||||
);
|
||||
for (const index of selected) {
|
||||
const pattern = prepared.custom[index];
|
||||
if (pattern != null) {
|
||||
candidates.push([pattern.regex, pattern.category]);
|
||||
}
|
||||
}
|
||||
for (const [expression, category] of candidates) {
|
||||
let regex = compiled.get(expression);
|
||||
if (regex == null) {
|
||||
regex = RE2JS.compile(expression);
|
||||
compiled.set(expression, regex);
|
||||
}
|
||||
const matcher = regex.matcher(text);
|
||||
while (matcher.find()) {
|
||||
const start = matcher.start();
|
||||
const end = matcher.end();
|
||||
if (end <= start || matches.length >= maxMatches) {
|
||||
throw configurationError('redaction match limit or empty match encountered');
|
||||
}
|
||||
matches.push({ start, end, category });
|
||||
}
|
||||
}
|
||||
return matches;
|
||||
};
|
||||
return createInspector(prepared.cost.active, inspectFragment, locate);
|
||||
}
|
||||
|
||||
function cacheLinearInspector(
|
||||
|
|
@ -480,6 +567,7 @@ export function createPatternContentInspector(
|
|||
id: pattern.id,
|
||||
label: pattern.label,
|
||||
pattern: new RegExp(pattern.regex),
|
||||
category: pattern.category,
|
||||
}));
|
||||
const inspector = createSequentialInspector([...prepared.starter, ...custom]);
|
||||
if (options.cacheResult !== false) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
export * from './types';
|
||||
export * from './runtime';
|
||||
export * from './transform';
|
||||
export * from './title';
|
||||
export * from './legacy';
|
||||
export * from './provenance';
|
||||
|
|
|
|||
227
packages/api/src/protection/transform.spec.ts
Normal file
227
packages/api/src/protection/transform.spec.ts
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
import type { FiltersConfig } from 'librechat-data-provider';
|
||||
import type { TextContentFragment } from './types';
|
||||
import { createPiiTextTransformer, PiiTransformationError } from './transform';
|
||||
import { inspectContent } from './runtime';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: { warn: jest.fn(), error: jest.fn(), info: jest.fn(), debug: jest.fn() },
|
||||
}));
|
||||
|
||||
const pii: NonNullable<NonNullable<FiltersConfig['messages']>['pii']> = {
|
||||
action: 'redact' as const,
|
||||
starterPatterns: [],
|
||||
fields: ['text'],
|
||||
customPatterns: [
|
||||
{
|
||||
id: 'email',
|
||||
label: 'Email address',
|
||||
regex: '(?i)[a-z]+@[a-z]+\\.[a-z]+',
|
||||
category: 'email' as const,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function fragment(text: string, extras: Partial<TextContentFragment> = {}): TextContentFragment {
|
||||
return {
|
||||
id: 'chat.text',
|
||||
path: '/text',
|
||||
text,
|
||||
source: 'message',
|
||||
field: 'text',
|
||||
format: 'plain',
|
||||
treatment: 'replaceable',
|
||||
provenance: 'user',
|
||||
...extras,
|
||||
} as TextContentFragment;
|
||||
}
|
||||
|
||||
describe('opt-in PII text transformation', () => {
|
||||
it('leaves the existing inspection path blocking until a caller integrates transformation', () => {
|
||||
const filters: FiltersConfig = { messages: { pii } };
|
||||
expect(inspectContent([fragment('Alice@Example.com')], { filters })).toMatchObject({
|
||||
detectorId: 'pii-pattern',
|
||||
ruleId: 'email',
|
||||
});
|
||||
});
|
||||
|
||||
it('uses typed placeholders and returns no matched values in metadata', () => {
|
||||
const session = createPiiTextTransformer(pii).createSession();
|
||||
const first = session.transform(fragment('😀 Alice@Example.com, Alice@Example.com'));
|
||||
const second = session.transform(fragment('Next: Bob@Example.com and Alice@Example.com'));
|
||||
|
||||
expect(first).toEqual({
|
||||
version: 1,
|
||||
content: '😀 [EMAIL_1], [EMAIL_1]',
|
||||
replacements: 2,
|
||||
categories: [{ category: 'EMAIL', count: 2 }],
|
||||
});
|
||||
expect(second.content).toBe('Next: [EMAIL_2] and [EMAIL_1]');
|
||||
expect(JSON.stringify(first.categories)).not.toContain('Alice');
|
||||
expect(JSON.stringify(second.categories)).not.toContain('Bob');
|
||||
const newTurn = createPiiTextTransformer(pii).createSession();
|
||||
expect(newTurn.transform(fragment('Bob@Example.com')).content).toBe('[EMAIL_1]');
|
||||
});
|
||||
|
||||
it('never aliases a user-supplied placeholder to another private value', () => {
|
||||
const session = createPiiTextTransformer(pii).createSession();
|
||||
expect(session.transform(fragment('[EMAIL_1] Alice@Example.com')).content).toBe(
|
||||
'[EMAIL_1] [EMAIL_2]',
|
||||
);
|
||||
expect(() => session.transform(fragment('A forged [EMAIL_2] marker'))).toThrow(
|
||||
PiiTransformationError,
|
||||
);
|
||||
});
|
||||
|
||||
it('reserves literal markers across selected fragments without sharing session state', () => {
|
||||
const transformer = createPiiTextTransformer(pii);
|
||||
const session = transformer.createSession();
|
||||
const independent = transformer.createSession();
|
||||
|
||||
expect(session.transform(fragment('[EMAIL_1] [EMAIL_3]'))).toEqual({
|
||||
version: 1,
|
||||
content: '[EMAIL_1] [EMAIL_3]',
|
||||
replacements: 0,
|
||||
categories: [],
|
||||
});
|
||||
expect(session.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_2]');
|
||||
expect(independent.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_1]');
|
||||
expect(session.transform(fragment('Bob@Example.com Alice@Example.com')).content).toBe(
|
||||
'[EMAIL_4] [EMAIL_2]',
|
||||
);
|
||||
expect(session.transform(fragment('[EMAIL_1]'))).toMatchObject({ replacements: 0 });
|
||||
expect(() => session.transform(fragment('[EMAIL_4]'))).toThrow(PiiTransformationError);
|
||||
});
|
||||
|
||||
it('still charges selected literal markers against the character budget', () => {
|
||||
const session = createPiiTextTransformer({ ...pii, maxCharacters: 18 }).createSession();
|
||||
expect(session.transform(fragment('[EMAIL_1]')).replacements).toBe(0);
|
||||
expect(session.transform(fragment('[EMAIL_2]')).replacements).toBe(0);
|
||||
expect(() => session.transform(fragment('[EMAIL_3]'))).toThrow(PiiTransformationError);
|
||||
});
|
||||
|
||||
it('does not charge an excluded field against the selected character budget', () => {
|
||||
const text = 'Alice@Example.com';
|
||||
const session = createPiiTextTransformer({
|
||||
...pii,
|
||||
maxCharacters: text.length,
|
||||
}).createSession();
|
||||
const excluded = fragment(text.repeat(100), { field: 'summary' });
|
||||
|
||||
expect(session.transform(excluded)).toEqual({
|
||||
version: 1,
|
||||
content: excluded.text,
|
||||
replacements: 0,
|
||||
categories: [],
|
||||
});
|
||||
expect(session.transform(fragment(text)).content).toBe('[EMAIL_1]');
|
||||
});
|
||||
|
||||
it('neither validates nor reserves excluded markers before or after selected fragments', () => {
|
||||
const session = createPiiTextTransformer(pii).createSession();
|
||||
const excluded = fragment('[EMAIL_1] [EMAIL_2]', { field: 'summary' });
|
||||
|
||||
expect(session.transform(excluded).content).toBe(excluded.text);
|
||||
expect(session.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_1]');
|
||||
expect(session.transform(excluded).content).toBe(excluded.text);
|
||||
expect(session.transform(fragment('Bob@Example.com')).content).toBe('[EMAIL_2]');
|
||||
});
|
||||
|
||||
it('accepts clean and empty fragments at the match limit but rejects one more match', () => {
|
||||
const session = createPiiTextTransformer({ ...pii, maxMatches: 1 }).createSession();
|
||||
|
||||
expect(session.transform(fragment('Alice@Example.com')).replacements).toBe(1);
|
||||
for (const text of ['No private details here.', '']) {
|
||||
expect(session.transform(fragment(text))).toEqual({
|
||||
version: 1,
|
||||
content: text,
|
||||
replacements: 0,
|
||||
categories: [],
|
||||
});
|
||||
}
|
||||
expect(() => session.transform(fragment('Alice@Example.com'))).toThrow(PiiTransformationError);
|
||||
});
|
||||
|
||||
it('redacts selected built-in credential headers without leaking the value', () => {
|
||||
const session = createPiiTextTransformer({
|
||||
action: 'redact',
|
||||
starterPatterns: ['bearer_header'],
|
||||
}).createSession();
|
||||
const result = session.transform(fragment('Authorization: bEaReR contract-token'));
|
||||
expect(result.content).toBe('Authorization: [CREDENTIAL_1]');
|
||||
expect(JSON.stringify(result.categories)).not.toContain('contract-token');
|
||||
});
|
||||
|
||||
it('merges overlapping patterns so partial sensitive values never survive', () => {
|
||||
const config = {
|
||||
action: 'redact' as const,
|
||||
starterPatterns: [],
|
||||
customPatterns: [
|
||||
{ id: 'early', label: 'early', regex: 'ORG-SECRET', category: 'credential' as const },
|
||||
{ id: 'later', label: 'later', regex: 'SECRET-DATA', category: 'custom' as const },
|
||||
],
|
||||
};
|
||||
const result = createPiiTextTransformer(config)
|
||||
.createSession()
|
||||
.transform(fragment('prefix ORG-SECRET-DATA suffix'));
|
||||
expect(result.content).toBe('prefix [CREDENTIAL_1] suffix');
|
||||
expect(result.replacements).toBe(1);
|
||||
expect(result.content).not.toContain('SECRET');
|
||||
});
|
||||
|
||||
it('respects configured fields without attempting to replace other fields', () => {
|
||||
const session = createPiiTextTransformer(pii).createSession();
|
||||
expect(session.transform(fragment('Alice@Example.com', { field: 'summary' })).content).toBe(
|
||||
'Alice@Example.com',
|
||||
);
|
||||
expect(session.transform(fragment('Alice@Example.com')).content).toBe('[EMAIL_1]');
|
||||
});
|
||||
|
||||
it('fails closed for non-replaceable matches and structured or URI content', () => {
|
||||
for (const extras of [
|
||||
{ treatment: 'inspect_only' as const },
|
||||
{ format: 'json' as const },
|
||||
{ format: 'uri' as const },
|
||||
]) {
|
||||
expect(() =>
|
||||
createPiiTextTransformer(pii)
|
||||
.createSession()
|
||||
.transform(fragment('Alice@Example.com', extras)),
|
||||
).toThrow(PiiTransformationError);
|
||||
}
|
||||
});
|
||||
|
||||
it('bounds characters and matches across all fragments in one session', () => {
|
||||
const config = {
|
||||
...pii,
|
||||
maxCharacters: 50,
|
||||
maxMatches: 2,
|
||||
};
|
||||
const session = createPiiTextTransformer(config).createSession();
|
||||
expect(session.transform(fragment('A@b.com A@b.com')).replacements).toBe(2);
|
||||
expect(() => session.transform(fragment('C@d.com'))).toThrow(PiiTransformationError);
|
||||
expect(() =>
|
||||
createPiiTextTransformer({ ...config, maxCharacters: 4 })
|
||||
.createSession()
|
||||
.transform(fragment('Alice@Example.com')),
|
||||
).toThrow(PiiTransformationError);
|
||||
});
|
||||
|
||||
it('rejects empty regex matches, invalid limits and missing redact action without echoing input', () => {
|
||||
expect(() => createPiiTextTransformer({ ...pii, maxMatches: -1 })).toThrow();
|
||||
expect(() => createPiiTextTransformer({ ...pii, maxMatches: 0 })).toThrow();
|
||||
expect(() => createPiiTextTransformer({ ...pii, action: 'block' })).toThrow();
|
||||
const session = createPiiTextTransformer({
|
||||
action: 'redact',
|
||||
starterPatterns: [],
|
||||
customPatterns: [{ id: 'empty', label: 'empty', regex: 'a*' }],
|
||||
}).createSession();
|
||||
let failure: Error | undefined;
|
||||
try {
|
||||
session.transform(fragment('secret value'));
|
||||
} catch (error) {
|
||||
failure = error as Error;
|
||||
}
|
||||
expect(failure).toBeInstanceOf(PiiTransformationError);
|
||||
expect(failure?.message).not.toContain('secret value');
|
||||
});
|
||||
});
|
||||
184
packages/api/src/protection/transform.ts
Normal file
184
packages/api/src/protection/transform.ts
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
import {
|
||||
DEFAULT_PII_REDACTION_MAX_CHARACTERS,
|
||||
DEFAULT_PII_REDACTION_MAX_MATCHES,
|
||||
MAX_PII_PATTERNS_PER_SOURCE,
|
||||
} from 'librechat-data-provider';
|
||||
import type { FilterPiiAction, FilterPiiCategory } from 'librechat-data-provider';
|
||||
import type { PatternContentInspectorConfig, PatternTextMatch } from './detectors/pattern';
|
||||
import type { TextContentFragment } from './types';
|
||||
import { createPatternContentInspector, PatternConfigurationError } from './detectors/pattern';
|
||||
|
||||
export type PiiPlaceholderType = Uppercase<FilterPiiCategory>;
|
||||
|
||||
export interface PiiRedactionConfig extends PatternContentInspectorConfig {
|
||||
readonly action?: FilterPiiAction;
|
||||
readonly fields?: readonly string[];
|
||||
readonly maxCharacters?: number;
|
||||
readonly maxMatches?: number;
|
||||
}
|
||||
|
||||
export interface PiiRedactionMetadata {
|
||||
readonly category: PiiPlaceholderType;
|
||||
readonly count: number;
|
||||
}
|
||||
|
||||
export interface PiiTransformationResult {
|
||||
readonly version: 1;
|
||||
readonly content: string;
|
||||
readonly replacements: number;
|
||||
readonly categories: readonly PiiRedactionMetadata[];
|
||||
}
|
||||
|
||||
export class PiiTransformationError extends Error {
|
||||
constructor(reason: 'limit' | 'unreplaceable' | 'inspection') {
|
||||
super(`PII transformation failed: ${reason}`);
|
||||
this.name = 'PiiTransformationError';
|
||||
}
|
||||
}
|
||||
|
||||
export interface PiiTransformationSession {
|
||||
/** Applies this rule to selected text fields; the caller must still enforce all other policies. */
|
||||
transform(fragment: TextContentFragment): PiiTransformationResult;
|
||||
}
|
||||
|
||||
function readLimit(value: number | undefined, fallback: number, maximum: number): number {
|
||||
if (value == null) {
|
||||
return fallback;
|
||||
}
|
||||
if (!Number.isSafeInteger(value) || value <= 0 || value > maximum) {
|
||||
throw new PatternConfigurationError('Invalid PII transformation limit');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function mergedMatches(matches: readonly PatternTextMatch[]): PatternTextMatch[] {
|
||||
const sorted = [...matches].sort(
|
||||
(left, right) =>
|
||||
left.start - right.start ||
|
||||
right.end - left.end ||
|
||||
left.category.localeCompare(right.category),
|
||||
);
|
||||
const merged: PatternTextMatch[] = [];
|
||||
for (const match of sorted) {
|
||||
const previous = merged[merged.length - 1];
|
||||
if (previous == null || match.start >= previous.end) {
|
||||
merged.push(match);
|
||||
} else if (match.end > previous.end) {
|
||||
merged[merged.length - 1] = { ...previous, end: match.end };
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit, opt-in transformation. Legacy inspectors still reject `redact` findings
|
||||
* until the authenticated submission and storage pipeline consumes this API.
|
||||
* A session owns only ephemeral, bounded mappings; never log or persist it.
|
||||
*/
|
||||
export function createPiiTextTransformer(config: PiiRedactionConfig): {
|
||||
createSession(): PiiTransformationSession;
|
||||
} {
|
||||
if (config.action !== 'redact') {
|
||||
throw new PatternConfigurationError('PII transformation requires the redact action');
|
||||
}
|
||||
const maxCharacters = readLimit(
|
||||
config.maxCharacters,
|
||||
DEFAULT_PII_REDACTION_MAX_CHARACTERS,
|
||||
262_144,
|
||||
);
|
||||
const maxMatches = readLimit(config.maxMatches, DEFAULT_PII_REDACTION_MAX_MATCHES, 4_096);
|
||||
let fields: ReadonlySet<string> | undefined;
|
||||
if (config.fields != null) {
|
||||
try {
|
||||
if (!Array.isArray(config.fields) || config.fields.length > MAX_PII_PATTERNS_PER_SOURCE) {
|
||||
throw new PatternConfigurationError('Invalid PII transformation fields');
|
||||
}
|
||||
fields = new Set(config.fields);
|
||||
if (fields.size === 0 || [...fields].some((field) => typeof field !== 'string')) {
|
||||
throw new PatternConfigurationError('Invalid PII transformation fields');
|
||||
}
|
||||
} catch {
|
||||
throw new PatternConfigurationError('Invalid PII transformation fields');
|
||||
}
|
||||
}
|
||||
const inspector = createPatternContentInspector(config, { linearTime: true });
|
||||
|
||||
return {
|
||||
createSession() {
|
||||
const placeholders = new Map<string, string>();
|
||||
const issued = new Set<string>();
|
||||
const reserved = new Set<string>();
|
||||
const nextByCategory = new Map<PiiPlaceholderType, number>();
|
||||
let remainingCharacters = maxCharacters;
|
||||
let remainingMatches = maxMatches;
|
||||
return {
|
||||
transform(fragment) {
|
||||
let matches: readonly PatternTextMatch[];
|
||||
try {
|
||||
if (fields != null && !fields.has(fragment.field)) {
|
||||
return { version: 1, content: fragment.text, replacements: 0, categories: [] };
|
||||
}
|
||||
if (typeof fragment.text !== 'string' || fragment.text.length > remainingCharacters) {
|
||||
throw new PiiTransformationError('limit');
|
||||
}
|
||||
remainingCharacters -= fragment.text.length;
|
||||
for (const marker of fragment.text.matchAll(
|
||||
/\[(?:EMAIL|PHONE|NAME|CREDENTIAL|CUSTOM)_[1-9]\d*\]/g,
|
||||
)) {
|
||||
reserved.add(marker[0]);
|
||||
if (issued.has(marker[0])) {
|
||||
throw new PiiTransformationError('inspection');
|
||||
}
|
||||
}
|
||||
matches = inspector.locate(fragment.text, remainingMatches);
|
||||
} catch (error) {
|
||||
if (error instanceof PiiTransformationError) {
|
||||
throw error;
|
||||
}
|
||||
throw new PiiTransformationError('inspection');
|
||||
}
|
||||
if (matches.length === 0) {
|
||||
return { version: 1, content: fragment.text, replacements: 0, categories: [] };
|
||||
}
|
||||
if (
|
||||
fragment.treatment !== 'replaceable' ||
|
||||
!['plain', 'markdown'].includes(fragment.format)
|
||||
) {
|
||||
throw new PiiTransformationError('unreplaceable');
|
||||
}
|
||||
const merged = mergedMatches(matches);
|
||||
remainingMatches -= matches.length;
|
||||
const categories = new Map<PiiPlaceholderType, number>();
|
||||
const parts: string[] = [];
|
||||
let offset = 0;
|
||||
for (const { start, end, category } of merged) {
|
||||
const type = category.toUpperCase() as PiiPlaceholderType;
|
||||
const value = fragment.text.slice(start, end);
|
||||
const key = JSON.stringify([type, value]);
|
||||
let placeholder = placeholders.get(key);
|
||||
if (placeholder == null) {
|
||||
let next = nextByCategory.get(type) ?? 0;
|
||||
do {
|
||||
next++;
|
||||
placeholder = `[${type}_${next}]`;
|
||||
} while (reserved.has(placeholder) || issued.has(placeholder));
|
||||
nextByCategory.set(type, next);
|
||||
placeholders.set(key, placeholder);
|
||||
issued.add(placeholder);
|
||||
}
|
||||
categories.set(type, (categories.get(type) ?? 0) + 1);
|
||||
parts.push(fragment.text.slice(offset, start), placeholder);
|
||||
offset = end;
|
||||
}
|
||||
parts.push(fragment.text.slice(offset));
|
||||
return {
|
||||
version: 1,
|
||||
content: parts.join(''),
|
||||
replacements: merged.length,
|
||||
categories: [...categories].map(([category, count]) => ({ category, count })),
|
||||
};
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -57,7 +57,7 @@ describe('filtersConfigSchema', () => {
|
|||
).toBe(false);
|
||||
});
|
||||
|
||||
it('accepts omitted, audit, and block actions while rejecting unsupported actions', () => {
|
||||
it('accepts opt-in redact, audit, and block actions while rejecting unsupported actions', () => {
|
||||
expect(filtersConfigSchema.parse({ messages: { pii: {} } })).toEqual({
|
||||
messages: { pii: {} },
|
||||
});
|
||||
|
|
@ -71,8 +71,44 @@ describe('filtersConfigSchema', () => {
|
|||
prompts: { pii: { action: 'block' } },
|
||||
});
|
||||
expect(filtersConfigSchema.safeParse({ messages: { pii: { action: 'redact' } } }).success).toBe(
|
||||
true,
|
||||
);
|
||||
expect(
|
||||
filtersConfigSchema.safeParse({ messages: { pii: { action: 'forward' } } }).success,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('bounds opt-in transformation controls and validates placeholder categories', () => {
|
||||
expect(filtersConfigSchema.parse({ messages: { pii: { action: 'redact' } } })).toEqual({
|
||||
messages: { pii: { action: 'redact' } },
|
||||
});
|
||||
expect(filtersConfigSchema.safeParse({ messages: { pii: { maxCharacters: 0 } } }).success).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
filtersConfigSchema.safeParse({ messages: { pii: { maxMatches: 4_097 } } }).success,
|
||||
).toBe(false);
|
||||
expect(
|
||||
filtersConfigSchema.safeParse({
|
||||
messages: {
|
||||
pii: {
|
||||
action: 'redact',
|
||||
customPatterns: [{ id: 'email', label: 'Email', regex: 'a@b', category: 'email' }],
|
||||
},
|
||||
},
|
||||
}).success,
|
||||
).toBe(true);
|
||||
expect(
|
||||
filtersConfigSchema.safeParse({
|
||||
messages: {
|
||||
pii: {
|
||||
customPatterns: [
|
||||
{ id: 'email', label: 'Email', regex: 'a@b', category: 'secret-value' },
|
||||
],
|
||||
},
|
||||
},
|
||||
}).success,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('combines active-pattern and selected-field checks', () => {
|
||||
|
|
|
|||
|
|
@ -162,7 +162,10 @@ export const fileFilterFieldSchema = z.enum(FILE_FILTER_FIELDS);
|
|||
export const toolArgumentFilterFieldSchema = z.enum(TOOL_ARGUMENT_FILTER_FIELDS);
|
||||
export const modelParameterFilterFieldSchema = z.enum(MODEL_PARAMETER_FILTER_FIELDS);
|
||||
export const filterPiiStarterPatternSchema = z.enum(FILTER_PII_STARTER_PATTERNS);
|
||||
export const filterPiiActionSchema = z.enum(['block', 'audit']);
|
||||
export const filterPiiActionSchema = z.enum(['block', 'audit', 'redact']);
|
||||
export const filterPiiCategorySchema = z.enum(['email', 'phone', 'name', 'credential', 'custom']);
|
||||
export const DEFAULT_PII_REDACTION_MAX_CHARACTERS = 65_536;
|
||||
export const DEFAULT_PII_REDACTION_MAX_MATCHES = 256;
|
||||
export const actionMetadataFilterFieldSchema = z.enum(ACTION_METADATA_FILTER_FIELDS);
|
||||
export const unattributedAssistantContentSchema = z.enum(['model_output', 'inspect']);
|
||||
export type UnattributedAssistantContent = z.infer<typeof unattributedAssistantContentSchema>;
|
||||
|
|
@ -180,6 +183,7 @@ export type ToolArgumentFilterField = z.infer<typeof toolArgumentFilterFieldSche
|
|||
export type ModelParameterFilterField = z.infer<typeof modelParameterFilterFieldSchema>;
|
||||
export type ActionMetadataFilterField = z.infer<typeof actionMetadataFilterFieldSchema>;
|
||||
export type FilterPiiAction = z.infer<typeof filterPiiActionSchema>;
|
||||
export type FilterPiiCategory = z.infer<typeof filterPiiCategorySchema>;
|
||||
|
||||
export const userSubmittedMessageFieldPathSchema = z
|
||||
.object({
|
||||
|
|
@ -279,6 +283,7 @@ export const filterPiiCustomPatternSchema = z
|
|||
id: z.string().min(1).max(MAX_PII_PATTERN_ID_LENGTH),
|
||||
label: z.string().min(1).max(MAX_PII_PATTERN_LABEL_LENGTH),
|
||||
regex: filterPiiRegexSchema,
|
||||
category: filterPiiCategorySchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
|
|
@ -288,6 +293,8 @@ function createPiiFilterSchema<Field extends z.ZodTypeAny>(fieldSchema: Field) {
|
|||
return z
|
||||
.object({
|
||||
action: filterPiiActionSchema.optional(),
|
||||
maxCharacters: z.number().int().positive().max(262_144).optional(),
|
||||
maxMatches: z.number().int().positive().max(4_096).optional(),
|
||||
fields: z.array(fieldSchema).min(1).max(MAX_PII_PATTERNS_PER_SOURCE).optional(),
|
||||
starterPatterns: z
|
||||
.array(filterPiiStarterPatternSchema)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue