mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
🛡️ fix: Run message-filter PII patterns on a linear-time regex engine
The messageFilter.pii middleware compiled admin-configured customPatterns with the native RegExp engine and ran them synchronously against every message on the shared event loop, so a catastrophic-backtracking pattern such as (a+)+$ could stall the entire process (native RegExp takes tens of seconds at roughly 32 characters) and take the instance down for every user. Compile these patterns with RE2JS, a linear-time RE2 port with no native addon, so catastrophic backtracking is impossible regardless of the pattern rather than something the code tries to detect. Patterns using features RE2 does not support, such as backreferences, fail to compile and are dropped and logged exactly as an invalid pattern already is. The filter only tests for a match, so this is a drop-in engine swap with no behavior change for valid patterns.
This commit is contained in:
parent
664290c653
commit
e55d27e4bb
4 changed files with 56 additions and 7 deletions
|
|
@ -122,6 +122,7 @@
|
|||
"pdfjs-dist": "^5.4.624",
|
||||
"prom-client": "^15.1.3",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"re2js": "^2.8.6",
|
||||
"sanitize-html": "^2.17.6",
|
||||
"sharp": "^0.35.3",
|
||||
"ua-parser-js": "^1.0.36",
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@
|
|||
"pdfjs-dist": "^5.4.624",
|
||||
"prom-client": "^15.1.3",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"re2js": "^2.8.6",
|
||||
"sanitize-html": "^2.17.6",
|
||||
"sharp": "^0.35.3",
|
||||
"undici": "^7.24.1",
|
||||
|
|
|
|||
|
|
@ -246,6 +246,45 @@ describe('messageFilterPii middleware', () => {
|
|||
expect(matching.nextCalls).toBe(0);
|
||||
expect(matching.capturedRes.status).toBe(400);
|
||||
});
|
||||
|
||||
it('evaluates a catastrophic-backtracking customPattern in bounded time', () => {
|
||||
// `(a+)+$` against a long non-terminating run is exponential on a backtracking
|
||||
// engine (native RegExp takes tens of seconds at ~32 chars); the linear-time
|
||||
// engine returns immediately, so this must not hang.
|
||||
const config = {
|
||||
starterPatterns: [],
|
||||
customPatterns: [{ id: 'evil', label: 'Evil', regex: '(a+)+$' }],
|
||||
} as unknown as MessageFilterPiiConfig;
|
||||
const adversarial = 'a'.repeat(60) + '!';
|
||||
const start = process.hrtime.bigint();
|
||||
const { nextCalls } = runMiddleware(config, { text: adversarial });
|
||||
const elapsedMs = Number(process.hrtime.bigint() - start) / 1e6;
|
||||
expect(nextCalls).toBe(1);
|
||||
expect(elapsedMs).toBeLessThan(1000);
|
||||
});
|
||||
|
||||
it('still matches a catastrophic-shaped pattern against matching input', () => {
|
||||
const config = {
|
||||
starterPatterns: [],
|
||||
customPatterns: [{ id: 'evil', label: 'Evil', regex: '(a+)+$' }],
|
||||
} as unknown as MessageFilterPiiConfig;
|
||||
const { capturedRes, nextCalls } = runMiddleware(config, { text: 'a'.repeat(20) });
|
||||
expect(nextCalls).toBe(0);
|
||||
expect(capturedRes.status).toBe(400);
|
||||
});
|
||||
|
||||
it('drops a customPattern using engine-unsupported syntax and keeps others active', () => {
|
||||
const config = {
|
||||
starterPatterns: [],
|
||||
customPatterns: [
|
||||
{ id: 'backref', label: 'Backref', regex: '(a)\\1' },
|
||||
{ id: 'org', label: 'Org token', regex: '\\bORG-[A-Z0-9]{6,}' },
|
||||
],
|
||||
} as unknown as MessageFilterPiiConfig;
|
||||
const matching = runMiddleware(config, { text: 'token ORG-DEADBEEF here' });
|
||||
expect(matching.nextCalls).toBe(0);
|
||||
expect(matching.capturedRes.status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findPiiMatchInMessages', () => {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { RE2JS } from 're2js';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import type {
|
||||
NextFunction,
|
||||
|
|
@ -8,12 +9,20 @@ import type {
|
|||
import type { MessageFilterPiiConfig } from 'librechat-data-provider';
|
||||
import { getReferencedQuotes, mergeQuotedText } from '../utils/quotes';
|
||||
|
||||
type CompiledPattern = { id: string; label: string; pattern: RegExp };
|
||||
type CompiledPattern = { id: string; label: string; pattern: RE2JS };
|
||||
|
||||
const STARTER_PATTERNS: CompiledPattern[] = [
|
||||
{ id: 'sk_prefix', label: 'sk- prefix token', pattern: /\b(sk-)[a-zA-Z0-9_-]+/g },
|
||||
{ id: 'bearer_header', label: 'Bearer token', pattern: /\b(Bearer )[^\s"']+/gi },
|
||||
{ id: 'api_key_header', label: 'api-key header', pattern: /\b(api-key:?\s+)[^\s"']+/gi },
|
||||
{ id: 'sk_prefix', label: 'sk- prefix token', pattern: RE2JS.compile('\\b(sk-)[a-zA-Z0-9_-]+') },
|
||||
{
|
||||
id: 'bearer_header',
|
||||
label: 'Bearer token',
|
||||
pattern: RE2JS.compile('\\b(Bearer )[^\\s"\']+', RE2JS.CASE_INSENSITIVE),
|
||||
},
|
||||
{
|
||||
id: 'api_key_header',
|
||||
label: 'api-key header',
|
||||
pattern: RE2JS.compile('\\b(api-key:?\\s+)[^\\s"\']+', RE2JS.CASE_INSENSITIVE),
|
||||
},
|
||||
];
|
||||
|
||||
const STARTER_BY_ID = new Map(STARTER_PATTERNS.map((p) => [p.id, p]));
|
||||
|
|
@ -43,10 +52,10 @@ function compile(config: MessageFilterPiiConfig): CompiledPattern[] {
|
|||
const custom: CompiledPattern[] = [];
|
||||
for (const p of config.customPatterns ?? []) {
|
||||
try {
|
||||
custom.push({ id: p.id, label: p.label, pattern: new RegExp(p.regex, 'g') });
|
||||
custom.push({ id: p.id, label: p.label, pattern: RE2JS.compile(p.regex) });
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
`[messageFilter.pii] dropping invalid customPattern ${JSON.stringify(p.id)}: ${(err as Error).message}`,
|
||||
`[messageFilter.pii] dropping invalid or unsupported customPattern ${JSON.stringify(p.id)}: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -57,7 +66,6 @@ function compile(config: MessageFilterPiiConfig): CompiledPattern[] {
|
|||
|
||||
function findMatch(text: string, patterns: CompiledPattern[]): CompiledPattern | null {
|
||||
for (const p of patterns) {
|
||||
p.pattern.lastIndex = 0;
|
||||
if (p.pattern.test(text)) {
|
||||
return p;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue