From 73b090e83f3aa53635b5c270ff95b1305357b850 Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Tue, 9 Jun 2026 10:53:38 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B9=20refactor:=20Namespace=20messageP?= =?UTF-8?q?iiFilter=20under=20messageFilter.pii=20+=20fix=20import=20order?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renames the yaml field `messagePiiFilter` to `messageFilter.pii`, the module to `messageFilterPii`, the factory to `createMessageFilterPii`, the type to `MessageFilterPiiConfig`, and the error code to `message_filter_pii_block`. The wrapper `messageFilter` namespace gives future safety filters (e.g. `messageFilter.toxicity`) a place to plug in without restructuring the config later. The `findPiiMatchInMessages` helper kept its name because it already describes what it does at the value level. Also fixes import order Danny flagged on the OpenAI-compatible and Responses controllers: `findPiiMatchInMessages` was appended at the bottom of two `require('@librechat/api')` destructures rather than placed in the length-sorted slot the house style expects. --- api/server/controllers/agents/openai.js | 6 ++-- api/server/controllers/agents/responses.js | 6 ++-- api/server/routes/agents/chat.js | 4 +-- librechat.example.yaml | 29 ++++++++++--------- packages/api/src/middleware/index.ts | 2 +- ...ilter.spec.ts => messageFilterPii.spec.ts} | 20 ++++++------- ...essagePiiFilter.ts => messageFilterPii.ts} | 18 ++++++------ packages/data-provider/src/config.ts | 16 ++++++---- packages/data-schemas/src/app/service.ts | 4 +-- packages/data-schemas/src/types/app.ts | 4 +-- 10 files changed, 59 insertions(+), 50 deletions(-) rename packages/api/src/middleware/{messagePiiFilter.spec.ts => messageFilterPii.spec.ts} (92%) rename packages/api/src/middleware/{messagePiiFilter.ts => messageFilterPii.ts} (85%) diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index 313b51ced8..aedc700204 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -25,6 +25,7 @@ const { recordCollectedUsage, getTransactionsConfig, resolveRecursionLimit, + findPiiMatchInMessages, discoverConnectedAgents, getRemoteAgentPermissions, createToolExecuteHandler, @@ -33,7 +34,6 @@ const { resolveAgentScopedSkillIds, createOpenAIContentAggregator, isChatCompletionValidationFailure, - findPiiMatchInMessages, } = require('@librechat/api'); const { buildSummarizationHandlers, @@ -178,14 +178,14 @@ const OpenAIChatCompletionController = async (req, res) => { ); } - const piiHit = findPiiMatchInMessages(request.messages, appConfig?.messagePiiFilter); + const piiHit = findPiiMatchInMessages(request.messages, appConfig?.messageFilter?.pii); if (piiHit != null) { return sendErrorResponse( res, 400, `Message contains a ${piiHit.label}. Remove it and try again.`, 'invalid_request_error', - 'message_pii_filter_block', + 'message_filter_pii_block', ); } diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index a230b8f988..a6cbc3c5a6 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -22,6 +22,7 @@ const { extractManualSkills, injectSkillPrimes, createToolExecuteHandler, + findPiiMatchInMessages, discoverConnectedAgents, getRemoteAgentPermissions, // Responses API @@ -41,7 +42,6 @@ const { sendResponsesErrorResponse, createResponsesEventHandlers, createAggregatorEventHandlers, - findPiiMatchInMessages, } = require('@librechat/api'); const { createResponsesToolEndCallback, @@ -576,14 +576,14 @@ const createResponse = async (req, res) => { typeof request.input === 'string' ? request.input : request.input, ); - const piiHit = findPiiMatchInMessages(inputMessages, appConfig?.messagePiiFilter); + const piiHit = findPiiMatchInMessages(inputMessages, appConfig?.messageFilter?.pii); if (piiHit != null) { return sendResponsesErrorResponse( res, 400, `Message contains a ${piiHit.label}. Remove it and try again.`, 'invalid_request', - 'message_pii_filter_block', + 'message_filter_pii_block', ); } diff --git a/api/server/routes/agents/chat.js b/api/server/routes/agents/chat.js index f560fe286a..8ffbde6552 100644 --- a/api/server/routes/agents/chat.js +++ b/api/server/routes/agents/chat.js @@ -1,5 +1,5 @@ const express = require('express'); -const { createMessagePiiFilter, generateCheckAccess, skipAgentCheck } = require('@librechat/api'); +const { createMessageFilterPii, generateCheckAccess, skipAgentCheck } = require('@librechat/api'); const { PermissionTypes, Permissions, PermissionBits } = require('librechat-data-provider'); const { moderateText, @@ -25,7 +25,7 @@ const checkAgentResourceAccess = canAccessAgentFromBody({ requiredPermission: PermissionBits.VIEW, }); -router.use(createMessagePiiFilter({ getConfig: (req) => req.config?.messagePiiFilter })); +router.use(createMessageFilterPii({ getConfig: (req) => req.config?.messageFilter?.pii })); router.use(moderateText); router.use(checkAgentAccess); router.use(checkAgentResourceAccess); diff --git a/librechat.example.yaml b/librechat.example.yaml index 2311a2e268..086171fef7 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -778,16 +778,19 @@ endpoints: # # temperature: 0.1 # Reject chat messages whose text matches credential-shaped patterns -# before they reach moderation, the model, or persistence. Omit the -# whole section to disable. -# messagePiiFilter: -# # (optional) Pick a subset of the starter catalog by id; omit to -# # enable all starters (sk_prefix, bearer_header, api_key_header). -# starterPatterns: [sk_prefix, bearer_header, api_key_header] -# # (optional) Operator-defined patterns. Each entry needs id, label, -# # and a JavaScript-flavor regex; the regex is validated at config -# # load time. -# customPatterns: -# - id: anthropic_api_key -# label: Anthropic API key -# regex: "sk-ant-[A-Za-z0-9_-]{20,}" +# before they reach moderation, the model, or persistence. Filter +# types live under `messageFilter.`; today only `pii` ships, but +# the namespace is structured so future filter types can plug in. +# Omit the whole section to disable. +# messageFilter: +# pii: +# # (optional) Pick a subset of the starter catalog by id; omit to +# # enable all starters (sk_prefix, bearer_header, api_key_header). +# starterPatterns: [sk_prefix, bearer_header, api_key_header] +# # (optional) Operator-defined patterns. Each entry needs id, +# # label, and a JavaScript-flavor regex; the regex is validated +# # at config load time. +# customPatterns: +# - id: anthropic_api_key +# label: Anthropic API key +# regex: "sk-ant-[A-Za-z0-9_-]{20,}" diff --git a/packages/api/src/middleware/index.ts b/packages/api/src/middleware/index.ts index da47770545..fbefaac54b 100644 --- a/packages/api/src/middleware/index.ts +++ b/packages/api/src/middleware/index.ts @@ -16,4 +16,4 @@ export * from './concurrency'; export * from './checkBalance'; export * from './remoteAgentAuth'; export * from './share'; -export * from './messagePiiFilter'; +export * from './messageFilterPii'; diff --git a/packages/api/src/middleware/messagePiiFilter.spec.ts b/packages/api/src/middleware/messageFilterPii.spec.ts similarity index 92% rename from packages/api/src/middleware/messagePiiFilter.spec.ts rename to packages/api/src/middleware/messageFilterPii.spec.ts index babc3642fa..7f9f8e049e 100644 --- a/packages/api/src/middleware/messagePiiFilter.spec.ts +++ b/packages/api/src/middleware/messageFilterPii.spec.ts @@ -1,20 +1,20 @@ -import type { MessagePiiFilterConfig } from 'librechat-data-provider'; +import type { MessageFilterPiiConfig } from 'librechat-data-provider'; import type { Request, Response, NextFunction } from 'express'; jest.mock('@librechat/data-schemas', () => ({ logger: { warn: jest.fn(), error: jest.fn(), info: jest.fn(), debug: jest.fn() }, })); -import { createMessagePiiFilter, findPiiMatchInMessages } from './messagePiiFilter'; +import { createMessageFilterPii, findPiiMatchInMessages } from './messageFilterPii'; type CapturedResponse = { status?: number; body?: unknown }; function runMiddleware( - config: MessagePiiFilterConfig | undefined, + config: MessageFilterPiiConfig | undefined, body: unknown, ): { capturedRes: CapturedResponse; nextCalls: number } { const captured: CapturedResponse = {}; let nextCalls = 0; - const mw = createMessagePiiFilter({ getConfig: () => config }); + const mw = createMessageFilterPii({ getConfig: () => config }); const req = { body } as unknown as Request; const res = { status(code: number) { @@ -33,7 +33,7 @@ function runMiddleware( return { capturedRes: captured, nextCalls }; } -describe('messagePiiFilter middleware', () => { +describe('messageFilterPii middleware', () => { it('passes through when no config is provided', () => { const { capturedRes, nextCalls } = runMiddleware(undefined, { text: 'my key is sk-proj-FAKE1234567890ABCDEF', @@ -68,7 +68,7 @@ describe('messagePiiFilter middleware', () => { expect(nextCalls).toBe(0); expect(capturedRes.status).toBe(400); expect(capturedRes.body).toEqual({ - error: 'message_pii_filter_block', + error: 'message_filter_pii_block', message: 'Message contains a sk- prefix token. Remove it and try again.', }); }); @@ -80,7 +80,7 @@ describe('messagePiiFilter middleware', () => { ); expect(nextCalls).toBe(0); expect(capturedRes.status).toBe(400); - expect(capturedRes.body).toMatchObject({ error: 'message_pii_filter_block' }); + expect(capturedRes.body).toMatchObject({ error: 'message_filter_pii_block' }); }); it('rejects with 400 when an api-key header is present', () => { @@ -118,7 +118,7 @@ describe('messagePiiFilter middleware', () => { expect(nextCalls).toBe(0); expect(capturedRes.status).toBe(400); expect(capturedRes.body).toEqual({ - error: 'message_pii_filter_block', + error: 'message_filter_pii_block', message: 'Message contains a Org token. Remove it and try again.', }); }); @@ -135,7 +135,7 @@ describe('messagePiiFilter middleware', () => { }); it('returns the same compiled pattern array for repeat calls with the same config (memoization)', () => { - const config: MessagePiiFilterConfig = { + const config: MessageFilterPiiConfig = { customPatterns: [{ id: 'org', label: 'Org token', regex: '\\bORG-[A-Z0-9]{6,}' }], }; const a = runMiddleware(config, { text: 'plain' }); @@ -151,7 +151,7 @@ describe('messagePiiFilter middleware', () => { { id: 'broken', label: 'Broken', regex: '(' }, { id: 'org', label: 'Org token', regex: '\\bORG-[A-Z0-9]{6,}' }, ], - } as unknown as MessagePiiFilterConfig; + } as unknown as MessageFilterPiiConfig; const benign = runMiddleware(config, { text: 'plain text' }); expect(benign.nextCalls).toBe(1); expect(benign.capturedRes.status).toBeUndefined(); diff --git a/packages/api/src/middleware/messagePiiFilter.ts b/packages/api/src/middleware/messageFilterPii.ts similarity index 85% rename from packages/api/src/middleware/messagePiiFilter.ts rename to packages/api/src/middleware/messageFilterPii.ts index 1ce893dd61..aa09caac5b 100644 --- a/packages/api/src/middleware/messagePiiFilter.ts +++ b/packages/api/src/middleware/messageFilterPii.ts @@ -5,7 +5,7 @@ import type { Request as ServerRequest, Response as ServerResponse, } from 'express'; -import type { MessagePiiFilterConfig } from 'librechat-data-provider'; +import type { MessageFilterPiiConfig } from 'librechat-data-provider'; type CompiledPattern = { id: string; label: string; pattern: RegExp }; @@ -33,7 +33,7 @@ function selectStarter(ids?: string[]): CompiledPattern[] { const COMPILE_CACHE = new WeakMap(); -function compile(config: MessagePiiFilterConfig): CompiledPattern[] { +function compile(config: MessageFilterPiiConfig): CompiledPattern[] { const cached = COMPILE_CACHE.get(config); if (cached != null) { return cached; @@ -45,7 +45,7 @@ function compile(config: MessagePiiFilterConfig): CompiledPattern[] { custom.push({ id: p.id, label: p.label, pattern: new RegExp(p.regex, 'g') }); } catch (err) { logger.warn( - `[messagePiiFilter] dropping invalid customPattern ${JSON.stringify(p.id)}: ${(err as Error).message}`, + `[messageFilter.pii] dropping invalid customPattern ${JSON.stringify(p.id)}: ${(err as Error).message}`, ); } } @@ -77,7 +77,7 @@ type ChatLikeMessage = { export function findPiiMatchInMessages( messages: ChatLikeMessage[] | undefined, - config: MessagePiiFilterConfig | undefined, + config: MessageFilterPiiConfig | undefined, ): PiiMatch | null { if (config == null || !Array.isArray(messages) || messages.length === 0) { return null; @@ -111,12 +111,12 @@ export function findPiiMatchInMessages( return null; } -export interface CreateMessagePiiFilterOptions { - getConfig: (req: ServerRequest) => MessagePiiFilterConfig | undefined; +export interface CreateMessageFilterPiiOptions { + getConfig: (req: ServerRequest) => MessageFilterPiiConfig | undefined; } -export function createMessagePiiFilter(options: CreateMessagePiiFilterOptions): RequestHandler { - return function messagePiiFilter(req: ServerRequest, res: ServerResponse, next: NextFunction) { +export function createMessageFilterPii(options: CreateMessageFilterPiiOptions): RequestHandler { + return function messageFilterPii(req: ServerRequest, res: ServerResponse, next: NextFunction) { const config = options.getConfig(req); if (config == null) { next(); @@ -138,7 +138,7 @@ export function createMessagePiiFilter(options: CreateMessagePiiFilterOptions): return; } res.status(400).json({ - error: 'message_pii_filter_block', + error: 'message_filter_pii_block', message: `Message contains a ${match.label}. Remove it and try again.`, }); }; diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index a54c36681e..71178a277e 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1420,7 +1420,7 @@ export type SummarizationConfig = z.infer; const customEndpointsSchema = z.array(endpointSchema.partial()).optional(); -const messagePiiCustomPatternSchema = z.object({ +const messageFilterPiiCustomPatternSchema = z.object({ id: z.string().min(1), label: z.string().min(1), regex: z @@ -1439,12 +1439,18 @@ const messagePiiCustomPatternSchema = z.object({ ), }); -export const messagePiiFilterSchema = z.object({ +export const messageFilterPiiSchema = z.object({ starterPatterns: z.array(z.string()).optional(), - customPatterns: z.array(messagePiiCustomPatternSchema).optional(), + customPatterns: z.array(messageFilterPiiCustomPatternSchema).optional(), }); -export type MessagePiiFilterConfig = z.infer; +export type MessageFilterPiiConfig = z.infer; + +export const messageFilterSchema = z.object({ + pii: messageFilterPiiSchema.optional(), +}); + +export type MessageFilterConfig = z.infer; export const configSchema = z.object({ version: z.string(), @@ -1493,7 +1499,7 @@ export const configSchema = z.object({ rateLimits: rateLimitSchema.optional(), fileConfig: fileConfigSchema.optional(), modelSpecs: specsConfigSchema.optional(), - messagePiiFilter: messagePiiFilterSchema.optional(), + messageFilter: messageFilterSchema.optional(), endpoints: z .object({ allowedAddresses: allowedAddressesSchema, diff --git a/packages/data-schemas/src/app/service.ts b/packages/data-schemas/src/app/service.ts index 19bb62c161..9c66240037 100644 --- a/packages/data-schemas/src/app/service.ts +++ b/packages/data-schemas/src/app/service.ts @@ -110,7 +110,7 @@ export const AppService = async (params?: { const interfaceConfig = await loadDefaultInterface({ config, configDefaults }); const turnstileConfig = loadTurnstileConfig(config, configDefaults); const speech = config.speech; - const messagePiiFilter = config.messagePiiFilter; + const messageFilter = config.messageFilter; const defaultConfig = { ocr, @@ -118,7 +118,7 @@ export const AppService = async (params?: { config, memory, speech, - messagePiiFilter, + messageFilter, balance, actions, webSearch, diff --git a/packages/data-schemas/src/types/app.ts b/packages/data-schemas/src/types/app.ts index 71b47c5874..2b47e6f011 100644 --- a/packages/data-schemas/src/types/app.ts +++ b/packages/data-schemas/src/types/app.ts @@ -62,8 +62,8 @@ export interface AppConfig { summarization?: SummarizationConfig; /** Web search configuration */ webSearch?: TCustomConfig['webSearch']; - /** Message PII filter configuration */ - messagePiiFilter?: TCustomConfig['messagePiiFilter']; + /** Message filter configuration (PII and future filter types) */ + messageFilter?: TCustomConfig['messageFilter']; /** File storage strategy ('local', 's3', 'firebase', 'azure_blob', 'cloudfront') */ fileStrategy: FileStorage; /** File strategies configuration */