mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
🧹 refactor: Namespace messagePiiFilter under messageFilter.pii + fix import order
Renames the yaml field `messagePiiFilter` to `messageFilter.pii`, the
module to `messageFilterPii`, the factory to `createMessageFilterPii`,
the type to `MessageFilterPiiConfig`, and the error code to
`message_filter_pii_block`. The wrapper `messageFilter` namespace
gives future safety filters (e.g. `messageFilter.toxicity`) a place
to plug in without restructuring the config later. The
`findPiiMatchInMessages` helper kept its name because it already
describes what it does at the value level.
Also fixes import order Danny flagged on the OpenAI-compatible and
Responses controllers: `findPiiMatchInMessages` was appended at the
bottom of two `require('@librechat/api')` destructures rather than
placed in the length-sorted slot the house style expects.
This commit is contained in:
parent
87f7bdeb1f
commit
73b090e83f
10 changed files with 59 additions and 50 deletions
|
|
@ -25,6 +25,7 @@ const {
|
|||
recordCollectedUsage,
|
||||
getTransactionsConfig,
|
||||
resolveRecursionLimit,
|
||||
findPiiMatchInMessages,
|
||||
discoverConnectedAgents,
|
||||
getRemoteAgentPermissions,
|
||||
createToolExecuteHandler,
|
||||
|
|
@ -33,7 +34,6 @@ const {
|
|||
resolveAgentScopedSkillIds,
|
||||
createOpenAIContentAggregator,
|
||||
isChatCompletionValidationFailure,
|
||||
findPiiMatchInMessages,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
buildSummarizationHandlers,
|
||||
|
|
@ -178,14 +178,14 @@ const OpenAIChatCompletionController = async (req, res) => {
|
|||
);
|
||||
}
|
||||
|
||||
const piiHit = findPiiMatchInMessages(request.messages, appConfig?.messagePiiFilter);
|
||||
const piiHit = findPiiMatchInMessages(request.messages, appConfig?.messageFilter?.pii);
|
||||
if (piiHit != null) {
|
||||
return sendErrorResponse(
|
||||
res,
|
||||
400,
|
||||
`Message contains a ${piiHit.label}. Remove it and try again.`,
|
||||
'invalid_request_error',
|
||||
'message_pii_filter_block',
|
||||
'message_filter_pii_block',
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ const {
|
|||
extractManualSkills,
|
||||
injectSkillPrimes,
|
||||
createToolExecuteHandler,
|
||||
findPiiMatchInMessages,
|
||||
discoverConnectedAgents,
|
||||
getRemoteAgentPermissions,
|
||||
// Responses API
|
||||
|
|
@ -41,7 +42,6 @@ const {
|
|||
sendResponsesErrorResponse,
|
||||
createResponsesEventHandlers,
|
||||
createAggregatorEventHandlers,
|
||||
findPiiMatchInMessages,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
createResponsesToolEndCallback,
|
||||
|
|
@ -576,14 +576,14 @@ const createResponse = async (req, res) => {
|
|||
typeof request.input === 'string' ? request.input : request.input,
|
||||
);
|
||||
|
||||
const piiHit = findPiiMatchInMessages(inputMessages, appConfig?.messagePiiFilter);
|
||||
const piiHit = findPiiMatchInMessages(inputMessages, appConfig?.messageFilter?.pii);
|
||||
if (piiHit != null) {
|
||||
return sendResponsesErrorResponse(
|
||||
res,
|
||||
400,
|
||||
`Message contains a ${piiHit.label}. Remove it and try again.`,
|
||||
'invalid_request',
|
||||
'message_pii_filter_block',
|
||||
'message_filter_pii_block',
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
const express = require('express');
|
||||
const { createMessagePiiFilter, generateCheckAccess, skipAgentCheck } = require('@librechat/api');
|
||||
const { createMessageFilterPii, generateCheckAccess, skipAgentCheck } = require('@librechat/api');
|
||||
const { PermissionTypes, Permissions, PermissionBits } = require('librechat-data-provider');
|
||||
const {
|
||||
moderateText,
|
||||
|
|
@ -25,7 +25,7 @@ const checkAgentResourceAccess = canAccessAgentFromBody({
|
|||
requiredPermission: PermissionBits.VIEW,
|
||||
});
|
||||
|
||||
router.use(createMessagePiiFilter({ getConfig: (req) => req.config?.messagePiiFilter }));
|
||||
router.use(createMessageFilterPii({ getConfig: (req) => req.config?.messageFilter?.pii }));
|
||||
router.use(moderateText);
|
||||
router.use(checkAgentAccess);
|
||||
router.use(checkAgentResourceAccess);
|
||||
|
|
|
|||
|
|
@ -778,16 +778,19 @@ endpoints:
|
|||
# # temperature: 0.1
|
||||
|
||||
# Reject chat messages whose text matches credential-shaped patterns
|
||||
# before they reach moderation, the model, or persistence. Omit the
|
||||
# whole section to disable.
|
||||
# messagePiiFilter:
|
||||
# # (optional) Pick a subset of the starter catalog by id; omit to
|
||||
# # enable all starters (sk_prefix, bearer_header, api_key_header).
|
||||
# starterPatterns: [sk_prefix, bearer_header, api_key_header]
|
||||
# # (optional) Operator-defined patterns. Each entry needs id, label,
|
||||
# # and a JavaScript-flavor regex; the regex is validated at config
|
||||
# # load time.
|
||||
# customPatterns:
|
||||
# - id: anthropic_api_key
|
||||
# label: Anthropic API key
|
||||
# regex: "sk-ant-[A-Za-z0-9_-]{20,}"
|
||||
# before they reach moderation, the model, or persistence. Filter
|
||||
# types live under `messageFilter.<type>`; today only `pii` ships, but
|
||||
# the namespace is structured so future filter types can plug in.
|
||||
# Omit the whole section to disable.
|
||||
# messageFilter:
|
||||
# pii:
|
||||
# # (optional) Pick a subset of the starter catalog by id; omit to
|
||||
# # enable all starters (sk_prefix, bearer_header, api_key_header).
|
||||
# starterPatterns: [sk_prefix, bearer_header, api_key_header]
|
||||
# # (optional) Operator-defined patterns. Each entry needs id,
|
||||
# # label, and a JavaScript-flavor regex; the regex is validated
|
||||
# # at config load time.
|
||||
# customPatterns:
|
||||
# - id: anthropic_api_key
|
||||
# label: Anthropic API key
|
||||
# regex: "sk-ant-[A-Za-z0-9_-]{20,}"
|
||||
|
|
|
|||
|
|
@ -16,4 +16,4 @@ export * from './concurrency';
|
|||
export * from './checkBalance';
|
||||
export * from './remoteAgentAuth';
|
||||
export * from './share';
|
||||
export * from './messagePiiFilter';
|
||||
export * from './messageFilterPii';
|
||||
|
|
|
|||
|
|
@ -1,20 +1,20 @@
|
|||
import type { MessagePiiFilterConfig } from 'librechat-data-provider';
|
||||
import type { MessageFilterPiiConfig } from 'librechat-data-provider';
|
||||
import type { Request, Response, NextFunction } from 'express';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: { warn: jest.fn(), error: jest.fn(), info: jest.fn(), debug: jest.fn() },
|
||||
}));
|
||||
import { createMessagePiiFilter, findPiiMatchInMessages } from './messagePiiFilter';
|
||||
import { createMessageFilterPii, findPiiMatchInMessages } from './messageFilterPii';
|
||||
|
||||
type CapturedResponse = { status?: number; body?: unknown };
|
||||
|
||||
function runMiddleware(
|
||||
config: MessagePiiFilterConfig | undefined,
|
||||
config: MessageFilterPiiConfig | undefined,
|
||||
body: unknown,
|
||||
): { capturedRes: CapturedResponse; nextCalls: number } {
|
||||
const captured: CapturedResponse = {};
|
||||
let nextCalls = 0;
|
||||
const mw = createMessagePiiFilter({ getConfig: () => config });
|
||||
const mw = createMessageFilterPii({ getConfig: () => config });
|
||||
const req = { body } as unknown as Request;
|
||||
const res = {
|
||||
status(code: number) {
|
||||
|
|
@ -33,7 +33,7 @@ function runMiddleware(
|
|||
return { capturedRes: captured, nextCalls };
|
||||
}
|
||||
|
||||
describe('messagePiiFilter middleware', () => {
|
||||
describe('messageFilterPii middleware', () => {
|
||||
it('passes through when no config is provided', () => {
|
||||
const { capturedRes, nextCalls } = runMiddleware(undefined, {
|
||||
text: 'my key is sk-proj-FAKE1234567890ABCDEF',
|
||||
|
|
@ -68,7 +68,7 @@ describe('messagePiiFilter middleware', () => {
|
|||
expect(nextCalls).toBe(0);
|
||||
expect(capturedRes.status).toBe(400);
|
||||
expect(capturedRes.body).toEqual({
|
||||
error: 'message_pii_filter_block',
|
||||
error: 'message_filter_pii_block',
|
||||
message: 'Message contains a sk- prefix token. Remove it and try again.',
|
||||
});
|
||||
});
|
||||
|
|
@ -80,7 +80,7 @@ describe('messagePiiFilter middleware', () => {
|
|||
);
|
||||
expect(nextCalls).toBe(0);
|
||||
expect(capturedRes.status).toBe(400);
|
||||
expect(capturedRes.body).toMatchObject({ error: 'message_pii_filter_block' });
|
||||
expect(capturedRes.body).toMatchObject({ error: 'message_filter_pii_block' });
|
||||
});
|
||||
|
||||
it('rejects with 400 when an api-key header is present', () => {
|
||||
|
|
@ -118,7 +118,7 @@ describe('messagePiiFilter middleware', () => {
|
|||
expect(nextCalls).toBe(0);
|
||||
expect(capturedRes.status).toBe(400);
|
||||
expect(capturedRes.body).toEqual({
|
||||
error: 'message_pii_filter_block',
|
||||
error: 'message_filter_pii_block',
|
||||
message: 'Message contains a Org token. Remove it and try again.',
|
||||
});
|
||||
});
|
||||
|
|
@ -135,7 +135,7 @@ describe('messagePiiFilter middleware', () => {
|
|||
});
|
||||
|
||||
it('returns the same compiled pattern array for repeat calls with the same config (memoization)', () => {
|
||||
const config: MessagePiiFilterConfig = {
|
||||
const config: MessageFilterPiiConfig = {
|
||||
customPatterns: [{ id: 'org', label: 'Org token', regex: '\\bORG-[A-Z0-9]{6,}' }],
|
||||
};
|
||||
const a = runMiddleware(config, { text: 'plain' });
|
||||
|
|
@ -151,7 +151,7 @@ describe('messagePiiFilter middleware', () => {
|
|||
{ id: 'broken', label: 'Broken', regex: '(' },
|
||||
{ id: 'org', label: 'Org token', regex: '\\bORG-[A-Z0-9]{6,}' },
|
||||
],
|
||||
} as unknown as MessagePiiFilterConfig;
|
||||
} as unknown as MessageFilterPiiConfig;
|
||||
const benign = runMiddleware(config, { text: 'plain text' });
|
||||
expect(benign.nextCalls).toBe(1);
|
||||
expect(benign.capturedRes.status).toBeUndefined();
|
||||
|
|
@ -5,7 +5,7 @@ import type {
|
|||
Request as ServerRequest,
|
||||
Response as ServerResponse,
|
||||
} from 'express';
|
||||
import type { MessagePiiFilterConfig } from 'librechat-data-provider';
|
||||
import type { MessageFilterPiiConfig } from 'librechat-data-provider';
|
||||
|
||||
type CompiledPattern = { id: string; label: string; pattern: RegExp };
|
||||
|
||||
|
|
@ -33,7 +33,7 @@ function selectStarter(ids?: string[]): CompiledPattern[] {
|
|||
|
||||
const COMPILE_CACHE = new WeakMap<object, CompiledPattern[]>();
|
||||
|
||||
function compile(config: MessagePiiFilterConfig): CompiledPattern[] {
|
||||
function compile(config: MessageFilterPiiConfig): CompiledPattern[] {
|
||||
const cached = COMPILE_CACHE.get(config);
|
||||
if (cached != null) {
|
||||
return cached;
|
||||
|
|
@ -45,7 +45,7 @@ function compile(config: MessagePiiFilterConfig): CompiledPattern[] {
|
|||
custom.push({ id: p.id, label: p.label, pattern: new RegExp(p.regex, 'g') });
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
`[messagePiiFilter] dropping invalid customPattern ${JSON.stringify(p.id)}: ${(err as Error).message}`,
|
||||
`[messageFilter.pii] dropping invalid customPattern ${JSON.stringify(p.id)}: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -77,7 +77,7 @@ type ChatLikeMessage = {
|
|||
|
||||
export function findPiiMatchInMessages(
|
||||
messages: ChatLikeMessage[] | undefined,
|
||||
config: MessagePiiFilterConfig | undefined,
|
||||
config: MessageFilterPiiConfig | undefined,
|
||||
): PiiMatch | null {
|
||||
if (config == null || !Array.isArray(messages) || messages.length === 0) {
|
||||
return null;
|
||||
|
|
@ -111,12 +111,12 @@ export function findPiiMatchInMessages(
|
|||
return null;
|
||||
}
|
||||
|
||||
export interface CreateMessagePiiFilterOptions {
|
||||
getConfig: (req: ServerRequest) => MessagePiiFilterConfig | undefined;
|
||||
export interface CreateMessageFilterPiiOptions {
|
||||
getConfig: (req: ServerRequest) => MessageFilterPiiConfig | undefined;
|
||||
}
|
||||
|
||||
export function createMessagePiiFilter(options: CreateMessagePiiFilterOptions): RequestHandler {
|
||||
return function messagePiiFilter(req: ServerRequest, res: ServerResponse, next: NextFunction) {
|
||||
export function createMessageFilterPii(options: CreateMessageFilterPiiOptions): RequestHandler {
|
||||
return function messageFilterPii(req: ServerRequest, res: ServerResponse, next: NextFunction) {
|
||||
const config = options.getConfig(req);
|
||||
if (config == null) {
|
||||
next();
|
||||
|
|
@ -138,7 +138,7 @@ export function createMessagePiiFilter(options: CreateMessagePiiFilterOptions):
|
|||
return;
|
||||
}
|
||||
res.status(400).json({
|
||||
error: 'message_pii_filter_block',
|
||||
error: 'message_filter_pii_block',
|
||||
message: `Message contains a ${match.label}. Remove it and try again.`,
|
||||
});
|
||||
};
|
||||
|
|
@ -1420,7 +1420,7 @@ export type SummarizationConfig = z.infer<typeof summarizationConfigSchema>;
|
|||
|
||||
const customEndpointsSchema = z.array(endpointSchema.partial()).optional();
|
||||
|
||||
const messagePiiCustomPatternSchema = z.object({
|
||||
const messageFilterPiiCustomPatternSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
label: z.string().min(1),
|
||||
regex: z
|
||||
|
|
@ -1439,12 +1439,18 @@ const messagePiiCustomPatternSchema = z.object({
|
|||
),
|
||||
});
|
||||
|
||||
export const messagePiiFilterSchema = z.object({
|
||||
export const messageFilterPiiSchema = z.object({
|
||||
starterPatterns: z.array(z.string()).optional(),
|
||||
customPatterns: z.array(messagePiiCustomPatternSchema).optional(),
|
||||
customPatterns: z.array(messageFilterPiiCustomPatternSchema).optional(),
|
||||
});
|
||||
|
||||
export type MessagePiiFilterConfig = z.infer<typeof messagePiiFilterSchema>;
|
||||
export type MessageFilterPiiConfig = z.infer<typeof messageFilterPiiSchema>;
|
||||
|
||||
export const messageFilterSchema = z.object({
|
||||
pii: messageFilterPiiSchema.optional(),
|
||||
});
|
||||
|
||||
export type MessageFilterConfig = z.infer<typeof messageFilterSchema>;
|
||||
|
||||
export const configSchema = z.object({
|
||||
version: z.string(),
|
||||
|
|
@ -1493,7 +1499,7 @@ export const configSchema = z.object({
|
|||
rateLimits: rateLimitSchema.optional(),
|
||||
fileConfig: fileConfigSchema.optional(),
|
||||
modelSpecs: specsConfigSchema.optional(),
|
||||
messagePiiFilter: messagePiiFilterSchema.optional(),
|
||||
messageFilter: messageFilterSchema.optional(),
|
||||
endpoints: z
|
||||
.object({
|
||||
allowedAddresses: allowedAddressesSchema,
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ export const AppService = async (params?: {
|
|||
const interfaceConfig = await loadDefaultInterface({ config, configDefaults });
|
||||
const turnstileConfig = loadTurnstileConfig(config, configDefaults);
|
||||
const speech = config.speech;
|
||||
const messagePiiFilter = config.messagePiiFilter;
|
||||
const messageFilter = config.messageFilter;
|
||||
|
||||
const defaultConfig = {
|
||||
ocr,
|
||||
|
|
@ -118,7 +118,7 @@ export const AppService = async (params?: {
|
|||
config,
|
||||
memory,
|
||||
speech,
|
||||
messagePiiFilter,
|
||||
messageFilter,
|
||||
balance,
|
||||
actions,
|
||||
webSearch,
|
||||
|
|
|
|||
|
|
@ -62,8 +62,8 @@ export interface AppConfig {
|
|||
summarization?: SummarizationConfig;
|
||||
/** Web search configuration */
|
||||
webSearch?: TCustomConfig['webSearch'];
|
||||
/** Message PII filter configuration */
|
||||
messagePiiFilter?: TCustomConfig['messagePiiFilter'];
|
||||
/** Message filter configuration (PII and future filter types) */
|
||||
messageFilter?: TCustomConfig['messageFilter'];
|
||||
/** File storage strategy ('local', 's3', 'firebase', 'azure_blob', 'cloudfront') */
|
||||
fileStrategy: FileStorage;
|
||||
/** File strategies configuration */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue