mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-10-21 09:32:24 +00:00
🚰 fix: Throttle Shared Link Reads and Aggregate Audit Findings (#15605)
* 🪵 fix: Aggregate Audit Findings and Rate Limit Shared Link Reads Audit-mode content filter rules wrote one `logger.info` per matching fragment, and inspection deliberately continues past every finding. A shared conversation full of distinct matching strings therefore produced one log write per fragment, regenerated on every retrieval of `GET /api/share/:shareId` — a public, unauthenticated route with no limiter. The 4,096-entry inspection dedupe set is a memory bound, not an event budget: once full, further values are inspected and logged again. Audit metadata carries no inspected text — only rule, source, field and provenance — so thousands of those writes were byte-identical lines. Findings are now counted inside an aggregation scope and reported once per distinct key with an `occurrences` count when the scope ends: - `assertModelBoundContent` opens a scope when audit rules are configured, bounding one inspection pass (a single message's fragments). - `assertConversationImportContentAllowed` opens the outer scope for import and shared-link snapshots, so per-message inspections nest into one report instead of one per message. - `createShareContentPreflight` spans the shared-file metadata pass too. Nested scopes reuse the outermost aggregation, and outside any scope the previous immediate write is kept. `GET /api/share/:shareId` re-inspects the whole snapshot on every request, so it now carries IP and user limiters like the neighbouring fork route (`SHARE_IP_MAX`/`SHARE_IP_WINDOW`, `SHARE_USER_MAX`/`SHARE_USER_WINDOW`, `SHARE_VIOLATION_SCORE`); the user limiter skips anonymous viewers. * 🔒 fix: Address Codex Findings on Share Audit Aggregation - The aggregation key joined rule fields with a space, but the filter schema allows spaces in custom pattern ids and labels, so `{id: 'a', label: 'b c'}` and `{id: 'a b', label: 'c'}` collided on the same source/field/provenance and merged two rules' counts under one identity. Serialize the tuple instead. - `SHARE_VIOLATION_SCORE` left unset reached `logViolation` as `undefined`, which selects that function's default score of 1 rather than the documented zero. With default ban settings a viewer refreshing a rate-limited shared link would be banned after 20 rejections. Default the score to 0 in code, not only in `.env.example`. - `removePorts` returns a full IPv6 address, so an attacker holding a prefix could rotate the host portion for a fresh bucket on every request — and for anonymous viewers the IP limiter is the only bound. Derive the key through `ipKeyGenerator` so IPv6 clients group by /56.
This commit is contained in:
parent
d07527ff93
commit
09da878a47
15 changed files with 632 additions and 9 deletions
|
|
@ -814,7 +814,15 @@ TTS_VIOLATION_SCORE=0
|
|||
STT_VIOLATION_SCORE=0
|
||||
FORK_VIOLATION_SCORE=0
|
||||
IMPORT_VIOLATION_SCORE=0
|
||||
SHARE_VIOLATION_SCORE=0
|
||||
FILE_UPLOAD_VIOLATION_SCORE=0
|
||||
# Shared link retrieval re-inspects the whole shared snapshot on every request
|
||||
# and is reachable without authentication (defaults: 100 per IP and 60 per user,
|
||||
# both per minute).
|
||||
# SHARE_IP_MAX=100
|
||||
# SHARE_IP_WINDOW=1
|
||||
# SHARE_USER_MAX=60
|
||||
# SHARE_USER_WINDOW=1
|
||||
# Per-user limiter for metadata-only /files/usage requests that renew the TTL
|
||||
# of attachments waiting in queued Agent messages (default: 120 per 15 minutes).
|
||||
# FILE_USAGE_USER_MAX=120
|
||||
|
|
|
|||
1
api/cache/getLogStores.js
vendored
1
api/cache/getLogStores.js
vendored
|
|
@ -30,6 +30,7 @@ const namespaces = {
|
|||
[ViolationTypes.TTS_LIMIT]: violationCache(ViolationTypes.TTS_LIMIT),
|
||||
[ViolationTypes.STT_LIMIT]: violationCache(ViolationTypes.STT_LIMIT),
|
||||
[ViolationTypes.CONVO_ACCESS]: violationCache(ViolationTypes.CONVO_ACCESS),
|
||||
[ViolationTypes.SHARE_LIMIT]: violationCache(ViolationTypes.SHARE_LIMIT),
|
||||
[ViolationTypes.TOOL_CALL_LIMIT]: violationCache(ViolationTypes.TOOL_CALL_LIMIT),
|
||||
[ViolationTypes.FILE_UPLOAD_LIMIT]: violationCache(ViolationTypes.FILE_UPLOAD_LIMIT),
|
||||
[ViolationTypes.VERIFY_EMAIL_LIMIT]: violationCache(ViolationTypes.VERIFY_EMAIL_LIMIT),
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ const loginLimiter = require('./loginLimiter');
|
|||
const importLimiters = require('./importLimiters');
|
||||
const uploadLimiters = require('./uploadLimiters');
|
||||
const forkLimiters = require('./forkLimiters');
|
||||
const shareLimiters = require('./shareLimiters');
|
||||
const registerLimiter = require('./registerLimiter');
|
||||
const toolCallLimiter = require('./toolCallLimiter');
|
||||
const messageLimiters = require('./messageLimiters');
|
||||
|
|
@ -20,6 +21,7 @@ module.exports = {
|
|||
...importLimiters,
|
||||
...messageLimiters,
|
||||
...forkLimiters,
|
||||
...shareLimiters,
|
||||
...promptUsageLimiter,
|
||||
loginLimiter,
|
||||
registerLimiter,
|
||||
|
|
|
|||
98
api/server/middleware/limiters/shareLimiters.js
Normal file
98
api/server/middleware/limiters/shareLimiters.js
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
const rateLimit = require('express-rate-limit');
|
||||
const { ipKeyGenerator } = rateLimit;
|
||||
const { ViolationTypes } = require('librechat-data-provider');
|
||||
const { limiterCache, removePorts } = require('@librechat/api');
|
||||
const logViolation = require('~/cache/logViolation');
|
||||
|
||||
const getEnvironmentVariables = () => {
|
||||
const SHARE_IP_MAX = parseInt(process.env.SHARE_IP_MAX) || 100;
|
||||
const SHARE_IP_WINDOW = parseInt(process.env.SHARE_IP_WINDOW) || 1;
|
||||
const SHARE_USER_MAX = parseInt(process.env.SHARE_USER_MAX) || 60;
|
||||
const SHARE_USER_WINDOW = parseInt(process.env.SHARE_USER_WINDOW) || 1;
|
||||
const SHARE_VIOLATION_SCORE = process.env.SHARE_VIOLATION_SCORE ?? 0;
|
||||
|
||||
const shareIpWindowMs = SHARE_IP_WINDOW * 60 * 1000;
|
||||
const shareIpMax = SHARE_IP_MAX;
|
||||
const shareIpWindowInMinutes = shareIpWindowMs / 60000;
|
||||
|
||||
const shareUserWindowMs = SHARE_USER_WINDOW * 60 * 1000;
|
||||
const shareUserMax = SHARE_USER_MAX;
|
||||
const shareUserWindowInMinutes = shareUserWindowMs / 60000;
|
||||
|
||||
return {
|
||||
shareIpWindowMs,
|
||||
shareIpMax,
|
||||
shareIpWindowInMinutes,
|
||||
shareUserWindowMs,
|
||||
shareUserMax,
|
||||
shareUserWindowInMinutes,
|
||||
shareViolationScore: SHARE_VIOLATION_SCORE,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Groups IPv6 clients by subnet so an address rotation cannot mint a fresh
|
||||
* bucket; this limiter is the only bound on anonymous shared-link retrieval.
|
||||
*/
|
||||
const shareIpKey = (req) => {
|
||||
const address = removePorts(req);
|
||||
return address == null ? address : ipKeyGenerator(address);
|
||||
};
|
||||
|
||||
const createShareHandler = (ip = true) => {
|
||||
const {
|
||||
shareIpMax,
|
||||
shareUserMax,
|
||||
shareViolationScore,
|
||||
shareIpWindowInMinutes,
|
||||
shareUserWindowInMinutes,
|
||||
} = getEnvironmentVariables();
|
||||
|
||||
return async (req, res) => {
|
||||
const type = ViolationTypes.SHARE_LIMIT;
|
||||
const errorMessage = {
|
||||
type,
|
||||
max: ip ? shareIpMax : shareUserMax,
|
||||
limiter: ip ? 'ip' : 'user',
|
||||
windowInMinutes: ip ? shareIpWindowInMinutes : shareUserWindowInMinutes,
|
||||
};
|
||||
|
||||
await logViolation(req, res, type, errorMessage, shareViolationScore);
|
||||
res.status(429).json({ message: 'Too many shared link requests. Try again later' });
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Bounds shared-link retrieval, which re-inspects the whole shared snapshot
|
||||
* on every request and is reachable without authentication.
|
||||
*/
|
||||
const createShareLimiters = () => {
|
||||
const { shareIpWindowMs, shareIpMax, shareUserWindowMs, shareUserMax } =
|
||||
getEnvironmentVariables();
|
||||
|
||||
const ipLimiterOptions = {
|
||||
windowMs: shareIpWindowMs,
|
||||
max: shareIpMax,
|
||||
handler: createShareHandler(),
|
||||
keyGenerator: shareIpKey,
|
||||
store: limiterCache('share_ip_limiter'),
|
||||
};
|
||||
const userLimiterOptions = {
|
||||
windowMs: shareUserWindowMs,
|
||||
max: shareUserMax,
|
||||
handler: createShareHandler(false),
|
||||
skip: function (req) {
|
||||
return req.user?.id == null;
|
||||
},
|
||||
keyGenerator: function (req) {
|
||||
return req.user.id;
|
||||
},
|
||||
store: limiterCache('share_user_limiter'),
|
||||
};
|
||||
|
||||
const shareIpLimiter = rateLimit(ipLimiterOptions);
|
||||
const shareUserLimiter = rateLimit(userLimiterOptions);
|
||||
return { shareIpLimiter, shareUserLimiter };
|
||||
};
|
||||
|
||||
module.exports = { createShareLimiters };
|
||||
127
api/server/middleware/limiters/shareLimiters.spec.js
Normal file
127
api/server/middleware/limiters/shareLimiters.spec.js
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
const express = require('express');
|
||||
const request = require('supertest');
|
||||
|
||||
const mockLogViolation = jest.fn();
|
||||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
limiterCache: jest.fn(() => undefined),
|
||||
removePorts: (req) => req.ip,
|
||||
}));
|
||||
jest.mock(
|
||||
'~/cache/logViolation',
|
||||
() =>
|
||||
(...args) =>
|
||||
mockLogViolation(...args),
|
||||
);
|
||||
|
||||
describe('shared link limiters', () => {
|
||||
let originalEnv;
|
||||
|
||||
beforeEach(() => {
|
||||
originalEnv = { ...process.env };
|
||||
jest.resetModules();
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv;
|
||||
});
|
||||
|
||||
const buildApp = (user, { trustProxy = false } = {}) => {
|
||||
const { createShareLimiters } = require('./shareLimiters');
|
||||
const { shareIpLimiter, shareUserLimiter } = createShareLimiters();
|
||||
const app = express();
|
||||
if (trustProxy) {
|
||||
app.set('trust proxy', true);
|
||||
}
|
||||
app.get(
|
||||
'/api/share/:shareId',
|
||||
(req, _res, next) => {
|
||||
req.user = user;
|
||||
next();
|
||||
},
|
||||
shareIpLimiter,
|
||||
shareUserLimiter,
|
||||
(_req, res) => res.status(200).json({ ok: true }),
|
||||
);
|
||||
return app;
|
||||
};
|
||||
|
||||
it('rejects an anonymous retrieval flood by IP', async () => {
|
||||
process.env.SHARE_IP_MAX = '2';
|
||||
process.env.SHARE_IP_WINDOW = '1';
|
||||
const app = buildApp(undefined);
|
||||
|
||||
await request(app).get('/api/share/share-1').expect(200);
|
||||
await request(app).get('/api/share/share-1').expect(200);
|
||||
const response = await request(app).get('/api/share/share-1').expect(429);
|
||||
|
||||
expect(response.body).toEqual({ message: 'Too many shared link requests. Try again later' });
|
||||
expect(mockLogViolation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
'share_limit',
|
||||
expect.objectContaining({ limiter: 'ip', max: 2, windowInMinutes: 1 }),
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps an unset violation score from banning shared link viewers', async () => {
|
||||
process.env.SHARE_IP_MAX = '1';
|
||||
delete process.env.SHARE_VIOLATION_SCORE;
|
||||
const app = buildApp({ id: 'user-1' });
|
||||
|
||||
await request(app).get('/api/share/share-1').expect(200);
|
||||
await request(app).get('/api/share/share-1').expect(429);
|
||||
|
||||
expect(mockLogViolation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
'share_limit',
|
||||
expect.anything(),
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('buckets IPv6 viewers by subnet so address rotation cannot mint a bucket', async () => {
|
||||
process.env.SHARE_IP_MAX = '1';
|
||||
const app = buildApp(undefined, { trustProxy: true });
|
||||
|
||||
const retrieve = (address) =>
|
||||
request(app).get('/api/share/share-1').set('X-Forwarded-For', address);
|
||||
|
||||
expect((await retrieve('2001:db8:1234:5600::1')).status).toBe(200);
|
||||
expect((await retrieve('2001:db8:1234:5622::9')).status).toBe(429);
|
||||
expect((await retrieve('2001:db8:1234:5700::1')).status).toBe(200);
|
||||
});
|
||||
|
||||
it('rejects an authenticated retrieval flood by user', async () => {
|
||||
process.env.SHARE_IP_MAX = '100';
|
||||
process.env.SHARE_USER_MAX = '1';
|
||||
process.env.SHARE_USER_WINDOW = '1';
|
||||
process.env.SHARE_VIOLATION_SCORE = '3';
|
||||
const app = buildApp({ id: 'user-1' });
|
||||
|
||||
await request(app).get('/api/share/share-1').expect(200);
|
||||
await request(app).get('/api/share/share-1').expect(429);
|
||||
|
||||
expect(mockLogViolation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
'share_limit',
|
||||
expect.objectContaining({ limiter: 'user', max: 1 }),
|
||||
'3',
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves the per-user bucket untouched for anonymous viewers', async () => {
|
||||
process.env.SHARE_IP_MAX = '100';
|
||||
process.env.SHARE_USER_MAX = '1';
|
||||
const app = buildApp(undefined);
|
||||
|
||||
await request(app).get('/api/share/share-1').expect(200);
|
||||
await request(app).get('/api/share/share-1').expect(200);
|
||||
|
||||
expect(mockLogViolation).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -21,6 +21,8 @@ const mockCanAccessSharedLink = jest.fn((req, _res, next) => {
|
|||
next();
|
||||
});
|
||||
const mockGetAppConfig = jest.fn();
|
||||
const mockShareIpLimiter = jest.fn((_req, _res, next) => next());
|
||||
const mockShareUserLimiter = jest.fn((_req, _res, next) => next());
|
||||
const mockParseSharedLinksPageSize = jest.fn(() => 10);
|
||||
const mockIsValidSharedLinksCursor = jest.fn(() => true);
|
||||
const mockAssertConversationContentAllowed = jest.fn();
|
||||
|
|
@ -205,6 +207,10 @@ jest.mock('~/server/middleware/limiters', () => ({
|
|||
forkIpLimiter: (_req, _res, next) => next(),
|
||||
forkUserLimiter: (_req, _res, next) => next(),
|
||||
}),
|
||||
createShareLimiters: () => ({
|
||||
shareIpLimiter: (req, res, next) => mockShareIpLimiter(req, res, next),
|
||||
shareUserLimiter: (req, res, next) => mockShareUserLimiter(req, res, next),
|
||||
}),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/utils/import/fork', () => ({
|
||||
|
|
@ -370,6 +376,15 @@ describe('share routes', () => {
|
|||
expect(mockGetAppConfig).toHaveBeenCalledWith({ baseOnly: true });
|
||||
});
|
||||
|
||||
it('rate limits shared message retrieval by IP and by user', async () => {
|
||||
mockSharedMessagesResult({ shareId: 'share-123', messages: [] });
|
||||
|
||||
await request(buildApp()).get('/api/share/share-123').expect(200);
|
||||
|
||||
expect(mockShareIpLimiter).toHaveBeenCalledTimes(1);
|
||||
expect(mockShareUserLimiter).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('prevents successful shared message responses from being cached', async () => {
|
||||
mockSharedMessagesResult({ shareId: 'share-123', messages: [] });
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ const { getStrategyFunctions } = require('~/server/services/Files/strategies');
|
|||
const { cleanFileName, getContentDisposition } = require('~/server/utils/files');
|
||||
const canAccessSharedLink = require('~/server/middleware/canAccessSharedLink');
|
||||
const { forkSharedConversation } = require('~/server/utils/import/fork');
|
||||
const { createForkLimiters } = require('~/server/middleware/limiters');
|
||||
const { createForkLimiters, createShareLimiters } = require('~/server/middleware/limiters');
|
||||
const optionalShareFileAuth = require('~/server/middleware/optionalShareFileAuth');
|
||||
const optionalJwtAuth = require('~/server/middleware/optionalJwtAuth');
|
||||
const requireJwtAuth = require('~/server/middleware/requireJwtAuth');
|
||||
|
|
@ -336,6 +336,7 @@ const streamSharedFile = async (req, res, file, requestedDisposition) => {
|
|||
|
||||
if (allowSharedLinks) {
|
||||
const { forkIpLimiter, forkUserLimiter } = createForkLimiters();
|
||||
const { shareIpLimiter, shareUserLimiter } = createShareLimiters();
|
||||
|
||||
router.get(
|
||||
'/:shareId/config',
|
||||
|
|
@ -357,6 +358,8 @@ if (allowSharedLinks) {
|
|||
router.get(
|
||||
'/:shareId',
|
||||
optionalJwtAuth,
|
||||
shareIpLimiter,
|
||||
shareUserLimiter,
|
||||
canAccessSharedLink,
|
||||
sharedLinkConfigMiddleware,
|
||||
async (req, res) => {
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import {
|
|||
import { createConfiguredContentInspector, inspectContent } from './protection/runtime';
|
||||
import { extractConversationImportContent } from './protection/adapters/submissions';
|
||||
import { ContentFilterError } from './middleware/contentFilter';
|
||||
import { aggregateAuditFindings } from './protection/audit';
|
||||
|
||||
export interface ConversationImportMessage extends StoredMessageContentInput {
|
||||
readonly isCreatedByUser?: boolean;
|
||||
|
|
@ -64,11 +65,21 @@ const hitlMessageFilterFieldSet = new Set<string>(HITL_MESSAGE_FILTER_FIELDS);
|
|||
* Applies the active policy to the complete normalized import snapshot before
|
||||
* the legacy importer starts any writes. Canonical resolution operates on an
|
||||
* inspection copy, so a rejected preflight never mutates the pending batch.
|
||||
* Audit findings are aggregated across the snapshot so a large conversation
|
||||
* reports each finding once instead of once per matching fragment.
|
||||
*/
|
||||
export async function assertConversationImportContentAllowed(
|
||||
filters: FiltersConfig | null | undefined,
|
||||
snapshot: ConversationImportSnapshot,
|
||||
context: ConversationImportProtectionContext = {},
|
||||
): Promise<void> {
|
||||
return aggregateAuditFindings(() => inspectConversationImportContent(filters, snapshot, context));
|
||||
}
|
||||
|
||||
async function inspectConversationImportContent(
|
||||
filters: FiltersConfig | null | undefined,
|
||||
snapshot: ConversationImportSnapshot,
|
||||
context: ConversationImportProtectionContext,
|
||||
): Promise<void> {
|
||||
const activeFilters = filters ?? undefined;
|
||||
const legacyPii = context.legacyPii ?? undefined;
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import type {
|
|||
} from '../protection/adapters/nested';
|
||||
import type { JsonPointer, TextContentFragment } from '../protection/types';
|
||||
import type { ExternalChatMessage } from '../protection/adapters/messages';
|
||||
import type { ConfiguredContentInspector } from '../protection/runtime';
|
||||
import type { CanonicalFileInspectionFile } from '../protection/files';
|
||||
import {
|
||||
CONTENT_TRAVERSAL_MAX_DEPTH,
|
||||
|
|
@ -68,6 +69,7 @@ import { extractMessageContent, snapshotExternalMessages } from '../protection/a
|
|||
import { ContentTraversalLimitError } from '../protection/adapters/nested';
|
||||
import { ContentFilterError, isContentFilterError } from './contentFilter';
|
||||
import { createConfiguredContentInspector } from '../protection/runtime';
|
||||
import { aggregateAuditFindingsSync } from '../protection/audit';
|
||||
|
||||
export type ModelBoundProviderAttribution = 'user' | 'model' | 'tool' | 'synthetic';
|
||||
|
||||
|
|
@ -3393,6 +3395,17 @@ export function assertModelBoundContent(input: ModelBoundContentInput): void {
|
|||
filters: input.filters,
|
||||
legacyPii: input.legacyPii,
|
||||
});
|
||||
if (inspector?.hasAuditRules !== true) {
|
||||
inspectModelBoundContent(input, inspector);
|
||||
return;
|
||||
}
|
||||
aggregateAuditFindingsSync(() => inspectModelBoundContent(input, inspector));
|
||||
}
|
||||
|
||||
function inspectModelBoundContent(
|
||||
input: ModelBoundContentInput,
|
||||
inspector: ConfiguredContentInspector | null,
|
||||
): void {
|
||||
const hasFileFailClose =
|
||||
getBlockedUninspectableFileField(input.filters, FILE_FILTER_FIELDS) != null;
|
||||
if (inspector == null && !hasFileFailClose) {
|
||||
|
|
|
|||
201
packages/api/src/protection/audit.spec.ts
Normal file
201
packages/api/src/protection/audit.spec.ts
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import type { FiltersConfig } from 'librechat-data-provider';
|
||||
import type { ShareContentPreflightMessage } from '../shared-links/protection';
|
||||
import { assertModelBoundContent } from '../middleware/modelBoundContent';
|
||||
import { createShareContentPreflight } from '../shared-links/protection';
|
||||
import { ContentFilterError } from '../middleware/contentFilter';
|
||||
import { inspectContent } from './runtime';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: { warn: jest.fn(), error: jest.fn(), info: jest.fn(), debug: jest.fn() },
|
||||
}));
|
||||
|
||||
const AUDIT_PATTERN = {
|
||||
id: 'org-token',
|
||||
label: 'organization token',
|
||||
regex: 'ORG-[A-Z0-9]+',
|
||||
};
|
||||
|
||||
const BLOCK_PATTERN = {
|
||||
id: 'private',
|
||||
label: 'private value',
|
||||
regex: 'PRIVATE-[A-Z]+',
|
||||
};
|
||||
|
||||
const auditFilters: FiltersConfig = {
|
||||
messages: {
|
||||
pii: {
|
||||
action: 'audit',
|
||||
starterPatterns: [],
|
||||
customPatterns: [AUDIT_PATTERN],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
function matchingMessages(count: number): ShareContentPreflightMessage[] {
|
||||
const messages: ShareContentPreflightMessage[] = [];
|
||||
for (let index = 0; index < count; index++) {
|
||||
messages.push({ role: 'user', isCreatedByUser: true, text: `ORG-${index}` });
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
|
||||
function auditCalls(): unknown[][] {
|
||||
return (logger.info as jest.Mock).mock.calls.filter(([message]) =>
|
||||
String(message).startsWith('[content-filter] Audit-only finding'),
|
||||
);
|
||||
}
|
||||
|
||||
describe('audit finding aggregation', () => {
|
||||
beforeEach(() => {
|
||||
(logger.info as jest.Mock).mockClear();
|
||||
});
|
||||
|
||||
it('reports one audit finding per rule for a snapshot that matches thousands of times', async () => {
|
||||
const preflight = createShareContentPreflight(auditFilters);
|
||||
|
||||
await expect(
|
||||
preflight?.({ title: 'Safe title', messages: matchingMessages(4_096), shareId: 'share-1' }),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(auditCalls()).toHaveLength(1);
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.stringContaining('"occurrences":4096'),
|
||||
expect.objectContaining({ action: 'audit', ruleId: 'org-token', occurrences: 4_096 }),
|
||||
);
|
||||
});
|
||||
|
||||
it('bounds a repeated retrieval to one aggregated finding per request', async () => {
|
||||
const preflight = createShareContentPreflight(auditFilters);
|
||||
const messages = matchingMessages(512);
|
||||
|
||||
await preflight?.({ title: 'Safe title', messages, shareId: 'share-1' });
|
||||
await preflight?.({ title: 'Safe title', messages, shareId: 'share-1' });
|
||||
|
||||
const calls = auditCalls();
|
||||
expect(calls).toHaveLength(2);
|
||||
for (const [, metadata] of calls) {
|
||||
expect(metadata).toMatchObject({ ruleId: 'org-token', occurrences: 512 });
|
||||
}
|
||||
});
|
||||
|
||||
it('counts each configured audit rule separately', async () => {
|
||||
const preflight = createShareContentPreflight({
|
||||
...auditFilters,
|
||||
conversationTitles: {
|
||||
pii: {
|
||||
action: 'audit',
|
||||
starterPatterns: [],
|
||||
customPatterns: [AUDIT_PATTERN],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await preflight?.({ title: 'ORG-TITLE', messages: matchingMessages(8), shareId: 'share-1' });
|
||||
|
||||
const calls = auditCalls();
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(calls.map(([, metadata]) => metadata)).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ source: 'message', occurrences: 8 }),
|
||||
expect.objectContaining({ source: 'conversation_title', occurrences: 1 }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('reports aggregated findings when a later message is blocked', async () => {
|
||||
const preflight = createShareContentPreflight(auditFilters, {
|
||||
legacyPii: { starterPatterns: [], customPatterns: [BLOCK_PATTERN] },
|
||||
});
|
||||
|
||||
await expect(
|
||||
preflight?.({
|
||||
title: 'Safe title',
|
||||
messages: [
|
||||
...matchingMessages(64),
|
||||
{ role: 'user', isCreatedByUser: true, text: 'PRIVATE-VALUE' },
|
||||
],
|
||||
shareId: 'share-1',
|
||||
}),
|
||||
).rejects.toBeInstanceOf(ContentFilterError);
|
||||
|
||||
expect(auditCalls()).toHaveLength(1);
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.stringContaining('"occurrences":64'),
|
||||
expect.objectContaining({ ruleId: 'org-token', occurrences: 64 }),
|
||||
);
|
||||
});
|
||||
|
||||
it('aggregates the fragments of one model-bound inspection', () => {
|
||||
const content: { type: string; text: string }[] = [];
|
||||
for (let index = 0; index < 512; index++) {
|
||||
content.push({ type: 'text', text: `ORG-${index}` });
|
||||
}
|
||||
|
||||
expect(() =>
|
||||
assertModelBoundContent({
|
||||
filters: auditFilters,
|
||||
storedMessages: [{ isCreatedByUser: true, role: 'user', content }],
|
||||
}),
|
||||
).not.toThrow();
|
||||
|
||||
expect(auditCalls().map(([, metadata]) => metadata)).toEqual([
|
||||
expect.objectContaining({ source: 'message', field: 'content_part', occurrences: 512 }),
|
||||
expect.objectContaining({ source: 'assembled_context', occurrences: 1 }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps rules whose ids and labels differ only by spacing separate', async () => {
|
||||
const preflight = createShareContentPreflight({
|
||||
messages: {
|
||||
pii: {
|
||||
action: 'audit',
|
||||
starterPatterns: [],
|
||||
customPatterns: [
|
||||
{ id: 'a', label: 'b c', regex: 'ALPHA' },
|
||||
{ id: 'a b', label: 'c', regex: 'BETA' },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await preflight?.({
|
||||
title: 'Safe title',
|
||||
messages: [
|
||||
{ role: 'user', isCreatedByUser: true, text: 'ALPHA' },
|
||||
{ role: 'user', isCreatedByUser: true, text: 'BETA' },
|
||||
],
|
||||
shareId: 'share-1',
|
||||
});
|
||||
|
||||
expect(auditCalls().map(([, metadata]) => metadata)).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ ruleId: 'a', label: 'b c', occurrences: 1 }),
|
||||
expect.objectContaining({ ruleId: 'a b', label: 'c', occurrences: 1 }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('reports findings immediately outside an aggregation scope', () => {
|
||||
const fragment = {
|
||||
id: 'message.text',
|
||||
text: 'ORG-SECRET',
|
||||
path: '/message/text',
|
||||
source: 'message',
|
||||
field: 'text',
|
||||
format: 'plain',
|
||||
treatment: 'replaceable',
|
||||
provenance: 'user',
|
||||
} as const;
|
||||
|
||||
expect(
|
||||
inspectContent([fragment, { ...fragment, text: 'ORG-OTHER' }], { filters: auditFilters }),
|
||||
).toBeNull();
|
||||
|
||||
const calls = auditCalls();
|
||||
expect(calls).toHaveLength(2);
|
||||
for (const [, metadata] of calls) {
|
||||
expect(metadata).toMatchObject({ occurrences: 1 });
|
||||
}
|
||||
});
|
||||
});
|
||||
134
packages/api/src/protection/audit.ts
Normal file
134
packages/api/src/protection/audit.ts
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
import { AsyncLocalStorage } from 'async_hooks';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import type { FilterPiiAction } from 'librechat-data-provider';
|
||||
import type { ContentFieldMap, ContentProvenance, ContentSource } from './types';
|
||||
|
||||
export interface AuditFindingMetadata {
|
||||
readonly action: FilterPiiAction;
|
||||
readonly detectorId: string;
|
||||
readonly ruleId: string;
|
||||
readonly label: string;
|
||||
readonly source: ContentSource;
|
||||
readonly field: ContentFieldMap[ContentSource];
|
||||
readonly provenance: ContentProvenance;
|
||||
}
|
||||
|
||||
interface AggregatedAuditFinding {
|
||||
readonly metadata: AuditFindingMetadata;
|
||||
occurrences: number;
|
||||
}
|
||||
|
||||
interface AuditFindingAggregation {
|
||||
readonly findings: Map<string, AggregatedAuditFinding>;
|
||||
untrackedOccurrences: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Audit metadata carries no inspected content, so its cardinality is bounded by
|
||||
* the configured rules rather than by the text they match. The cap only guards
|
||||
* against an unforeseen source of distinct keys.
|
||||
*/
|
||||
const MAX_AGGREGATED_AUDIT_FINDINGS = 1_024;
|
||||
|
||||
const auditAggregationStorage = new AsyncLocalStorage<AuditFindingAggregation>();
|
||||
|
||||
/** Serialized rather than joined: configured rule ids and labels may contain any character. */
|
||||
function aggregationKey(metadata: AuditFindingMetadata): string {
|
||||
return JSON.stringify([
|
||||
metadata.action,
|
||||
metadata.detectorId,
|
||||
metadata.ruleId,
|
||||
metadata.label,
|
||||
metadata.source,
|
||||
metadata.field,
|
||||
metadata.provenance,
|
||||
]);
|
||||
}
|
||||
|
||||
function emitAuditFinding(metadata: AuditFindingMetadata, occurrences: number): void {
|
||||
const auditMetadata = { ...metadata, occurrences };
|
||||
logger.info(
|
||||
`[content-filter] Audit-only finding ${JSON.stringify(auditMetadata)}`,
|
||||
auditMetadata,
|
||||
);
|
||||
}
|
||||
|
||||
function flushAuditFindings(aggregation: AuditFindingAggregation): void {
|
||||
for (const { metadata, occurrences } of aggregation.findings.values()) {
|
||||
emitAuditFinding(metadata, occurrences);
|
||||
}
|
||||
aggregation.findings.clear();
|
||||
const { untrackedOccurrences } = aggregation;
|
||||
if (untrackedOccurrences === 0) {
|
||||
return;
|
||||
}
|
||||
aggregation.untrackedOccurrences = 0;
|
||||
const untrackedMetadata = {
|
||||
action: 'audit',
|
||||
untrackedOccurrences,
|
||||
trackedFindings: MAX_AGGREGATED_AUDIT_FINDINGS,
|
||||
};
|
||||
logger.info(
|
||||
`[content-filter] Audit-only findings untracked ${JSON.stringify(untrackedMetadata)}`,
|
||||
untrackedMetadata,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports an audit-only finding. Within an aggregation scope the finding is
|
||||
* counted and reported once per distinct rule/source/field/provenance key when
|
||||
* the scope ends, so caller-supplied content cannot turn a single request into
|
||||
* one log write per matching fragment.
|
||||
*/
|
||||
export function recordAuditFinding(metadata: AuditFindingMetadata): void {
|
||||
const aggregation = auditAggregationStorage.getStore();
|
||||
if (aggregation == null) {
|
||||
emitAuditFinding(metadata, 1);
|
||||
return;
|
||||
}
|
||||
const key = aggregationKey(metadata);
|
||||
const aggregated = aggregation.findings.get(key);
|
||||
if (aggregated != null) {
|
||||
aggregated.occurrences += 1;
|
||||
return;
|
||||
}
|
||||
if (aggregation.findings.size >= MAX_AGGREGATED_AUDIT_FINDINGS) {
|
||||
aggregation.untrackedOccurrences += 1;
|
||||
return;
|
||||
}
|
||||
aggregation.findings.set(key, { metadata, occurrences: 1 });
|
||||
}
|
||||
|
||||
function createAuditFindingAggregation(): AuditFindingAggregation {
|
||||
return { findings: new Map<string, AggregatedAuditFinding>(), untrackedOccurrences: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounds audit logging for one inspection pass over caller-supplied content.
|
||||
* Nested scopes reuse the outermost aggregation, so a single request reports
|
||||
* each distinct audit finding once, with its occurrence count, on completion.
|
||||
*/
|
||||
export async function aggregateAuditFindings<T>(inspect: () => Promise<T>): Promise<T> {
|
||||
if (auditAggregationStorage.getStore() != null) {
|
||||
return inspect();
|
||||
}
|
||||
const aggregation = createAuditFindingAggregation();
|
||||
try {
|
||||
return await auditAggregationStorage.run(aggregation, inspect);
|
||||
} finally {
|
||||
flushAuditFindings(aggregation);
|
||||
}
|
||||
}
|
||||
|
||||
/** Synchronous counterpart of {@link aggregateAuditFindings}. */
|
||||
export function aggregateAuditFindingsSync<T>(inspect: () => T): T {
|
||||
if (auditAggregationStorage.getStore() != null) {
|
||||
return inspect();
|
||||
}
|
||||
const aggregation = createAuditFindingAggregation();
|
||||
try {
|
||||
return auditAggregationStorage.run(aggregation, inspect);
|
||||
} finally {
|
||||
flushAuditFindings(aggregation);
|
||||
}
|
||||
}
|
||||
|
|
@ -180,6 +180,7 @@ describe('configured content inspection', () => {
|
|||
source: 'message',
|
||||
field: 'text',
|
||||
provenance: 'user',
|
||||
occurrences: 1,
|
||||
};
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
`[content-filter] Audit-only finding ${JSON.stringify(metadata)}`,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import {
|
||||
MAX_PII_CUSTOM_PATTERNS_TOTAL,
|
||||
MAX_PII_CUSTOM_REGEX_CHARACTERS,
|
||||
|
|
@ -31,6 +30,7 @@ import {
|
|||
isContentTraversalProtected,
|
||||
} from './adapters/nested';
|
||||
import { isLegacyPiiFragment } from './legacy';
|
||||
import { recordAuditFinding } from './audit';
|
||||
|
||||
interface CompiledFilter {
|
||||
readonly detectorId: string;
|
||||
|
|
@ -47,6 +47,7 @@ export interface ContentInspectionConfig {
|
|||
}
|
||||
|
||||
export interface ConfiguredContentInspector {
|
||||
readonly hasAuditRules: boolean;
|
||||
createSession(): ConfiguredContentInspectionSession;
|
||||
inspect(fragments: Iterable<TextContentFragment>): ProtectionFinding | null;
|
||||
}
|
||||
|
|
@ -347,7 +348,7 @@ function createInspector(rules: readonly CompiledFilter[]): ConfiguredContentIns
|
|||
detectorId: rule.detectorId,
|
||||
};
|
||||
if (rule.action === 'audit') {
|
||||
const auditMetadata = {
|
||||
recordAuditFinding({
|
||||
action: rule.action,
|
||||
detectorId: configuredFinding.detectorId,
|
||||
ruleId: configuredFinding.ruleId,
|
||||
|
|
@ -355,11 +356,7 @@ function createInspector(rules: readonly CompiledFilter[]): ConfiguredContentIns
|
|||
source: configuredFinding.source,
|
||||
field: configuredFinding.field,
|
||||
provenance: configuredFinding.provenance,
|
||||
};
|
||||
logger.info(
|
||||
`[content-filter] Audit-only finding ${JSON.stringify(auditMetadata)}`,
|
||||
auditMetadata,
|
||||
);
|
||||
});
|
||||
continue;
|
||||
}
|
||||
firstBlockingFinding ??= configuredFinding;
|
||||
|
|
@ -398,6 +395,7 @@ function createInspector(rules: readonly CompiledFilter[]): ConfiguredContentIns
|
|||
};
|
||||
|
||||
return {
|
||||
hasAuditRules,
|
||||
createSession,
|
||||
inspect(fragments) {
|
||||
return inspect(fragments, new Array(rules.length));
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import { assertModelBoundContent } from '../middleware/modelBoundContent';
|
|||
import { getUserSubmittedPathState } from '../protection/provenance';
|
||||
import { assertConversationImportContentAllowed } from '../imports';
|
||||
import { ContentFilterError } from '../middleware/contentFilter';
|
||||
import { aggregateAuditFindings } from '../protection/audit';
|
||||
import { inspectContent } from '../protection/runtime';
|
||||
|
||||
export interface SerializedSharedFile extends FileContentInput {
|
||||
|
|
@ -139,7 +140,11 @@ export function createShareContentPreflight(
|
|||
return undefined;
|
||||
}
|
||||
|
||||
return async ({ title, messages, shareId }) => {
|
||||
const inspectSharedContent = async ({
|
||||
title,
|
||||
messages,
|
||||
shareId,
|
||||
}: ShareContentPreflightInput): Promise<void> => {
|
||||
const inspectSharedFileMetadata = options.sharedFileMetadata === true;
|
||||
const inspectSharedFiles =
|
||||
inspectSharedFileMetadata && options.sharedFileMetadataFiles !== false;
|
||||
|
|
@ -169,6 +174,8 @@ export function createShareContentPreflight(
|
|||
...(options.sharedFileMetadataFiles === false && { includeFiles: false }),
|
||||
});
|
||||
};
|
||||
|
||||
return (input) => aggregateAuditFindings(() => inspectSharedContent(input));
|
||||
}
|
||||
|
||||
const SERIALIZED_LOCATOR_KEYS = [
|
||||
|
|
|
|||
|
|
@ -3185,6 +3185,10 @@ export enum ViolationTypes {
|
|||
* Registration violations.
|
||||
*/
|
||||
REGISTRATIONS = 'registrations',
|
||||
/**
|
||||
* Shared link retrieval limit violations.
|
||||
*/
|
||||
SHARE_LIMIT = 'share_limit',
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue