mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-10-23 10:36:09 +00:00
🛡️ feat: Cap audit-log CSV exports at 100k rows; cover stream error path
Introduces MAX_AUDIT_EXPORT_ROWS (100k) and threads a `maxRows` option through streamAuditLogEntries. The handler now passes the cap into the stream so a careless admin script or a hostile auditor cannot pin a Node worker and a Mongo cursor by exporting unbounded result sets. Beyond 100k rows, callers should slice exports by from / to date. Adds a methods-layer spec for the cap behavior, a handler-layer spec that asserts the option is plumbed through, and a handler-layer spec that exercises the streamAuditLogEntries-throws-after-headers-sent path (catch block falls through to res.end instead of attempting JSON). Documents on buildFilter that case-insensitive substring regex filters (actorName, targetName, capability, search) cannot use a B-tree index and degrade to a tenant-scoped partition scan, so deployments with hundreds of thousands of audit rows per tenant should constrain those queries with a date window.
This commit is contained in:
parent
ddc69efb04
commit
e65dff3bb3
6 changed files with 95 additions and 7 deletions
|
|
@ -534,5 +534,40 @@ describe('createAdminAuditLogHandlers', () => {
|
|||
expect(json.mock.calls[0][0].error).toMatch(/from/);
|
||||
expect(deps.streamAuditLogEntries).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('passes a maxRows cap into the stream call', async () => {
|
||||
const deps = createDeps();
|
||||
const handlers = createAdminAuditLogHandlers(deps);
|
||||
const ctx = createCsvContext();
|
||||
|
||||
await handlers.exportAuditLogCsv(ctx.req, ctx.res);
|
||||
|
||||
const optionsArg = (deps.streamAuditLogEntries as jest.Mock).mock.calls[0][3];
|
||||
expect(optionsArg).toEqual(
|
||||
expect.objectContaining({ maxRows: expect.any(Number) }),
|
||||
);
|
||||
expect(optionsArg.maxRows).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('closes the response cleanly when the stream throws after headers are sent', async () => {
|
||||
const entry = mockEntry();
|
||||
const deps = createDeps({
|
||||
streamAuditLogEntries: jest.fn(async (_t, _f, onEntry) => {
|
||||
await (onEntry as (e: AdminAuditLogEntry) => Promise<void>)(entry);
|
||||
throw new Error('cursor exploded mid-stream');
|
||||
}),
|
||||
});
|
||||
const handlers = createAdminAuditLogHandlers(deps);
|
||||
const ctx = createCsvContext();
|
||||
/** Headers were already sent by the BOM / header write before the throw,
|
||||
* so the catch block must use res.end() instead of trying to send JSON. */
|
||||
(ctx.res as unknown as { headersSent: boolean }).headersSent = true;
|
||||
|
||||
await handlers.exportAuditLogCsv(ctx.req, ctx.res);
|
||||
|
||||
expect(ctx.endCalled()).toBe(true);
|
||||
const statusMock = (ctx.res as unknown as { status: jest.Mock }).status;
|
||||
expect(statusMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
import { PrincipalType } from 'librechat-data-provider';
|
||||
import { AUDIT_ACTIONS, logger, MAX_AUDIT_LOG_LIMIT } from '@librechat/data-schemas';
|
||||
import {
|
||||
AUDIT_ACTIONS,
|
||||
logger,
|
||||
MAX_AUDIT_EXPORT_ROWS,
|
||||
MAX_AUDIT_LOG_LIMIT,
|
||||
} from '@librechat/data-schemas';
|
||||
import type {
|
||||
AdminAuditLogEntry,
|
||||
AuditAction,
|
||||
|
|
@ -37,7 +42,7 @@ export interface AdminAuditLogDeps {
|
|||
tenantId: string | undefined,
|
||||
filters: Omit<AuditLogFilters, 'offset' | 'limit'>,
|
||||
onEntry: (entry: AdminAuditLogEntry) => void | Promise<void>,
|
||||
options?: { isCancelled?: () => boolean },
|
||||
options?: { isCancelled?: () => boolean; maxRows?: number },
|
||||
) => Promise<number>;
|
||||
}
|
||||
|
||||
|
|
@ -360,7 +365,7 @@ export function createAdminAuditLogHandlers(deps: AdminAuditLogDeps) {
|
|||
await writeChunk(formatCsvRow(entry));
|
||||
await writeChunk('\r\n');
|
||||
},
|
||||
{ isCancelled },
|
||||
{ isCancelled, maxRows: MAX_AUDIT_EXPORT_ROWS },
|
||||
);
|
||||
|
||||
res.removeListener('close', markAborted);
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ export {
|
|||
validateSkillFrontmatter,
|
||||
validateSkillDescription,
|
||||
deriveStructuredFrontmatterFields,
|
||||
MAX_AUDIT_EXPORT_ROWS,
|
||||
MAX_AUDIT_LOG_LIMIT,
|
||||
} from './methods';
|
||||
export type * from './types';
|
||||
|
|
|
|||
|
|
@ -249,5 +249,24 @@ describe('auditLog methods', () => {
|
|||
expect(count).toBe(3);
|
||||
expect(seen.length).toBe(3);
|
||||
});
|
||||
|
||||
it('stops after maxRows entries even if more remain', async () => {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await methods.recordAuditEntry(baseInput({ capability: `manage:cap-${i}` }));
|
||||
}
|
||||
|
||||
const seen: string[] = [];
|
||||
const count = await methods.streamAuditLogEntries(
|
||||
'tenant-a',
|
||||
{},
|
||||
(entry) => {
|
||||
seen.push(entry.capability);
|
||||
},
|
||||
{ maxRows: 4 },
|
||||
);
|
||||
|
||||
expect(count).toBe(4);
|
||||
expect(seen.length).toBe(4);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,6 +10,13 @@ import logger from '~/config/winston';
|
|||
|
||||
const DEFAULT_LIMIT = 100;
|
||||
export const MAX_AUDIT_LOG_LIMIT = 500;
|
||||
/**
|
||||
* Upper bound on rows emitted by the CSV export stream. At 100k rows per
|
||||
* tenant per export request, a careless admin script (or a hostile auditor)
|
||||
* can keep a cursor and a Node worker busy without saturating either; beyond
|
||||
* that, exports should be sliced by `from`/`to`.
|
||||
*/
|
||||
export const MAX_AUDIT_EXPORT_ROWS = 100_000;
|
||||
const MAX_SEARCH_LENGTH = 200;
|
||||
|
||||
export interface AuditLogMethods {
|
||||
|
|
@ -26,7 +33,7 @@ export interface AuditLogMethods {
|
|||
tenantId: string | undefined,
|
||||
filters: Omit<AuditLogFilters, 'offset' | 'limit'>,
|
||||
onEntry: (entry: AdminAuditLogEntry) => void | Promise<void>,
|
||||
options?: { isCancelled?: () => boolean },
|
||||
options?: { isCancelled?: () => boolean; maxRows?: number },
|
||||
) => Promise<number>;
|
||||
}
|
||||
|
||||
|
|
@ -54,6 +61,17 @@ function tenantFilter(tenantId?: string): FilterQuery<IAuditLog> {
|
|||
: { tenantId: { $exists: false } };
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the Mongo `find` filter for audit-log queries. Tenant scoping is
|
||||
* always applied first and uses the compound `{ tenantId, createdAt, _id }`
|
||||
* index. The substring regex filters below (`actorName`, `targetName`,
|
||||
* `capability`, `search`) are case-insensitive and therefore cannot use a
|
||||
* standard B-tree index: within the tenant slice they degrade to a partition
|
||||
* scan. For deployments where the audit log grows past hundreds of thousands
|
||||
* of entries per tenant, consider a text index or storing lowercased shadow
|
||||
* fields. Today's primary listing combines `tenantId` with `createdAt`
|
||||
* pagination, so the scan stays bounded by the user-chosen date window.
|
||||
*/
|
||||
function buildFilter(
|
||||
tenantId: string | undefined,
|
||||
filters: AuditLogFilters,
|
||||
|
|
@ -177,7 +195,7 @@ export function createAuditLogMethods(mongoose: typeof import('mongoose')): Audi
|
|||
tenantId: string | undefined,
|
||||
filters: Omit<AuditLogFilters, 'offset' | 'limit'>,
|
||||
onEntry: (entry: AdminAuditLogEntry) => void | Promise<void>,
|
||||
options?: { isCancelled?: () => boolean },
|
||||
options?: { isCancelled?: () => boolean; maxRows?: number },
|
||||
): Promise<number> {
|
||||
const AuditLog = mongoose.models.AuditLog as Model<IAuditLog>;
|
||||
const query = buildFilter(tenantId, filters);
|
||||
|
|
@ -187,6 +205,7 @@ export function createAuditLogMethods(mongoose: typeof import('mongoose')): Audi
|
|||
.cursor({ batchSize: 500 });
|
||||
|
||||
const isCancelled = options?.isCancelled;
|
||||
const maxRows = options?.maxRows;
|
||||
let count = 0;
|
||||
try {
|
||||
for await (const doc of cursor) {
|
||||
|
|
@ -194,6 +213,10 @@ export function createAuditLogMethods(mongoose: typeof import('mongoose')): Audi
|
|||
await cursor.close();
|
||||
break;
|
||||
}
|
||||
if (maxRows != null && count >= maxRows) {
|
||||
await cursor.close();
|
||||
break;
|
||||
}
|
||||
await onEntry(toWire(doc));
|
||||
count++;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,12 @@ import { createAccessRoleMethods, type AccessRoleMethods } from './accessRole';
|
|||
import { createUserGroupMethods, type UserGroupMethods } from './userGroup';
|
||||
import { createAclEntryMethods, permissionBitSupersets, type AclEntryMethods } from './aclEntry';
|
||||
import { createSystemGrantMethods, type SystemGrantMethods } from './systemGrant';
|
||||
import { createAuditLogMethods, MAX_AUDIT_LOG_LIMIT, type AuditLogMethods } from './auditLog';
|
||||
import {
|
||||
createAuditLogMethods,
|
||||
MAX_AUDIT_EXPORT_ROWS,
|
||||
MAX_AUDIT_LOG_LIMIT,
|
||||
type AuditLogMethods,
|
||||
} from './auditLog';
|
||||
import { createShareMethods, type ShareMethods } from './share';
|
||||
/* Tier 1 — Simple CRUD */
|
||||
import { createActionMethods, type ActionMethods } from './action';
|
||||
|
|
@ -102,7 +107,7 @@ export {
|
|||
deriveStructuredFrontmatterFields,
|
||||
inferSkillFileCategory,
|
||||
};
|
||||
export { MAX_AUDIT_LOG_LIMIT };
|
||||
export { MAX_AUDIT_EXPORT_ROWS, MAX_AUDIT_LOG_LIMIT };
|
||||
|
||||
export type AllMethods = UserMethods &
|
||||
SessionMethods &
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue