mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-21 23:43:35 +00:00
🛂 feat: Filter Model-Bound Content by Source (#14425)
* feat: introduce optional content protection seam * feat: enforce source-aware content filters * feat: complete source-aware content enforcement * test: activate skill file-text fail-close fixtures * fix: harden source-aware content filters * fix: harden model-bound content filtering * fix: preserve legacy filters and generated files * fix: inspect shared scalar metadata * test: align mocks with current dev dependencies * feat: add persisted content filter safeguards * feat: complete source-aware content filter enforcement * fix: move resume content preflight into TypeScript * fix: close content inspection edge cases * fix: harden content protection boundaries * fix: complete content protection safeguards * test: align persisted memory filter coverage * fix: reconcile content protection with current dev * fix: reconcile content protection with latest dev * fix: close content protection review gaps * fix: enforce source-aware provider boundaries * fix: preserve legacy PII preflight semantics * test: stabilize stored branch preflight fixture * fix: defer agent writes until protected model admission * perf: harden source-aware model-bound filtering * fix: canonicalize provider lineage before validation * fix: satisfy model-bound callback type checks * perf: Bound content protection filtering work * fix: Bound submission array traversal * fix: Stabilize bounded content snapshots * fix: Scope model-bound traversal overflows * fix: Preserve scoped content inspection * fix: Accumulate aggregate traversal scopes * fix: centralize content policy boundaries * test: align deferred tool policy context * test: align controller policy mocks * style: normalize content protection imports * fix: close content policy review gaps * fix: narrow active skill policy config * fix: address content protection review boundaries * fix: retain exact provenance overflow sentinel * fix: preserve literal and scoped provenance updates * fix: narrow persisted edit provenance * fix: isolate exact overflow attribution * fix: centralize stored prompt protection * fix: fail closed on incomplete transcript evidence * fix: align canonical transcript routing * refactor: centralize content policy preflights * fix: isolate upload policy error typing * style: sort policy preflight imports * refactor: centralize content policy boundaries
This commit is contained in:
parent
10f95c0ce9
commit
67b7b441b2
303 changed files with 75546 additions and 2327 deletions
|
|
@ -1,12 +1,24 @@
|
|||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import {
|
||||
extractCodeArtifactRawText,
|
||||
extractCodeArtifactInspectionText,
|
||||
extractCodeArtifactText,
|
||||
getExtractedTextFormat,
|
||||
resolveMaxTextExtractBytes,
|
||||
MAX_TEXT_CACHE_BYTES,
|
||||
MAX_TEXT_EXTRACT_BYTES,
|
||||
} from './extract';
|
||||
import { parseDocument } from '~/files/documents/crud';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: {
|
||||
debug: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const docxText = '__DOCX_PARSED__';
|
||||
/* parseDocument throws on any originalname containing this token, so
|
||||
|
|
@ -516,6 +528,119 @@ describe('extractCodeArtifactText', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('extractCodeArtifactRawText', () => {
|
||||
it('returns the full inspectable text beyond the preview cache limit', () => {
|
||||
const suffix = 'BLOCK-LATE';
|
||||
const buffer = Buffer.from(`${'a'.repeat(MAX_TEXT_CACHE_BYTES + 1024)}${suffix}`);
|
||||
|
||||
const text = extractCodeArtifactRawText(buffer, 'utf8-text');
|
||||
|
||||
expect(text).toHaveLength(buffer.length);
|
||||
expect(text?.endsWith(suffix)).toBe(true);
|
||||
expect(text).not.toContain('…[truncated]');
|
||||
});
|
||||
|
||||
it('rejects binary, unsupported, and oversized raw content', () => {
|
||||
expect(extractCodeArtifactRawText(Buffer.from([0x00, 0x01]), 'utf8-text')).toBeNull();
|
||||
expect(extractCodeArtifactRawText(Buffer.from('text'), 'document')).toBeNull();
|
||||
expect(
|
||||
extractCodeArtifactRawText(Buffer.alloc(MAX_TEXT_EXTRACT_BYTES + 1, 'a'), 'utf8-text'),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractCodeArtifactInspectionText', () => {
|
||||
beforeEach(() => {
|
||||
mockOfficeHtml.mockReset();
|
||||
parseDocumentCalls.length = 0;
|
||||
});
|
||||
|
||||
it('returns complete plain text beyond the persisted preview cache limit', async () => {
|
||||
const suffix = 'BLOCK-LATE';
|
||||
const buffer = Buffer.from(`${'a'.repeat(MAX_TEXT_CACHE_BYTES + 1024)}${suffix}`);
|
||||
|
||||
const result = await extractCodeArtifactInspectionText(
|
||||
buffer,
|
||||
'output.txt',
|
||||
'text/plain',
|
||||
'utf8-text',
|
||||
);
|
||||
|
||||
expect(result.complete).toBe(true);
|
||||
expect(result.text?.endsWith(suffix)).toBe(true);
|
||||
expect(result.text).not.toContain('…[truncated]');
|
||||
});
|
||||
|
||||
it('returns complete parsed document text when the derived output fits the inspection limit', async () => {
|
||||
const result = await extractCodeArtifactInspectionText(
|
||||
Buffer.from('%PDF'),
|
||||
'report.pdf',
|
||||
'application/pdf',
|
||||
'document',
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
text: docxText,
|
||||
complete: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('marks parsed document text incomplete when the derived output exceeds the inspection limit', async () => {
|
||||
const suffix = 'BLOCK-DOCUMENT-SUFFIX';
|
||||
const text = `${'a'.repeat(MAX_TEXT_EXTRACT_BYTES + 1)}${suffix}`;
|
||||
jest.mocked(parseDocument).mockResolvedValueOnce({
|
||||
filename: 'large.pdf',
|
||||
bytes: Buffer.byteLength(text),
|
||||
filepath: 'document_parser',
|
||||
text,
|
||||
images: [],
|
||||
});
|
||||
|
||||
const result = await extractCodeArtifactInspectionText(
|
||||
Buffer.from('%PDF'),
|
||||
'large.pdf',
|
||||
'application/pdf',
|
||||
'document',
|
||||
);
|
||||
|
||||
expect(result.complete).toBe(false);
|
||||
expect(result.text).toContain('…[truncated]');
|
||||
expect(result.text).not.toContain(suffix);
|
||||
});
|
||||
|
||||
it('marks an oversized office preview banner as incomplete', async () => {
|
||||
mockOfficeHtml.mockResolvedValueOnce('x'.repeat(MAX_TEXT_CACHE_BYTES + 1));
|
||||
|
||||
const result = await extractCodeArtifactInspectionText(
|
||||
Buffer.from('PK'),
|
||||
'slides.pptx',
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||
'presentation',
|
||||
);
|
||||
|
||||
expect(result.complete).toBe(false);
|
||||
expect(result.text).toContain('Preview exceeds the size limit');
|
||||
});
|
||||
|
||||
it('keeps parser diagnostics free of generated filenames and parser errors', async () => {
|
||||
jest.mocked(parseDocument).mockRejectedValueOnce(new Error('PRIVATE-PARSER-ERROR'));
|
||||
|
||||
await expect(
|
||||
extractCodeArtifactInspectionText(
|
||||
Buffer.from('%PDF'),
|
||||
'PRIVATE-FILENAME.pdf',
|
||||
'application/pdf',
|
||||
'document',
|
||||
),
|
||||
).resolves.toEqual({ text: null, complete: false });
|
||||
|
||||
expect(logger.debug).toHaveBeenCalledWith(
|
||||
'[extractCodeArtifactInspectionText] Artifact inspection failed',
|
||||
);
|
||||
expect(JSON.stringify(jest.mocked(logger.debug).mock.calls)).not.toContain('PRIVATE-');
|
||||
});
|
||||
});
|
||||
|
||||
/* `getExtractedTextFormat` is the trust-flag classifier consumed by
|
||||
* `processCodeOutput` (api/server/services/Files/Code/process.js) to
|
||||
* persist `textFormat` on the file record. The client's security gate
|
||||
|
|
|
|||
|
|
@ -140,6 +140,20 @@ const extractUtf8 = (buffer: Buffer): string | null => {
|
|||
return truncate(buffer.toString('utf-8'), buffer);
|
||||
};
|
||||
|
||||
export function extractCodeArtifactRawText(
|
||||
buffer: Buffer,
|
||||
category: CodeArtifactCategory,
|
||||
): string | null {
|
||||
if (
|
||||
buffer.length > MAX_TEXT_EXTRACT_BYTES ||
|
||||
category !== 'utf8-text' ||
|
||||
isBinaryBuffer(buffer)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return buffer.toString('utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a known office-document extension back to its canonical MIME so we can
|
||||
* route through `parseDocument` even when buffer-sniffing yielded a generic
|
||||
|
|
@ -165,7 +179,7 @@ const documentMimeFromExtension = (name: string): string | null => {
|
|||
}
|
||||
};
|
||||
|
||||
const extractDocument = async (
|
||||
const extractDocumentText = async (
|
||||
buffer: Buffer,
|
||||
name: string,
|
||||
mimeType: string,
|
||||
|
|
@ -189,12 +203,21 @@ const extractDocument = async (
|
|||
if (!result?.text) {
|
||||
return null;
|
||||
}
|
||||
return truncate(result.text);
|
||||
return result.text;
|
||||
} finally {
|
||||
fs.unlink(tempPath).catch(() => {});
|
||||
}
|
||||
};
|
||||
|
||||
const extractDocument = async (
|
||||
buffer: Buffer,
|
||||
name: string,
|
||||
mimeType: string,
|
||||
): Promise<string | null> => {
|
||||
const text = await extractDocumentText(buffer, name, mimeType);
|
||||
return text == null ? null : truncate(text);
|
||||
};
|
||||
|
||||
/**
|
||||
* Minimal valid HTML document substituted when a producer's output
|
||||
* exceeds `MAX_TEXT_CACHE_BYTES`. Byte-truncating the producer's HTML
|
||||
|
|
@ -329,3 +352,69 @@ export async function extractCodeArtifactText(
|
|||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface CodeArtifactInspectionText {
|
||||
readonly text: string | null;
|
||||
readonly complete: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts text for content inspection independently of the persisted preview
|
||||
* cache. Plain text and parsed documents are complete only when their full
|
||||
* derived text fits inside the inspection ceiling. Parseable office documents
|
||||
* use their full semantic extraction; office preview-only formats retain the
|
||||
* available HTML for compatibility-mode inspection but stay marked partial
|
||||
* because their producers have independent row, slide, and output caps.
|
||||
*/
|
||||
export async function extractCodeArtifactInspectionText(
|
||||
buffer: Buffer,
|
||||
name: string,
|
||||
mimeType: string,
|
||||
category: CodeArtifactCategory,
|
||||
): Promise<CodeArtifactInspectionText> {
|
||||
const incomplete = (text: string | null = null): CodeArtifactInspectionText => ({
|
||||
text,
|
||||
complete: false,
|
||||
});
|
||||
const bounded = (text: string | null): CodeArtifactInspectionText => {
|
||||
if (text == null) {
|
||||
return incomplete();
|
||||
}
|
||||
if (Buffer.byteLength(text, 'utf-8') > MAX_TEXT_EXTRACT_BYTES) {
|
||||
return incomplete(truncate(text));
|
||||
}
|
||||
return {
|
||||
text,
|
||||
complete: true,
|
||||
};
|
||||
};
|
||||
if (buffer.length > MAX_TEXT_EXTRACT_BYTES) {
|
||||
return incomplete();
|
||||
}
|
||||
try {
|
||||
if (hasOfficeHtmlPath(name, mimeType)) {
|
||||
if (category === 'utf8-text') {
|
||||
return bounded(extractCodeArtifactRawText(buffer, category));
|
||||
}
|
||||
if (category === 'document') {
|
||||
try {
|
||||
return bounded(await extractDocumentText(buffer, name, mimeType));
|
||||
} catch {
|
||||
// Compatibility mode can still inspect the available preview below.
|
||||
}
|
||||
}
|
||||
return incomplete(await renderOfficeHtml(buffer, name, mimeType));
|
||||
}
|
||||
if (category === 'utf8-text') {
|
||||
return bounded(extractCodeArtifactRawText(buffer, category));
|
||||
}
|
||||
if (category !== 'document') {
|
||||
return incomplete();
|
||||
}
|
||||
|
||||
return bounded(await extractDocumentText(buffer, name, mimeType));
|
||||
} catch {
|
||||
logger.debug('[extractCodeArtifactInspectionText] Artifact inspection failed');
|
||||
return incomplete();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,3 +2,4 @@ export * from './classify';
|
|||
export * from './extract';
|
||||
export * from './form';
|
||||
export * from './identity';
|
||||
export * from './preflight';
|
||||
|
|
|
|||
317
packages/api/src/files/code/preflight.spec.ts
Normal file
317
packages/api/src/files/code/preflight.spec.ts
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
import type { FiltersConfig } from 'librechat-data-provider';
|
||||
import type { CodeOutputArtifact, PrepareCodeOutputInput, PreparedCodeOutput } from './preflight';
|
||||
import { CODE_OUTPUT_PREFLIGHT_MAX_BYTES, preflightCodeOutputBatch } from './preflight';
|
||||
|
||||
const BLOCK_PATTERN = {
|
||||
id: 'private-token',
|
||||
label: 'private token',
|
||||
regex: 'PRIVATE-[A-Z]+',
|
||||
};
|
||||
|
||||
const limits = {
|
||||
fileLimit: 10,
|
||||
fileSizeLimit: 16,
|
||||
totalSizeLimit: 32,
|
||||
};
|
||||
|
||||
function artifact(count = 1): CodeOutputArtifact {
|
||||
return {
|
||||
session_id: 'artifact-session',
|
||||
files: Array.from({ length: count }, (_, index) => ({
|
||||
id: `file-${index}`,
|
||||
name: `file-${index}.txt`,
|
||||
storage_session_id: `storage-${index}`,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function prepared(
|
||||
input: PrepareCodeOutputInput,
|
||||
overrides: Partial<PreparedCodeOutput> = {},
|
||||
): PreparedCodeOutput {
|
||||
return {
|
||||
buffer: Buffer.from('safe'),
|
||||
extractedTextComplete: true,
|
||||
file: {
|
||||
...input.file,
|
||||
type: 'text/plain',
|
||||
content: 'safe',
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('preflightCodeOutputBatch', () => {
|
||||
it('bounded-downloads default-off artifacts and reuses the prepared bytes', async () => {
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) => prepared(input));
|
||||
|
||||
const result = await preflightCodeOutputBatch({
|
||||
artifact: artifact(2),
|
||||
limits,
|
||||
prepare,
|
||||
});
|
||||
|
||||
expect(prepare).toHaveBeenCalledTimes(2);
|
||||
expect(prepare).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ maxBytes: 16, inspectContent: false }),
|
||||
);
|
||||
expect(prepare).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({ maxBytes: 16, inspectContent: false }),
|
||||
);
|
||||
expect(result.map((entry) => entry.preparedBuffer?.toString())).toEqual(['safe', 'safe']);
|
||||
expect(result.every((entry) => entry.downloadFallback !== true)).toBe(true);
|
||||
});
|
||||
|
||||
it('treats zero byte limits as unlimited within the hard transport ceiling', async () => {
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
|
||||
prepared(input, { buffer: Buffer.alloc(4) }),
|
||||
);
|
||||
|
||||
const result = await preflightCodeOutputBatch({
|
||||
artifact: artifact(2),
|
||||
limits: { ...limits, fileSizeLimit: 0, totalSizeLimit: 0 },
|
||||
prepare,
|
||||
});
|
||||
|
||||
expect(prepare).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ maxBytes: CODE_OUTPUT_PREFLIGHT_MAX_BYTES }),
|
||||
);
|
||||
expect(prepare).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({ maxBytes: CODE_OUTPUT_PREFLIGHT_MAX_BYTES - 4 }),
|
||||
);
|
||||
expect(result.map((entry) => entry.preparedBuffer?.length)).toEqual([4, 4]);
|
||||
expect(result.every((entry) => entry.downloadFallback !== true)).toBe(true);
|
||||
});
|
||||
|
||||
it('uses explicit URL fallbacks for default-off count overflow without downloading', async () => {
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) => prepared(input));
|
||||
|
||||
const result = await preflightCodeOutputBatch({
|
||||
artifact: artifact(3),
|
||||
limits: { ...limits, fileLimit: 2 },
|
||||
prepare,
|
||||
});
|
||||
|
||||
expect(prepare).not.toHaveBeenCalled();
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result.every((entry) => entry.downloadFallback === true)).toBe(true);
|
||||
});
|
||||
|
||||
it('preserves configured files above the inspection count as URL fallbacks', async () => {
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) => prepared(input));
|
||||
|
||||
const result = await preflightCodeOutputBatch({
|
||||
artifact: artifact(12),
|
||||
limits: { ...limits, fileLimit: 12 },
|
||||
prepare,
|
||||
});
|
||||
|
||||
expect(prepare).not.toHaveBeenCalled();
|
||||
expect(result.map((entry) => entry.file.id)).toEqual(
|
||||
Array.from({ length: 12 }, (_, index) => `file-${index}`),
|
||||
);
|
||||
expect(result.every((entry) => entry.downloadFallback === true)).toBe(true);
|
||||
});
|
||||
|
||||
it('stops default-off downloads at the aggregate budget and falls back without retrying', async () => {
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
|
||||
prepared(input, { buffer: Buffer.alloc(4) }),
|
||||
);
|
||||
|
||||
const result = await preflightCodeOutputBatch({
|
||||
artifact: artifact(2),
|
||||
limits: { ...limits, fileSizeLimit: 4, totalSizeLimit: 4 },
|
||||
prepare,
|
||||
});
|
||||
|
||||
expect(prepare).toHaveBeenCalledTimes(1);
|
||||
expect(prepare).toHaveBeenCalledWith(expect.objectContaining({ maxBytes: 4 }));
|
||||
expect(result[0]).toMatchObject({ preparedBuffer: Buffer.alloc(4) });
|
||||
expect(result[1]).toMatchObject({ downloadFallback: true });
|
||||
});
|
||||
|
||||
it('reserves a failed transport budget and never starts another default-off download', async () => {
|
||||
const prepare = jest.fn(async () => {
|
||||
throw new Error('transport limit');
|
||||
});
|
||||
|
||||
const result = await preflightCodeOutputBatch({
|
||||
artifact: artifact(2),
|
||||
limits: { ...limits, fileSizeLimit: 4, totalSizeLimit: 4 },
|
||||
prepare,
|
||||
});
|
||||
|
||||
expect(prepare).toHaveBeenCalledTimes(1);
|
||||
expect(result.every((entry) => entry.downloadFallback === true)).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects active-policy count overflow instead of bypassing inspection', async () => {
|
||||
const filters: FiltersConfig = {
|
||||
files: {
|
||||
pii: {
|
||||
fields: ['content'],
|
||||
starterPatterns: [],
|
||||
customPatterns: [BLOCK_PATTERN],
|
||||
uninspectable: 'allow',
|
||||
},
|
||||
},
|
||||
};
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) => prepared(input));
|
||||
|
||||
await expect(
|
||||
preflightCodeOutputBatch({
|
||||
filters,
|
||||
artifact: artifact(3),
|
||||
limits: { ...limits, fileLimit: 2 },
|
||||
prepare,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: 'content_filter_uninspectable',
|
||||
body: { source: 'file', field: 'content' },
|
||||
});
|
||||
expect(prepare).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects active inspection above the hard count even when the configured limit allows it', async () => {
|
||||
const filters: FiltersConfig = {
|
||||
files: {
|
||||
pii: {
|
||||
fields: ['content'],
|
||||
starterPatterns: [],
|
||||
customPatterns: [BLOCK_PATTERN],
|
||||
uninspectable: 'allow',
|
||||
},
|
||||
},
|
||||
};
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) => prepared(input));
|
||||
const inputArtifact = artifact(12);
|
||||
const overflowFile = inputArtifact.files?.[10];
|
||||
if (overflowFile == null) {
|
||||
throw new Error('Expected an overflow file');
|
||||
}
|
||||
const readOverflowContent = jest.fn(() => 'safe');
|
||||
Object.defineProperty(overflowFile, 'content', { get: readOverflowContent });
|
||||
|
||||
await expect(
|
||||
preflightCodeOutputBatch({
|
||||
filters,
|
||||
artifact: inputArtifact,
|
||||
limits: { ...limits, fileLimit: 12 },
|
||||
prepare,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: 'content_filter_uninspectable',
|
||||
body: { source: 'file', field: 'content' },
|
||||
});
|
||||
expect(readOverflowContent).not.toHaveBeenCalled();
|
||||
expect(prepare).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects a late blocked file only after preparing the complete no-write batch', async () => {
|
||||
const filters: FiltersConfig = {
|
||||
files: {
|
||||
pii: {
|
||||
fields: ['content'],
|
||||
starterPatterns: [],
|
||||
customPatterns: [BLOCK_PATTERN],
|
||||
},
|
||||
},
|
||||
};
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
|
||||
prepared(input, {
|
||||
file: {
|
||||
...input.file,
|
||||
type: 'text/plain',
|
||||
content: input.file.id === 'file-1' ? 'PRIVATE-SECRET' : 'safe',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
preflightCodeOutputBatch({
|
||||
filters,
|
||||
artifact: artifact(2),
|
||||
limits,
|
||||
prepare,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: 'content_filter_block',
|
||||
body: { source: 'file', field: 'content' },
|
||||
});
|
||||
expect(prepare).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('sniffs actual non-audio bytes before treating transcript policy as inapplicable', async () => {
|
||||
const filters: FiltersConfig = {
|
||||
files: {
|
||||
pii: {
|
||||
fields: ['transcript'],
|
||||
starterPatterns: [],
|
||||
uninspectable: 'block',
|
||||
},
|
||||
},
|
||||
};
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
|
||||
prepared(input, {
|
||||
file: { ...input.file, type: 'text/plain', content: 'safe' },
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
preflightCodeOutputBatch({ filters, artifact: artifact(), limits, prepare }),
|
||||
).resolves.toHaveLength(1);
|
||||
expect(prepare).toHaveBeenCalledWith(expect.objectContaining({ inspectContent: true }));
|
||||
});
|
||||
|
||||
it('fails closed when a text-named artifact sniffs as audio without a transcript', async () => {
|
||||
const filters: FiltersConfig = {
|
||||
files: {
|
||||
pii: {
|
||||
fields: ['transcript'],
|
||||
starterPatterns: [],
|
||||
uninspectable: 'block',
|
||||
},
|
||||
},
|
||||
};
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
|
||||
prepared(input, {
|
||||
file: { ...input.file, type: 'audio/wav' },
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
preflightCodeOutputBatch({ filters, artifact: artifact(), limits, prepare }),
|
||||
).rejects.toMatchObject({
|
||||
code: 'content_filter_uninspectable',
|
||||
body: { source: 'file', field: 'transcript' },
|
||||
});
|
||||
});
|
||||
|
||||
it('fails closed for unknown bytes under transcript policy', async () => {
|
||||
const filters: FiltersConfig = {
|
||||
files: {
|
||||
pii: {
|
||||
fields: ['transcript'],
|
||||
starterPatterns: [],
|
||||
uninspectable: 'block',
|
||||
},
|
||||
},
|
||||
};
|
||||
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
|
||||
prepared(input, {
|
||||
file: { ...input.file, type: 'application/octet-stream' },
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
preflightCodeOutputBatch({ filters, artifact: artifact(), limits, prepare }),
|
||||
).rejects.toMatchObject({
|
||||
code: 'content_filter_uninspectable',
|
||||
body: { source: 'file', field: 'transcript' },
|
||||
});
|
||||
});
|
||||
});
|
||||
243
packages/api/src/files/code/preflight.ts
Normal file
243
packages/api/src/files/code/preflight.ts
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
import { hasActivePiiPatterns } from 'librechat-data-provider';
|
||||
import type { FileFilterField, FiltersConfig } from 'librechat-data-provider';
|
||||
import type { FileContentInput } from '../../protection/adapters/submissions';
|
||||
import {
|
||||
UninspectableFileError,
|
||||
assertHydratedFileInspectable,
|
||||
getBlockedUninspectableFileField,
|
||||
} from '../../protection/files';
|
||||
import { extractFileContent } from '../../protection/adapters/submissions';
|
||||
import { ContentFilterError } from '../../middleware/contentFilter';
|
||||
import { inspectContent } from '../../protection/runtime';
|
||||
|
||||
const GENERATED_FILE_CONTENT_FIELDS = [
|
||||
'content',
|
||||
'extracted_text',
|
||||
'transcript',
|
||||
] as const satisfies readonly FileFilterField[];
|
||||
|
||||
export const CODE_OUTPUT_PREFLIGHT_MAX_COUNT: number = 10;
|
||||
export const CODE_OUTPUT_PREFLIGHT_MAX_BYTES: number = 64 * 1024 * 1024;
|
||||
|
||||
export interface CodeOutputArtifactFile extends FileContentInput {
|
||||
readonly id: string;
|
||||
readonly name: string;
|
||||
readonly storage_session_id?: string;
|
||||
readonly inherited?: boolean;
|
||||
}
|
||||
|
||||
export interface CodeOutputArtifact {
|
||||
readonly session_id?: string;
|
||||
readonly files?: readonly CodeOutputArtifactFile[];
|
||||
}
|
||||
|
||||
export interface PreparedCodeOutput {
|
||||
readonly buffer: Buffer;
|
||||
readonly extractedTextComplete?: boolean;
|
||||
readonly file: FileContentInput;
|
||||
}
|
||||
|
||||
export interface PreparedCodeOutputEntry {
|
||||
readonly file: CodeOutputArtifactFile;
|
||||
readonly sessionId?: string;
|
||||
readonly preparedBuffer?: Buffer;
|
||||
/** Skip a second download and use the existing code-output URL fallback. */
|
||||
readonly downloadFallback?: boolean;
|
||||
}
|
||||
|
||||
export interface CodeOutputPreflightLimits {
|
||||
readonly fileLimit: number;
|
||||
readonly fileSizeLimit: number;
|
||||
readonly totalSizeLimit: number;
|
||||
}
|
||||
|
||||
export interface PrepareCodeOutputInput {
|
||||
readonly file: CodeOutputArtifactFile;
|
||||
readonly sessionId?: string;
|
||||
readonly maxBytes: number;
|
||||
readonly inspectContent: boolean;
|
||||
}
|
||||
|
||||
export interface PreflightCodeOutputBatchInput {
|
||||
readonly filters?: FiltersConfig;
|
||||
readonly artifact?: CodeOutputArtifact | null;
|
||||
readonly limits: CodeOutputPreflightLimits;
|
||||
readonly prepare: (input: PrepareCodeOutputInput) => Promise<PreparedCodeOutput>;
|
||||
readonly onInspectionUnavailable?: (index: number) => void;
|
||||
}
|
||||
|
||||
function throwIfContentBlocked(
|
||||
filters: FiltersConfig | undefined,
|
||||
fragments: Parameters<typeof inspectContent>[0],
|
||||
): void {
|
||||
const finding = inspectContent(fragments, { filters });
|
||||
if (finding != null) {
|
||||
throw new ContentFilterError(finding);
|
||||
}
|
||||
}
|
||||
|
||||
function getSelectedContentFields(filters: FiltersConfig | undefined): FileFilterField[] {
|
||||
const pii = filters?.files?.pii;
|
||||
if (pii == null) {
|
||||
return [];
|
||||
}
|
||||
return GENERATED_FILE_CONTENT_FIELDS.filter(
|
||||
(field) => pii.fields == null || pii.fields.includes(field),
|
||||
);
|
||||
}
|
||||
|
||||
function normalizedPreparedMimeType(prepared: PreparedCodeOutput): string {
|
||||
if (typeof prepared.file.type !== 'string') {
|
||||
return '';
|
||||
}
|
||||
return prepared.file.type.split(';', 1)[0].trim().toLowerCase();
|
||||
}
|
||||
|
||||
function getConfiguredCountLimit(configured: number, fallback: number): number {
|
||||
if (!Number.isFinite(configured) || configured < 0) {
|
||||
return fallback;
|
||||
}
|
||||
return configured;
|
||||
}
|
||||
|
||||
/**
|
||||
* LibreChat file-size configuration uses zero to mean unlimited. Generated
|
||||
* output inspection preserves that contract while still applying its
|
||||
* process-wide safety ceiling.
|
||||
*/
|
||||
export function getBoundedCodeOutputByteLimit(
|
||||
configured: number | null | undefined,
|
||||
hardLimit: number = CODE_OUTPUT_PREFLIGHT_MAX_BYTES,
|
||||
): number {
|
||||
if (typeof configured !== 'number' || !Number.isFinite(configured) || configured <= 0) {
|
||||
return hardLimit;
|
||||
}
|
||||
return Math.min(configured, hardLimit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares an all-or-nothing inspection batch. No persistence callback is
|
||||
* accepted here by design: callers cannot write one artifact before every
|
||||
* active content check has passed.
|
||||
*/
|
||||
export async function preflightCodeOutputBatch(
|
||||
input: PreflightCodeOutputBatchInput,
|
||||
): Promise<PreparedCodeOutputEntry[]> {
|
||||
const selectedFields = getSelectedContentFields(input.filters);
|
||||
const pii = input.filters?.files?.pii;
|
||||
const inspectionActive =
|
||||
selectedFields.length > 0 && (hasActivePiiPatterns(pii) || pii?.uninspectable === 'block');
|
||||
|
||||
const resourceField = selectedFields[0] ?? 'content';
|
||||
const configuredMaxCount = getConfiguredCountLimit(
|
||||
Math.floor(input.limits.fileLimit),
|
||||
CODE_OUTPUT_PREFLIGHT_MAX_COUNT,
|
||||
);
|
||||
const maxBytes = getBoundedCodeOutputByteLimit(input.limits.totalSizeLimit);
|
||||
const fileSizeLimit = getBoundedCodeOutputByteLimit(input.limits.fileSizeLimit, maxBytes);
|
||||
const entries: PreparedCodeOutputEntry[] = [];
|
||||
let configuredCountExceeded = false;
|
||||
for (const file of input.artifact?.files ?? []) {
|
||||
if (file.inherited === true) {
|
||||
continue;
|
||||
}
|
||||
if (entries.length >= configuredMaxCount) {
|
||||
configuredCountExceeded = true;
|
||||
break;
|
||||
}
|
||||
if (inspectionActive && entries.length >= CODE_OUTPUT_PREFLIGHT_MAX_COUNT) {
|
||||
throw new UninspectableFileError(resourceField);
|
||||
}
|
||||
throwIfContentBlocked(input.filters, extractFileContent(file));
|
||||
entries.push({
|
||||
file,
|
||||
sessionId: file.storage_session_id ?? input.artifact?.session_id,
|
||||
});
|
||||
}
|
||||
const inspectionCountExceeded = entries.length > CODE_OUTPUT_PREFLIGHT_MAX_COUNT;
|
||||
if (configuredCountExceeded || inspectionCountExceeded) {
|
||||
if (inspectionActive) {
|
||||
throw new UninspectableFileError(resourceField);
|
||||
}
|
||||
return entries.map((entry) => ({ ...entry, downloadFallback: true }));
|
||||
}
|
||||
|
||||
let inspectedBytes = 0;
|
||||
for (let index = 0; index < entries.length; index++) {
|
||||
const entry = entries[index];
|
||||
const remainingBytes = maxBytes - inspectedBytes;
|
||||
const transportBytes = Math.min(remainingBytes, fileSizeLimit);
|
||||
if (!Number.isFinite(transportBytes) || transportBytes <= 0) {
|
||||
if (inspectionActive) {
|
||||
throw new UninspectableFileError(resourceField);
|
||||
}
|
||||
entries[index] = { ...entry, downloadFallback: true };
|
||||
continue;
|
||||
}
|
||||
|
||||
let prepared: PreparedCodeOutput;
|
||||
try {
|
||||
prepared = await input.prepare({
|
||||
file: entry.file,
|
||||
sessionId: entry.sessionId,
|
||||
maxBytes: transportBytes,
|
||||
inspectContent: inspectionActive,
|
||||
});
|
||||
} catch (_error) {
|
||||
inspectedBytes += transportBytes;
|
||||
if (inspectionActive) {
|
||||
const blockedField = getBlockedUninspectableFileField(input.filters, selectedFields);
|
||||
if (blockedField != null) {
|
||||
throw new UninspectableFileError(blockedField);
|
||||
}
|
||||
}
|
||||
input.onInspectionUnavailable?.(index);
|
||||
entries[index] = { ...entry, downloadFallback: true };
|
||||
continue;
|
||||
}
|
||||
|
||||
const preparedBytes = prepared.buffer.length;
|
||||
if (
|
||||
!Number.isFinite(preparedBytes) ||
|
||||
preparedBytes < 0 ||
|
||||
preparedBytes > transportBytes ||
|
||||
preparedBytes > remainingBytes
|
||||
) {
|
||||
if (inspectionActive) {
|
||||
throw new UninspectableFileError(resourceField);
|
||||
}
|
||||
entries[index] = { ...entry, downloadFallback: true };
|
||||
inspectedBytes += transportBytes;
|
||||
continue;
|
||||
}
|
||||
|
||||
inspectedBytes += preparedBytes;
|
||||
entries[index] = { ...entry, preparedBuffer: prepared.buffer };
|
||||
if (!inspectionActive) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const mimeType = normalizedPreparedMimeType(prepared);
|
||||
const transcriptUnknown =
|
||||
selectedFields.includes('transcript') &&
|
||||
(!mimeType || mimeType === 'application/octet-stream');
|
||||
if (
|
||||
transcriptUnknown &&
|
||||
getBlockedUninspectableFileField(input.filters, ['transcript']) != null
|
||||
) {
|
||||
throw new UninspectableFileError('transcript');
|
||||
}
|
||||
|
||||
const inspectionFile = { ...entry.file, ...prepared.file };
|
||||
const incompleteExtractedTextField =
|
||||
prepared.extractedTextComplete === true
|
||||
? null
|
||||
: getBlockedUninspectableFileField(input.filters, ['extracted_text']);
|
||||
if (incompleteExtractedTextField != null) {
|
||||
throw new UninspectableFileError(incompleteExtractedTextField);
|
||||
}
|
||||
assertHydratedFileInspectable(input.filters, inspectionFile);
|
||||
throwIfContentBlocked(input.filters, extractFileContent(inspectionFile));
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue