🛂 feat: Filter Model-Bound Content by Source (#14425)

* feat: introduce optional content protection seam

* feat: enforce source-aware content filters

* feat: complete source-aware content enforcement

* test: activate skill file-text fail-close fixtures

* fix: harden source-aware content filters

* fix: harden model-bound content filtering

* fix: preserve legacy filters and generated files

* fix: inspect shared scalar metadata

* test: align mocks with current dev dependencies

* feat: add persisted content filter safeguards

* feat: complete source-aware content filter enforcement

* fix: move resume content preflight into TypeScript

* fix: close content inspection edge cases

* fix: harden content protection boundaries

* fix: complete content protection safeguards

* test: align persisted memory filter coverage

* fix: reconcile content protection with current dev

* fix: reconcile content protection with latest dev

* fix: close content protection review gaps

* fix: enforce source-aware provider boundaries

* fix: preserve legacy PII preflight semantics

* test: stabilize stored branch preflight fixture

* fix: defer agent writes until protected model admission

* perf: harden source-aware model-bound filtering

* fix: canonicalize provider lineage before validation

* fix: satisfy model-bound callback type checks

* perf: Bound content protection filtering work

* fix: Bound submission array traversal

* fix: Stabilize bounded content snapshots

* fix: Scope model-bound traversal overflows

* fix: Preserve scoped content inspection

* fix: Accumulate aggregate traversal scopes

* fix: centralize content policy boundaries

* test: align deferred tool policy context

* test: align controller policy mocks

* style: normalize content protection imports

* fix: close content policy review gaps

* fix: narrow active skill policy config

* fix: address content protection review boundaries

* fix: retain exact provenance overflow sentinel

* fix: preserve literal and scoped provenance updates

* fix: narrow persisted edit provenance

* fix: isolate exact overflow attribution

* fix: centralize stored prompt protection

* fix: fail closed on incomplete transcript evidence

* fix: align canonical transcript routing

* refactor: centralize content policy preflights

* fix: isolate upload policy error typing

* style: sort policy preflight imports

* refactor: centralize content policy boundaries
This commit is contained in:
Danny Avila 2026-08-21 22:43:32 -04:00 • committed by GitHub
parent 10f95c0ce9
commit 67b7b441b2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
303 changed files with 75546 additions and 2327 deletions

View file

@ -1,12 +1,24 @@
import * as os from 'os';
import * as path from 'path';
import { logger } from '@librechat/data-schemas';
import {
extractCodeArtifactRawText,
extractCodeArtifactInspectionText,
extractCodeArtifactText,
getExtractedTextFormat,
resolveMaxTextExtractBytes,
MAX_TEXT_CACHE_BYTES,
MAX_TEXT_EXTRACT_BYTES,
} from './extract';
import { parseDocument } from '~/files/documents/crud';
jest.mock('@librechat/data-schemas', () => ({
logger: {
debug: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
},
}));
const docxText = '__DOCX_PARSED__';
/* parseDocument throws on any originalname containing this token, so
@ -516,6 +528,119 @@ describe('extractCodeArtifactText', () => {
});
});
describe('extractCodeArtifactRawText', () => {
it('returns the full inspectable text beyond the preview cache limit', () => {
const suffix = 'BLOCK-LATE';
const buffer = Buffer.from(`${'a'.repeat(MAX_TEXT_CACHE_BYTES + 1024)}${suffix}`);
const text = extractCodeArtifactRawText(buffer, 'utf8-text');
expect(text).toHaveLength(buffer.length);
expect(text?.endsWith(suffix)).toBe(true);
expect(text).not.toContain('…[truncated]');
});
it('rejects binary, unsupported, and oversized raw content', () => {
expect(extractCodeArtifactRawText(Buffer.from([0x00, 0x01]), 'utf8-text')).toBeNull();
expect(extractCodeArtifactRawText(Buffer.from('text'), 'document')).toBeNull();
expect(
extractCodeArtifactRawText(Buffer.alloc(MAX_TEXT_EXTRACT_BYTES + 1, 'a'), 'utf8-text'),
).toBeNull();
});
});
describe('extractCodeArtifactInspectionText', () => {
beforeEach(() => {
mockOfficeHtml.mockReset();
parseDocumentCalls.length = 0;
});
it('returns complete plain text beyond the persisted preview cache limit', async () => {
const suffix = 'BLOCK-LATE';
const buffer = Buffer.from(`${'a'.repeat(MAX_TEXT_CACHE_BYTES + 1024)}${suffix}`);
const result = await extractCodeArtifactInspectionText(
buffer,
'output.txt',
'text/plain',
'utf8-text',
);
expect(result.complete).toBe(true);
expect(result.text?.endsWith(suffix)).toBe(true);
expect(result.text).not.toContain('…[truncated]');
});
it('returns complete parsed document text when the derived output fits the inspection limit', async () => {
const result = await extractCodeArtifactInspectionText(
Buffer.from('%PDF'),
'report.pdf',
'application/pdf',
'document',
);
expect(result).toEqual({
text: docxText,
complete: true,
});
});
it('marks parsed document text incomplete when the derived output exceeds the inspection limit', async () => {
const suffix = 'BLOCK-DOCUMENT-SUFFIX';
const text = `${'a'.repeat(MAX_TEXT_EXTRACT_BYTES + 1)}${suffix}`;
jest.mocked(parseDocument).mockResolvedValueOnce({
filename: 'large.pdf',
bytes: Buffer.byteLength(text),
filepath: 'document_parser',
text,
images: [],
});
const result = await extractCodeArtifactInspectionText(
Buffer.from('%PDF'),
'large.pdf',
'application/pdf',
'document',
);
expect(result.complete).toBe(false);
expect(result.text).toContain('…[truncated]');
expect(result.text).not.toContain(suffix);
});
it('marks an oversized office preview banner as incomplete', async () => {
mockOfficeHtml.mockResolvedValueOnce('x'.repeat(MAX_TEXT_CACHE_BYTES + 1));
const result = await extractCodeArtifactInspectionText(
Buffer.from('PK'),
'slides.pptx',
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'presentation',
);
expect(result.complete).toBe(false);
expect(result.text).toContain('Preview exceeds the size limit');
});
it('keeps parser diagnostics free of generated filenames and parser errors', async () => {
jest.mocked(parseDocument).mockRejectedValueOnce(new Error('PRIVATE-PARSER-ERROR'));
await expect(
extractCodeArtifactInspectionText(
Buffer.from('%PDF'),
'PRIVATE-FILENAME.pdf',
'application/pdf',
'document',
),
).resolves.toEqual({ text: null, complete: false });
expect(logger.debug).toHaveBeenCalledWith(
'[extractCodeArtifactInspectionText] Artifact inspection failed',
);
expect(JSON.stringify(jest.mocked(logger.debug).mock.calls)).not.toContain('PRIVATE-');
});
});
/* `getExtractedTextFormat` is the trust-flag classifier consumed by
* `processCodeOutput` (api/server/services/Files/Code/process.js) to
* persist `textFormat` on the file record. The client's security gate

View file

@ -140,6 +140,20 @@ const extractUtf8 = (buffer: Buffer): string | null => {
return truncate(buffer.toString('utf-8'), buffer);
};
export function extractCodeArtifactRawText(
buffer: Buffer,
category: CodeArtifactCategory,
): string | null {
if (
buffer.length > MAX_TEXT_EXTRACT_BYTES ||
category !== 'utf8-text' ||
isBinaryBuffer(buffer)
) {
return null;
}
return buffer.toString('utf-8');
}
/**
* Map a known office-document extension back to its canonical MIME so we can
* route through `parseDocument` even when buffer-sniffing yielded a generic
@ -165,7 +179,7 @@ const documentMimeFromExtension = (name: string): string | null => {
}
};
const extractDocument = async (
const extractDocumentText = async (
buffer: Buffer,
name: string,
mimeType: string,
@ -189,12 +203,21 @@ const extractDocument = async (
if (!result?.text) {
return null;
}
return truncate(result.text);
return result.text;
} finally {
fs.unlink(tempPath).catch(() => {});
}
};
const extractDocument = async (
buffer: Buffer,
name: string,
mimeType: string,
): Promise<string | null> => {
const text = await extractDocumentText(buffer, name, mimeType);
return text == null ? null : truncate(text);
};
/**
* Minimal valid HTML document substituted when a producer's output
* exceeds `MAX_TEXT_CACHE_BYTES`. Byte-truncating the producer's HTML
@ -329,3 +352,69 @@ export async function extractCodeArtifactText(
return null;
}
}
export interface CodeArtifactInspectionText {
readonly text: string | null;
readonly complete: boolean;
}
/**
* Extracts text for content inspection independently of the persisted preview
* cache. Plain text and parsed documents are complete only when their full
* derived text fits inside the inspection ceiling. Parseable office documents
* use their full semantic extraction; office preview-only formats retain the
* available HTML for compatibility-mode inspection but stay marked partial
* because their producers have independent row, slide, and output caps.
*/
export async function extractCodeArtifactInspectionText(
buffer: Buffer,
name: string,
mimeType: string,
category: CodeArtifactCategory,
): Promise<CodeArtifactInspectionText> {
const incomplete = (text: string | null = null): CodeArtifactInspectionText => ({
text,
complete: false,
});
const bounded = (text: string | null): CodeArtifactInspectionText => {
if (text == null) {
return incomplete();
}
if (Buffer.byteLength(text, 'utf-8') > MAX_TEXT_EXTRACT_BYTES) {
return incomplete(truncate(text));
}
return {
text,
complete: true,
};
};
if (buffer.length > MAX_TEXT_EXTRACT_BYTES) {
return incomplete();
}
try {
if (hasOfficeHtmlPath(name, mimeType)) {
if (category === 'utf8-text') {
return bounded(extractCodeArtifactRawText(buffer, category));
}
if (category === 'document') {
try {
return bounded(await extractDocumentText(buffer, name, mimeType));
} catch {
// Compatibility mode can still inspect the available preview below.
}
}
return incomplete(await renderOfficeHtml(buffer, name, mimeType));
}
if (category === 'utf8-text') {
return bounded(extractCodeArtifactRawText(buffer, category));
}
if (category !== 'document') {
return incomplete();
}
return bounded(await extractDocumentText(buffer, name, mimeType));
} catch {
logger.debug('[extractCodeArtifactInspectionText] Artifact inspection failed');
return incomplete();
}
}

View file

@ -2,3 +2,4 @@ export * from './classify';
export * from './extract';
export * from './form';
export * from './identity';
export * from './preflight';

View file

@ -0,0 +1,317 @@
import type { FiltersConfig } from 'librechat-data-provider';
import type { CodeOutputArtifact, PrepareCodeOutputInput, PreparedCodeOutput } from './preflight';
import { CODE_OUTPUT_PREFLIGHT_MAX_BYTES, preflightCodeOutputBatch } from './preflight';
const BLOCK_PATTERN = {
id: 'private-token',
label: 'private token',
regex: 'PRIVATE-[A-Z]+',
};
const limits = {
fileLimit: 10,
fileSizeLimit: 16,
totalSizeLimit: 32,
};
function artifact(count = 1): CodeOutputArtifact {
return {
session_id: 'artifact-session',
files: Array.from({ length: count }, (_, index) => ({
id: `file-${index}`,
name: `file-${index}.txt`,
storage_session_id: `storage-${index}`,
})),
};
}
function prepared(
input: PrepareCodeOutputInput,
overrides: Partial<PreparedCodeOutput> = {},
): PreparedCodeOutput {
return {
buffer: Buffer.from('safe'),
extractedTextComplete: true,
file: {
...input.file,
type: 'text/plain',
content: 'safe',
},
...overrides,
};
}
describe('preflightCodeOutputBatch', () => {
it('bounded-downloads default-off artifacts and reuses the prepared bytes', async () => {
const prepare = jest.fn(async (input: PrepareCodeOutputInput) => prepared(input));
const result = await preflightCodeOutputBatch({
artifact: artifact(2),
limits,
prepare,
});
expect(prepare).toHaveBeenCalledTimes(2);
expect(prepare).toHaveBeenNthCalledWith(
1,
expect.objectContaining({ maxBytes: 16, inspectContent: false }),
);
expect(prepare).toHaveBeenNthCalledWith(
2,
expect.objectContaining({ maxBytes: 16, inspectContent: false }),
);
expect(result.map((entry) => entry.preparedBuffer?.toString())).toEqual(['safe', 'safe']);
expect(result.every((entry) => entry.downloadFallback !== true)).toBe(true);
});
it('treats zero byte limits as unlimited within the hard transport ceiling', async () => {
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
prepared(input, { buffer: Buffer.alloc(4) }),
);
const result = await preflightCodeOutputBatch({
artifact: artifact(2),
limits: { ...limits, fileSizeLimit: 0, totalSizeLimit: 0 },
prepare,
});
expect(prepare).toHaveBeenNthCalledWith(
1,
expect.objectContaining({ maxBytes: CODE_OUTPUT_PREFLIGHT_MAX_BYTES }),
);
expect(prepare).toHaveBeenNthCalledWith(
2,
expect.objectContaining({ maxBytes: CODE_OUTPUT_PREFLIGHT_MAX_BYTES - 4 }),
);
expect(result.map((entry) => entry.preparedBuffer?.length)).toEqual([4, 4]);
expect(result.every((entry) => entry.downloadFallback !== true)).toBe(true);
});
it('uses explicit URL fallbacks for default-off count overflow without downloading', async () => {
const prepare = jest.fn(async (input: PrepareCodeOutputInput) => prepared(input));
const result = await preflightCodeOutputBatch({
artifact: artifact(3),
limits: { ...limits, fileLimit: 2 },
prepare,
});
expect(prepare).not.toHaveBeenCalled();
expect(result).toHaveLength(2);
expect(result.every((entry) => entry.downloadFallback === true)).toBe(true);
});
it('preserves configured files above the inspection count as URL fallbacks', async () => {
const prepare = jest.fn(async (input: PrepareCodeOutputInput) => prepared(input));
const result = await preflightCodeOutputBatch({
artifact: artifact(12),
limits: { ...limits, fileLimit: 12 },
prepare,
});
expect(prepare).not.toHaveBeenCalled();
expect(result.map((entry) => entry.file.id)).toEqual(
Array.from({ length: 12 }, (_, index) => `file-${index}`),
);
expect(result.every((entry) => entry.downloadFallback === true)).toBe(true);
});
it('stops default-off downloads at the aggregate budget and falls back without retrying', async () => {
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
prepared(input, { buffer: Buffer.alloc(4) }),
);
const result = await preflightCodeOutputBatch({
artifact: artifact(2),
limits: { ...limits, fileSizeLimit: 4, totalSizeLimit: 4 },
prepare,
});
expect(prepare).toHaveBeenCalledTimes(1);
expect(prepare).toHaveBeenCalledWith(expect.objectContaining({ maxBytes: 4 }));
expect(result[0]).toMatchObject({ preparedBuffer: Buffer.alloc(4) });
expect(result[1]).toMatchObject({ downloadFallback: true });
});
it('reserves a failed transport budget and never starts another default-off download', async () => {
const prepare = jest.fn(async () => {
throw new Error('transport limit');
});
const result = await preflightCodeOutputBatch({
artifact: artifact(2),
limits: { ...limits, fileSizeLimit: 4, totalSizeLimit: 4 },
prepare,
});
expect(prepare).toHaveBeenCalledTimes(1);
expect(result.every((entry) => entry.downloadFallback === true)).toBe(true);
});
it('rejects active-policy count overflow instead of bypassing inspection', async () => {
const filters: FiltersConfig = {
files: {
pii: {
fields: ['content'],
starterPatterns: [],
customPatterns: [BLOCK_PATTERN],
uninspectable: 'allow',
},
},
};
const prepare = jest.fn(async (input: PrepareCodeOutputInput) => prepared(input));
await expect(
preflightCodeOutputBatch({
filters,
artifact: artifact(3),
limits: { ...limits, fileLimit: 2 },
prepare,
}),
).rejects.toMatchObject({
code: 'content_filter_uninspectable',
body: { source: 'file', field: 'content' },
});
expect(prepare).not.toHaveBeenCalled();
});
it('rejects active inspection above the hard count even when the configured limit allows it', async () => {
const filters: FiltersConfig = {
files: {
pii: {
fields: ['content'],
starterPatterns: [],
customPatterns: [BLOCK_PATTERN],
uninspectable: 'allow',
},
},
};
const prepare = jest.fn(async (input: PrepareCodeOutputInput) => prepared(input));
const inputArtifact = artifact(12);
const overflowFile = inputArtifact.files?.[10];
if (overflowFile == null) {
throw new Error('Expected an overflow file');
}
const readOverflowContent = jest.fn(() => 'safe');
Object.defineProperty(overflowFile, 'content', { get: readOverflowContent });
await expect(
preflightCodeOutputBatch({
filters,
artifact: inputArtifact,
limits: { ...limits, fileLimit: 12 },
prepare,
}),
).rejects.toMatchObject({
code: 'content_filter_uninspectable',
body: { source: 'file', field: 'content' },
});
expect(readOverflowContent).not.toHaveBeenCalled();
expect(prepare).not.toHaveBeenCalled();
});
it('rejects a late blocked file only after preparing the complete no-write batch', async () => {
const filters: FiltersConfig = {
files: {
pii: {
fields: ['content'],
starterPatterns: [],
customPatterns: [BLOCK_PATTERN],
},
},
};
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
prepared(input, {
file: {
...input.file,
type: 'text/plain',
content: input.file.id === 'file-1' ? 'PRIVATE-SECRET' : 'safe',
},
}),
);
await expect(
preflightCodeOutputBatch({
filters,
artifact: artifact(2),
limits,
prepare,
}),
).rejects.toMatchObject({
code: 'content_filter_block',
body: { source: 'file', field: 'content' },
});
expect(prepare).toHaveBeenCalledTimes(2);
});
it('sniffs actual non-audio bytes before treating transcript policy as inapplicable', async () => {
const filters: FiltersConfig = {
files: {
pii: {
fields: ['transcript'],
starterPatterns: [],
uninspectable: 'block',
},
},
};
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
prepared(input, {
file: { ...input.file, type: 'text/plain', content: 'safe' },
}),
);
await expect(
preflightCodeOutputBatch({ filters, artifact: artifact(), limits, prepare }),
).resolves.toHaveLength(1);
expect(prepare).toHaveBeenCalledWith(expect.objectContaining({ inspectContent: true }));
});
it('fails closed when a text-named artifact sniffs as audio without a transcript', async () => {
const filters: FiltersConfig = {
files: {
pii: {
fields: ['transcript'],
starterPatterns: [],
uninspectable: 'block',
},
},
};
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
prepared(input, {
file: { ...input.file, type: 'audio/wav' },
}),
);
await expect(
preflightCodeOutputBatch({ filters, artifact: artifact(), limits, prepare }),
).rejects.toMatchObject({
code: 'content_filter_uninspectable',
body: { source: 'file', field: 'transcript' },
});
});
it('fails closed for unknown bytes under transcript policy', async () => {
const filters: FiltersConfig = {
files: {
pii: {
fields: ['transcript'],
starterPatterns: [],
uninspectable: 'block',
},
},
};
const prepare = jest.fn(async (input: PrepareCodeOutputInput) =>
prepared(input, {
file: { ...input.file, type: 'application/octet-stream' },
}),
);
await expect(
preflightCodeOutputBatch({ filters, artifact: artifact(), limits, prepare }),
).rejects.toMatchObject({
code: 'content_filter_uninspectable',
body: { source: 'file', field: 'transcript' },
});
});
});

View file

@ -0,0 +1,243 @@
import { hasActivePiiPatterns } from 'librechat-data-provider';
import type { FileFilterField, FiltersConfig } from 'librechat-data-provider';
import type { FileContentInput } from '../../protection/adapters/submissions';
import {
UninspectableFileError,
assertHydratedFileInspectable,
getBlockedUninspectableFileField,
} from '../../protection/files';
import { extractFileContent } from '../../protection/adapters/submissions';
import { ContentFilterError } from '../../middleware/contentFilter';
import { inspectContent } from '../../protection/runtime';
const GENERATED_FILE_CONTENT_FIELDS = [
'content',
'extracted_text',
'transcript',
] as const satisfies readonly FileFilterField[];
export const CODE_OUTPUT_PREFLIGHT_MAX_COUNT: number = 10;
export const CODE_OUTPUT_PREFLIGHT_MAX_BYTES: number = 64 * 1024 * 1024;
export interface CodeOutputArtifactFile extends FileContentInput {
readonly id: string;
readonly name: string;
readonly storage_session_id?: string;
readonly inherited?: boolean;
}
export interface CodeOutputArtifact {
readonly session_id?: string;
readonly files?: readonly CodeOutputArtifactFile[];
}
export interface PreparedCodeOutput {
readonly buffer: Buffer;
readonly extractedTextComplete?: boolean;
readonly file: FileContentInput;
}
export interface PreparedCodeOutputEntry {
readonly file: CodeOutputArtifactFile;
readonly sessionId?: string;
readonly preparedBuffer?: Buffer;
/** Skip a second download and use the existing code-output URL fallback. */
readonly downloadFallback?: boolean;
}
export interface CodeOutputPreflightLimits {
readonly fileLimit: number;
readonly fileSizeLimit: number;
readonly totalSizeLimit: number;
}
export interface PrepareCodeOutputInput {
readonly file: CodeOutputArtifactFile;
readonly sessionId?: string;
readonly maxBytes: number;
readonly inspectContent: boolean;
}
export interface PreflightCodeOutputBatchInput {
readonly filters?: FiltersConfig;
readonly artifact?: CodeOutputArtifact | null;
readonly limits: CodeOutputPreflightLimits;
readonly prepare: (input: PrepareCodeOutputInput) => Promise<PreparedCodeOutput>;
readonly onInspectionUnavailable?: (index: number) => void;
}
function throwIfContentBlocked(
filters: FiltersConfig | undefined,
fragments: Parameters<typeof inspectContent>[0],
): void {
const finding = inspectContent(fragments, { filters });
if (finding != null) {
throw new ContentFilterError(finding);
}
}
function getSelectedContentFields(filters: FiltersConfig | undefined): FileFilterField[] {
const pii = filters?.files?.pii;
if (pii == null) {
return [];
}
return GENERATED_FILE_CONTENT_FIELDS.filter(
(field) => pii.fields == null || pii.fields.includes(field),
);
}
function normalizedPreparedMimeType(prepared: PreparedCodeOutput): string {
if (typeof prepared.file.type !== 'string') {
return '';
}
return prepared.file.type.split(';', 1)[0].trim().toLowerCase();
}
function getConfiguredCountLimit(configured: number, fallback: number): number {
if (!Number.isFinite(configured) || configured < 0) {
return fallback;
}
return configured;
}
/**
* LibreChat file-size configuration uses zero to mean unlimited. Generated
* output inspection preserves that contract while still applying its
* process-wide safety ceiling.
*/
export function getBoundedCodeOutputByteLimit(
configured: number | null | undefined,
hardLimit: number = CODE_OUTPUT_PREFLIGHT_MAX_BYTES,
): number {
if (typeof configured !== 'number' || !Number.isFinite(configured) || configured <= 0) {
return hardLimit;
}
return Math.min(configured, hardLimit);
}
/**
* Prepares an all-or-nothing inspection batch. No persistence callback is
* accepted here by design: callers cannot write one artifact before every
* active content check has passed.
*/
export async function preflightCodeOutputBatch(
input: PreflightCodeOutputBatchInput,
): Promise<PreparedCodeOutputEntry[]> {
const selectedFields = getSelectedContentFields(input.filters);
const pii = input.filters?.files?.pii;
const inspectionActive =
selectedFields.length > 0 && (hasActivePiiPatterns(pii) || pii?.uninspectable === 'block');
const resourceField = selectedFields[0] ?? 'content';
const configuredMaxCount = getConfiguredCountLimit(
Math.floor(input.limits.fileLimit),
CODE_OUTPUT_PREFLIGHT_MAX_COUNT,
);
const maxBytes = getBoundedCodeOutputByteLimit(input.limits.totalSizeLimit);
const fileSizeLimit = getBoundedCodeOutputByteLimit(input.limits.fileSizeLimit, maxBytes);
const entries: PreparedCodeOutputEntry[] = [];
let configuredCountExceeded = false;
for (const file of input.artifact?.files ?? []) {
if (file.inherited === true) {
continue;
}
if (entries.length >= configuredMaxCount) {
configuredCountExceeded = true;
break;
}
if (inspectionActive && entries.length >= CODE_OUTPUT_PREFLIGHT_MAX_COUNT) {
throw new UninspectableFileError(resourceField);
}
throwIfContentBlocked(input.filters, extractFileContent(file));
entries.push({
file,
sessionId: file.storage_session_id ?? input.artifact?.session_id,
});
}
const inspectionCountExceeded = entries.length > CODE_OUTPUT_PREFLIGHT_MAX_COUNT;
if (configuredCountExceeded || inspectionCountExceeded) {
if (inspectionActive) {
throw new UninspectableFileError(resourceField);
}
return entries.map((entry) => ({ ...entry, downloadFallback: true }));
}
let inspectedBytes = 0;
for (let index = 0; index < entries.length; index++) {
const entry = entries[index];
const remainingBytes = maxBytes - inspectedBytes;
const transportBytes = Math.min(remainingBytes, fileSizeLimit);
if (!Number.isFinite(transportBytes) || transportBytes <= 0) {
if (inspectionActive) {
throw new UninspectableFileError(resourceField);
}
entries[index] = { ...entry, downloadFallback: true };
continue;
}
let prepared: PreparedCodeOutput;
try {
prepared = await input.prepare({
file: entry.file,
sessionId: entry.sessionId,
maxBytes: transportBytes,
inspectContent: inspectionActive,
});
} catch (_error) {
inspectedBytes += transportBytes;
if (inspectionActive) {
const blockedField = getBlockedUninspectableFileField(input.filters, selectedFields);
if (blockedField != null) {
throw new UninspectableFileError(blockedField);
}
}
input.onInspectionUnavailable?.(index);
entries[index] = { ...entry, downloadFallback: true };
continue;
}
const preparedBytes = prepared.buffer.length;
if (
!Number.isFinite(preparedBytes) ||
preparedBytes < 0 ||
preparedBytes > transportBytes ||
preparedBytes > remainingBytes
) {
if (inspectionActive) {
throw new UninspectableFileError(resourceField);
}
entries[index] = { ...entry, downloadFallback: true };
inspectedBytes += transportBytes;
continue;
}
inspectedBytes += preparedBytes;
entries[index] = { ...entry, preparedBuffer: prepared.buffer };
if (!inspectionActive) {
continue;
}
const mimeType = normalizedPreparedMimeType(prepared);
const transcriptUnknown =
selectedFields.includes('transcript') &&
(!mimeType || mimeType === 'application/octet-stream');
if (
transcriptUnknown &&
getBlockedUninspectableFileField(input.filters, ['transcript']) != null
) {
throw new UninspectableFileError('transcript');
}
const inspectionFile = { ...entry.file, ...prepared.file };
const incompleteExtractedTextField =
prepared.extractedTextComplete === true
? null
: getBlockedUninspectableFileField(input.filters, ['extracted_text']);
if (incompleteExtractedTextField != null) {
throw new UninspectableFileError(incompleteExtractedTextField);
}
assertHydratedFileInspectable(input.filters, inspectionFile);
throwIfContentBlocked(input.filters, extractFileContent(inspectionFile));
}
return entries;
}