mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
fix(import): reject an oversized bare JSON upload up front
The upload limit is 1 GiB because it is sized for a zip, whose shards are each well under the 512 MiB per-entry cap. A bare .json between the two cleared the client-side check and multer, then failed after being streamed in full, reported as an oversized archive. It now fails on the stat, with a message naming the workaround. The cap itself stays where it is: V8 refuses to build a string longer than 536,870,888 characters, so a larger entry could never be parsed anyway.
This commit is contained in:
parent
fdd0a9ce65
commit
959b356a1c
3 changed files with 58 additions and 4 deletions
|
|
@ -2,7 +2,7 @@ import fs from 'fs';
|
|||
import os from 'os';
|
||||
import path from 'path';
|
||||
import JSZip from 'jszip';
|
||||
|
||||
import { ImportFileTooLargeError, sanitizeImportError } from './errors';
|
||||
import { assertSafeName, openArchive, ZipBombError } from './archive';
|
||||
|
||||
const createdDirs: string[] = [];
|
||||
|
|
@ -171,11 +171,23 @@ describe('openArchive (bare / non-zip upload)', () => {
|
|||
archive.close();
|
||||
});
|
||||
|
||||
it('rejects a bare JSON file larger than the per-entry cap while reading', async () => {
|
||||
/** The upload limit is sized for a `.zip`, so a bare JSON between the two
|
||||
* caps clears both the client-side check and multer. Rejecting on the stat
|
||||
* means it fails in milliseconds and as its own error, rather than after
|
||||
* streaming the whole file and reporting it as an oversized archive. */
|
||||
it('rejects a bare JSON file larger than the per-entry cap without reading it', async () => {
|
||||
const filepath = writeBareFile('x'.repeat(5000), 'big.json');
|
||||
const archive = await openArchive(filepath, { maxEntryBytes: 100 });
|
||||
|
||||
await expect(archive.read('big.json')).rejects.toThrow(ZipBombError);
|
||||
await expect(openArchive(filepath, { maxEntryBytes: 100 })).rejects.toThrow(
|
||||
ImportFileTooLargeError,
|
||||
);
|
||||
});
|
||||
|
||||
it('maps an oversized bare JSON file onto a message naming the workaround', async () => {
|
||||
const filepath = writeBareFile('x'.repeat(5000), 'big.json');
|
||||
const error = await openArchive(filepath, { maxEntryBytes: 100 }).catch((e: unknown) => e);
|
||||
|
||||
expect(sanitizeImportError(error, 'test')).toMatch(/compress it into a \.zip/i);
|
||||
});
|
||||
|
||||
it('rejects immediately when a bare JSON file exceeds the total-bytes cap by its own size', async () => {
|
||||
|
|
|
|||
|
|
@ -7,10 +7,18 @@ import { megabyte } from 'librechat-data-provider';
|
|||
import type { Readable } from 'stream';
|
||||
|
||||
import { ZipBombError } from '~/files/documents/zipSafety';
|
||||
import { ImportFileTooLargeError } from './errors';
|
||||
|
||||
export { ZipBombError };
|
||||
|
||||
const DEFAULT_MAX_ENTRIES = 20000;
|
||||
/**
|
||||
* Also the ceiling on what any single shard can be parsed as. V8 caps a
|
||||
* string at 536,870,888 characters (`buffer.constants.MAX_STRING_LENGTH`),
|
||||
* just under this, so an entry larger than it can never survive the
|
||||
* `.toString('utf8')` every reader performs — raising this cap would only
|
||||
* trade a clear rejection for an opaque V8 allocation failure.
|
||||
*/
|
||||
const DEFAULT_MAX_ENTRY_BYTES = 512 * megabyte;
|
||||
const DEFAULT_MAX_TOTAL_BYTES = 4096 * megabyte;
|
||||
/** Local file header signature every ZIP file begins with. Its absence
|
||||
|
|
@ -261,6 +269,20 @@ async function openSingleFileArchive(
|
|||
const name = path.basename(filepath);
|
||||
const stat = await fs.promises.stat(filepath);
|
||||
|
||||
/**
|
||||
* Checked up front rather than after streaming the file: the upload limit
|
||||
* (`CONVERSATION_IMPORT_MAX_FILE_SIZE_BYTES`, 1 GiB by default) is sized for
|
||||
* a `.zip`, whose individual shards are each far below the per-entry cap, so
|
||||
* a bare `.json` between the two limits passes both the client-side check
|
||||
* and multer and only fails here. Failing on the stat means it fails
|
||||
* immediately and for the real reason, instead of after reading half a
|
||||
* gigabyte and reporting it as an oversized archive.
|
||||
*/
|
||||
if (stat.size > limits.maxEntryBytes) {
|
||||
throw new ImportFileTooLargeError(
|
||||
`Upload of ${stat.size} bytes exceeds the ${limits.maxEntryBytes}-byte single-file limit`,
|
||||
);
|
||||
}
|
||||
if (stat.size > limits.maxTotalBytes) {
|
||||
throw new ZipBombError('Archive exceeds the maximum decompressed size');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import { logger } from '@librechat/data-schemas';
|
|||
|
||||
const UNSUPPORTED_IMPORT_TYPE = 'Unsupported import type';
|
||||
const ARCHIVE_TOO_LARGE_MESSAGE = 'The uploaded archive exceeds the allowed size limits';
|
||||
const FILE_TOO_LARGE_MESSAGE =
|
||||
'This JSON file is too large to import on its own. Compress it into a .zip and upload that instead';
|
||||
const ARCHIVE_CORRUPT_MESSAGE = 'The uploaded archive is corrupt or could not be read';
|
||||
const STORAGE_FAILURE_MESSAGE = 'A storage error occurred while processing the import';
|
||||
const IMPORT_FAILED_MESSAGE = 'The import could not be completed';
|
||||
|
|
@ -29,6 +31,21 @@ const FS_ERROR_CODES = new Set([
|
|||
const CORRUPT_ARCHIVE_PATTERN =
|
||||
/invalid relative path|absolute path|traversal|central directory|not a valid zip|unable to read archive|entry not found in archive|unable to read entry/i;
|
||||
|
||||
/**
|
||||
* A bare (un-zipped) upload larger than the per-entry cap. Distinct from
|
||||
* `ZipBombError` because it is not a bomb and not the user's mistake: the
|
||||
* upload limit is sized for a `.zip`, whose shards are each well under the
|
||||
* cap, so this file passed every check before the one that can reject it.
|
||||
* The message it maps to therefore names the workaround.
|
||||
*/
|
||||
export class ImportFileTooLargeError extends Error {
|
||||
readonly code = 'IMPORT_FILE_TOO_LARGE';
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'ImportFileTooLargeError';
|
||||
}
|
||||
}
|
||||
|
||||
function errorCode(error: Error): string | undefined {
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
return typeof code === 'string' ? code : undefined;
|
||||
|
|
@ -48,6 +65,9 @@ export function sanitizeImportError(error: unknown, context: string): string {
|
|||
logger.error(`[import] ${context}`, normalized);
|
||||
|
||||
const code = errorCode(normalized);
|
||||
if (code === 'IMPORT_FILE_TOO_LARGE') {
|
||||
return FILE_TOO_LARGE_MESSAGE;
|
||||
}
|
||||
if (code === 'ZIP_BOMB') {
|
||||
return ARCHIVE_TOO_LARGE_MESSAGE;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue