mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
🌐 fix: Preserve Unicode Filenames (#12977)
* fix: Preserve unicode filenames * fix: Cap unicode filenames by bytes * fix: Preserve clean artifact directories * fix: Disambiguate normalized artifact names
This commit is contained in:
parent
56b87f70bd
commit
f2de3a219c
8 changed files with 268 additions and 62 deletions
|
|
@ -29,7 +29,7 @@ const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
|
|||
const { hasCapability } = require('~/server/middleware/roles/capabilities');
|
||||
const { checkPermission } = require('~/server/services/PermissionService');
|
||||
const { hasAccessToFilesViaAgent } = require('~/server/services/Files');
|
||||
const { cleanFileName } = require('~/server/utils/files');
|
||||
const { getContentDisposition } = require('~/server/utils/files');
|
||||
const { getLogStores } = require('~/cache');
|
||||
const { Readable } = require('stream');
|
||||
const db = require('~/models');
|
||||
|
|
@ -397,8 +397,7 @@ router.get('/download/:userId/:file_id', fileAccess, async (req, res) => {
|
|||
}
|
||||
|
||||
const setHeaders = () => {
|
||||
const cleanedFilename = cleanFileName(file.filename);
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${cleanedFilename}"`);
|
||||
res.setHeader('Content-Disposition', getContentDisposition(file.filename));
|
||||
res.setHeader('Content-Type', 'application/octet-stream');
|
||||
res.setHeader('X-File-Metadata', JSON.stringify(file));
|
||||
};
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ jest.mock('~/server/services/Files', () => ({
|
|||
|
||||
jest.mock('~/server/utils/files', () => ({
|
||||
cleanFileName: (name) => name,
|
||||
getContentDisposition: (name) => `attachment; filename="${name}"`,
|
||||
}));
|
||||
|
||||
jest.mock('~/cache', () => ({
|
||||
|
|
|
|||
|
|
@ -64,4 +64,32 @@ const cleanFileName = (fileName) => {
|
|||
return cleaned;
|
||||
};
|
||||
|
||||
module.exports = { determineFileType, getBufferMetadata, cleanFileName };
|
||||
const encodeRFC5987ValueChars = (value) =>
|
||||
encodeURIComponent(value).replace(
|
||||
/['()*]/g,
|
||||
(char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`,
|
||||
);
|
||||
|
||||
const getAsciiFilenameFallback = (fileName) => {
|
||||
const fallback = fileName
|
||||
.normalize('NFKD')
|
||||
.replace(/[^\x20-\x7e]/g, '_')
|
||||
.replace(/["\\\r\n]/g, '_');
|
||||
|
||||
return fallback || 'download';
|
||||
};
|
||||
|
||||
const getContentDisposition = (fileName, disposition = 'attachment') => {
|
||||
const cleanedFilename = cleanFileName(fileName) || 'download';
|
||||
const asciiFallback = getAsciiFilenameFallback(cleanedFilename);
|
||||
const encodedFilename = encodeRFC5987ValueChars(cleanedFilename);
|
||||
|
||||
return `${disposition}; filename="${asciiFallback}"; filename*=UTF-8''${encodedFilename}`;
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
determineFileType,
|
||||
getBufferMetadata,
|
||||
cleanFileName,
|
||||
getContentDisposition,
|
||||
};
|
||||
|
|
|
|||
31
api/server/utils/files.spec.js
Normal file
31
api/server/utils/files.spec.js
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
jest.mock('sharp', () => jest.fn(), { virtual: true });
|
||||
|
||||
const { cleanFileName, getContentDisposition } = require('./files');
|
||||
|
||||
describe('file utilities', () => {
|
||||
describe('cleanFileName', () => {
|
||||
it('removes storage UUID prefixes', () => {
|
||||
expect(cleanFileName('123e4567-e89b-12d3-a456-426614174000__report.txt')).toBe('report.txt');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getContentDisposition', () => {
|
||||
it('adds RFC 8187 encoding for Unicode filenames', () => {
|
||||
const filename = '日本語レポート.xlsx';
|
||||
const header = getContentDisposition(`123e4567-e89b-12d3-a456-426614174000__${filename}`);
|
||||
|
||||
expect(header).toMatch(/^attachment; filename=".*"; filename\*=UTF-8''/);
|
||||
expect(header).not.toContain('123e4567-e89b-12d3-a456-426614174000__');
|
||||
expect(header).toContain(`filename*=UTF-8''${encodeURIComponent(filename)}`);
|
||||
});
|
||||
|
||||
it('escapes the ASCII fallback without dropping the encoded filename', () => {
|
||||
const filename = 'bad"name\r\n.txt';
|
||||
const header = getContentDisposition(filename);
|
||||
|
||||
expect(header).toContain('filename="bad_name__.txt"');
|
||||
expect(header).toContain("filename*=UTF-8''bad%22name%0D%0A.txt");
|
||||
expect(header).not.toMatch(/[\r\n]/);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -38,6 +38,13 @@ describe('code env FormData filenames', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('uses filepath for nested Unicode filenames', () => {
|
||||
expect(getCodeEnvFileOptions('分析/結果📊.csv')).toEqual({
|
||||
filename: '結果📊.csv',
|
||||
filepath: '分析/結果📊.csv',
|
||||
});
|
||||
});
|
||||
|
||||
it('documents the form-data string overload regression', async () => {
|
||||
const disposition = await renderMultipartDisposition((form) => {
|
||||
form.append('file', Readable.from(['x']), 'pptx/pptx.py');
|
||||
|
|
|
|||
|
|
@ -7,11 +7,30 @@ export interface CodeEnvFileOptions {
|
|||
filepath?: string;
|
||||
}
|
||||
|
||||
const CODE_ENV_SAFE_FILEPATH_PATTERN = /^[a-zA-Z0-9._\-/]+$/;
|
||||
const CODE_ENV_SAFE_ASCII_FILEPATH_CHAR_PATTERN = /^[a-zA-Z0-9._\-/]$/;
|
||||
const CODE_ENV_UNSAFE_UNICODE_FILEPATH_CHAR_PATTERN =
|
||||
/[^\p{L}\p{M}\p{N}\p{Emoji}\u200d._\-/]/u;
|
||||
const CODE_ENV_FILENAME_CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
|
||||
|
||||
function hasUnsafeCodeEnvFilepathChar(filepath: string): boolean {
|
||||
for (const char of filepath) {
|
||||
if (char.charCodeAt(0) <= 0x7f) {
|
||||
if (!CODE_ENV_SAFE_ASCII_FILEPATH_CHAR_PATTERN.test(char)) {
|
||||
return true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (CODE_ENV_UNSAFE_UNICODE_FILEPATH_CHAR_PATTERN.test(char)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function isSafeCodeEnvFilepath(filepath: string): boolean {
|
||||
if (!filepath || filepath.startsWith('/') || !CODE_ENV_SAFE_FILEPATH_PATTERN.test(filepath)) {
|
||||
if (!filepath || filepath.startsWith('/') || hasUnsafeCodeEnvFilepathChar(filepath)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,10 @@ function expectedHexSuffix(input: string): string {
|
|||
return createHash('sha256').update(input).digest('hex').slice(0, 6);
|
||||
}
|
||||
|
||||
function utf8ByteLength(input: string): number {
|
||||
return Buffer.byteLength(input, 'utf8');
|
||||
}
|
||||
|
||||
describe('sanitizeFilename', () => {
|
||||
test('removes directory components (1/2)', () => {
|
||||
expect(sanitizeFilename('/path/to/file.txt')).toBe('file.txt');
|
||||
|
|
@ -35,6 +39,17 @@ describe('sanitizeFilename', () => {
|
|||
expect(sanitizeFilename('file name@#$.txt')).toBe('file_name___.txt');
|
||||
});
|
||||
|
||||
test('preserves Unicode filenames', () => {
|
||||
expect(sanitizeFilename('日本語レポート.xlsx')).toBe('日本語レポート.xlsx');
|
||||
expect(sanitizeFilename('résumé-данные-تقرير-보고서📊.csv')).toBe(
|
||||
'résumé-данные-تقرير-보고서📊.csv',
|
||||
);
|
||||
});
|
||||
|
||||
test('normalizes decomposed Unicode marks before sanitizing', () => {
|
||||
expect(sanitizeFilename('Cafe\u0301.txt')).toBe('Café.txt');
|
||||
});
|
||||
|
||||
test('preserves dots and hyphens', () => {
|
||||
expect(sanitizeFilename('file-name.with.dots.txt')).toBe('file-name.with.dots.txt');
|
||||
});
|
||||
|
|
@ -50,6 +65,13 @@ describe('sanitizeFilename', () => {
|
|||
expect(result).toMatch(/^a+-abc123\.txt$/);
|
||||
});
|
||||
|
||||
test('truncates Unicode filenames by UTF-8 bytes, preserving the extension', () => {
|
||||
const longName = '界'.repeat(100) + '.txt';
|
||||
const result = sanitizeFilename(longName);
|
||||
expect(utf8ByteLength(result)).toBeLessThanOrEqual(255);
|
||||
expect(result.endsWith('-abc123.txt')).toBe(true);
|
||||
});
|
||||
|
||||
test('handles filenames with no extension', () => {
|
||||
const longName = 'a'.repeat(300);
|
||||
const result = sanitizeFilename(longName);
|
||||
|
|
@ -75,6 +97,10 @@ describe('sanitizeArtifactPath', () => {
|
|||
expect(sanitizeArtifactPath('a/b/c/file.txt')).toBe('a/b/c/file.txt');
|
||||
});
|
||||
|
||||
test('preserves Unicode path segments', () => {
|
||||
expect(sanitizeArtifactPath('分析/結果📊.csv')).toBe('分析/結果📊.csv');
|
||||
});
|
||||
|
||||
test('replaces non-alphanumeric characters per segment + adds raw-input disambiguator', () => {
|
||||
/* Different raw inputs that sanitize to the same form (`out 1.csv`
|
||||
* vs `out_1.csv`, `out@1.csv` vs `out#1.csv`) would otherwise share
|
||||
|
|
@ -146,6 +172,13 @@ describe('sanitizeArtifactPath', () => {
|
|||
expect(result).toMatch(new RegExp(`^a+-${expectedHexSuffix(longName)}\\.txt$`));
|
||||
});
|
||||
|
||||
test('caps Unicode leaf segments at 255 UTF-8 bytes with extension-preserving truncation', () => {
|
||||
const longName = '界'.repeat(100) + '.txt';
|
||||
const result = sanitizeArtifactPath(longName);
|
||||
expect(utf8ByteLength(result)).toBeLessThanOrEqual(255);
|
||||
expect(result.endsWith(`-${expectedHexSuffix(longName)}.txt`)).toBe(true);
|
||||
});
|
||||
|
||||
test('caps the leaf when nested under a directory, preserving the directory verbatim', () => {
|
||||
const longLeaf = 'b'.repeat(300) + '.csv';
|
||||
const result = sanitizeArtifactPath(`reports/${longLeaf}`);
|
||||
|
|
@ -167,6 +200,15 @@ describe('sanitizeArtifactPath', () => {
|
|||
expect(leaf).toBe('notes.txt');
|
||||
});
|
||||
|
||||
test('caps Unicode non-leaf directory segments at 255 UTF-8 bytes', () => {
|
||||
const longDir = '界'.repeat(100);
|
||||
const result = sanitizeArtifactPath(`${longDir}/notes.txt`);
|
||||
const [dir, leaf] = result.split('/');
|
||||
expect(utf8ByteLength(dir)).toBeLessThanOrEqual(255);
|
||||
expect(dir.endsWith(`-${expectedHexSuffix(longDir)}`)).toBe(true);
|
||||
expect(leaf).toBe('notes.txt');
|
||||
});
|
||||
|
||||
test('produces deterministic output across calls (no orphaned uploads on re-truncation)', () => {
|
||||
/* Codex review P2: `sanitizeFilename`'s `crypto.randomBytes(3)` made
|
||||
* the truncated form non-deterministic — re-uploading the same long
|
||||
|
|
@ -221,6 +263,14 @@ describe('sanitizeArtifactPath', () => {
|
|||
expect(result).toBe('file.txt');
|
||||
});
|
||||
|
||||
test('falls back to leaf-only when Unicode path bytes exceed the DB-index cap', () => {
|
||||
const segA = '界'.repeat(80);
|
||||
const segB = '分'.repeat(80);
|
||||
const segC = '析'.repeat(80);
|
||||
const result = sanitizeArtifactPath(`${segA}/${segB}/${segC}/file.txt`);
|
||||
expect(result).toBe('file.txt');
|
||||
});
|
||||
|
||||
test('keeps the nested path when total length is within the DB-index cap', () => {
|
||||
/* The cap doesn't fire for realistic outputs — typical artifact
|
||||
* depth is ≤ 3 segments × short names. */
|
||||
|
|
@ -276,6 +326,16 @@ describe('sanitizeArtifactPath', () => {
|
|||
expect(a).toBe(`_.hidden-${expectedHexSuffix('.hidden')}`);
|
||||
});
|
||||
|
||||
test('normalization-only collisions get distinct safe forms', () => {
|
||||
const composed = 'reports/Café.csv';
|
||||
const decomposed = 'reports/Cafe\u0301.csv';
|
||||
const normalizedDecomposed = `reports/Café-${expectedHexSuffix(decomposed)}.csv`;
|
||||
|
||||
expect(sanitizeArtifactPath(composed)).toBe(composed);
|
||||
expect(sanitizeArtifactPath(decomposed)).toBe(normalizedDecomposed);
|
||||
expect(normalizedDecomposed).not.toBe(composed);
|
||||
});
|
||||
|
||||
test('idempotent: same raw input always produces the same safe form', () => {
|
||||
/* Disambiguator is deterministic (SHA-256 prefix of raw input),
|
||||
* so re-uploading the same long-or-mutated name lands at the
|
||||
|
|
@ -372,6 +432,14 @@ describe('flattenArtifactPath', () => {
|
|||
expect(result).toMatch(new RegExp(`-${expectedHexSuffix(safePath)}\\.txt$`));
|
||||
});
|
||||
|
||||
test('truncates Unicode flat forms by UTF-8 bytes', () => {
|
||||
const safePath = `${'界'.repeat(80)}/結果.csv`;
|
||||
const result = flattenArtifactPath(safePath, 100);
|
||||
expect(utf8ByteLength(result)).toBeLessThanOrEqual(100);
|
||||
expect(result.endsWith('.csv')).toBe(true);
|
||||
expect(result).toMatch(new RegExp(`-${expectedHexSuffix(safePath)}\\.csv$`));
|
||||
});
|
||||
|
||||
test('preserves the extension even when only the leaf overflows', () => {
|
||||
const longLeaf = 'L'.repeat(300);
|
||||
const result = flattenArtifactPath(`${longLeaf}.json`, 200);
|
||||
|
|
|
|||
|
|
@ -9,6 +9,56 @@ const USER_FACING_UPLOAD_ERRORS = [
|
|||
'Unable to extract text from',
|
||||
] as const;
|
||||
|
||||
const ASCII_FILENAME_SAFE_PATTERN = /^[a-zA-Z0-9._-]$/;
|
||||
const UNSAFE_UNICODE_FILENAME_PATTERN = /[^\p{L}\p{M}\p{N}\p{Emoji}\u200d._-]/gu;
|
||||
const FILENAME_SEGMENT_MAX_BYTES = 255;
|
||||
|
||||
function sanitizeFilenameSegment(segment: string): string {
|
||||
return segment
|
||||
.normalize('NFC')
|
||||
.replace(/[\u0000-\u007f]/g, (char) => (ASCII_FILENAME_SAFE_PATTERN.test(char) ? char : '_'))
|
||||
.replace(UNSAFE_UNICODE_FILENAME_PATTERN, '_');
|
||||
}
|
||||
|
||||
function utf8ByteLength(value: string): number {
|
||||
return Buffer.byteLength(value, 'utf8');
|
||||
}
|
||||
|
||||
function truncateUtf8Bytes(value: string, maxBytes: number): string {
|
||||
if (maxBytes <= 0) return '';
|
||||
let bytes = 0;
|
||||
let result = '';
|
||||
|
||||
for (const char of value) {
|
||||
const charBytes = utf8ByteLength(char);
|
||||
if (bytes + charBytes > maxBytes) break;
|
||||
result += char;
|
||||
bytes += charBytes;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function truncateWithSuffix(value: string, suffix: string, maxBytes: number): string {
|
||||
const suffixBytes = utf8ByteLength(suffix);
|
||||
const stemBudget = Math.max(0, maxBytes - suffixBytes);
|
||||
const result = truncateUtf8Bytes(value, stemBudget) + suffix;
|
||||
return utf8ByteLength(result) <= maxBytes ? result : truncateUtf8Bytes(result, maxBytes);
|
||||
}
|
||||
|
||||
function truncateLeafWithSuffix(leaf: string, suffix: string, maxBytes: number): string {
|
||||
if (utf8ByteLength(leaf) <= maxBytes) return leaf;
|
||||
const ext = path.extname(leaf);
|
||||
const stem = path.basename(leaf, ext);
|
||||
const suffixBytes = utf8ByteLength(suffix);
|
||||
const extBytes = utf8ByteLength(ext);
|
||||
if (extBytes > maxBytes - suffixBytes - 1) {
|
||||
return truncateWithSuffix(leaf, suffix, maxBytes);
|
||||
}
|
||||
const stemBudget = maxBytes - extBytes - suffixBytes;
|
||||
return truncateUtf8Bytes(stem, stemBudget) + suffix + ext;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a user-facing error message from a file upload error.
|
||||
* Returns the error's own message if it matches a known user-facing pattern,
|
||||
|
|
@ -37,42 +87,37 @@ export function resolveUploadErrorMessage(
|
|||
}
|
||||
|
||||
/**
|
||||
* Sanitize a filename by removing any directory components, replacing non-alphanumeric characters
|
||||
* Sanitize a filename by removing any directory components, replacing unsafe characters
|
||||
* @param inputName
|
||||
*/
|
||||
export function sanitizeFilename(inputName: string): string {
|
||||
// Remove any directory components
|
||||
let name = path.basename(inputName);
|
||||
|
||||
// Replace any non-alphanumeric characters except for '.' and '-'
|
||||
name = name.replace(/[^a-zA-Z0-9.-]/g, '_');
|
||||
// Preserve Unicode word characters and emoji while replacing unsafe ASCII punctuation.
|
||||
name = sanitizeFilenameSegment(name);
|
||||
|
||||
// Ensure the name doesn't start with a dot (hidden file in Unix-like systems)
|
||||
if (name.startsWith('.') || name === '') {
|
||||
name = '_' + name;
|
||||
}
|
||||
|
||||
// Limit the length of the filename
|
||||
const MAX_LENGTH = 255;
|
||||
if (name.length > MAX_LENGTH) {
|
||||
const ext = path.extname(name);
|
||||
const nameWithoutExt = path.basename(name, ext);
|
||||
name =
|
||||
nameWithoutExt.slice(0, MAX_LENGTH - ext.length - 7) +
|
||||
'-' +
|
||||
crypto.randomBytes(3).toString('hex') +
|
||||
ext;
|
||||
}
|
||||
// Limit the filename to filesystem NAME_MAX, which is byte-based on Linux/APFS.
|
||||
name = truncateLeafWithSuffix(
|
||||
name,
|
||||
'-' + crypto.randomBytes(3).toString('hex'),
|
||||
FILENAME_SEGMENT_MAX_BYTES,
|
||||
);
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
/** Per-path-component length cap. Mirrors `sanitizeFilename`'s 255-char
|
||||
/** Per-path-component byte cap. Mirrors `sanitizeFilename`'s 255-byte
|
||||
* basename cap and matches filesystem `NAME_MAX` (255 bytes on Linux/ext4,
|
||||
* 255 chars on Windows/NTFS) — without it, `saveBuffer` writes
|
||||
* `${file_id}__${flatName}` and a long artifact name surfaces as
|
||||
* `ENAMETOOLONG` and falls back to a download URL instead of persisting. */
|
||||
const ARTIFACT_PATH_SEGMENT_MAX = 255;
|
||||
const ARTIFACT_PATH_SEGMENT_MAX_BYTES = FILENAME_SEGMENT_MAX_BYTES;
|
||||
|
||||
/** Whole-path length cap for the path-preserving form. The DB stores this
|
||||
* value in `filename`, which participates in a compound unique index on
|
||||
|
|
@ -81,7 +126,7 @@ const ARTIFACT_PATH_SEGMENT_MAX = 255;
|
|||
* (`packages/data-schemas/src/schema/file.ts`). MongoDB 4.0 and earlier
|
||||
* reject indexed values past 1024 bytes; even on 4.2+ where the limit
|
||||
* is configurable, runaway nested paths bloat the index for no real
|
||||
* benefit. 512 chars is plenty for realistic code-execution outputs
|
||||
* benefit. 512 bytes is plenty for realistic code-execution outputs
|
||||
* (typical depth ≤ 3 segments × short names) and gives headroom for
|
||||
* BSON / index-overhead encoding.
|
||||
*
|
||||
|
|
@ -92,7 +137,7 @@ const ARTIFACT_PATH_SEGMENT_MAX = 255;
|
|||
* with a missing artifact is strictly worse than a flat-name fallback.
|
||||
* Pre-PR every artifact got this treatment regardless of depth, so the
|
||||
* cap is monotonically better than the prior behavior. */
|
||||
const ARTIFACT_PATH_TOTAL_MAX = 512;
|
||||
const ARTIFACT_PATH_TOTAL_MAX_BYTES = 512;
|
||||
|
||||
/**
|
||||
* Deterministic disambiguator suffix for truncated names. The original
|
||||
|
|
@ -126,23 +171,23 @@ function deterministicHexSuffix(input: string): string {
|
|||
* still blow past the cap.
|
||||
*/
|
||||
function truncateLeafSegment(leaf: string): string {
|
||||
if (leaf.length <= ARTIFACT_PATH_SEGMENT_MAX) return leaf;
|
||||
const ext = path.extname(leaf);
|
||||
const stem = path.basename(leaf, ext);
|
||||
// 8 = 1 (`-`) + 6 (hex disambiguator) + 1 (minimum 1-char stem)
|
||||
if (ext.length > ARTIFACT_PATH_SEGMENT_MAX - 8) {
|
||||
return truncateDirSegment(leaf);
|
||||
}
|
||||
const stemBudget = ARTIFACT_PATH_SEGMENT_MAX - ext.length - 7;
|
||||
return stem.slice(0, stemBudget) + '-' + deterministicHexSuffix(leaf) + ext;
|
||||
return truncateLeafWithSuffix(
|
||||
leaf,
|
||||
'-' + deterministicHexSuffix(leaf),
|
||||
ARTIFACT_PATH_SEGMENT_MAX_BYTES,
|
||||
);
|
||||
}
|
||||
|
||||
/** Truncates a non-leaf (directory) segment. Directory segments don't
|
||||
* carry semantic extensions, so we just slice and append the same 6-hex
|
||||
* disambiguation suffix. */
|
||||
function truncateDirSegment(seg: string): string {
|
||||
if (seg.length <= ARTIFACT_PATH_SEGMENT_MAX) return seg;
|
||||
return seg.slice(0, ARTIFACT_PATH_SEGMENT_MAX - 7) + '-' + deterministicHexSuffix(seg);
|
||||
if (utf8ByteLength(seg) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES) return seg;
|
||||
return truncateWithSuffix(
|
||||
seg,
|
||||
'-' + deterministicHexSuffix(seg),
|
||||
ARTIFACT_PATH_SEGMENT_MAX_BYTES,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -154,7 +199,7 @@ function truncateDirSegment(seg: string): string {
|
|||
* first record and overwrite the first artifact's bytes.
|
||||
*
|
||||
* The disambiguator survives length capping: if appending pushes the
|
||||
* segment past `ARTIFACT_PATH_SEGMENT_MAX`, the stem is trimmed to make
|
||||
* segment past `ARTIFACT_PATH_SEGMENT_MAX_BYTES`, the stem is trimmed to make
|
||||
* room (the hash + extension are load-bearing for collision avoidance
|
||||
* and MIME inference, the stem is just the human-readable prefix).
|
||||
*/
|
||||
|
|
@ -172,30 +217,33 @@ function embedDisambiguatorInLeaf(segment: string, hashSource: string): string {
|
|||
if (dot <= 1) {
|
||||
const proposed = segment + suffix;
|
||||
result =
|
||||
proposed.length <= ARTIFACT_PATH_SEGMENT_MAX
|
||||
utf8ByteLength(proposed) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES
|
||||
? proposed
|
||||
: segment.slice(0, ARTIFACT_PATH_SEGMENT_MAX - suffix.length) + suffix;
|
||||
: truncateWithSuffix(segment, suffix, ARTIFACT_PATH_SEGMENT_MAX_BYTES);
|
||||
} else {
|
||||
const stem = segment.slice(0, dot);
|
||||
const ext = segment.slice(dot);
|
||||
const proposed = stem + suffix + ext;
|
||||
if (proposed.length <= ARTIFACT_PATH_SEGMENT_MAX) {
|
||||
if (utf8ByteLength(proposed) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES) {
|
||||
result = proposed;
|
||||
} else {
|
||||
// Trim stem to make room while preserving disambiguator + extension.
|
||||
const stemBudget = Math.max(0, ARTIFACT_PATH_SEGMENT_MAX - suffix.length - ext.length);
|
||||
result = stem.slice(0, stemBudget) + suffix + ext;
|
||||
const stemBudget = Math.max(
|
||||
0,
|
||||
ARTIFACT_PATH_SEGMENT_MAX_BYTES - utf8ByteLength(suffix) - utf8ByteLength(ext),
|
||||
);
|
||||
result = truncateUtf8Bytes(stem, stemBudget) + suffix + ext;
|
||||
}
|
||||
}
|
||||
/* Defensive final clamp. The branches above already keep the result
|
||||
* within `ARTIFACT_PATH_SEGMENT_MAX` for any input where the
|
||||
* within `ARTIFACT_PATH_SEGMENT_MAX_BYTES` for any input where the
|
||||
* extension fits the segment, but a pathological extension (e.g. the
|
||||
* `.aaaaa…` shape `path.extname` returns for contrived inputs) could
|
||||
* still produce a longer result. Hard-clamp so the segment cap holds
|
||||
* unconditionally. */
|
||||
return result.length <= ARTIFACT_PATH_SEGMENT_MAX
|
||||
return utf8ByteLength(result) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES
|
||||
? result
|
||||
: result.slice(0, ARTIFACT_PATH_SEGMENT_MAX);
|
||||
: truncateUtf8Bytes(result, ARTIFACT_PATH_SEGMENT_MAX_BYTES);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -205,7 +253,7 @@ function embedDisambiguatorInLeaf(segment: string, hashSource: string): string {
|
|||
* same rules as `sanitizeFilename`. Falls back to the basename for absolute
|
||||
* paths or names containing `..` traversal.
|
||||
*
|
||||
* Each path component is capped at `ARTIFACT_PATH_SEGMENT_MAX` (255) chars
|
||||
* Each path component is capped at `ARTIFACT_PATH_SEGMENT_MAX_BYTES` (255 bytes)
|
||||
* — the leaf with extension preservation (matching `sanitizeFilename`),
|
||||
* non-leaf segments with a plain truncate-and-disambiguate. Without the
|
||||
* cap, long artifact names flow into `saveBuffer`'s storage key
|
||||
|
|
@ -226,7 +274,7 @@ export function sanitizeArtifactPath(inputName: string): string {
|
|||
}
|
||||
const segments = normalized
|
||||
.split('/')
|
||||
.map((seg) => seg.replace(/[^a-zA-Z0-9.-]/g, '_'))
|
||||
.map(sanitizeFilenameSegment)
|
||||
.filter((seg) => seg.length > 0 && seg !== '.');
|
||||
if (segments.length === 0) return '_';
|
||||
const leafIdx = segments.length - 1;
|
||||
|
|
@ -256,7 +304,7 @@ export function sanitizeArtifactPath(inputName: string): string {
|
|||
* input → same safe form (idempotent for re-uploads); different raw
|
||||
* inputs that would have collided → different safe forms.
|
||||
*
|
||||
* Clean inputs (where the regex/normalize pass was a no-op) skip
|
||||
* Clean inputs (where the regex/normalize pass was a raw-string no-op) skip
|
||||
* the disambiguator — no collision is possible because they're
|
||||
* already distinct strings, and we don't want to clutter human-
|
||||
* readable filenames with a hash when nothing was at risk. */
|
||||
|
|
@ -271,21 +319,21 @@ export function sanitizeArtifactPath(inputName: string): string {
|
|||
* the (filename, conversationId, context, tenantId) compound unique
|
||||
* index never sees an oversized key. The leaf is already capped by
|
||||
* `truncateLeafSegment` + `embedDisambiguatorInLeaf`, so this
|
||||
* guarantees ≤ ARTIFACT_PATH_SEGMENT_MAX (255) chars. Same shape as
|
||||
* guarantees ≤ ARTIFACT_PATH_SEGMENT_MAX_BYTES (255 bytes). Same shape as
|
||||
* the absolute-path / `..`-traversal fallback above. */
|
||||
if (joined.length > ARTIFACT_PATH_TOTAL_MAX) {
|
||||
if (utf8ByteLength(joined) > ARTIFACT_PATH_TOTAL_MAX_BYTES) {
|
||||
return capped[leafIdx];
|
||||
}
|
||||
return joined;
|
||||
}
|
||||
|
||||
/** Limit on `path.extname`'s "extension" length we'll honor when
|
||||
/** Limit on `path.extname`'s "extension" byte length we'll honor when
|
||||
* truncating a flat key. Real file extensions cap out around 8 chars
|
||||
* (`.parquet`, `.tsv`, `.html`); a 16-char ceiling keeps us tolerant of
|
||||
* legitimate edge cases (`.openshift`) while ignoring pathological
|
||||
* outputs from `path.extname` on contrived inputs. Above that we treat
|
||||
* the trailing `.foo` as part of the stem and just hard-truncate. */
|
||||
const FLAT_KEY_MAX_EXT_LENGTH = 16;
|
||||
const FLAT_KEY_MAX_EXT_BYTES = 16;
|
||||
|
||||
/**
|
||||
* Map a (sanitized) artifact path to a flat storage-safe key. The local
|
||||
|
|
@ -293,7 +341,7 @@ const FLAT_KEY_MAX_EXT_LENGTH = 16;
|
|||
* unintended subdirectories on disk while the DB record retains the
|
||||
* nested path for the next prime().
|
||||
*
|
||||
* Optionally caps the result at `maxLength` characters. Per-segment caps
|
||||
* Optionally caps the result at `maxLength` UTF-8 bytes. Per-segment caps
|
||||
* applied by `sanitizeArtifactPath` aren't enough on their own —
|
||||
* `${file_id}__${flatName}` has to fit in one filesystem path component
|
||||
* (NAME_MAX = 255 on most filesystems), so a deeply-nested path whose
|
||||
|
|
@ -308,32 +356,37 @@ const FLAT_KEY_MAX_EXT_LENGTH = 16;
|
|||
*/
|
||||
export function flattenArtifactPath(safePath: string, maxLength?: number): string {
|
||||
const flat = safePath.replace(/\//g, '__');
|
||||
if (maxLength == null || flat.length <= maxLength) return flat;
|
||||
if (maxLength == null || utf8ByteLength(flat) <= maxLength) return flat;
|
||||
if (maxLength <= 0) return '';
|
||||
|
||||
/* Find the leaf's extension (last `.`) — segment separators are `__`,
|
||||
* never `.`, so the last dot is always inside the leaf. Ignore
|
||||
* "extensions" longer than `FLAT_KEY_MAX_EXT_LENGTH` so a pathological
|
||||
* "extensions" longer than `FLAT_KEY_MAX_EXT_BYTES` so a pathological
|
||||
* input doesn't leave us with no stem budget. */
|
||||
const lastDot = flat.lastIndexOf('.');
|
||||
const candidateExt = lastDot >= 0 ? flat.slice(lastDot) : '';
|
||||
const ext =
|
||||
candidateExt.length > 0 && candidateExt.length <= FLAT_KEY_MAX_EXT_LENGTH ? candidateExt : '';
|
||||
candidateExt.length > 0 && utf8ByteLength(candidateExt) <= FLAT_KEY_MAX_EXT_BYTES
|
||||
? candidateExt
|
||||
: '';
|
||||
const stem = ext ? flat.slice(0, lastDot) : flat;
|
||||
// 7 = '-' + 6 hex disambiguator. Stem budget can collapse to 0 when
|
||||
// `ext.length > maxLength - 7` — that's fine; the stem just disappears
|
||||
// `ext` byte length > maxLength - 7 — that's fine; the stem just disappears
|
||||
// and we fall back to `-<hash><ext>` (still bounded). The hash is a
|
||||
// SHA-256 prefix of `safePath` so re-flattening the same input
|
||||
// produces the same key (same storage location across re-uploads).
|
||||
const stemBudget = Math.max(0, maxLength - ext.length - 7);
|
||||
const truncated = stem.slice(0, stemBudget) + '-' + deterministicHexSuffix(safePath) + ext;
|
||||
const stemBudget = Math.max(0, maxLength - utf8ByteLength(ext) - 7);
|
||||
const truncated =
|
||||
truncateUtf8Bytes(stem, stemBudget) + '-' + deterministicHexSuffix(safePath) + ext;
|
||||
/* Final clamp. The stemBudget formula above keeps `truncated.length`
|
||||
* exactly at `maxLength` for any maxLength ≥ ext.length + 7, and at
|
||||
* `7 + ext.length` otherwise. The latter can still exceed maxLength
|
||||
* for absurdly small budgets (maxLength < ext.length + 7) — clamp
|
||||
* exactly at `maxLength` for any maxLength ≥ ext byte length + 7, and at
|
||||
* `7 + ext byte length` otherwise. The latter can still exceed maxLength
|
||||
* for absurdly small budgets (maxLength < ext byte length + 7) — clamp
|
||||
* defensively so callers always get a key ≤ maxLength regardless of
|
||||
* what they passed in. */
|
||||
return truncated.length <= maxLength ? truncated : truncated.slice(0, maxLength);
|
||||
return utf8ByteLength(truncated) <= maxLength
|
||||
? truncated
|
||||
: truncateUtf8Bytes(truncated, maxLength);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue