diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js index 285f783909..0e3273efe2 100644 --- a/api/server/routes/files/files.js +++ b/api/server/routes/files/files.js @@ -29,7 +29,7 @@ const { getOpenAIClient } = require('~/server/controllers/assistants/helpers'); const { hasCapability } = require('~/server/middleware/roles/capabilities'); const { checkPermission } = require('~/server/services/PermissionService'); const { hasAccessToFilesViaAgent } = require('~/server/services/Files'); -const { cleanFileName } = require('~/server/utils/files'); +const { getContentDisposition } = require('~/server/utils/files'); const { getLogStores } = require('~/cache'); const { Readable } = require('stream'); const db = require('~/models'); @@ -397,8 +397,7 @@ router.get('/download/:userId/:file_id', fileAccess, async (req, res) => { } const setHeaders = () => { - const cleanedFilename = cleanFileName(file.filename); - res.setHeader('Content-Disposition', `attachment; filename="${cleanedFilename}"`); + res.setHeader('Content-Disposition', getContentDisposition(file.filename)); res.setHeader('Content-Type', 'application/octet-stream'); res.setHeader('X-File-Metadata', JSON.stringify(file)); }; diff --git a/api/server/routes/files/preview.spec.js b/api/server/routes/files/preview.spec.js index f86c128d4b..84790c1e5f 100644 --- a/api/server/routes/files/preview.spec.js +++ b/api/server/routes/files/preview.spec.js @@ -62,6 +62,7 @@ jest.mock('~/server/services/Files', () => ({ jest.mock('~/server/utils/files', () => ({ cleanFileName: (name) => name, + getContentDisposition: (name) => `attachment; filename="${name}"`, })); jest.mock('~/cache', () => ({ diff --git a/api/server/utils/files.js b/api/server/utils/files.js index e4825b5ac8..3129c8ef6a 100644 --- a/api/server/utils/files.js +++ b/api/server/utils/files.js @@ -64,4 +64,32 @@ const cleanFileName = (fileName) => { return cleaned; }; -module.exports = { determineFileType, getBufferMetadata, cleanFileName }; +const encodeRFC5987ValueChars = (value) => + encodeURIComponent(value).replace( + /['()*]/g, + (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`, + ); + +const getAsciiFilenameFallback = (fileName) => { + const fallback = fileName + .normalize('NFKD') + .replace(/[^\x20-\x7e]/g, '_') + .replace(/["\\\r\n]/g, '_'); + + return fallback || 'download'; +}; + +const getContentDisposition = (fileName, disposition = 'attachment') => { + const cleanedFilename = cleanFileName(fileName) || 'download'; + const asciiFallback = getAsciiFilenameFallback(cleanedFilename); + const encodedFilename = encodeRFC5987ValueChars(cleanedFilename); + + return `${disposition}; filename="${asciiFallback}"; filename*=UTF-8''${encodedFilename}`; +}; + +module.exports = { + determineFileType, + getBufferMetadata, + cleanFileName, + getContentDisposition, +}; diff --git a/api/server/utils/files.spec.js b/api/server/utils/files.spec.js new file mode 100644 index 0000000000..41332c682e --- /dev/null +++ b/api/server/utils/files.spec.js @@ -0,0 +1,31 @@ +jest.mock('sharp', () => jest.fn(), { virtual: true }); + +const { cleanFileName, getContentDisposition } = require('./files'); + +describe('file utilities', () => { + describe('cleanFileName', () => { + it('removes storage UUID prefixes', () => { + expect(cleanFileName('123e4567-e89b-12d3-a456-426614174000__report.txt')).toBe('report.txt'); + }); + }); + + describe('getContentDisposition', () => { + it('adds RFC 8187 encoding for Unicode filenames', () => { + const filename = '日本語レポート.xlsx'; + const header = getContentDisposition(`123e4567-e89b-12d3-a456-426614174000__${filename}`); + + expect(header).toMatch(/^attachment; filename=".*"; filename\*=UTF-8''/); + expect(header).not.toContain('123e4567-e89b-12d3-a456-426614174000__'); + expect(header).toContain(`filename*=UTF-8''${encodeURIComponent(filename)}`); + }); + + it('escapes the ASCII fallback without dropping the encoded filename', () => { + const filename = 'bad"name\r\n.txt'; + const header = getContentDisposition(filename); + + expect(header).toContain('filename="bad_name__.txt"'); + expect(header).toContain("filename*=UTF-8''bad%22name%0D%0A.txt"); + expect(header).not.toMatch(/[\r\n]/); + }); + }); +}); diff --git a/packages/api/src/files/code/form.spec.ts b/packages/api/src/files/code/form.spec.ts index 14f8d577b9..da8d2d815e 100644 --- a/packages/api/src/files/code/form.spec.ts +++ b/packages/api/src/files/code/form.spec.ts @@ -38,6 +38,13 @@ describe('code env FormData filenames', () => { }); }); + it('uses filepath for nested Unicode filenames', () => { + expect(getCodeEnvFileOptions('分析/結果📊.csv')).toEqual({ + filename: '結果📊.csv', + filepath: '分析/結果📊.csv', + }); + }); + it('documents the form-data string overload regression', async () => { const disposition = await renderMultipartDisposition((form) => { form.append('file', Readable.from(['x']), 'pptx/pptx.py'); diff --git a/packages/api/src/files/code/form.ts b/packages/api/src/files/code/form.ts index 1565721b64..6f9abab7d4 100644 --- a/packages/api/src/files/code/form.ts +++ b/packages/api/src/files/code/form.ts @@ -7,11 +7,30 @@ export interface CodeEnvFileOptions { filepath?: string; } -const CODE_ENV_SAFE_FILEPATH_PATTERN = /^[a-zA-Z0-9._\-/]+$/; +const CODE_ENV_SAFE_ASCII_FILEPATH_CHAR_PATTERN = /^[a-zA-Z0-9._\-/]$/; +const CODE_ENV_UNSAFE_UNICODE_FILEPATH_CHAR_PATTERN = + /[^\p{L}\p{M}\p{N}\p{Emoji}\u200d._\-/]/u; const CODE_ENV_FILENAME_CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g; +function hasUnsafeCodeEnvFilepathChar(filepath: string): boolean { + for (const char of filepath) { + if (char.charCodeAt(0) <= 0x7f) { + if (!CODE_ENV_SAFE_ASCII_FILEPATH_CHAR_PATTERN.test(char)) { + return true; + } + continue; + } + + if (CODE_ENV_UNSAFE_UNICODE_FILEPATH_CHAR_PATTERN.test(char)) { + return true; + } + } + + return false; +} + function isSafeCodeEnvFilepath(filepath: string): boolean { - if (!filepath || filepath.startsWith('/') || !CODE_ENV_SAFE_FILEPATH_PATTERN.test(filepath)) { + if (!filepath || filepath.startsWith('/') || hasUnsafeCodeEnvFilepathChar(filepath)) { return false; } diff --git a/packages/api/src/utils/files.spec.ts b/packages/api/src/utils/files.spec.ts index fc41b00a11..89fd3a35c2 100644 --- a/packages/api/src/utils/files.spec.ts +++ b/packages/api/src/utils/files.spec.ts @@ -22,6 +22,10 @@ function expectedHexSuffix(input: string): string { return createHash('sha256').update(input).digest('hex').slice(0, 6); } +function utf8ByteLength(input: string): number { + return Buffer.byteLength(input, 'utf8'); +} + describe('sanitizeFilename', () => { test('removes directory components (1/2)', () => { expect(sanitizeFilename('/path/to/file.txt')).toBe('file.txt'); @@ -35,6 +39,17 @@ describe('sanitizeFilename', () => { expect(sanitizeFilename('file name@#$.txt')).toBe('file_name___.txt'); }); + test('preserves Unicode filenames', () => { + expect(sanitizeFilename('日本語レポート.xlsx')).toBe('日本語レポート.xlsx'); + expect(sanitizeFilename('résumé-данные-تقرير-보고서📊.csv')).toBe( + 'résumé-данные-تقرير-보고서📊.csv', + ); + }); + + test('normalizes decomposed Unicode marks before sanitizing', () => { + expect(sanitizeFilename('Cafe\u0301.txt')).toBe('Café.txt'); + }); + test('preserves dots and hyphens', () => { expect(sanitizeFilename('file-name.with.dots.txt')).toBe('file-name.with.dots.txt'); }); @@ -50,6 +65,13 @@ describe('sanitizeFilename', () => { expect(result).toMatch(/^a+-abc123\.txt$/); }); + test('truncates Unicode filenames by UTF-8 bytes, preserving the extension', () => { + const longName = '界'.repeat(100) + '.txt'; + const result = sanitizeFilename(longName); + expect(utf8ByteLength(result)).toBeLessThanOrEqual(255); + expect(result.endsWith('-abc123.txt')).toBe(true); + }); + test('handles filenames with no extension', () => { const longName = 'a'.repeat(300); const result = sanitizeFilename(longName); @@ -75,6 +97,10 @@ describe('sanitizeArtifactPath', () => { expect(sanitizeArtifactPath('a/b/c/file.txt')).toBe('a/b/c/file.txt'); }); + test('preserves Unicode path segments', () => { + expect(sanitizeArtifactPath('分析/結果📊.csv')).toBe('分析/結果📊.csv'); + }); + test('replaces non-alphanumeric characters per segment + adds raw-input disambiguator', () => { /* Different raw inputs that sanitize to the same form (`out 1.csv` * vs `out_1.csv`, `out@1.csv` vs `out#1.csv`) would otherwise share @@ -146,6 +172,13 @@ describe('sanitizeArtifactPath', () => { expect(result).toMatch(new RegExp(`^a+-${expectedHexSuffix(longName)}\\.txt$`)); }); + test('caps Unicode leaf segments at 255 UTF-8 bytes with extension-preserving truncation', () => { + const longName = '界'.repeat(100) + '.txt'; + const result = sanitizeArtifactPath(longName); + expect(utf8ByteLength(result)).toBeLessThanOrEqual(255); + expect(result.endsWith(`-${expectedHexSuffix(longName)}.txt`)).toBe(true); + }); + test('caps the leaf when nested under a directory, preserving the directory verbatim', () => { const longLeaf = 'b'.repeat(300) + '.csv'; const result = sanitizeArtifactPath(`reports/${longLeaf}`); @@ -167,6 +200,15 @@ describe('sanitizeArtifactPath', () => { expect(leaf).toBe('notes.txt'); }); + test('caps Unicode non-leaf directory segments at 255 UTF-8 bytes', () => { + const longDir = '界'.repeat(100); + const result = sanitizeArtifactPath(`${longDir}/notes.txt`); + const [dir, leaf] = result.split('/'); + expect(utf8ByteLength(dir)).toBeLessThanOrEqual(255); + expect(dir.endsWith(`-${expectedHexSuffix(longDir)}`)).toBe(true); + expect(leaf).toBe('notes.txt'); + }); + test('produces deterministic output across calls (no orphaned uploads on re-truncation)', () => { /* Codex review P2: `sanitizeFilename`'s `crypto.randomBytes(3)` made * the truncated form non-deterministic — re-uploading the same long @@ -221,6 +263,14 @@ describe('sanitizeArtifactPath', () => { expect(result).toBe('file.txt'); }); + test('falls back to leaf-only when Unicode path bytes exceed the DB-index cap', () => { + const segA = '界'.repeat(80); + const segB = '分'.repeat(80); + const segC = '析'.repeat(80); + const result = sanitizeArtifactPath(`${segA}/${segB}/${segC}/file.txt`); + expect(result).toBe('file.txt'); + }); + test('keeps the nested path when total length is within the DB-index cap', () => { /* The cap doesn't fire for realistic outputs — typical artifact * depth is ≤ 3 segments × short names. */ @@ -276,6 +326,16 @@ describe('sanitizeArtifactPath', () => { expect(a).toBe(`_.hidden-${expectedHexSuffix('.hidden')}`); }); + test('normalization-only collisions get distinct safe forms', () => { + const composed = 'reports/Café.csv'; + const decomposed = 'reports/Cafe\u0301.csv'; + const normalizedDecomposed = `reports/Café-${expectedHexSuffix(decomposed)}.csv`; + + expect(sanitizeArtifactPath(composed)).toBe(composed); + expect(sanitizeArtifactPath(decomposed)).toBe(normalizedDecomposed); + expect(normalizedDecomposed).not.toBe(composed); + }); + test('idempotent: same raw input always produces the same safe form', () => { /* Disambiguator is deterministic (SHA-256 prefix of raw input), * so re-uploading the same long-or-mutated name lands at the @@ -372,6 +432,14 @@ describe('flattenArtifactPath', () => { expect(result).toMatch(new RegExp(`-${expectedHexSuffix(safePath)}\\.txt$`)); }); + test('truncates Unicode flat forms by UTF-8 bytes', () => { + const safePath = `${'界'.repeat(80)}/結果.csv`; + const result = flattenArtifactPath(safePath, 100); + expect(utf8ByteLength(result)).toBeLessThanOrEqual(100); + expect(result.endsWith('.csv')).toBe(true); + expect(result).toMatch(new RegExp(`-${expectedHexSuffix(safePath)}\\.csv$`)); + }); + test('preserves the extension even when only the leaf overflows', () => { const longLeaf = 'L'.repeat(300); const result = flattenArtifactPath(`${longLeaf}.json`, 200); diff --git a/packages/api/src/utils/files.ts b/packages/api/src/utils/files.ts index 37c993a3f3..e98797b97b 100644 --- a/packages/api/src/utils/files.ts +++ b/packages/api/src/utils/files.ts @@ -9,6 +9,56 @@ const USER_FACING_UPLOAD_ERRORS = [ 'Unable to extract text from', ] as const; +const ASCII_FILENAME_SAFE_PATTERN = /^[a-zA-Z0-9._-]$/; +const UNSAFE_UNICODE_FILENAME_PATTERN = /[^\p{L}\p{M}\p{N}\p{Emoji}\u200d._-]/gu; +const FILENAME_SEGMENT_MAX_BYTES = 255; + +function sanitizeFilenameSegment(segment: string): string { + return segment + .normalize('NFC') + .replace(/[\u0000-\u007f]/g, (char) => (ASCII_FILENAME_SAFE_PATTERN.test(char) ? char : '_')) + .replace(UNSAFE_UNICODE_FILENAME_PATTERN, '_'); +} + +function utf8ByteLength(value: string): number { + return Buffer.byteLength(value, 'utf8'); +} + +function truncateUtf8Bytes(value: string, maxBytes: number): string { + if (maxBytes <= 0) return ''; + let bytes = 0; + let result = ''; + + for (const char of value) { + const charBytes = utf8ByteLength(char); + if (bytes + charBytes > maxBytes) break; + result += char; + bytes += charBytes; + } + + return result; +} + +function truncateWithSuffix(value: string, suffix: string, maxBytes: number): string { + const suffixBytes = utf8ByteLength(suffix); + const stemBudget = Math.max(0, maxBytes - suffixBytes); + const result = truncateUtf8Bytes(value, stemBudget) + suffix; + return utf8ByteLength(result) <= maxBytes ? result : truncateUtf8Bytes(result, maxBytes); +} + +function truncateLeafWithSuffix(leaf: string, suffix: string, maxBytes: number): string { + if (utf8ByteLength(leaf) <= maxBytes) return leaf; + const ext = path.extname(leaf); + const stem = path.basename(leaf, ext); + const suffixBytes = utf8ByteLength(suffix); + const extBytes = utf8ByteLength(ext); + if (extBytes > maxBytes - suffixBytes - 1) { + return truncateWithSuffix(leaf, suffix, maxBytes); + } + const stemBudget = maxBytes - extBytes - suffixBytes; + return truncateUtf8Bytes(stem, stemBudget) + suffix + ext; +} + /** * Resolves a user-facing error message from a file upload error. * Returns the error's own message if it matches a known user-facing pattern, @@ -37,42 +87,37 @@ export function resolveUploadErrorMessage( } /** - * Sanitize a filename by removing any directory components, replacing non-alphanumeric characters + * Sanitize a filename by removing any directory components, replacing unsafe characters * @param inputName */ export function sanitizeFilename(inputName: string): string { // Remove any directory components let name = path.basename(inputName); - // Replace any non-alphanumeric characters except for '.' and '-' - name = name.replace(/[^a-zA-Z0-9.-]/g, '_'); + // Preserve Unicode word characters and emoji while replacing unsafe ASCII punctuation. + name = sanitizeFilenameSegment(name); // Ensure the name doesn't start with a dot (hidden file in Unix-like systems) if (name.startsWith('.') || name === '') { name = '_' + name; } - // Limit the length of the filename - const MAX_LENGTH = 255; - if (name.length > MAX_LENGTH) { - const ext = path.extname(name); - const nameWithoutExt = path.basename(name, ext); - name = - nameWithoutExt.slice(0, MAX_LENGTH - ext.length - 7) + - '-' + - crypto.randomBytes(3).toString('hex') + - ext; - } + // Limit the filename to filesystem NAME_MAX, which is byte-based on Linux/APFS. + name = truncateLeafWithSuffix( + name, + '-' + crypto.randomBytes(3).toString('hex'), + FILENAME_SEGMENT_MAX_BYTES, + ); return name; } -/** Per-path-component length cap. Mirrors `sanitizeFilename`'s 255-char +/** Per-path-component byte cap. Mirrors `sanitizeFilename`'s 255-byte * basename cap and matches filesystem `NAME_MAX` (255 bytes on Linux/ext4, * 255 chars on Windows/NTFS) — without it, `saveBuffer` writes * `${file_id}__${flatName}` and a long artifact name surfaces as * `ENAMETOOLONG` and falls back to a download URL instead of persisting. */ -const ARTIFACT_PATH_SEGMENT_MAX = 255; +const ARTIFACT_PATH_SEGMENT_MAX_BYTES = FILENAME_SEGMENT_MAX_BYTES; /** Whole-path length cap for the path-preserving form. The DB stores this * value in `filename`, which participates in a compound unique index on @@ -81,7 +126,7 @@ const ARTIFACT_PATH_SEGMENT_MAX = 255; * (`packages/data-schemas/src/schema/file.ts`). MongoDB 4.0 and earlier * reject indexed values past 1024 bytes; even on 4.2+ where the limit * is configurable, runaway nested paths bloat the index for no real - * benefit. 512 chars is plenty for realistic code-execution outputs + * benefit. 512 bytes is plenty for realistic code-execution outputs * (typical depth ≤ 3 segments × short names) and gives headroom for * BSON / index-overhead encoding. * @@ -92,7 +137,7 @@ const ARTIFACT_PATH_SEGMENT_MAX = 255; * with a missing artifact is strictly worse than a flat-name fallback. * Pre-PR every artifact got this treatment regardless of depth, so the * cap is monotonically better than the prior behavior. */ -const ARTIFACT_PATH_TOTAL_MAX = 512; +const ARTIFACT_PATH_TOTAL_MAX_BYTES = 512; /** * Deterministic disambiguator suffix for truncated names. The original @@ -126,23 +171,23 @@ function deterministicHexSuffix(input: string): string { * still blow past the cap. */ function truncateLeafSegment(leaf: string): string { - if (leaf.length <= ARTIFACT_PATH_SEGMENT_MAX) return leaf; - const ext = path.extname(leaf); - const stem = path.basename(leaf, ext); - // 8 = 1 (`-`) + 6 (hex disambiguator) + 1 (minimum 1-char stem) - if (ext.length > ARTIFACT_PATH_SEGMENT_MAX - 8) { - return truncateDirSegment(leaf); - } - const stemBudget = ARTIFACT_PATH_SEGMENT_MAX - ext.length - 7; - return stem.slice(0, stemBudget) + '-' + deterministicHexSuffix(leaf) + ext; + return truncateLeafWithSuffix( + leaf, + '-' + deterministicHexSuffix(leaf), + ARTIFACT_PATH_SEGMENT_MAX_BYTES, + ); } /** Truncates a non-leaf (directory) segment. Directory segments don't * carry semantic extensions, so we just slice and append the same 6-hex * disambiguation suffix. */ function truncateDirSegment(seg: string): string { - if (seg.length <= ARTIFACT_PATH_SEGMENT_MAX) return seg; - return seg.slice(0, ARTIFACT_PATH_SEGMENT_MAX - 7) + '-' + deterministicHexSuffix(seg); + if (utf8ByteLength(seg) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES) return seg; + return truncateWithSuffix( + seg, + '-' + deterministicHexSuffix(seg), + ARTIFACT_PATH_SEGMENT_MAX_BYTES, + ); } /** @@ -154,7 +199,7 @@ function truncateDirSegment(seg: string): string { * first record and overwrite the first artifact's bytes. * * The disambiguator survives length capping: if appending pushes the - * segment past `ARTIFACT_PATH_SEGMENT_MAX`, the stem is trimmed to make + * segment past `ARTIFACT_PATH_SEGMENT_MAX_BYTES`, the stem is trimmed to make * room (the hash + extension are load-bearing for collision avoidance * and MIME inference, the stem is just the human-readable prefix). */ @@ -172,30 +217,33 @@ function embedDisambiguatorInLeaf(segment: string, hashSource: string): string { if (dot <= 1) { const proposed = segment + suffix; result = - proposed.length <= ARTIFACT_PATH_SEGMENT_MAX + utf8ByteLength(proposed) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES ? proposed - : segment.slice(0, ARTIFACT_PATH_SEGMENT_MAX - suffix.length) + suffix; + : truncateWithSuffix(segment, suffix, ARTIFACT_PATH_SEGMENT_MAX_BYTES); } else { const stem = segment.slice(0, dot); const ext = segment.slice(dot); const proposed = stem + suffix + ext; - if (proposed.length <= ARTIFACT_PATH_SEGMENT_MAX) { + if (utf8ByteLength(proposed) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES) { result = proposed; } else { // Trim stem to make room while preserving disambiguator + extension. - const stemBudget = Math.max(0, ARTIFACT_PATH_SEGMENT_MAX - suffix.length - ext.length); - result = stem.slice(0, stemBudget) + suffix + ext; + const stemBudget = Math.max( + 0, + ARTIFACT_PATH_SEGMENT_MAX_BYTES - utf8ByteLength(suffix) - utf8ByteLength(ext), + ); + result = truncateUtf8Bytes(stem, stemBudget) + suffix + ext; } } /* Defensive final clamp. The branches above already keep the result - * within `ARTIFACT_PATH_SEGMENT_MAX` for any input where the + * within `ARTIFACT_PATH_SEGMENT_MAX_BYTES` for any input where the * extension fits the segment, but a pathological extension (e.g. the * `.aaaaa…` shape `path.extname` returns for contrived inputs) could * still produce a longer result. Hard-clamp so the segment cap holds * unconditionally. */ - return result.length <= ARTIFACT_PATH_SEGMENT_MAX + return utf8ByteLength(result) <= ARTIFACT_PATH_SEGMENT_MAX_BYTES ? result - : result.slice(0, ARTIFACT_PATH_SEGMENT_MAX); + : truncateUtf8Bytes(result, ARTIFACT_PATH_SEGMENT_MAX_BYTES); } /** @@ -205,7 +253,7 @@ function embedDisambiguatorInLeaf(segment: string, hashSource: string): string { * same rules as `sanitizeFilename`. Falls back to the basename for absolute * paths or names containing `..` traversal. * - * Each path component is capped at `ARTIFACT_PATH_SEGMENT_MAX` (255) chars + * Each path component is capped at `ARTIFACT_PATH_SEGMENT_MAX_BYTES` (255 bytes) * — the leaf with extension preservation (matching `sanitizeFilename`), * non-leaf segments with a plain truncate-and-disambiguate. Without the * cap, long artifact names flow into `saveBuffer`'s storage key @@ -226,7 +274,7 @@ export function sanitizeArtifactPath(inputName: string): string { } const segments = normalized .split('/') - .map((seg) => seg.replace(/[^a-zA-Z0-9.-]/g, '_')) + .map(sanitizeFilenameSegment) .filter((seg) => seg.length > 0 && seg !== '.'); if (segments.length === 0) return '_'; const leafIdx = segments.length - 1; @@ -256,7 +304,7 @@ export function sanitizeArtifactPath(inputName: string): string { * input → same safe form (idempotent for re-uploads); different raw * inputs that would have collided → different safe forms. * - * Clean inputs (where the regex/normalize pass was a no-op) skip + * Clean inputs (where the regex/normalize pass was a raw-string no-op) skip * the disambiguator — no collision is possible because they're * already distinct strings, and we don't want to clutter human- * readable filenames with a hash when nothing was at risk. */ @@ -271,21 +319,21 @@ export function sanitizeArtifactPath(inputName: string): string { * the (filename, conversationId, context, tenantId) compound unique * index never sees an oversized key. The leaf is already capped by * `truncateLeafSegment` + `embedDisambiguatorInLeaf`, so this - * guarantees ≤ ARTIFACT_PATH_SEGMENT_MAX (255) chars. Same shape as + * guarantees ≤ ARTIFACT_PATH_SEGMENT_MAX_BYTES (255 bytes). Same shape as * the absolute-path / `..`-traversal fallback above. */ - if (joined.length > ARTIFACT_PATH_TOTAL_MAX) { + if (utf8ByteLength(joined) > ARTIFACT_PATH_TOTAL_MAX_BYTES) { return capped[leafIdx]; } return joined; } -/** Limit on `path.extname`'s "extension" length we'll honor when +/** Limit on `path.extname`'s "extension" byte length we'll honor when * truncating a flat key. Real file extensions cap out around 8 chars * (`.parquet`, `.tsv`, `.html`); a 16-char ceiling keeps us tolerant of * legitimate edge cases (`.openshift`) while ignoring pathological * outputs from `path.extname` on contrived inputs. Above that we treat * the trailing `.foo` as part of the stem and just hard-truncate. */ -const FLAT_KEY_MAX_EXT_LENGTH = 16; +const FLAT_KEY_MAX_EXT_BYTES = 16; /** * Map a (sanitized) artifact path to a flat storage-safe key. The local @@ -293,7 +341,7 @@ const FLAT_KEY_MAX_EXT_LENGTH = 16; * unintended subdirectories on disk while the DB record retains the * nested path for the next prime(). * - * Optionally caps the result at `maxLength` characters. Per-segment caps + * Optionally caps the result at `maxLength` UTF-8 bytes. Per-segment caps * applied by `sanitizeArtifactPath` aren't enough on their own — * `${file_id}__${flatName}` has to fit in one filesystem path component * (NAME_MAX = 255 on most filesystems), so a deeply-nested path whose @@ -308,32 +356,37 @@ const FLAT_KEY_MAX_EXT_LENGTH = 16; */ export function flattenArtifactPath(safePath: string, maxLength?: number): string { const flat = safePath.replace(/\//g, '__'); - if (maxLength == null || flat.length <= maxLength) return flat; + if (maxLength == null || utf8ByteLength(flat) <= maxLength) return flat; if (maxLength <= 0) return ''; /* Find the leaf's extension (last `.`) — segment separators are `__`, * never `.`, so the last dot is always inside the leaf. Ignore - * "extensions" longer than `FLAT_KEY_MAX_EXT_LENGTH` so a pathological + * "extensions" longer than `FLAT_KEY_MAX_EXT_BYTES` so a pathological * input doesn't leave us with no stem budget. */ const lastDot = flat.lastIndexOf('.'); const candidateExt = lastDot >= 0 ? flat.slice(lastDot) : ''; const ext = - candidateExt.length > 0 && candidateExt.length <= FLAT_KEY_MAX_EXT_LENGTH ? candidateExt : ''; + candidateExt.length > 0 && utf8ByteLength(candidateExt) <= FLAT_KEY_MAX_EXT_BYTES + ? candidateExt + : ''; const stem = ext ? flat.slice(0, lastDot) : flat; // 7 = '-' + 6 hex disambiguator. Stem budget can collapse to 0 when - // `ext.length > maxLength - 7` — that's fine; the stem just disappears + // `ext` byte length > maxLength - 7 — that's fine; the stem just disappears // and we fall back to `-` (still bounded). The hash is a // SHA-256 prefix of `safePath` so re-flattening the same input // produces the same key (same storage location across re-uploads). - const stemBudget = Math.max(0, maxLength - ext.length - 7); - const truncated = stem.slice(0, stemBudget) + '-' + deterministicHexSuffix(safePath) + ext; + const stemBudget = Math.max(0, maxLength - utf8ByteLength(ext) - 7); + const truncated = + truncateUtf8Bytes(stem, stemBudget) + '-' + deterministicHexSuffix(safePath) + ext; /* Final clamp. The stemBudget formula above keeps `truncated.length` - * exactly at `maxLength` for any maxLength ≥ ext.length + 7, and at - * `7 + ext.length` otherwise. The latter can still exceed maxLength - * for absurdly small budgets (maxLength < ext.length + 7) — clamp + * exactly at `maxLength` for any maxLength ≥ ext byte length + 7, and at + * `7 + ext byte length` otherwise. The latter can still exceed maxLength + * for absurdly small budgets (maxLength < ext byte length + 7) — clamp * defensively so callers always get a key ≤ maxLength regardless of * what they passed in. */ - return truncated.length <= maxLength ? truncated : truncated.slice(0, maxLength); + return utf8ByteLength(truncated) <= maxLength + ? truncated + : truncateUtf8Bytes(truncated, maxLength); } /**