From ea3c61d69aa53f715385181583ec9af4e5daf0c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigur=C3=B0ur=20Gu=C3=B0brandsson?= <1608474+SGudbrandsson@users.noreply.github.com> Date: Mon, 7 Sep 2026 04:23:36 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=AA=A3=20fix:=20Decode=20S3=20Object=20Ke?= =?UTF-8?q?ys=20from=20URLs=20(#15426)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🔀 fix: Decode percent-encoded S3 keys so non-ASCII filenames are readable extractKeyFromS3Url returns `URL.pathname` as the S3 object key without decoding it. The AWS SDK percent-encodes the Key again when it signs the request, so a file stored under `Ársreikningur.pdf` is fetched as `%C3%81rsreikningur.pdf` and every read fails with NoSuchKey. ASCII keys are byte-identical either way, which is why this only appears for non-English filenames. Decode keys derived from a URL path in all three branches (path-style endpoint, bucket-in-path, virtual-hosted). A key passed in raw — not a URL — still returns untouched, and a malformed escape sequence falls back to the raw value with a warning rather than throwing. * 🔗 fix: Percent-encode CloudFront URL keys so both URL forms agree buildCloudFrontUrl interpolated the raw S3 key into the URL while SDK-generated S3 URLs carry an encoded one, so the two forms of `file.filepath` disagreed about what a `%` means. `assertS3FileName` permits `%`, so a key containing the literal text `report%20final.pdf` produced a CloudFront URL indistinguishable from one for a key containing a space — and decoding on extraction would then target the wrong object on read, re-sign, and delete. Encoding each path segment here (separators stay literal) makes both producers consistent, which is what lets extractKeyFromS3Url decode unconditionally. * style: sort imports in cloudfront/crud.ts (pre-existing drift) The changed-file import-sort gate flags this file; the drift predates this PR (the untouched upstream version fails the same check). Kept as its own commit so it does not obscure the fix. * 🔏 fix: Encode the CloudFront Invalidation Path Like the Viewer URL `buildCloudFrontUrl` now percent-encodes each key segment, so the cached viewer path for a key with a literal `%` or a non-ASCII character is the encoded form. `deleteFileFromCloudFront` still handed the raw key to `CreateInvalidationCommand`, so the invalidation no longer matched the cached object and deleted content stayed served until the entry expired. Both producers now share one `encodeKeyPath` helper. Also asserts that `getS3FileStream` sends the decoded key to `GetObjectCommand`, which is the call that actually failed with `NoSuchKey`, rather than only checking the extractor. Co-authored-by: dinershtein <228485+dinershtein@users.noreply.github.com> --------- Co-authored-by: Danny Avila Co-authored-by: dinershtein <228485+dinershtein@users.noreply.github.com> --- .../storage/cloudfront/__tests__/crud.test.ts | 42 +++++++++++++++ packages/api/src/storage/cloudfront/crud.ts | 30 +++++++---- .../api/src/storage/s3/__tests__/crud.test.ts | 52 ++++++++++++++++++- packages/api/src/storage/s3/crud.ts | 30 +++++++++-- 4 files changed, 138 insertions(+), 16 deletions(-) diff --git a/packages/api/src/storage/cloudfront/__tests__/crud.test.ts b/packages/api/src/storage/cloudfront/__tests__/crud.test.ts index 5990cceac5..bb40b2cb3e 100644 --- a/packages/api/src/storage/cloudfront/__tests__/crud.test.ts +++ b/packages/api/src/storage/cloudfront/__tests__/crud.test.ts @@ -92,6 +92,28 @@ describe('CloudFront CRUD', () => { expect(mockGetSignedUrl).not.toHaveBeenCalled(); }); + it('percent-encodes non-ASCII characters in the key', async () => { + const { getCloudFrontURL } = await import('~/storage/cloudfront/crud'); + const url = await getCloudFrontURL({ userId: 'user1', fileName: 'Ársreikningur.pdf' }); + expect(url).toBe('https://d123.cloudfront.net/i/images/user1/%C3%81rsreikningur.pdf'); + }); + + it('percent-encodes a literal % so it cannot be read back as an escape', async () => { + const { getCloudFrontURL } = await import('~/storage/cloudfront/crud'); + const url = await getCloudFrontURL({ userId: 'user1', fileName: 'report%20final.pdf' }); + expect(url).toBe('https://d123.cloudfront.net/i/images/user1/report%2520final.pdf'); + }); + + it('leaves path separators literal', async () => { + const { getCloudFrontURL } = await import('~/storage/cloudfront/crud'); + const url = await getCloudFrontURL({ + userId: 'user1', + fileName: 'doc.pdf', + basePath: 'documents', + }); + expect(url).toBe('https://d123.cloudfront.net/documents/user1/doc.pdf'); + }); + it('uses custom basePath when provided', async () => { const { getCloudFrontURL } = await import('~/storage/cloudfront/crud'); const url = await getCloudFrontURL({ @@ -512,6 +534,26 @@ describe('CloudFront CRUD', () => { ); }); + it('encodes the invalidation path the same way as the viewer URL', async () => { + mockResolveStoredS3Key.mockReturnValue('images/u/report%20final \u0151.webp'); + mockGetCloudFrontConfig.mockReturnValue( + makeConfig({ invalidateOnDelete: true, distributionId: 'E123' }), + ); + mockCloudFrontSend.mockResolvedValue({}); + + const { deleteFileFromCloudFront } = await import('~/storage/cloudfront/crud'); + await deleteFileFromCloudFront(mockReq, mockFile); + + const { CreateInvalidationCommand } = await import('@aws-sdk/client-cloudfront'); + expect(CreateInvalidationCommand).toHaveBeenCalledWith( + expect.objectContaining({ + InvalidationBatch: expect.objectContaining({ + Paths: { Quantity: 1, Items: ['/images/u/report%2520final%20%C5%91.webp'] }, + }), + }), + ); + }); + it('prefixes key with / for invalidation path', async () => { mockResolveStoredS3Key.mockReturnValue('images/u/file.webp'); // no leading slash mockGetCloudFrontConfig.mockReturnValue( diff --git a/packages/api/src/storage/cloudfront/crud.ts b/packages/api/src/storage/cloudfront/crud.ts index b532bbd909..5636704b6e 100644 --- a/packages/api/src/storage/cloudfront/crud.ts +++ b/packages/api/src/storage/cloudfront/crud.ts @@ -1,10 +1,9 @@ import crypto from 'crypto'; +import { logger } from '@librechat/data-schemas'; import { getSignedUrl } from '@aws-sdk/cloudfront-signer'; import { CloudFrontClient, CreateInvalidationCommand } from '@aws-sdk/client-cloudfront'; -import { logger } from '@librechat/data-schemas'; import type { TFile } from 'librechat-data-provider'; import type { Readable } from 'stream'; -import type { ServerRequest } from '~/types'; import type { SaveBufferParams, GetURLParams, @@ -14,10 +13,7 @@ import type { SaveURLResult, UploadResult, } from '~/storage/types'; -import { getCloudFrontConfig } from '~/cdn/cloudfront'; -import { s3Config } from '~/storage/s3/s3Config'; -import { AVATAR_BASE_PATH, DEFAULT_BASE_PATH as defaultBasePath } from '~/storage/constants'; -import { sanitizeContentDispositionFilename } from '~/storage/validation'; +import type { ServerRequest } from '~/types'; import { getS3Key, saveBufferToS3, @@ -27,6 +23,10 @@ import { getS3FileStream, resolveStoredS3Key, } from '~/storage/s3/crud'; +import { AVATAR_BASE_PATH, DEFAULT_BASE_PATH as defaultBasePath } from '~/storage/constants'; +import { sanitizeContentDispositionFilename } from '~/storage/validation'; +import { getCloudFrontConfig } from '~/cdn/cloudfront'; +import { s3Config } from '~/storage/s3/s3Config'; let _cloudFrontClient: CloudFrontClient | null = null; @@ -77,14 +77,25 @@ function isInlineFileUpload({ basePath, file, useInlinePath }: UploadFileParams) return (basePath ?? defaultBasePath) === defaultBasePath && file.mimetype?.startsWith('image/'); } +/** + * Percent-encodes each path segment of an S3 key (the separators stay literal). Without this a + * CloudFront URL carries the raw key while an SDK-generated S3 URL carries an encoded one, so the + * two forms of `file.filepath` disagree about what a `%` means: a key containing the literal text + * `%20` would be indistinguishable from a key containing a space. Encoding here keeps both + * producers consistent, which is what lets `extractKeyFromS3Url` decode unconditionally. The + * invalidation path must use the same encoding, or it no longer matches the cached viewer URL. + */ +function encodeKeyPath(s3Key: string): string { + return s3Key.replace(/^\/+/, '').split('/').map(encodeURIComponent).join('/'); +} + function buildCloudFrontUrl(s3Key: string): string { const config = getCloudFrontConfig(); if (!config?.domain) { throw new Error('[buildCloudFrontUrl] CloudFront not initialized.'); } const cleanDomain = config.domain.replace(/\/+$/, ''); - const cleanKey = s3Key.replace(/^\/+/, ''); - return `${cleanDomain}/${cleanKey}`; + return `${cleanDomain}/${encodeKeyPath(s3Key)}`; } function signUrl(url: string | URL): string { @@ -235,8 +246,7 @@ export async function deleteFileFromCloudFront(req: ServerRequest, file: TFile): try { const client = getOrCreateCloudFrontClient(); // CloudFront URL pathname matches S3 key when no origin path prefix is configured - const key = resolveStoredS3Key(file); - const path = key.startsWith('/') ? key : `/${key}`; + const path = `/${encodeKeyPath(resolveStoredS3Key(file))}`; await client.send( new CreateInvalidationCommand({ diff --git a/packages/api/src/storage/s3/__tests__/crud.test.ts b/packages/api/src/storage/s3/__tests__/crud.test.ts index ecf60b8bc1..715e80fcf7 100644 --- a/packages/api/src/storage/s3/__tests__/crud.test.ts +++ b/packages/api/src/storage/s3/__tests__/crud.test.ts @@ -1118,6 +1118,19 @@ describe('S3 CRUD', () => { expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(1); }); + it('requests the decoded key for a non-ASCII file name', async () => { + const { getS3FileStream } = await import('../crud'); + await getS3FileStream( + {} as ServerRequest, + 'https://test-bucket.s3.amazonaws.com/images/user123/%D0%94%D0%BE%D0%B3%D0%BE%D0%B2%D0%BE%D1%80.pdf', + ); + + const [call] = s3Mock.commandCalls(GetObjectCommand); + expect(call.args[0].input.Key).toBe( + 'images/user123/\u0414\u043e\u0433\u043e\u0432\u043e\u0440.pdf', + ); + }); + it('handles errors when retrieving stream', async () => { s3Mock.on(GetObjectCommand).rejects(new Error('Stream error')); @@ -1463,6 +1476,40 @@ describe('S3 CRUD', () => { expect(key).toBe('images/user123/file.png'); }); + it('decodes percent-encoded keys from virtual-hosted-style URLs', async () => { + const { extractKeyFromS3Url } = await import('../crud'); + const key = extractKeyFromS3Url( + 'https://bucket.s3.amazonaws.com/uploads/user123/abc__%C3%81rsreikningur_2025.pdf', + ); + expect(key).toBe('uploads/user123/abc__Ársreikningur_2025.pdf'); + }); + + it('decodes percent-encoded keys from path-style URLs', async () => { + const { extractKeyFromS3Url } = await import('../crud'); + const key = extractKeyFromS3Url( + 'https://s3.us-west-2.amazonaws.com/test-bucket/uploads/user123/%E6%97%A5%E6%9C%AC%E8%AA%9E.pdf', + ); + expect(key).toBe('uploads/user123/日本語.pdf'); + }); + + it('preserves a literal percent in a filename (round-trip through %25)', async () => { + const { extractKeyFromS3Url } = await import('../crud'); + const key = extractKeyFromS3Url('https://bucket.s3.amazonaws.com/uploads/100%25_done.pdf'); + expect(key).toBe('uploads/100%_done.pdf'); + }); + + it('returns a raw key untouched even when it contains a percent sign', async () => { + const { extractKeyFromS3Url } = await import('../crud'); + const key = 'uploads/user123/100%_done.pdf'; + expect(extractKeyFromS3Url(key)).toBe(key); + }); + + it('falls back to the raw value on a malformed escape sequence', async () => { + const { extractKeyFromS3Url } = await import('../crud'); + const key = extractKeyFromS3Url('https://bucket.s3.amazonaws.com/uploads/bad%E0%A4A.pdf'); + expect(key).toBe('uploads/bad%E0%A4A.pdf'); + }); + it('extracts key from path-style regional endpoint', async () => { const { extractKeyFromS3Url } = await import('../crud'); const key = extractKeyFromS3Url( @@ -1511,12 +1558,13 @@ describe('S3 CRUD', () => { expect(key).toBe('folder/file.txt'); }); - it('handles URLs with encoded characters', async () => { + it('decodes URLs with encoded characters back to the stored key', async () => { const { extractKeyFromS3Url } = await import('../crud'); const key = extractKeyFromS3Url( 'https://bucket.s3.amazonaws.com/test-bucket/images/user123/my%20file%20name.jpg', ); - expect(key).toBe('images/user123/my%20file%20name.jpg'); + /** The object was stored under `my file name.jpg`; `%20` is URL transport, not part of the key. */ + expect(key).toBe('images/user123/my file name.jpg'); }); it('handles deep nested paths', async () => { diff --git a/packages/api/src/storage/s3/crud.ts b/packages/api/src/storage/s3/crud.ts index 271801a51a..48305793e9 100644 --- a/packages/api/src/storage/s3/crud.ts +++ b/packages/api/src/storage/s3/crud.ts @@ -638,6 +638,27 @@ export async function saveURLToS3( return filepath; } +/** + * Decodes a key taken from a URL path. + * + * `URL.pathname` is percent-encoded, but S3 object keys are raw UTF-8 and the AWS SDK encodes + * them again when it signs the request. Returning the encoded pathname therefore asks S3 for a + * literally different key (`%C3%81rsreikningur.pdf` instead of `Ársreikningur.pdf`) and every + * read of a file whose name contains a non-ASCII character fails with `NoSuchKey`. Keys made of + * ASCII are unaffected, which is why this only shows up for non-English filenames. + * + * Malformed sequences fall back to the raw value rather than throwing — a key that cannot be + * decoded is still worth attempting. + */ +function decodeKeyFromUrlPath(key: string): string { + try { + return decodeURIComponent(key); + } catch { + logger.warn(`[extractKeyFromS3Url] Could not decode key, using it as-is: ${key}`); + return key; + } +} + export function extractKeyFromS3Url(fileUrlOrKey: string): string { if (!fileUrlOrKey) { throw new Error('Invalid input: URL or key is empty'); @@ -659,7 +680,7 @@ export function extractKeyFromS3Url(fileUrlOrKey: string): string { (endpointUrl.pathname.endsWith('/') ? 0 : 1) + bucketName.length + 1; - const key = url.pathname.substring(startPos); + const key = decodeKeyFromUrlPath(url.pathname.substring(startPos)); if (!key) { logger.warn( `[extractKeyFromS3Url] Extracted key is empty for endpoint path-style URL: ${fileUrlOrKey}`, @@ -677,7 +698,7 @@ export function extractKeyFromS3Url(fileUrlOrKey: string): string { ) { const firstSlashIndex = pathname.indexOf('/'); if (firstSlashIndex > 0) { - const key = pathname.substring(firstSlashIndex + 1); + const key = decodeKeyFromUrlPath(pathname.substring(firstSlashIndex + 1)); if (key === '') { logger.warn( `[extractKeyFromS3Url] Extracted key is empty after removing bucket name from URL: ${fileUrlOrKey}`, @@ -695,8 +716,9 @@ export function extractKeyFromS3Url(fileUrlOrKey: string): string { return ''; } - logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${pathname}`); - return pathname; + const key = decodeKeyFromUrlPath(pathname); + logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`); + return key; } catch (error) { if (fileUrlOrKey.startsWith('http://') || fileUrlOrKey.startsWith('https://')) { logger.error(