From 58de77619ec7ede5a97a03cf12c136996ebc49da Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 01:28:35 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=97=9D=EF=B8=8F=20refactor:=20Prefer=20th?= =?UTF-8?q?e=20Recorded=20Storage=20Key=20for=20Every=20Download=20Stream?= =?UTF-8?q?=20(#15695)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit S3 and CloudFront records have carried `storageKey` since #12987, yet six readers still handed `file.filepath` to `getDownloadStream` and re-derived the key by parsing a presigned or CDN URL, while four others had grown their own `storageKey || filepath` expression. One resolver now serves all ten: `resolveDownloadPath` returns the recorded key when present and the path otherwise, so records without a key (local, Firebase, Azure, code output) behave exactly as before, and the share route keeps its local-only query-string strip on top. `resolveStoredS3Key` reuses the same `StoredFileRef` type. Covered by unit cases for the resolver and an S3 case proving a record with a key streams correctly even when its stored URL no longer parses. Closes #15693 --- api/server/routes/__tests__/share.spec.js | 1 + api/server/routes/files/files.js | 3 +- api/server/routes/share.js | 3 +- .../Code/__tests__/process-traversal.spec.js | 1 + api/server/services/Files/Code/process.js | 3 +- .../services/Files/Code/process.spec.js | 1 + packages/api/src/agents/handlers.ts | 5 ++-- packages/api/src/agents/skillFiles.ts | 5 ++-- packages/api/src/files/encode/utils.ts | 3 +- packages/api/src/skills/handlers.ts | 5 ++-- .../api/src/storage/__tests__/path.test.ts | 30 +++++++++++++++++++ packages/api/src/storage/index.ts | 1 + packages/api/src/storage/path.ts | 14 +++++++++ .../api/src/storage/s3/__tests__/crud.test.ts | 15 ++++++++++ packages/api/src/storage/s3/crud.ts | 5 ++-- 15 files changed, 82 insertions(+), 13 deletions(-) create mode 100644 packages/api/src/storage/__tests__/path.test.ts create mode 100644 packages/api/src/storage/path.ts diff --git a/api/server/routes/__tests__/share.spec.js b/api/server/routes/__tests__/share.spec.js index c8aa9833e3..47d8f7b753 100644 --- a/api/server/routes/__tests__/share.spec.js +++ b/api/server/routes/__tests__/share.spec.js @@ -78,6 +78,7 @@ const mockCreateShareContentPreflight = jest.fn((filters, options = {}) => { }); jest.mock('@librechat/api', () => ({ + resolveDownloadPath: (file) => file.storageKey || file.filepath, assertModelBoundContent: (...args) => mockAssertModelBoundContent(...args), assertSharedFileMetadataAllowed: (...args) => mockAssertSharedFileMetadataAllowed(...args), createShareContentPreflight: (...args) => mockCreateShareContentPreflight(...args), diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js index c3b76df66b..a945d657e7 100644 --- a/api/server/routes/files/files.js +++ b/api/server/routes/files/files.js @@ -18,6 +18,7 @@ const { assertUploadContentAllowed, hasActiveFilePolicy, sanitizeFilename, + resolveDownloadPath, } = require('@librechat/api'); const { Time, @@ -700,7 +701,7 @@ router.get('/download/:userId/:file_id', fileAccess, async (req, res) => { return res.status(501).send('Not Implemented'); } - const fileStream = await getDownloadStream(req, file.storageKey || file.filepath); + const fileStream = await getDownloadStream(req, resolveDownloadPath(file)); fileStream.on('error', (streamError) => { logger.error('[DOWNLOAD ROUTE] Stream error:', streamError); diff --git a/api/server/routes/share.js b/api/server/routes/share.js index 9ff04aba33..040ce5a8ad 100644 --- a/api/server/routes/share.js +++ b/api/server/routes/share.js @@ -24,6 +24,7 @@ const { createSharedLangfuseSessionResolver, recordShareLinkRejection, traceIdForMessage, + resolveDownloadPath, } = require('@librechat/api'); const { logger, @@ -297,7 +298,7 @@ const streamSharedFile = async (req, res, file, requestedDisposition) => { // Strip any cache-busting query string (e.g. code-output images add `?v=...`) so // the local stream resolves the real filename, not a literal `*.png?v=...` path. - const streamPath = (file.storageKey || file.filepath || '').split('?')[0]; + const streamPath = (resolveDownloadPath(file) || '').split('?')[0]; const fileStream = await getDownloadStream(req, streamPath); res.setHeader('X-Content-Type-Options', 'nosniff'); diff --git a/api/server/services/Files/Code/__tests__/process-traversal.spec.js b/api/server/services/Files/Code/__tests__/process-traversal.spec.js index 6866e3e309..ec28e907ab 100644 --- a/api/server/services/Files/Code/__tests__/process-traversal.spec.js +++ b/api/server/services/Files/Code/__tests__/process-traversal.spec.js @@ -20,6 +20,7 @@ jest.mock('@librechat/api', () => { const http = require('http'); const https = require('https'); return { + resolveDownloadPath: (file) => file.storageKey || file.filepath, logAxiosError: jest.fn(), getBasePath: jest.fn(() => ''), sanitizeArtifactPath: mockSanitizeArtifactPath, diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js index 41cc6f456c..9de8f27e25 100644 --- a/api/server/services/Files/Code/process.js +++ b/api/server/services/Files/Code/process.js @@ -35,6 +35,7 @@ const { CODE_OUTPUT_PREFLIGHT_MAX_COUNT, sortCodeFilesByDestinationPriority, normalizeArtifactDeliveryFailure, + resolveDownloadPath, } = require('@librechat/api'); const { Tools, @@ -1430,7 +1431,7 @@ const primeFiles = async (options) => { const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions( FileSources.execute_code, ); - const stream = await getDownloadStream(options.req, file.filepath); + const stream = await getDownloadStream(options.req, resolveDownloadPath(file)); /* Reupload preserves the resource identity from the existing * ref so codeapi re-buckets under the same sessionKey shape * (skill stays skill, user stays user). Without this, a diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js index cfe2dcf775..91473eb74f 100644 --- a/api/server/services/Files/Code/process.spec.js +++ b/api/server/services/Files/Code/process.spec.js @@ -66,6 +66,7 @@ jest.mock('@librechat/api', () => { const http = require('http'); const https = require('https'); return { + resolveDownloadPath: (file) => file.storageKey || file.filepath, logAxiosError: jest.fn(), /* Behaviourally identical to the real predicate in * `packages/api/src/files/code/errors.ts`, which owns the contract and diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index b27c4e5208..7802e2611b 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -108,6 +108,7 @@ import { buildSkillPrimeMessage, isSkillFilePath, SKILL_FILE_PREFIX } from './sk import { resolveCallerCapabilityProjectionSnapshot } from './callerCapabilities'; import { createSkillContentDigest } from './compatibility'; import { isMissingSandboxPathError } from '~/files/code'; +import { resolveDownloadPath } from '~/storage/path'; import { parseFrontmatter } from '../skills/import'; import { cleanCodeToolOutput } from './cleanup'; import { primeSkillFiles } from './skillFiles'; @@ -3269,7 +3270,7 @@ async function loadSkillFileTextForAuthoring({ return { status: 'error', message: 'Download is not supported for this storage backend.' }; } - const stream = await strategy.getDownloadStream(req, file.filepath); + const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file)); const chunks: Uint8Array[] = []; let streamedBytes = 0; for await (const chunk of stream as AsyncIterable) { @@ -4673,7 +4674,7 @@ async function handleReadFileCall( }; } - const stream = await strategy.getDownloadStream(req, file.filepath); + const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file)); const chunks: Uint8Array[] = []; // Use the larger binary limit as streaming cap; cheaper type-specific // checks happen after binary detection on the assembled buffer. diff --git a/packages/api/src/agents/skillFiles.ts b/packages/api/src/agents/skillFiles.ts index 6344c30b83..8afb75cf64 100644 --- a/packages/api/src/agents/skillFiles.ts +++ b/packages/api/src/agents/skillFiles.ts @@ -25,6 +25,7 @@ import { getCodeExecutionRouteKey, type CodeExecutionContext } from './execution import { assertSkillFileContentAllowed } from '~/skills/protection'; import { createSkillContentDigest } from './compatibility'; import { extractInvokedSkillsFromPayload } from './run'; +import { resolveDownloadPath } from '~/storage/path'; import { SKILL_FILE_PREFIX } from './skills'; const MAX_INSPECTABLE_SKILL_FILE_BYTES = 10 * 1024 * 1024; @@ -197,7 +198,7 @@ async function collectSkillUploadFiles( ); return null; } - const stream = await strategy.getDownloadStream(req, file.filepath); + const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file)); return { stream, filename: `${SKILL_FILE_PREFIX}${skill.name}/${file.relativePath}` }; }), ); @@ -402,7 +403,7 @@ async function executePrimeSkillFiles( logger.warn('[primeSkillFiles] No download stream for stored skill file'); continue; } - const sourceStream = await strategy.getDownloadStream(req, file.filepath); + const sourceStream = await strategy.getDownloadStream(req, resolveDownloadPath(file)); const buffer = await bufferSkillFileStream(sourceStream); if (buffer == null) { throwIfStoredSkillFileMustBeInspectable(req); diff --git a/packages/api/src/files/encode/utils.ts b/packages/api/src/files/encode/utils.ts index 899f3592a2..be1116519b 100644 --- a/packages/api/src/files/encode/utils.ts +++ b/packages/api/src/files/encode/utils.ts @@ -3,6 +3,7 @@ import { Providers } from '@librechat/agents'; import { FileSources, mergeFileConfig, getEndpointFileConfig } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest, StrategyFunctions, ProcessedFile } from '~/types'; +import { resolveDownloadPath } from '~/storage/path'; /** * Extracts the configured file size limit for a specific provider from fileConfig @@ -55,7 +56,7 @@ export async function getFileStream( } const { getDownloadStream } = encodingMethods[source]; - const stream = await getDownloadStream(req, file.filepath); + const stream = await getDownloadStream(req, resolveDownloadPath(file)); let buffer: Buffer | null = await getStream.buffer(stream); const content = buffer.toString('base64'); buffer = null; diff --git a/packages/api/src/skills/handlers.ts b/packages/api/src/skills/handlers.ts index 3a9fefc903..7a75d0d199 100644 --- a/packages/api/src/skills/handlers.ts +++ b/packages/api/src/skills/handlers.ts @@ -36,6 +36,7 @@ import type { ServerRequest, StrategyFunctions } from '~/types'; import { extractSkillContent, inspectContentWithTraversal } from '~/protection'; import { contentFilterBlockResponse } from '~/middleware/contentFilter'; import { getDeploymentSkillIds } from './deployment'; +import { resolveDownloadPath } from '~/storage/path'; import { resolveSkillFilePathParam } from './path'; import { parseSkillMarkdown } from './parse'; import { isBinaryBuffer } from './binary'; @@ -702,7 +703,7 @@ export function createSkillsHandlers(deps: SkillsHandlersDeps): { 'Content-Disposition', `${isImageMime ? 'inline' : 'attachment'}; filename="${safeName}"`, ); - const stream = await strategy.getDownloadStream(req, file.storageKey || file.filepath); + const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file)); stream.on('error', (err: Error) => { logger.error('[downloadFile] Stream error:', err); if (!res.headersSent) { @@ -736,7 +737,7 @@ export function createSkillsHandlers(deps: SkillsHandlersDeps): { // destroy (binary) or continue reading (text) in the same iteration. // N.B. breaking out of `for await...of` destroys the stream via // iterator.return(), so we must NOT use break + a second loop. - const stream = await strategy.getDownloadStream(req, file.storageKey || file.filepath); + const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file)); const chunks: Buffer[] = []; let totalBytes = 0; let binaryChecked = false; diff --git a/packages/api/src/storage/__tests__/path.test.ts b/packages/api/src/storage/__tests__/path.test.ts new file mode 100644 index 0000000000..3ffaa5e41e --- /dev/null +++ b/packages/api/src/storage/__tests__/path.test.ts @@ -0,0 +1,30 @@ +import { resolveDownloadPath } from '../path'; + +describe('resolveDownloadPath', () => { + it('prefers the recorded object key over the stored URL', () => { + expect( + resolveDownloadPath({ + filepath: 'https://bucket.s3.amazonaws.com/uploads/u1/f.pdf?X-Amz-Expires=900', + storageKey: 'uploads/u1/f.pdf', + }), + ).toBe('uploads/u1/f.pdf'); + }); + + it('falls back to the path when no key was recorded', () => { + expect(resolveDownloadPath({ filepath: '/uploads/u1/f.pdf' })).toBe('/uploads/u1/f.pdf'); + }); + + it('treats an empty or null key as absent', () => { + expect(resolveDownloadPath({ filepath: '/uploads/u1/f.pdf', storageKey: '' })).toBe( + '/uploads/u1/f.pdf', + ); + expect(resolveDownloadPath({ filepath: '/uploads/u1/f.pdf', storageKey: null })).toBe( + '/uploads/u1/f.pdf', + ); + }); + + it('keeps a remote URL intact for strategies that fetch it directly', () => { + const url = 'https://firebasestorage.googleapis.com/v0/b/x/o/f.pdf?alt=media&token=t'; + expect(resolveDownloadPath({ filepath: url })).toBe(url); + }); +}); diff --git a/packages/api/src/storage/index.ts b/packages/api/src/storage/index.ts index 53ccbe2eab..718f3437aa 100644 --- a/packages/api/src/storage/index.ts +++ b/packages/api/src/storage/index.ts @@ -5,3 +5,4 @@ export * from './images'; export * from './avatar'; export * from './metadata'; export * from './url'; +export * from './path'; diff --git a/packages/api/src/storage/path.ts b/packages/api/src/storage/path.ts new file mode 100644 index 0000000000..36ef64a030 --- /dev/null +++ b/packages/api/src/storage/path.ts @@ -0,0 +1,14 @@ +import type { TFile } from 'librechat-data-provider'; + +/** A stored file as a download needs it: the recorded object key when one exists, else the URL or path. */ +export type StoredFileRef = Pick & { storageKey?: string | null }; + +/** + * Resolves the argument for a strategy's `getDownloadStream`. S3 and CloudFront records carry + * the object key since the region-aware storage keys landed; handing it over directly skips + * re-deriving it from a presigned or CDN URL, which is only sound while that URL still parses. + * Records without a key (local, Firebase, Azure, code output) fall through to `filepath` as before. + */ +export function resolveDownloadPath(file: StoredFileRef): string { + return file.storageKey || file.filepath; +} diff --git a/packages/api/src/storage/s3/__tests__/crud.test.ts b/packages/api/src/storage/s3/__tests__/crud.test.ts index 715e80fcf7..d1c12e8371 100644 --- a/packages/api/src/storage/s3/__tests__/crud.test.ts +++ b/packages/api/src/storage/s3/__tests__/crud.test.ts @@ -1131,6 +1131,21 @@ describe('S3 CRUD', () => { ); }); + it('streams by the recorded key even when the stored URL no longer parses', async () => { + const { getS3FileStream } = await import('../crud'); + const { resolveDownloadPath } = await import('~/storage/path'); + const file = { + filepath: 'not a url: presigned link expired and was overwritten', + storageKey: 'uploads/user123/Ársreikningur 2025.pdf', + }; + + await getS3FileStream({} as ServerRequest, resolveDownloadPath(file)); + + const [call] = s3Mock.commandCalls(GetObjectCommand); + expect(call.args[0].input.Key).toBe(file.storageKey); + expect(logger.error).not.toHaveBeenCalled(); + }); + it('handles errors when retrieving stream', async () => { s3Mock.on(GetObjectCommand).rejects(new Error('Stream error')); diff --git a/packages/api/src/storage/s3/crud.ts b/packages/api/src/storage/s3/crud.ts index 48305793e9..c11f3b0286 100644 --- a/packages/api/src/storage/s3/crud.ts +++ b/packages/api/src/storage/s3/crud.ts @@ -31,6 +31,7 @@ import type { UrlBuilder, S3FileRef, } from '~/storage/types'; +import type { StoredFileRef } from '~/storage/path'; import type { ServerRequest } from '~/types'; import { assertRemoteFileURL, @@ -280,9 +281,7 @@ export function getStorageMetadataForKey( }; } -export function resolveStoredS3Key( - file: Pick & { storageKey?: string | null }, -): string { +export function resolveStoredS3Key(file: StoredFileRef): string { return file.storageKey || extractKeyFromS3Url(file.filepath); }