mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-19 14:33:38 +00:00
🗝️ refactor: Prefer the Recorded Storage Key for Every Download Stream (#15695)
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
S3 and CloudFront records have carried `storageKey` since #12987, yet six readers still handed `file.filepath` to `getDownloadStream` and re-derived the key by parsing a presigned or CDN URL, while four others had grown their own `storageKey || filepath` expression. One resolver now serves all ten: `resolveDownloadPath` returns the recorded key when present and the path otherwise, so records without a key (local, Firebase, Azure, code output) behave exactly as before, and the share route keeps its local-only query-string strip on top. `resolveStoredS3Key` reuses the same `StoredFileRef` type. Covered by unit cases for the resolver and an S3 case proving a record with a key streams correctly even when its stored URL no longer parses. Closes #15693
This commit is contained in:
parent
45bb99ab04
commit
58de77619e
15 changed files with 82 additions and 13 deletions
|
|
@ -78,6 +78,7 @@ const mockCreateShareContentPreflight = jest.fn((filters, options = {}) => {
|
|||
});
|
||||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
resolveDownloadPath: (file) => file.storageKey || file.filepath,
|
||||
assertModelBoundContent: (...args) => mockAssertModelBoundContent(...args),
|
||||
assertSharedFileMetadataAllowed: (...args) => mockAssertSharedFileMetadataAllowed(...args),
|
||||
createShareContentPreflight: (...args) => mockCreateShareContentPreflight(...args),
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ const {
|
|||
assertUploadContentAllowed,
|
||||
hasActiveFilePolicy,
|
||||
sanitizeFilename,
|
||||
resolveDownloadPath,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
Time,
|
||||
|
|
@ -700,7 +701,7 @@ router.get('/download/:userId/:file_id', fileAccess, async (req, res) => {
|
|||
return res.status(501).send('Not Implemented');
|
||||
}
|
||||
|
||||
const fileStream = await getDownloadStream(req, file.storageKey || file.filepath);
|
||||
const fileStream = await getDownloadStream(req, resolveDownloadPath(file));
|
||||
|
||||
fileStream.on('error', (streamError) => {
|
||||
logger.error('[DOWNLOAD ROUTE] Stream error:', streamError);
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ const {
|
|||
createSharedLangfuseSessionResolver,
|
||||
recordShareLinkRejection,
|
||||
traceIdForMessage,
|
||||
resolveDownloadPath,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
logger,
|
||||
|
|
@ -297,7 +298,7 @@ const streamSharedFile = async (req, res, file, requestedDisposition) => {
|
|||
|
||||
// Strip any cache-busting query string (e.g. code-output images add `?v=...`) so
|
||||
// the local stream resolves the real filename, not a literal `*.png?v=...` path.
|
||||
const streamPath = (file.storageKey || file.filepath || '').split('?')[0];
|
||||
const streamPath = (resolveDownloadPath(file) || '').split('?')[0];
|
||||
const fileStream = await getDownloadStream(req, streamPath);
|
||||
|
||||
res.setHeader('X-Content-Type-Options', 'nosniff');
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ jest.mock('@librechat/api', () => {
|
|||
const http = require('http');
|
||||
const https = require('https');
|
||||
return {
|
||||
resolveDownloadPath: (file) => file.storageKey || file.filepath,
|
||||
logAxiosError: jest.fn(),
|
||||
getBasePath: jest.fn(() => ''),
|
||||
sanitizeArtifactPath: mockSanitizeArtifactPath,
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ const {
|
|||
CODE_OUTPUT_PREFLIGHT_MAX_COUNT,
|
||||
sortCodeFilesByDestinationPriority,
|
||||
normalizeArtifactDeliveryFailure,
|
||||
resolveDownloadPath,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
Tools,
|
||||
|
|
@ -1430,7 +1431,7 @@ const primeFiles = async (options) => {
|
|||
const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(
|
||||
FileSources.execute_code,
|
||||
);
|
||||
const stream = await getDownloadStream(options.req, file.filepath);
|
||||
const stream = await getDownloadStream(options.req, resolveDownloadPath(file));
|
||||
/* Reupload preserves the resource identity from the existing
|
||||
* ref so codeapi re-buckets under the same sessionKey shape
|
||||
* (skill stays skill, user stays user). Without this, a
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ jest.mock('@librechat/api', () => {
|
|||
const http = require('http');
|
||||
const https = require('https');
|
||||
return {
|
||||
resolveDownloadPath: (file) => file.storageKey || file.filepath,
|
||||
logAxiosError: jest.fn(),
|
||||
/* Behaviourally identical to the real predicate in
|
||||
* `packages/api/src/files/code/errors.ts`, which owns the contract and
|
||||
|
|
|
|||
|
|
@ -108,6 +108,7 @@ import { buildSkillPrimeMessage, isSkillFilePath, SKILL_FILE_PREFIX } from './sk
|
|||
import { resolveCallerCapabilityProjectionSnapshot } from './callerCapabilities';
|
||||
import { createSkillContentDigest } from './compatibility';
|
||||
import { isMissingSandboxPathError } from '~/files/code';
|
||||
import { resolveDownloadPath } from '~/storage/path';
|
||||
import { parseFrontmatter } from '../skills/import';
|
||||
import { cleanCodeToolOutput } from './cleanup';
|
||||
import { primeSkillFiles } from './skillFiles';
|
||||
|
|
@ -3269,7 +3270,7 @@ async function loadSkillFileTextForAuthoring({
|
|||
return { status: 'error', message: 'Download is not supported for this storage backend.' };
|
||||
}
|
||||
|
||||
const stream = await strategy.getDownloadStream(req, file.filepath);
|
||||
const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file));
|
||||
const chunks: Uint8Array[] = [];
|
||||
let streamedBytes = 0;
|
||||
for await (const chunk of stream as AsyncIterable<Uint8Array>) {
|
||||
|
|
@ -4673,7 +4674,7 @@ async function handleReadFileCall(
|
|||
};
|
||||
}
|
||||
|
||||
const stream = await strategy.getDownloadStream(req, file.filepath);
|
||||
const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file));
|
||||
const chunks: Uint8Array[] = [];
|
||||
// Use the larger binary limit as streaming cap; cheaper type-specific
|
||||
// checks happen after binary detection on the assembled buffer.
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import { getCodeExecutionRouteKey, type CodeExecutionContext } from './execution
|
|||
import { assertSkillFileContentAllowed } from '~/skills/protection';
|
||||
import { createSkillContentDigest } from './compatibility';
|
||||
import { extractInvokedSkillsFromPayload } from './run';
|
||||
import { resolveDownloadPath } from '~/storage/path';
|
||||
import { SKILL_FILE_PREFIX } from './skills';
|
||||
|
||||
const MAX_INSPECTABLE_SKILL_FILE_BYTES = 10 * 1024 * 1024;
|
||||
|
|
@ -197,7 +198,7 @@ async function collectSkillUploadFiles(
|
|||
);
|
||||
return null;
|
||||
}
|
||||
const stream = await strategy.getDownloadStream(req, file.filepath);
|
||||
const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file));
|
||||
return { stream, filename: `${SKILL_FILE_PREFIX}${skill.name}/${file.relativePath}` };
|
||||
}),
|
||||
);
|
||||
|
|
@ -402,7 +403,7 @@ async function executePrimeSkillFiles(
|
|||
logger.warn('[primeSkillFiles] No download stream for stored skill file');
|
||||
continue;
|
||||
}
|
||||
const sourceStream = await strategy.getDownloadStream(req, file.filepath);
|
||||
const sourceStream = await strategy.getDownloadStream(req, resolveDownloadPath(file));
|
||||
const buffer = await bufferSkillFileStream(sourceStream);
|
||||
if (buffer == null) {
|
||||
throwIfStoredSkillFileMustBeInspectable(req);
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { Providers } from '@librechat/agents';
|
|||
import { FileSources, mergeFileConfig, getEndpointFileConfig } from 'librechat-data-provider';
|
||||
import type { IMongoFile } from '@librechat/data-schemas';
|
||||
import type { ServerRequest, StrategyFunctions, ProcessedFile } from '~/types';
|
||||
import { resolveDownloadPath } from '~/storage/path';
|
||||
|
||||
/**
|
||||
* Extracts the configured file size limit for a specific provider from fileConfig
|
||||
|
|
@ -55,7 +56,7 @@ export async function getFileStream(
|
|||
}
|
||||
|
||||
const { getDownloadStream } = encodingMethods[source];
|
||||
const stream = await getDownloadStream(req, file.filepath);
|
||||
const stream = await getDownloadStream(req, resolveDownloadPath(file));
|
||||
let buffer: Buffer | null = await getStream.buffer(stream);
|
||||
const content = buffer.toString('base64');
|
||||
buffer = null;
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import type { ServerRequest, StrategyFunctions } from '~/types';
|
|||
import { extractSkillContent, inspectContentWithTraversal } from '~/protection';
|
||||
import { contentFilterBlockResponse } from '~/middleware/contentFilter';
|
||||
import { getDeploymentSkillIds } from './deployment';
|
||||
import { resolveDownloadPath } from '~/storage/path';
|
||||
import { resolveSkillFilePathParam } from './path';
|
||||
import { parseSkillMarkdown } from './parse';
|
||||
import { isBinaryBuffer } from './binary';
|
||||
|
|
@ -702,7 +703,7 @@ export function createSkillsHandlers(deps: SkillsHandlersDeps): {
|
|||
'Content-Disposition',
|
||||
`${isImageMime ? 'inline' : 'attachment'}; filename="${safeName}"`,
|
||||
);
|
||||
const stream = await strategy.getDownloadStream(req, file.storageKey || file.filepath);
|
||||
const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file));
|
||||
stream.on('error', (err: Error) => {
|
||||
logger.error('[downloadFile] Stream error:', err);
|
||||
if (!res.headersSent) {
|
||||
|
|
@ -736,7 +737,7 @@ export function createSkillsHandlers(deps: SkillsHandlersDeps): {
|
|||
// destroy (binary) or continue reading (text) in the same iteration.
|
||||
// N.B. breaking out of `for await...of` destroys the stream via
|
||||
// iterator.return(), so we must NOT use break + a second loop.
|
||||
const stream = await strategy.getDownloadStream(req, file.storageKey || file.filepath);
|
||||
const stream = await strategy.getDownloadStream(req, resolveDownloadPath(file));
|
||||
const chunks: Buffer[] = [];
|
||||
let totalBytes = 0;
|
||||
let binaryChecked = false;
|
||||
|
|
|
|||
30
packages/api/src/storage/__tests__/path.test.ts
Normal file
30
packages/api/src/storage/__tests__/path.test.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import { resolveDownloadPath } from '../path';
|
||||
|
||||
describe('resolveDownloadPath', () => {
|
||||
it('prefers the recorded object key over the stored URL', () => {
|
||||
expect(
|
||||
resolveDownloadPath({
|
||||
filepath: 'https://bucket.s3.amazonaws.com/uploads/u1/f.pdf?X-Amz-Expires=900',
|
||||
storageKey: 'uploads/u1/f.pdf',
|
||||
}),
|
||||
).toBe('uploads/u1/f.pdf');
|
||||
});
|
||||
|
||||
it('falls back to the path when no key was recorded', () => {
|
||||
expect(resolveDownloadPath({ filepath: '/uploads/u1/f.pdf' })).toBe('/uploads/u1/f.pdf');
|
||||
});
|
||||
|
||||
it('treats an empty or null key as absent', () => {
|
||||
expect(resolveDownloadPath({ filepath: '/uploads/u1/f.pdf', storageKey: '' })).toBe(
|
||||
'/uploads/u1/f.pdf',
|
||||
);
|
||||
expect(resolveDownloadPath({ filepath: '/uploads/u1/f.pdf', storageKey: null })).toBe(
|
||||
'/uploads/u1/f.pdf',
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps a remote URL intact for strategies that fetch it directly', () => {
|
||||
const url = 'https://firebasestorage.googleapis.com/v0/b/x/o/f.pdf?alt=media&token=t';
|
||||
expect(resolveDownloadPath({ filepath: url })).toBe(url);
|
||||
});
|
||||
});
|
||||
|
|
@ -5,3 +5,4 @@ export * from './images';
|
|||
export * from './avatar';
|
||||
export * from './metadata';
|
||||
export * from './url';
|
||||
export * from './path';
|
||||
|
|
|
|||
14
packages/api/src/storage/path.ts
Normal file
14
packages/api/src/storage/path.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import type { TFile } from 'librechat-data-provider';
|
||||
|
||||
/** A stored file as a download needs it: the recorded object key when one exists, else the URL or path. */
|
||||
export type StoredFileRef = Pick<TFile, 'filepath'> & { storageKey?: string | null };
|
||||
|
||||
/**
|
||||
* Resolves the argument for a strategy's `getDownloadStream`. S3 and CloudFront records carry
|
||||
* the object key since the region-aware storage keys landed; handing it over directly skips
|
||||
* re-deriving it from a presigned or CDN URL, which is only sound while that URL still parses.
|
||||
* Records without a key (local, Firebase, Azure, code output) fall through to `filepath` as before.
|
||||
*/
|
||||
export function resolveDownloadPath(file: StoredFileRef): string {
|
||||
return file.storageKey || file.filepath;
|
||||
}
|
||||
|
|
@ -1131,6 +1131,21 @@ describe('S3 CRUD', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('streams by the recorded key even when the stored URL no longer parses', async () => {
|
||||
const { getS3FileStream } = await import('../crud');
|
||||
const { resolveDownloadPath } = await import('~/storage/path');
|
||||
const file = {
|
||||
filepath: 'not a url: presigned link expired and was overwritten',
|
||||
storageKey: 'uploads/user123/Ársreikningur 2025.pdf',
|
||||
};
|
||||
|
||||
await getS3FileStream({} as ServerRequest, resolveDownloadPath(file));
|
||||
|
||||
const [call] = s3Mock.commandCalls(GetObjectCommand);
|
||||
expect(call.args[0].input.Key).toBe(file.storageKey);
|
||||
expect(logger.error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('handles errors when retrieving stream', async () => {
|
||||
s3Mock.on(GetObjectCommand).rejects(new Error('Stream error'));
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import type {
|
|||
UrlBuilder,
|
||||
S3FileRef,
|
||||
} from '~/storage/types';
|
||||
import type { StoredFileRef } from '~/storage/path';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import {
|
||||
assertRemoteFileURL,
|
||||
|
|
@ -280,9 +281,7 @@ export function getStorageMetadataForKey(
|
|||
};
|
||||
}
|
||||
|
||||
export function resolveStoredS3Key(
|
||||
file: Pick<TFile, 'filepath'> & { storageKey?: string | null },
|
||||
): string {
|
||||
export function resolveStoredS3Key(file: StoredFileRef): string {
|
||||
return file.storageKey || extractKeyFromS3Url(file.filepath);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue