From 0fa313443d51e630a534e02ea703ef63ac9b2de0 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 6 Sep 2026 14:35:34 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=85=20feat:=20Add=20Agent=20File=20Man?= =?UTF-8?q?agement=20APIs=20(#15666)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: expose agent file list and unlink API * feat: add agent file upload management route * fix: harden agent management file uploads * fix: coordinate agent upload limits across replicas * fix: keep agent upload coordination active * fix: validate agent uploads before processing * fix: reject unusable agent file uploads * refactor: move agent upload lock into api package --- .../middleware/limiters/uploadLimiters.js | 11 +- .../agents/__tests__/management.spec.js | 188 ++++- api/server/routes/agents/management.js | 112 ++- api/server/routes/files/files.js | 7 +- api/server/routes/files/multer.js | 66 +- api/server/routes/files/multer.spec.js | 42 +- packages/api/src/agents/files.spec.ts | 648 ++++++++++++++++++ packages/api/src/agents/files.ts | 592 ++++++++++++++++ packages/api/src/agents/index.ts | 1 + 9 files changed, 1633 insertions(+), 34 deletions(-) create mode 100644 packages/api/src/agents/files.spec.ts create mode 100644 packages/api/src/agents/files.ts diff --git a/api/server/middleware/limiters/uploadLimiters.js b/api/server/middleware/limiters/uploadLimiters.js index ab138b8679..17b5593e8a 100644 --- a/api/server/middleware/limiters/uploadLimiters.js +++ b/api/server/middleware/limiters/uploadLimiters.js @@ -29,7 +29,7 @@ const getEnvironmentVariables = () => { }; }; -const createFileUploadHandler = (ip = true) => { +const createFileUploadHandler = (ip = true, onLimit) => { const { fileUploadIpMax, fileUploadIpWindowInMinutes, @@ -48,18 +48,21 @@ const createFileUploadHandler = (ip = true) => { }; await logViolation(req, res, type, errorMessage, fileUploadViolationScore); + if (onLimit) { + return onLimit(req, res); + } res.status(429).json({ message: 'Too many file upload requests. Try again later' }); }; }; -const createFileLimiters = () => { +const createFileLimiters = ({ onLimit } = {}) => { const { fileUploadIpWindowMs, fileUploadIpMax, fileUploadUserWindowMs, fileUploadUserMax } = getEnvironmentVariables(); const ipLimiterOptions = { windowMs: fileUploadIpWindowMs, max: fileUploadIpMax, - handler: createFileUploadHandler(), + handler: createFileUploadHandler(true, onLimit), keyGenerator: removePorts, store: limiterCache('file_upload_ip_limiter'), }; @@ -67,7 +70,7 @@ const createFileLimiters = () => { const userLimiterOptions = { windowMs: fileUploadUserWindowMs, max: fileUploadUserMax, - handler: createFileUploadHandler(false), + handler: createFileUploadHandler(false, onLimit), keyGenerator: function (req) { return req.user?.id; }, diff --git a/api/server/routes/agents/__tests__/management.spec.js b/api/server/routes/agents/__tests__/management.spec.js index 3a022079b5..5d1e244103 100644 --- a/api/server/routes/agents/__tests__/management.spec.js +++ b/api/server/routes/agents/__tests__/management.spec.js @@ -31,11 +31,51 @@ const mockCreateAgentManagementDeleteHandler = jest.fn((deps) => { mockDeleteDeps = deps; return mockDelete; }); +const mockFileList = jest.fn((_req, res) => res.status(200).json({ object: 'list', data: [] })); +const mockFileRemove = jest.fn((_req, res) => + res.status(200).json({ id: 'file-one', deleted: true }), +); +const mockFileUpload = jest.fn((_req, res) => res.status(200).json({ id: 'file-uploaded' })); +const mockFileAuthorizeUpload = jest.fn((_req, _res, next) => next()); +const mockGetFileUploadConfig = jest.fn(() => ({ endpoint: 'openAI' })); +let mockFileDeps; +const mockCreateAgentManagementFileHandlers = jest.fn((deps) => { + mockFileDeps = deps; + return { + authorizeUpload: mockFileAuthorizeUpload, + getUploadConfig: mockGetFileUploadConfig, + upload: mockFileUpload, + list: mockFileList, + remove: mockFileRemove, + }; +}); const mockBrowserCreate = jest.fn(); const mockBrowserUpdate = jest.fn(); const mockCheckBan = jest.fn((_req, _res, next) => next()); -const mockConfigMiddleware = jest.fn((_req, _res, next) => next()); +const mockRequestFileConfig = { endpoints: { Moonshot: { fileLimit: 3 } } }; +const mockConfigMiddleware = jest.fn((req, _res, next) => { + req.config = { fileConfig: mockRequestFileConfig }; + next(); +}); const mockUaParser = jest.fn((_req, _res, next) => next()); +const mockUploadMiddleware = jest.fn((req, _res, next) => { + req.file = { path: '/tmp/upload', originalname: 'input.txt' }; + next(); +}); +const mockCreateMulterInstance = jest.fn().mockResolvedValue({ + single: jest.fn(() => mockUploadMiddleware), +}); +const mockGetEndpointsConfig = jest.fn().mockResolvedValue({ + Moonshot: { type: 'custom' }, +}); +const mockCheckCapability = jest.fn().mockResolvedValue(true); +const mockRunUploadExclusive = jest.fn(async (_key, task) => await task()); +const mockCreateAgentUploadLock = jest.fn(() => mockRunUploadExclusive); +let mockRateLimitIp = false; +const mockCreateFileLimiters = jest.fn(({ onLimit } = {}) => ({ + fileUploadIpLimiter: jest.fn((req, res, next) => (mockRateLimitIp ? onLimit(req, res) : next())), + fileUploadUserLimiter: jest.fn((_req, _res, next) => next()), +})); const mockRequireAgentManagementAuth = jest.fn((req, res, next) => { if (req.headers.authorization !== 'Bearer valid-token') { @@ -49,17 +89,30 @@ jest.mock('../middleware', () => ({ requireAgentManagementAuth: mockRequireAgentManagementAuth, })); jest.mock('@librechat/api', () => ({ + ioredisClient: {}, mapAgentManagementError: mockMapAgentManagementError, + restoreTenantContextFromReq: jest.fn((_req, _res, next) => next()), + createAgentUploadLock: mockCreateAgentUploadLock, createAgentManagementCreateHandler: mockCreateAgentManagementCreateHandler, createAgentManagementDeleteHandler: mockCreateAgentManagementDeleteHandler, + createAgentManagementFileHandlers: mockCreateAgentManagementFileHandlers, createAgentManagementReadHandlers: mockCreateAgentManagementReadHandlers, createAgentManagementUpdateHandler: mockCreateAgentManagementUpdateHandler, })); jest.mock('~/server/middleware', () => ({ checkBan: mockCheckBan, configMiddleware: mockConfigMiddleware, + createFileLimiters: mockCreateFileLimiters, uaParser: mockUaParser, })); +jest.mock('~/server/routes/files/multer', () => ({ + createMulterInstance: mockCreateMulterInstance, +})); +jest.mock('~/server/routes/files/files', () => ({ handleFileUpload: jest.fn() })); +jest.mock('~/server/services/Config', () => ({ + checkCapability: mockCheckCapability, + getEndpointsConfig: mockGetEndpointsConfig, +})); jest.mock('~/server/controllers/agents/v1', () => ({ createAgent: mockBrowserCreate, updateAgent: mockBrowserUpdate, @@ -73,6 +126,8 @@ jest.mock('~/models', () => ({ getRoleByName: jest.fn(), getAgentWithVersionCount: jest.fn(), getAgentManagementListByAccess: jest.fn(), + getFiles: jest.fn(), + removeAgentResourceFiles: jest.fn(), deleteAgent: jest.fn(), })); @@ -88,6 +143,11 @@ describe('Agent Management route boundary', () => { mockConfigMiddleware.mockClear(); mockUaParser.mockClear(); mockMapAgentManagementError.mockClear(); + mockFileAuthorizeUpload.mockClear(); + mockFileUpload.mockClear(); + mockUploadMiddleware.mockClear(); + mockRunUploadExclusive.mockClear(); + mockRateLimitIp = false; }); it('rejects a request before reaching management routes without machine authentication', async () => { @@ -174,4 +234,130 @@ describe('Agent Management route boundary', () => { expect(mockDeleteDeps.hasCapability).toEqual(expect.any(Function)); expect(mockDeleteDeps.deleteAgent).toEqual(expect.any(Function)); }); + + it('dispatches authenticated Agent file list and unlink requests', async () => { + const listResponse = await request(app) + .get('/api/agents/v1/agents/agent-one/files') + .set('Authorization', 'Bearer valid-token'); + const deleteResponse = await request(app) + .delete('/api/agents/v1/agents/agent-one/files/file-one') + .set('Authorization', 'Bearer valid-token'); + + expect(listResponse.status).toBe(200); + expect(deleteResponse.status).toBe(200); + expect(mockFileList).toHaveBeenCalledTimes(1); + expect(mockFileRemove).toHaveBeenCalledTimes(1); + expect(mockFileDeps.getAgentWithVersionCount).toEqual(expect.any(Function)); + expect(mockFileDeps.getFiles).toEqual(expect.any(Function)); + expect(mockFileDeps.removeAgentResourceFiles).toEqual(expect.any(Function)); + }); + + it('retries multipart initialization after a transient failure', async () => { + mockCreateMulterInstance.mockRejectedValueOnce(new Error('temporary config failure')); + mockMapAgentManagementError.mockReturnValueOnce({ + status: 500, + body: { error: { code: 'internal_error', message: 'Internal server error' } }, + }); + + const failed = await request(app) + .post('/api/agents/v1/agents/agent-one/files') + .set('Authorization', 'Bearer valid-token') + .send({ purpose: 'context' }); + const retried = await request(app) + .post('/api/agents/v1/agents/agent-one/files') + .set('Authorization', 'Bearer valid-token') + .send({ purpose: 'context' }); + + expect(failed.status).toBe(500); + expect(retried.status).toBe(200); + expect(mockCreateMulterInstance).toHaveBeenCalledTimes(2); + expect(mockCreateMulterInstance).toHaveBeenLastCalledWith({ + fileConfig: mockRequestFileConfig, + resolveEndpoint: mockGetFileUploadConfig, + uniqueTempPath: true, + }); + expect(mockFileUpload).toHaveBeenCalledTimes(1); + }); + + it('loads file configuration and dispatches authenticated multipart uploads', async () => { + const response = await request(app) + .post('/api/agents/v1/agents/agent-one/files') + .set('Authorization', 'Bearer valid-token') + .send({ purpose: 'context' }); + + expect(response.status).toBe(200); + expect(mockConfigMiddleware).toHaveBeenCalledTimes(1); + expect(mockFileAuthorizeUpload).toHaveBeenCalledTimes(1); + expect(mockFileUpload).toHaveBeenCalledTimes(1); + expect(mockFileAuthorizeUpload.mock.invocationCallOrder[0]).toBeLessThan( + mockUploadMiddleware.mock.invocationCallOrder[0], + ); + expect(mockFileDeps.processUpload).toEqual(expect.any(Function)); + expect(mockFileDeps.deleteTempFile).toEqual(expect.any(Function)); + expect(mockFileDeps.getUploadConfig).toEqual(expect.any(Function)); + expect(mockFileDeps.isUploadPurposeEnabled).toEqual(expect.any(Function)); + }); + + it('resolves upload limits from the target Agent provider', async () => { + const config = await mockFileDeps.getUploadConfig( + { + config: { + fileConfig: { + endpoints: { + Moonshot: { fileLimit: 3, totalSizeLimit: 20 }, + }, + }, + }, + }, + { provider: 'Moonshot' }, + ); + + expect(config).toMatchObject({ + endpoint: 'Moonshot', + endpointType: 'custom', + fileLimit: 3, + totalSizeLimit: 20 * 1024 * 1024, + }); + }); + + it('maps upload quota failures into the management error contract', async () => { + mockRateLimitIp = true; + + const response = await request(app) + .post('/api/agents/v1/agents/agent-one/files') + .set('Authorization', 'Bearer valid-token') + .send({ purpose: 'context' }); + + expect(response.status).toBe(429); + expect(response.body).toEqual({ + error: { + code: 'invalid_request', + message: 'Too many file upload requests. Try again later', + }, + }); + expect(mockFileAuthorizeUpload).not.toHaveBeenCalled(); + expect(mockFileUpload).not.toHaveBeenCalled(); + }); + + it('maps multipart failures into the management error contract', async () => { + mockMapAgentManagementError.mockReturnValueOnce({ + status: 400, + body: { error: { code: 'invalid_request', message: 'Invalid request' } }, + }); + mockUploadMiddleware.mockImplementationOnce((_req, _res, next) => { + next(Object.assign(new Error('File too large'), { code: 'LIMIT_FILE_SIZE' })); + }); + + const response = await request(app) + .post('/api/agents/v1/agents/agent-one/files') + .set('Authorization', 'Bearer valid-token') + .send({ purpose: 'context' }); + + expect(response.status).toBe(400); + expect(response.body).toEqual({ + error: { code: 'invalid_request', message: 'Invalid request' }, + }); + expect(mockFileUpload).not.toHaveBeenCalled(); + expect(mockMapAgentManagementError).toHaveBeenCalledWith('invalid_request'); + }); }); diff --git a/api/server/routes/agents/management.js b/api/server/routes/agents/management.js index f44bd8721e..1673c70b77 100644 --- a/api/server/routes/agents/management.js +++ b/api/server/routes/agents/management.js @@ -1,14 +1,31 @@ const express = require('express'); +const fs = require('fs').promises; const { + EModelEndpoint, + EToolResources, + AgentCapabilities, + getEndpointFileConfig, + mergeFileConfig, + resolveEndpointType, +} = require('librechat-data-provider'); +const { + createAgentUploadLock, createAgentManagementCreateHandler, createAgentManagementDeleteHandler, + createAgentManagementFileHandlers, + createAgentManagementUploadResponse, createAgentManagementReadHandlers, createAgentManagementUpdateHandler, + ioredisClient, mapAgentManagementError, + restoreTenantContextFromReq, } = require('@librechat/api'); -const { checkBan, configMiddleware } = require('~/server/middleware'); +const { checkBan, configMiddleware, createFileLimiters } = require('~/server/middleware'); const { hasCapability } = require('~/server/middleware/roles/capabilities'); const { checkPermission, findAccessibleResources } = require('~/server/services/PermissionService'); +const { createMulterInstance } = require('~/server/routes/files/multer'); +const { handleFileUpload } = require('~/server/routes/files/files'); +const { checkCapability, getEndpointsConfig } = require('~/server/services/Config'); const v1 = require('~/server/controllers/agents/v1'); const db = require('~/models'); const { requireAgentManagementAuth } = require('./middleware'); @@ -40,12 +57,105 @@ const deleteHandler = createAgentManagementDeleteHandler({ hasCapability, deleteAgent: db.deleteAgent, }); +const fileHandlers = createAgentManagementFileHandlers({ + getRoleByName: db.getRoleByName, + getAgentWithVersionCount: db.getAgentWithVersionCount, + getFiles: db.getFiles, + checkPermission, + hasCapability, + removeAgentResourceFiles: db.removeAgentResourceFiles, + processUpload: (req, res) => + handleFileUpload( + req, + createAgentManagementUploadResponse(res, req.file, req.body.tool_resource), + ), + deleteTempFile: fs.unlink, + getUploadConfig: async (req, agent) => { + const endpoint = agent.provider || EModelEndpoint.agents; + const endpointType = resolveEndpointType( + await getEndpointsConfig(req), + EModelEndpoint.agents, + endpoint, + ); + const endpointConfig = getEndpointFileConfig({ + fileConfig: mergeFileConfig(req.config?.fileConfig), + endpoint, + endpointType, + }); + return { + endpoint, + endpointType, + disabled: endpointConfig.disabled, + fileSizeLimit: endpointConfig.fileSizeLimit, + fileLimit: endpointConfig.fileLimit, + totalSizeLimit: endpointConfig.totalSizeLimit, + }; + }, + isUploadPurposeEnabled: async (req, purpose) => { + if (purpose === EToolResources.context) { + return await checkCapability(req, AgentCapabilities.context); + } + if (purpose === EToolResources.execute_code) { + return await checkCapability(req, AgentCapabilities.execute_code); + } + if (purpose === EToolResources.file_search) { + return await checkCapability(req, AgentCapabilities.file_search); + } + return true; + }, + runUploadExclusive: createAgentUploadLock({ redisClient: ioredisClient }), +}); +const sendUploadRateLimit = (_req, res) => + res.status(429).json({ + error: { + code: 'invalid_request', + message: 'Too many file upload requests. Try again later', + }, + }); +const { fileUploadIpLimiter, fileUploadUserLimiter } = createFileLimiters({ + onLimit: sendUploadRateLimit, +}); +const uploadSingleFile = async (req, res, next) => { + try { + const upload = await createMulterInstance({ + fileConfig: req.config?.fileConfig ?? null, + resolveEndpoint: fileHandlers.getUploadConfig, + uniqueTempPath: true, + }); + return upload.single('file')(req, res, next); + } catch (error) { + return next(error); + } +}; +const handleUploadError = (error, _req, res, _next) => { + const status = Number(error?.statusCode); + const isMultipartRequestError = + error?.name === 'MulterError' || + error?.code?.startsWith?.('LIMIT_') || + (Number.isInteger(status) && status >= 400 && status < 500); + const code = isMultipartRequestError ? 'invalid_request' : 'internal_error'; + const mapped = mapAgentManagementError(code); + return res.status(mapped.status).json(mapped.body); +}; router.use(requireAgentManagementAuth); router.use(checkBan); router.post('/', configMiddleware, createHandler); router.get('/', readHandlers.list); +router.post( + '/:id/files', + configMiddleware, + fileUploadIpLimiter, + fileUploadUserLimiter, + fileHandlers.authorizeUpload, + uploadSingleFile, + restoreTenantContextFromReq, + fileHandlers.upload, + handleUploadError, +); +router.get('/:id/files', fileHandlers.list); +router.delete('/:id/files/:fileId', fileHandlers.remove); router.get('/:id', readHandlers.get); router.patch('/:id', configMiddleware, updateHandler); router.delete('/:id', deleteHandler); diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js index daa13a8a9a..c3b76df66b 100644 --- a/api/server/routes/files/files.js +++ b/api/server/routes/files/files.js @@ -725,7 +725,7 @@ router.get('/download/:userId/:file_id', fileAccess, async (req, res) => { } }); -router.post('/', async (req, res) => { +const handleFileUpload = async (req, res) => { const metadata = req.body; let cleanup = true; @@ -845,6 +845,9 @@ router.post('/', async (req, res) => { sseStream.close(); } } -}); +}; + +router.post('/', handleFileUpload); module.exports = router; +module.exports.handleFileUpload = handleFileUpload; diff --git a/api/server/routes/files/multer.js b/api/server/routes/files/multer.js index 7b044f0b51..835bba56f2 100644 --- a/api/server/routes/files/multer.js +++ b/api/server/routes/files/multer.js @@ -11,22 +11,28 @@ const { } = require('librechat-data-provider'); const { getAppConfig } = require('~/server/services/Config'); -const storage = multer.diskStorage({ - destination: function (req, file, cb) { - const appConfig = req.config; - const outputPath = path.join(appConfig.paths.uploads, 'temp', req.user.id); - if (!fs.existsSync(outputPath)) { - fs.mkdirSync(outputPath, { recursive: true }); - } - cb(null, outputPath); - }, - filename: function (req, file, cb) { - req.file_id = crypto.randomUUID(); - file.originalname = decodeURIComponent(file.originalname); - const sanitizedFilename = sanitizeFilename(file.originalname); - cb(null, sanitizedFilename); - }, -}); +const createStorage = ({ uniqueTempPath = false } = {}) => + multer.diskStorage({ + destination: function (req, file, cb) { + const appConfig = req.config; + const outputPath = path.join(appConfig.paths.uploads, 'temp', req.user.id); + if (!fs.existsSync(outputPath)) { + fs.mkdirSync(outputPath, { recursive: true }); + } + cb(null, outputPath); + }, + filename: function (req, file, cb) { + req.file_id = crypto.randomUUID(); + file.originalname = decodeURIComponent(file.originalname); + const sanitizedFilename = sanitizeFilename(file.originalname); + const stagedFilename = uniqueTempPath + ? sanitizeFilename(`${req.file_id}-${sanitizedFilename}`) + : sanitizedFilename; + cb(null, stagedFilename); + }, + }); + +const storage = createStorage(); const importFileFilter = (req, file, cb) => { if (file.mimetype === 'application/json') { @@ -50,7 +56,7 @@ const normalizeUploadMimeType = (file) => { * * @param {import('librechat-data-provider').FileConfig | undefined} customFileConfig */ -const createFileFilter = (customFileConfig) => { +const createFileFilter = (customFileConfig, resolveEndpoint) => { /** * @param {ServerRequest} req * @param {Express.Multer.File} @@ -67,8 +73,9 @@ const createFileFilter = (customFileConfig) => { return cb(null, true); } - const endpoint = req.body.endpoint; - const endpointType = req.body.endpointType; + const resolved = resolveEndpoint?.(req); + const endpoint = resolved?.endpoint ?? req.body.endpoint; + const endpointType = resolved?.endpointType ?? req.body.endpointType; const endpointFileConfig = getEndpointFileConfig({ fileConfig: customFileConfig, endpoint, @@ -88,15 +95,24 @@ const createFileFilter = (customFileConfig) => { return fileFilter; }; -const createMulterInstance = async () => { - const appConfig = await getAppConfig(); - const fileConfig = mergeFileConfig(appConfig?.fileConfig); - const fileFilter = createFileFilter(fileConfig); +const createMulterInstance = async (options = {}) => { + const { resolveEndpoint, uniqueTempPath = false } = options; + const appConfig = Object.prototype.hasOwnProperty.call(options, 'fileConfig') + ? null + : await getAppConfig(); + const fileConfig = mergeFileConfig(options.fileConfig ?? appConfig?.fileConfig); + const fileFilter = createFileFilter(fileConfig, resolveEndpoint); return multer({ - storage, + storage: uniqueTempPath ? createStorage({ uniqueTempPath: true }) : storage, fileFilter, limits: { fileSize: fileConfig.serverFileSizeLimit }, }); }; -module.exports = { createMulterInstance, storage, importFileFilter, createFileFilter }; +module.exports = { + createMulterInstance, + createStorage, + storage, + importFileFilter, + createFileFilter, +}; diff --git a/api/server/routes/files/multer.spec.js b/api/server/routes/files/multer.spec.js index a8b9e386b6..368edbe72d 100644 --- a/api/server/routes/files/multer.spec.js +++ b/api/server/routes/files/multer.spec.js @@ -4,7 +4,13 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); const crypto = require('crypto'); -const { createMulterInstance, storage, importFileFilter, createFileFilter } = require('./multer'); +const { + createMulterInstance, + createStorage, + storage, + importFileFilter, + createFileFilter, +} = require('./multer'); // Mock only the config service that requires external dependencies jest.mock('~/server/services/Config', () => ({ @@ -77,6 +83,16 @@ describe('Multer Configuration', () => { }); describe('filename function', () => { + it('uses the request file ID to make management staging paths unique', (done) => { + const uniqueStorage = createStorage({ uniqueTempPath: true }); + + uniqueStorage.getFilename(mockReq, mockFile, (err, filename) => { + expect(err).toBeNull(); + expect(filename).toBe(`${mockReq.file_id}-test-file.jpg`); + done(); + }); + }); + it('should generate a UUID for req.file_id', (done) => { const cb = jest.fn((err, filename) => { expect(err).toBeNull(); @@ -302,6 +318,30 @@ describe('Multer Configuration', () => { fileFilter(mockReq, zipFile, cb); }); + it('uses a server-selected endpoint before multipart fields are parsed', (done) => { + const { mergeFileConfig } = require('librechat-data-provider'); + const fileFilter = createFileFilter( + mergeFileConfig({ + endpoints: { + agents: { supportedMimeTypes: ['text/plain'] }, + default: { supportedMimeTypes: ['application/pdf'] }, + }, + }), + () => ({ endpoint: 'agents' }), + ); + const textFile = { + ...mockFile, + originalname: 'notes.txt', + mimetype: 'text/plain', + }; + + fileFilter({ ...mockReq, body: {} }, textFile, (err, result) => { + expect(err).toBeNull(); + expect(result).toBe(true); + done(); + }); + }); + it.each(['application/x-shellscript', 'text/x-shellscript'])( 'should normalize %s to application/x-sh and accept the upload', (reportedType) => { diff --git a/packages/api/src/agents/files.spec.ts b/packages/api/src/agents/files.spec.ts new file mode 100644 index 0000000000..35272464c8 --- /dev/null +++ b/packages/api/src/agents/files.spec.ts @@ -0,0 +1,648 @@ +import { Types } from 'mongoose'; +import { SystemCapabilities } from '@librechat/data-schemas'; +import { + EToolResources, + PermissionBits, + Permissions, + PermissionTypes, + ResourceType, +} from 'librechat-data-provider'; +import type { IRole, IUser } from '@librechat/data-schemas'; +import type { Request, Response } from 'express'; +import type { AgentManagementFileDeps } from './files'; +import { + createAgentManagementFileHandlers, + createAgentManagementUploadResponse, + createAgentUploadLock, +} from './files'; + +jest.mock('@librechat/data-schemas', () => { + return { + ResourceCapabilityMap: { agent: 'MANAGE_AGENTS' }, + SystemCapabilities: { MANAGE_AGENTS: 'MANAGE_AGENTS' }, + logger: { warn: jest.fn(), error: jest.fn() }, + }; +}); + +const tenantId = 'tenant-a'; +const user = { + id: new Types.ObjectId().toString(), + tenantId, + role: 'USER', +} as IUser; +const objectId = new Types.ObjectId(); +const agent = { + _id: objectId, + id: 'agent-one', + provider: 'Moonshot', + tool_resources: { + context: { file_ids: ['file-context', 'file-shared'] }, + file_search: { file_ids: ['file-search', 'file-shared'] }, + execute_code: { file_ids: ['file-code'] }, + image_edit: { file_ids: ['file-image'] }, + ocr: { file_ids: ['file-ocr'] }, + }, +}; + +function makeRequest(params: Record): Request { + return { user, params, headers: {} } as unknown as Request; +} + +function makeResponse(): Response { + const response = { + status: jest.fn(), + json: jest.fn(), + }; + response.status.mockReturnValue(response); + response.json.mockReturnValue(response); + return response as unknown as Response; +} + +function makeDeps(overrides: Partial = {}): AgentManagementFileDeps { + return { + getRoleByName: jest.fn().mockResolvedValue({ + permissions: { + [PermissionTypes.AGENTS]: { + [Permissions.USE]: true, + [Permissions.CREATE]: true, + }, + }, + } as unknown as IRole), + getAgentWithVersionCount: jest.fn().mockResolvedValue(agent), + getFiles: jest.fn().mockResolvedValue([ + { + file_id: 'file-shared', + filename: 'shared.txt', + bytes: 12, + type: 'text/plain', + createdAt: new Date('2026-09-01T10:00:00.000Z'), + }, + ]), + checkPermission: jest.fn().mockResolvedValue(true), + hasCapability: jest.fn().mockResolvedValue(false), + removeAgentResourceFiles: jest.fn().mockResolvedValue(agent), + processUpload: jest.fn().mockResolvedValue(undefined), + deleteTempFile: jest.fn().mockResolvedValue(undefined), + getUploadConfig: jest.fn().mockResolvedValue({ + endpoint: 'Moonshot', + endpointType: 'custom', + fileLimit: 100, + totalSizeLimit: 1_000_000, + }), + isUploadPurposeEnabled: jest.fn().mockResolvedValue(true), + runUploadExclusive: async (_key, task) => await task(), + ...overrides, + }; +} + +async function authorizeUpload( + handlers: ReturnType, + request: Request, + response: Response, +) { + const next = jest.fn(); + await handlers.authorizeUpload(request, response, next); + expect(next).toHaveBeenCalledTimes(1); +} + +describe('Agent Management file handlers', () => { + it('holds and releases the shared upload lock around processing', async () => { + const redisClient = { + set: jest.fn().mockResolvedValue('OK'), + eval: jest.fn().mockResolvedValue(1), + }; + const task = jest.fn().mockResolvedValue('uploaded'); + + await expect( + createAgentUploadLock({ redisClient })('tenant-a:agent-one:context', task), + ).resolves.toBe('uploaded'); + + expect(redisClient.set).toHaveBeenCalledWith( + 'agent-management:file-upload:tenant-a:agent-one:context', + expect.any(String), + 'PX', + 10 * 60 * 1000, + 'NX', + ); + expect(task).toHaveBeenCalledTimes(1); + expect(redisClient.eval).toHaveBeenCalledWith( + expect.stringContaining("redis.call('DEL', KEYS[1])"), + 1, + 'agent-management:file-upload:tenant-a:agent-one:context', + expect.any(String), + ); + }); + + it('renews the shared upload lock while processing is still running', async () => { + jest.useFakeTimers(); + const redisClient = { + set: jest.fn().mockResolvedValue('OK'), + eval: jest.fn().mockResolvedValue(1), + }; + let completeUpload: (value: string) => void = () => {}; + const task = jest.fn( + () => + new Promise((resolve) => { + completeUpload = resolve; + }), + ); + + try { + const pendingUpload = createAgentUploadLock({ redisClient })( + 'tenant-a:agent-one:context', + task, + ); + await Promise.resolve(); + jest.advanceTimersByTime((10 * 60 * 1000) / 3); + await Promise.resolve(); + + expect(redisClient.eval).toHaveBeenCalledWith( + expect.stringContaining("redis.call('PEXPIRE', KEYS[1], ARGV[2])"), + 1, + 'agent-management:file-upload:tenant-a:agent-one:context', + expect.any(String), + 10 * 60 * 1000, + ); + + completeUpload('uploaded'); + await expect(pendingUpload).resolves.toBe('uploaded'); + } finally { + jest.useRealTimers(); + } + }); + + it('projects upload success through the management metadata allowlist', () => { + const response = makeResponse(); + const file = { + originalname: 'input.txt', + size: 12, + mimetype: 'text/plain', + } as Express.Multer.File; + + createAgentManagementUploadResponse(response, file, EToolResources.context).status(200).json({ + file_id: 'file-one', + filename: 'stored.txt', + filepath: '/private/storage/path', + storageKey: 'private-key', + tenantId: 'tenant-a', + bytes: 10, + type: 'text/plain', + }); + + expect(response.json).toHaveBeenCalledWith({ + id: 'file-one', + object: 'agent.file', + filename: 'stored.txt', + bytes: 10, + mime_type: 'text/plain', + purposes: [EToolResources.context], + created_at: null, + }); + }); + + it('rejects an incomplete shared-uploader success response', () => { + const response = makeResponse(); + const file = { + originalname: 'input.txt', + size: 12, + mimetype: 'text/plain', + } as Express.Multer.File; + + createAgentManagementUploadResponse(response, file, EToolResources.context).status(200).json({ + filename: 'stored.txt', + }); + + expect(response.status).toHaveBeenCalledWith(500); + expect(response.json).toHaveBeenCalledWith({ + error: { code: 'internal_error', message: 'Internal server error' }, + }); + }); + + it.each([EToolResources.context, EToolResources.file_search, EToolResources.execute_code])( + 'routes a %s upload through the shared browser pipeline', + async (purpose) => { + const deps = makeDeps(); + const response = makeResponse(); + const request = makeRequest({ id: 'agent-one' }); + const handlers = createAgentManagementFileHandlers(deps); + await authorizeUpload(handlers, request, response); + request.body = { purpose }; + request.file = { + path: '/tmp/upload', + originalname: 'input.txt', + size: 12, + } as Express.Multer.File; + + await handlers.upload(request, response); + + expect(deps.processUpload).toHaveBeenCalledWith(request, response); + expect(request.body).toEqual({ + endpoint: 'Moonshot', + endpointType: 'custom', + agent_id: 'agent-one', + tool_resource: purpose, + }); + expect(request.headers.accept).toBe('application/json'); + expect(deps.deleteTempFile).not.toHaveBeenCalled(); + }, + ); + + it('rejects unsupported purposes and removes the temporary upload', async () => { + const deps = makeDeps(); + const response = makeResponse(); + const request = makeRequest({ id: 'agent-one' }); + request.body = { purpose: 'provider_storage' }; + request.file = { path: '/tmp/rejected', originalname: 'input.txt' } as Express.Multer.File; + + await createAgentManagementFileHandlers(deps).upload(request, response); + + expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/rejected'); + expect(deps.processUpload).not.toHaveBeenCalled(); + expect(response.status).toHaveBeenCalledWith(400); + }); + + it('rejects a cross-tenant Agent before staging an upload', async () => { + const deps = makeDeps({ getAgentWithVersionCount: jest.fn().mockResolvedValue(null) }); + const response = makeResponse(); + const request = makeRequest({ id: 'agent-other-tenant' }); + const next = jest.fn(); + + await createAgentManagementFileHandlers(deps).authorizeUpload(request, response, next); + + expect(next).not.toHaveBeenCalled(); + expect(deps.deleteTempFile).not.toHaveBeenCalled(); + expect(deps.processUpload).not.toHaveBeenCalled(); + expect(response.status).toHaveBeenCalledWith(404); + }); + + it('reports cleanup failures with the management error contract', async () => { + const deps = makeDeps({ + deleteTempFile: jest.fn().mockRejectedValue(new Error('cleanup failed')), + getFiles: jest.fn().mockResolvedValue([ + { file_id: 'file-context', bytes: 8 }, + { file_id: 'file-shared', bytes: 12 }, + ]), + getUploadConfig: jest.fn().mockResolvedValue({ + endpoint: 'Moonshot', + endpointType: 'custom', + fileLimit: 2, + }), + }); + const response = makeResponse(); + const request = makeRequest({ id: 'agent-one' }); + const handlers = createAgentManagementFileHandlers(deps); + await authorizeUpload(handlers, request, response); + request.body = { purpose: EToolResources.context }; + request.file = { + path: '/tmp/rejected', + originalname: 'input.txt', + size: 12, + } as Express.Multer.File; + + await handlers.upload(request, response); + + expect(response.status).toHaveBeenCalledWith(500); + expect(response.json).toHaveBeenCalledWith({ + error: { code: 'internal_error', message: 'Internal server error' }, + }); + }); + + it('serializes uploads when enforcing the per-purpose aggregate file limit', async () => { + let attachedCount = 1; + const getAgentWithVersionCount = jest.fn().mockImplementation(async () => ({ + ...agent, + tool_resources: { + ...agent.tool_resources, + context: { + file_ids: Array.from({ length: attachedCount }, (_, index) => `file-${index}`), + }, + }, + })); + const processUpload = jest.fn().mockImplementation(async () => { + attachedCount += 1; + }); + const deps = makeDeps({ + getAgentWithVersionCount, + getFiles: jest.fn().mockImplementation(async () => + Array.from({ length: attachedCount }, (_, index) => ({ + file_id: `file-${index}`, + bytes: 1, + })), + ), + processUpload, + getUploadConfig: jest.fn().mockResolvedValue({ endpoint: 'Moonshot', fileLimit: 2 }), + }); + const handlers = createAgentManagementFileHandlers(deps); + const firstResponse = makeResponse(); + const secondResponse = makeResponse(); + const firstRequest = makeRequest({ id: 'agent-one' }); + const secondRequest = makeRequest({ id: 'agent-one' }); + await authorizeUpload(handlers, firstRequest, firstResponse); + await authorizeUpload(handlers, secondRequest, secondResponse); + firstRequest.body = { purpose: EToolResources.context }; + secondRequest.body = { purpose: EToolResources.context }; + firstRequest.file = { + path: '/tmp/first', + originalname: 'first.txt', + size: 1, + } as Express.Multer.File; + secondRequest.file = { + path: '/tmp/second', + originalname: 'second.txt', + size: 1, + } as Express.Multer.File; + + await Promise.all([ + handlers.upload(firstRequest, firstResponse), + handlers.upload(secondRequest, secondResponse), + ]); + + expect(processUpload).toHaveBeenCalledTimes(1); + expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/second'); + expect(secondResponse.status).toHaveBeenCalledWith(400); + }); + + it('rejects an upload that would exceed the per-purpose aggregate byte limit', async () => { + const deps = makeDeps({ + getFiles: jest.fn().mockResolvedValue([ + { file_id: 'file-context', bytes: 8 }, + { file_id: 'file-shared', bytes: 12 }, + ]), + getUploadConfig: jest.fn().mockResolvedValue({ + endpoint: 'Moonshot', + totalSizeLimit: 24, + }), + }); + const response = makeResponse(); + const request = makeRequest({ id: 'agent-one' }); + const handlers = createAgentManagementFileHandlers(deps); + await authorizeUpload(handlers, request, response); + request.body = { purpose: EToolResources.context }; + request.file = { + path: '/tmp/too-large-in-aggregate', + originalname: 'input.txt', + size: 5, + } as Express.Multer.File; + + await handlers.upload(request, response); + + expect(deps.processUpload).not.toHaveBeenCalled(); + expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/too-large-in-aggregate'); + expect(response.status).toHaveBeenCalledWith(400); + }); + + it('does not count dangling legacy file references toward the aggregate file limit', async () => { + const deps = makeDeps({ + getAgentWithVersionCount: jest.fn().mockResolvedValue({ + ...agent, + tool_resources: { context: { file_ids: ['file-live', 'file-missing'] } }, + }), + getFiles: jest.fn().mockResolvedValue([{ file_id: 'file-live', bytes: 8 }]), + getUploadConfig: jest.fn().mockResolvedValue({ endpoint: 'Moonshot', fileLimit: 2 }), + }); + const response = makeResponse(); + const request = makeRequest({ id: 'agent-one' }); + const handlers = createAgentManagementFileHandlers(deps); + await authorizeUpload(handlers, request, response); + request.body = { purpose: EToolResources.context }; + request.file = { + path: '/tmp/accepted-with-dangling-reference', + originalname: 'input.txt', + size: 5, + } as Express.Multer.File; + + await handlers.upload(request, response); + + expect(deps.processUpload).toHaveBeenCalledTimes(1); + expect(deps.deleteTempFile).not.toHaveBeenCalled(); + }); + + it('rejects provider file-size violations before shared upload processing', async () => { + const deps = makeDeps({ + getUploadConfig: jest.fn().mockResolvedValue({ + endpoint: 'Moonshot', + fileSizeLimit: 4, + }), + }); + const response = makeResponse(); + const request = makeRequest({ id: 'agent-one' }); + const handlers = createAgentManagementFileHandlers(deps); + await authorizeUpload(handlers, request, response); + request.body = { purpose: EToolResources.context }; + request.file = { + path: '/tmp/provider-file-too-large', + originalname: 'input.txt', + size: 5, + } as Express.Multer.File; + + await handlers.upload(request, response); + + expect(deps.processUpload).not.toHaveBeenCalled(); + expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/provider-file-too-large'); + expect(response.status).toHaveBeenCalledWith(400); + }); + + it('rejects empty files before shared upload processing', async () => { + const deps = makeDeps(); + const response = makeResponse(); + const request = makeRequest({ id: 'agent-one' }); + const handlers = createAgentManagementFileHandlers(deps); + await authorizeUpload(handlers, request, response); + request.body = { purpose: EToolResources.context }; + request.file = { + path: '/tmp/empty-file', + originalname: 'empty.txt', + size: 0, + } as Express.Multer.File; + + await handlers.upload(request, response); + + expect(deps.processUpload).not.toHaveBeenCalled(); + expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/empty-file'); + expect(response.status).toHaveBeenCalledWith(400); + }); + + it.each([EToolResources.context, EToolResources.execute_code])( + 'rejects disabled %s Agent uploads before shared processing', + async (purpose) => { + const deps = makeDeps({ isUploadPurposeEnabled: jest.fn().mockResolvedValue(false) }); + const response = makeResponse(); + const request = makeRequest({ id: 'agent-one' }); + const handlers = createAgentManagementFileHandlers(deps); + await authorizeUpload(handlers, request, response); + request.body = { purpose }; + request.file = { + path: '/tmp/disabled-purpose', + originalname: 'input.txt', + size: 5, + } as Express.Multer.File; + + await handlers.upload(request, response); + + expect(deps.processUpload).not.toHaveBeenCalled(); + expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/disabled-purpose'); + expect(response.status).toHaveBeenCalledWith(400); + }, + ); + + it('lists safe file metadata with every attached purpose in the authenticated tenant', async () => { + const deps = makeDeps(); + const response = makeResponse(); + + await createAgentManagementFileHandlers(deps).list(makeRequest({ id: 'agent-one' }), response); + + expect(deps.getAgentWithVersionCount).toHaveBeenCalledWith({ id: 'agent-one', tenantId }); + expect(deps.checkPermission).toHaveBeenCalledWith({ + userId: user.id, + role: user.role, + resourceType: ResourceType.AGENT, + resourceId: objectId, + requiredPermission: PermissionBits.EDIT, + }); + expect(deps.getFiles).toHaveBeenCalledWith( + { + file_id: { + $in: [ + 'file-context', + 'file-shared', + 'file-search', + 'file-code', + 'file-image', + 'file-ocr', + ], + }, + tenantId, + }, + null, + { text: 0 }, + ); + expect(response.json).toHaveBeenCalledWith({ + object: 'list', + data: [ + { + id: 'file-shared', + object: 'agent.file', + filename: 'shared.txt', + bytes: 12, + mime_type: 'text/plain', + purposes: [EToolResources.context, EToolResources.file_search], + created_at: '2026-09-01T10:00:00.000Z', + }, + ], + }); + }); + + it('unlinks a file from every purpose without deleting shared storage', async () => { + const deps = makeDeps(); + const response = makeResponse(); + + await createAgentManagementFileHandlers(deps).remove( + makeRequest({ id: 'agent-one', fileId: 'file-shared' }), + response, + ); + + expect(deps.removeAgentResourceFiles).toHaveBeenCalledWith({ + agent_id: 'agent-one', + files: [ + { tool_resource: EToolResources.context, file_id: 'file-shared' }, + { tool_resource: EToolResources.file_search, file_id: 'file-shared' }, + ], + }); + expect(response.status).toHaveBeenCalledWith(200); + expect(response.json).toHaveBeenCalledWith({ id: 'file-shared', deleted: true }); + }); + + it.each([ + [EToolResources.image_edit, 'file-image'], + [EToolResources.ocr, 'file-ocr'], + ])('unlinks legacy %s attachments', async (purpose, fileId) => { + const deps = makeDeps(); + const response = makeResponse(); + + await createAgentManagementFileHandlers(deps).remove( + makeRequest({ id: 'agent-one', fileId }), + response, + ); + + expect(deps.removeAgentResourceFiles).toHaveBeenCalledWith({ + agent_id: 'agent-one', + files: [{ tool_resource: purpose, file_id: fileId }], + }); + expect(response.status).toHaveBeenCalledWith(200); + }); + + it('checks mutation capability before looking up the agent', async () => { + const getAgentWithVersionCount = jest.fn().mockResolvedValue(agent); + const deps = makeDeps({ + getAgentWithVersionCount, + getRoleByName: jest.fn().mockResolvedValue({ + permissions: { + [PermissionTypes.AGENTS]: { + [Permissions.USE]: true, + [Permissions.CREATE]: false, + }, + }, + } as unknown as IRole), + }); + const response = makeResponse(); + + await createAgentManagementFileHandlers(deps).remove( + makeRequest({ id: 'agent-one', fileId: 'file-shared' }), + response, + ); + + expect(response.status).toHaveBeenCalledWith(403); + expect(getAgentWithVersionCount).not.toHaveBeenCalled(); + expect(deps.removeAgentResourceFiles).not.toHaveBeenCalled(); + }); + + it('fails closed when the agent is outside the authenticated tenant', async () => { + const deps = makeDeps({ getAgentWithVersionCount: jest.fn().mockResolvedValue(null) }); + const response = makeResponse(); + + await createAgentManagementFileHandlers(deps).list( + makeRequest({ id: 'agent-other-tenant' }), + response, + ); + + expect(response.status).toHaveBeenCalledWith(404); + expect(deps.getFiles).not.toHaveBeenCalled(); + }); + + it('requires EDIT access unless the caller has the management capability', async () => { + const deps = makeDeps({ checkPermission: jest.fn().mockResolvedValue(false) }); + const response = makeResponse(); + + await createAgentManagementFileHandlers(deps).list(makeRequest({ id: 'agent-one' }), response); + + expect(response.status).toHaveBeenCalledWith(403); + expect(deps.getFiles).not.toHaveBeenCalled(); + }); + + it('uses the manage-agents capability as the existing ACL bypass', async () => { + const deps = makeDeps({ hasCapability: jest.fn().mockResolvedValue(true) }); + const response = makeResponse(); + + await createAgentManagementFileHandlers(deps).list(makeRequest({ id: 'agent-one' }), response); + + expect(deps.hasCapability).toHaveBeenCalledWith(user, SystemCapabilities.MANAGE_AGENTS); + expect(deps.checkPermission).not.toHaveBeenCalled(); + expect(response.status).toHaveBeenCalledWith(200); + }); + + it('returns not found without mutating when the file is not attached', async () => { + const deps = makeDeps(); + const response = makeResponse(); + + await createAgentManagementFileHandlers(deps).remove( + makeRequest({ id: 'agent-one', fileId: 'file-missing' }), + response, + ); + + expect(response.status).toHaveBeenCalledWith(404); + expect(response.json).toHaveBeenCalledWith({ + error: { code: 'not_found', message: 'File not found' }, + }); + expect(deps.removeAgentResourceFiles).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/agents/files.ts b/packages/api/src/agents/files.ts new file mode 100644 index 0000000000..1614940bf5 --- /dev/null +++ b/packages/api/src/agents/files.ts @@ -0,0 +1,592 @@ +import { randomUUID } from 'node:crypto'; +import { logger, ResourceCapabilityMap } from '@librechat/data-schemas'; +import { + EToolResources, + PermissionBits, + Permissions, + PermissionTypes, + ResourceType, +} from 'librechat-data-provider'; +import type { IRole, IUser, SystemCapability } from '@librechat/data-schemas'; +import type { NextFunction, Request, Response } from 'express'; +import type { Types } from 'mongoose'; +import type { AgentManagementProjectionSource } from './management'; +import { checkAccessWithRequestCache } from '../middleware/access'; +import { mapAgentManagementError } from './management'; + +type AgentUploadPurpose = + | EToolResources.context + | EToolResources.file_search + | EToolResources.execute_code; + +type AgentFilePurpose = AgentUploadPurpose | EToolResources.image_edit | EToolResources.ocr; + +const UPLOAD_PURPOSES: readonly AgentUploadPurpose[] = [ + EToolResources.context, + EToolResources.file_search, + EToolResources.execute_code, +] as const; + +const FILE_PURPOSES: readonly AgentFilePurpose[] = [ + ...UPLOAD_PURPOSES, + EToolResources.image_edit, + EToolResources.ocr, +] as const; + +type AgentManagementFileRecord = { + file_id: string; + filename: string; + bytes: number; + type: string; + createdAt?: Date; +}; +type AgentManagementFile = { + id: string; + object: 'agent.file'; + filename: string; + bytes: number; + mime_type: string; + purposes: AgentFilePurpose[]; + created_at: string | null; +}; +type AgentManagementUploadBody = { + file_id?: string; + filename?: string; + bytes?: number; + type?: string; + createdAt?: string | Date; +}; +type AgentManagementFileAgent = AgentManagementProjectionSource & { + _id: Types.ObjectId; + provider?: string; + tool_resources?: Partial>; +}; +type AgentUploadConfig = { + endpoint: string; + endpointType?: string; + disabled?: boolean; + fileSizeLimit?: number; + fileLimit?: number; + totalSizeLimit?: number; +}; +type AgentUploadLockRedisClient = { + set: ( + key: string, + value: string, + expiryMode: 'PX', + ttlMs: number, + condition: 'NX', + ) => Promise; + eval: ( + script: string, + numberOfKeys: number, + key: string, + ...args: Array + ) => Promise; +}; + +const AGENT_UPLOAD_LOCK_TTL_MS = 10 * 60 * 1000; +const AGENT_UPLOAD_LOCK_WAIT_MS = 2 * 60 * 1000; +const AGENT_UPLOAD_LOCK_RENEW_MS = Math.floor(AGENT_UPLOAD_LOCK_TTL_MS / 3); +const releaseUploadLockScript = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +`; +const renewUploadLockScript = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('PEXPIRE', KEYS[1], ARGV[2]) +end +return 0 +`; + +export interface AgentManagementFileDeps { + getRoleByName: (roleName: string, fieldsToSelect?: string | string[]) => Promise; + getAgentWithVersionCount: (search: { + id: string; + tenantId: string; + }) => Promise; + getFiles: ( + filter: { file_id: { $in: string[] }; tenantId: string }, + sort?: null, + projection?: Record, + ) => Promise; + checkPermission: (params: { + userId: string; + role?: string; + resourceType: ResourceType; + resourceId: Types.ObjectId; + requiredPermission: PermissionBits; + }) => Promise; + hasCapability: (user: IUser, capability: SystemCapability) => Promise; + removeAgentResourceFiles: (params: { + agent_id: string; + files: Array<{ tool_resource: AgentFilePurpose; file_id: string }>; + }) => Promise; + processUpload: (req: Request, res: Response) => Promise; + deleteTempFile: (path: string) => Promise; + getUploadConfig: (req: Request, agent: AgentManagementFileAgent) => Promise; + isUploadPurposeEnabled: (req: Request, purpose: AgentUploadPurpose) => Promise; + runUploadExclusive: (key: string, task: () => Promise) => Promise; +} + +/** Serialize an Agent purpose's aggregate-limit check and upload across API replicas. */ +export function createAgentUploadLock({ + redisClient, +}: { + redisClient: AgentUploadLockRedisClient | null; +}): AgentManagementFileDeps['runUploadExclusive'] { + return async function withAgentUploadLock(key: string, task: () => Promise): Promise { + if (!redisClient) { + return await task(); + } + const lockKey = `agent-management:file-upload:${key}`; + const token = randomUUID(); + const deadline = Date.now() + AGENT_UPLOAD_LOCK_WAIT_MS; + while ((await redisClient.set(lockKey, token, 'PX', AGENT_UPLOAD_LOCK_TTL_MS, 'NX')) !== 'OK') { + if (Date.now() >= deadline) { + throw new Error('Timed out waiting for Agent file upload lock'); + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + let stopped = false; + let renewalTimer: ReturnType; + const renewLease = async () => { + try { + const renewed = await redisClient.eval( + renewUploadLockScript, + 1, + lockKey, + token, + AGENT_UPLOAD_LOCK_TTL_MS, + ); + if (renewed !== 1) { + logger.warn('[AgentManagement] Lost Agent file upload lock before processing completed'); + } + } catch (error) { + logger.warn('[AgentManagement] Failed to renew Agent file upload lock', error); + } finally { + if (!stopped) { + renewalTimer = setTimeout(renewLease, AGENT_UPLOAD_LOCK_RENEW_MS); + renewalTimer.unref?.(); + } + } + }; + renewalTimer = setTimeout(renewLease, AGENT_UPLOAD_LOCK_RENEW_MS); + renewalTimer.unref?.(); + + try { + return await task(); + } finally { + stopped = true; + clearTimeout(renewalTimer); + try { + await redisClient.eval(releaseUploadLockScript, 1, lockKey, token); + } catch (error) { + logger.warn('[AgentManagement] Failed to release Agent file upload lock', error); + } + } + }; +} + +function sendError(res: Response, code: Parameters[0]) { + const mapped = mapAgentManagementError(code); + return res.status(mapped.status).json(mapped.body); +} + +function sendFileNotFound(res: Response) { + return res.status(404).json({ error: { code: 'not_found', message: 'File not found' } }); +} + +function getUploadErrorCode(status: number): Parameters[0] { + if (status === 403) { + return 'permission_denied'; + } + if (status === 404) { + return 'not_found'; + } + if (status >= 400 && status < 500) { + return 'invalid_request'; + } + return 'internal_error'; +} + +function getUploadCreatedAt(value: string | Date | undefined): string | null { + if (value instanceof Date) { + return value.toISOString(); + } + if (typeof value === 'string') { + return value; + } + return null; +} + +/** Restrict the shared browser uploader's response to the management file contract. */ +export function createAgentManagementUploadResponse( + res: Response, + file: Express.Multer.File, + purpose: AgentUploadPurpose, +): Response { + let status = 200; + const response = Object.create(res) as Response; + response.status = (code: number) => { + status = code; + return response; + }; + response.json = (body: AgentManagementUploadBody) => { + if (status < 200 || status >= 300) { + return sendError(res, getUploadErrorCode(status)); + } + if (typeof body.file_id !== 'string' || body.file_id.length === 0) { + return sendError(res, 'internal_error'); + } + return res.status(status).json({ + id: body.file_id, + object: 'agent.file', + filename: body.filename ?? file.originalname, + bytes: body.bytes ?? file.size, + mime_type: body.type ?? file.mimetype, + purposes: [purpose], + created_at: getUploadCreatedAt(body.createdAt), + }); + }; + return response; +} + +async function canUseAgents( + req: Request, + user: IUser, + permissions: Permissions[], + deps: AgentManagementFileDeps, +) { + return await checkAccessWithRequestCache({ + req, + user, + permissionType: PermissionTypes.AGENTS, + permissions, + getRoleByName: deps.getRoleByName, + }); +} + +async function hasManageAgentsCapability(user: IUser, deps: AgentManagementFileDeps) { + const capability = ResourceCapabilityMap[ResourceType.AGENT]; + try { + return capability != null && (await deps.hasCapability(user, capability)); + } catch (error) { + logger.warn( + `[AgentManagement] Agent capability check failed, denying file access bypass: ${(error as Error).message}`, + ); + return false; + } +} + +async function canEditAgentFiles( + user: IUser, + agent: AgentManagementFileAgent, + deps: AgentManagementFileDeps, +) { + if (await hasManageAgentsCapability(user, deps)) { + return true; + } + return await deps.checkPermission({ + userId: user.id, + role: user.role, + resourceType: ResourceType.AGENT, + resourceId: agent._id, + requiredPermission: PermissionBits.EDIT, + }); +} + +function getFilePurposes(agent: AgentManagementFileAgent): Map { + const purposes = new Map(); + for (const purpose of FILE_PURPOSES) { + for (const fileId of agent.tool_resources?.[purpose]?.file_ids ?? []) { + purposes.set(fileId, [...(purposes.get(fileId) ?? []), purpose]); + } + } + return purposes; +} + +/** Machine-authenticated Agent file listing and unlink handlers. */ +export function createAgentManagementFileHandlers(deps: AgentManagementFileDeps): { + authorizeUpload: (req: Request, res: Response, next: NextFunction) => Promise; + getUploadConfig: ( + req: Request, + ) => Pick | undefined; + upload: (req: Request, res: Response) => Promise; + list: (req: Request, res: Response) => Promise; + remove: (req: Request, res: Response) => Promise; +} { + const authorizedUploads = new WeakMap< + Request, + { + agent: AgentManagementFileAgent; + tenantId: string; + uploadConfig: AgentUploadConfig; + } + >(); + const uploadQueues = new Map>(); + + async function getAuthorizedAgent(req: Request, permissions: Permissions[]) { + const user = req.user as IUser | undefined; + if (!user?.id || !user.tenantId) { + return { allowed: false as const, code: 'permission_denied' as const }; + } + + if (!(await canUseAgents(req, user, permissions, deps))) { + return { allowed: false as const, code: 'permission_denied' as const }; + } + + const agent = await deps.getAgentWithVersionCount({ + id: req.params.id, + tenantId: user.tenantId, + }); + if (!agent) { + return { allowed: false as const, code: 'not_found' as const }; + } + if (!(await canEditAgentFiles(user, agent, deps))) { + return { allowed: false as const, code: 'permission_denied' as const }; + } + return { allowed: true as const, agent, user, tenantId: user.tenantId }; + } + + async function authorizeUpload( + req: Request, + res: Response, + next: NextFunction, + ): Promise { + try { + const authorized = await getAuthorizedAgent(req, [Permissions.USE, Permissions.CREATE]); + if (!authorized.allowed) { + return sendError(res, authorized.code); + } + const uploadConfig = await deps.getUploadConfig(req, authorized.agent); + if (uploadConfig.disabled === true) { + return sendError(res, 'invalid_request'); + } + authorizedUploads.set(req, { + agent: authorized.agent, + tenantId: authorized.tenantId, + uploadConfig, + }); + next(); + } catch (error) { + logger.error('[AgentManagement] Error authorizing Agent file upload', error); + return sendError(res, 'internal_error'); + } + } + + function getAuthorizedUploadConfig(req: Request) { + const config = authorizedUploads.get(req)?.uploadConfig; + if (!config) { + return undefined; + } + return { endpoint: config.endpoint, endpointType: config.endpointType }; + } + + async function withUploadQueue(key: string, task: () => Promise): Promise { + const previous = uploadQueues.get(key) ?? Promise.resolve(); + let release = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + const queued = previous.then(() => gate); + uploadQueues.set(key, queued); + await previous; + try { + return await task(); + } finally { + release(); + if (uploadQueues.get(key) === queued) { + uploadQueues.delete(key); + } + } + } + + async function isWithinAggregateLimits( + req: Request, + purpose: AgentUploadPurpose, + tenantId: string, + config: AgentUploadConfig, + ): Promise { + const currentAgent = await deps.getAgentWithVersionCount({ + id: req.params.id, + tenantId, + }); + if (!currentAgent) { + return false; + } + const fileIds = [...new Set(currentAgent.tool_resources?.[purpose]?.file_ids ?? [])]; + if (!config.fileLimit && !config.totalSizeLimit) { + return true; + } + const files = + fileIds.length === 0 + ? [] + : ((await deps.getFiles({ file_id: { $in: fileIds }, tenantId }, null, { text: 0 })) ?? []); + const persistedFileCount = new Set(files.map((file) => file.file_id)).size; + if (config.fileLimit && persistedFileCount + 1 > config.fileLimit) { + return false; + } + if (!config.totalSizeLimit) { + return true; + } + const currentBytes = files.reduce((total, file) => total + file.bytes, 0); + return currentBytes + (req.file?.size ?? 0) <= config.totalSizeLimit; + } + + async function isValidUpload( + req: Request, + purpose: AgentUploadPurpose, + config: AgentUploadConfig, + ): Promise { + if (req.file?.size === 0) { + return false; + } + if (config.fileSizeLimit && (req.file?.size ?? 0) > config.fileSizeLimit) { + return false; + } + if (purpose === EToolResources.file_search && req.file?.mimetype?.startsWith('image')) { + return false; + } + return await deps.isUploadPurposeEnabled(req, purpose); + } + + async function cleanupRejectedUpload(req: Request): Promise { + if (!req.file?.path) { + return true; + } + try { + await deps.deleteTempFile(req.file.path); + return true; + } catch (error) { + logger.error('[AgentManagement] Error cleaning up rejected Agent file upload', error); + return false; + } + } + + async function upload(req: Request, res: Response): Promise { + try { + const purpose = req.body?.purpose as string | undefined; + if (!req.file || !UPLOAD_PURPOSES.includes(purpose as AgentUploadPurpose)) { + const cleaned = await cleanupRejectedUpload(req); + return sendError(res, cleaned ? 'invalid_request' : 'internal_error'); + } + + const authorized = authorizedUploads.get(req); + if (!authorized) { + const cleaned = await cleanupRejectedUpload(req); + return sendError(res, cleaned ? 'permission_denied' : 'internal_error'); + } + + const queueKey = `${authorized.tenantId}:${req.params.id}:${purpose}`; + return await deps.runUploadExclusive(queueKey, async () => + withUploadQueue(queueKey, async () => { + if (!(await isValidUpload(req, purpose as AgentUploadPurpose, authorized.uploadConfig))) { + const cleaned = await cleanupRejectedUpload(req); + return sendError(res, cleaned ? 'invalid_request' : 'internal_error'); + } + if ( + !(await isWithinAggregateLimits( + req, + purpose as AgentUploadPurpose, + authorized.tenantId, + authorized.uploadConfig, + )) + ) { + const cleaned = await cleanupRejectedUpload(req); + return sendError(res, cleaned ? 'invalid_request' : 'internal_error'); + } + + req.body = { + endpoint: authorized.uploadConfig.endpoint, + endpointType: authorized.uploadConfig.endpointType, + agent_id: req.params.id, + tool_resource: purpose, + }; + req.headers.accept = 'application/json'; + await deps.processUpload(req, res); + return res; + }), + ); + } catch (error) { + logger.error('[AgentManagement] Error preparing Agent file upload', error); + await cleanupRejectedUpload(req); + return sendError(res, 'internal_error'); + } + } + + async function list(req: Request, res: Response): Promise { + try { + const authorized = await getAuthorizedAgent(req, [Permissions.USE]); + if (!authorized.allowed) { + return sendError(res, authorized.code); + } + + const purposes = getFilePurposes(authorized.agent); + const fileIds = [...purposes.keys()]; + const records = + fileIds.length === 0 + ? [] + : ((await deps.getFiles( + { file_id: { $in: fileIds }, tenantId: authorized.tenantId }, + null, + { text: 0 }, + )) ?? []); + const data: AgentManagementFile[] = records.map((file) => ({ + id: file.file_id, + object: 'agent.file', + filename: file.filename, + bytes: file.bytes, + mime_type: file.type, + purposes: purposes.get(file.file_id) ?? [], + created_at: file.createdAt?.toISOString() ?? null, + })); + + return res.status(200).json({ object: 'list', data }); + } catch (error) { + logger.error('[AgentManagement] Error listing Agent files', error); + return sendError(res, 'internal_error'); + } + } + + async function remove(req: Request, res: Response): Promise { + try { + const fileId = req.params.fileId; + if (!fileId) { + return sendError(res, 'invalid_request'); + } + const authorized = await getAuthorizedAgent(req, [Permissions.USE, Permissions.CREATE]); + if (!authorized.allowed) { + return sendError(res, authorized.code); + } + + const purposes = getFilePurposes(authorized.agent).get(fileId) ?? []; + if (purposes.length === 0) { + return sendFileNotFound(res); + } + await deps.removeAgentResourceFiles({ + agent_id: req.params.id, + files: purposes.map((purpose) => ({ + tool_resource: purpose, + file_id: fileId, + })), + }); + + return res.status(200).json({ id: fileId, deleted: true }); + } catch (error) { + logger.error('[AgentManagement] Error unlinking Agent file', error); + return sendError(res, 'internal_error'); + } + } + + return { + authorizeUpload, + getUploadConfig: getAuthorizedUploadConfig, + upload, + list, + remove, + }; +} diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts index 7342529123..c440f38003 100644 --- a/packages/api/src/agents/index.ts +++ b/packages/api/src/agents/index.ts @@ -19,6 +19,7 @@ export * from './errors'; export * from './eventRetention'; export * from './envelope'; export * from './execution'; +export * from './files'; export * from './handlers'; export * from './guard'; export * from './harvest';