mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-19 14:33:38 +00:00
🛅 feat: Add Agent File Management APIs (#15666)
* feat: expose agent file list and unlink API * feat: add agent file upload management route * fix: harden agent management file uploads * fix: coordinate agent upload limits across replicas * fix: keep agent upload coordination active * fix: validate agent uploads before processing * fix: reject unusable agent file uploads * refactor: move agent upload lock into api package
This commit is contained in:
parent
a5932cb039
commit
0fa313443d
9 changed files with 1633 additions and 34 deletions
|
|
@ -29,7 +29,7 @@ const getEnvironmentVariables = () => {
|
|||
};
|
||||
};
|
||||
|
||||
const createFileUploadHandler = (ip = true) => {
|
||||
const createFileUploadHandler = (ip = true, onLimit) => {
|
||||
const {
|
||||
fileUploadIpMax,
|
||||
fileUploadIpWindowInMinutes,
|
||||
|
|
@ -48,18 +48,21 @@ const createFileUploadHandler = (ip = true) => {
|
|||
};
|
||||
|
||||
await logViolation(req, res, type, errorMessage, fileUploadViolationScore);
|
||||
if (onLimit) {
|
||||
return onLimit(req, res);
|
||||
}
|
||||
res.status(429).json({ message: 'Too many file upload requests. Try again later' });
|
||||
};
|
||||
};
|
||||
|
||||
const createFileLimiters = () => {
|
||||
const createFileLimiters = ({ onLimit } = {}) => {
|
||||
const { fileUploadIpWindowMs, fileUploadIpMax, fileUploadUserWindowMs, fileUploadUserMax } =
|
||||
getEnvironmentVariables();
|
||||
|
||||
const ipLimiterOptions = {
|
||||
windowMs: fileUploadIpWindowMs,
|
||||
max: fileUploadIpMax,
|
||||
handler: createFileUploadHandler(),
|
||||
handler: createFileUploadHandler(true, onLimit),
|
||||
keyGenerator: removePorts,
|
||||
store: limiterCache('file_upload_ip_limiter'),
|
||||
};
|
||||
|
|
@ -67,7 +70,7 @@ const createFileLimiters = () => {
|
|||
const userLimiterOptions = {
|
||||
windowMs: fileUploadUserWindowMs,
|
||||
max: fileUploadUserMax,
|
||||
handler: createFileUploadHandler(false),
|
||||
handler: createFileUploadHandler(false, onLimit),
|
||||
keyGenerator: function (req) {
|
||||
return req.user?.id;
|
||||
},
|
||||
|
|
|
|||
|
|
@ -31,11 +31,51 @@ const mockCreateAgentManagementDeleteHandler = jest.fn((deps) => {
|
|||
mockDeleteDeps = deps;
|
||||
return mockDelete;
|
||||
});
|
||||
const mockFileList = jest.fn((_req, res) => res.status(200).json({ object: 'list', data: [] }));
|
||||
const mockFileRemove = jest.fn((_req, res) =>
|
||||
res.status(200).json({ id: 'file-one', deleted: true }),
|
||||
);
|
||||
const mockFileUpload = jest.fn((_req, res) => res.status(200).json({ id: 'file-uploaded' }));
|
||||
const mockFileAuthorizeUpload = jest.fn((_req, _res, next) => next());
|
||||
const mockGetFileUploadConfig = jest.fn(() => ({ endpoint: 'openAI' }));
|
||||
let mockFileDeps;
|
||||
const mockCreateAgentManagementFileHandlers = jest.fn((deps) => {
|
||||
mockFileDeps = deps;
|
||||
return {
|
||||
authorizeUpload: mockFileAuthorizeUpload,
|
||||
getUploadConfig: mockGetFileUploadConfig,
|
||||
upload: mockFileUpload,
|
||||
list: mockFileList,
|
||||
remove: mockFileRemove,
|
||||
};
|
||||
});
|
||||
const mockBrowserCreate = jest.fn();
|
||||
const mockBrowserUpdate = jest.fn();
|
||||
const mockCheckBan = jest.fn((_req, _res, next) => next());
|
||||
const mockConfigMiddleware = jest.fn((_req, _res, next) => next());
|
||||
const mockRequestFileConfig = { endpoints: { Moonshot: { fileLimit: 3 } } };
|
||||
const mockConfigMiddleware = jest.fn((req, _res, next) => {
|
||||
req.config = { fileConfig: mockRequestFileConfig };
|
||||
next();
|
||||
});
|
||||
const mockUaParser = jest.fn((_req, _res, next) => next());
|
||||
const mockUploadMiddleware = jest.fn((req, _res, next) => {
|
||||
req.file = { path: '/tmp/upload', originalname: 'input.txt' };
|
||||
next();
|
||||
});
|
||||
const mockCreateMulterInstance = jest.fn().mockResolvedValue({
|
||||
single: jest.fn(() => mockUploadMiddleware),
|
||||
});
|
||||
const mockGetEndpointsConfig = jest.fn().mockResolvedValue({
|
||||
Moonshot: { type: 'custom' },
|
||||
});
|
||||
const mockCheckCapability = jest.fn().mockResolvedValue(true);
|
||||
const mockRunUploadExclusive = jest.fn(async (_key, task) => await task());
|
||||
const mockCreateAgentUploadLock = jest.fn(() => mockRunUploadExclusive);
|
||||
let mockRateLimitIp = false;
|
||||
const mockCreateFileLimiters = jest.fn(({ onLimit } = {}) => ({
|
||||
fileUploadIpLimiter: jest.fn((req, res, next) => (mockRateLimitIp ? onLimit(req, res) : next())),
|
||||
fileUploadUserLimiter: jest.fn((_req, _res, next) => next()),
|
||||
}));
|
||||
|
||||
const mockRequireAgentManagementAuth = jest.fn((req, res, next) => {
|
||||
if (req.headers.authorization !== 'Bearer valid-token') {
|
||||
|
|
@ -49,17 +89,30 @@ jest.mock('../middleware', () => ({
|
|||
requireAgentManagementAuth: mockRequireAgentManagementAuth,
|
||||
}));
|
||||
jest.mock('@librechat/api', () => ({
|
||||
ioredisClient: {},
|
||||
mapAgentManagementError: mockMapAgentManagementError,
|
||||
restoreTenantContextFromReq: jest.fn((_req, _res, next) => next()),
|
||||
createAgentUploadLock: mockCreateAgentUploadLock,
|
||||
createAgentManagementCreateHandler: mockCreateAgentManagementCreateHandler,
|
||||
createAgentManagementDeleteHandler: mockCreateAgentManagementDeleteHandler,
|
||||
createAgentManagementFileHandlers: mockCreateAgentManagementFileHandlers,
|
||||
createAgentManagementReadHandlers: mockCreateAgentManagementReadHandlers,
|
||||
createAgentManagementUpdateHandler: mockCreateAgentManagementUpdateHandler,
|
||||
}));
|
||||
jest.mock('~/server/middleware', () => ({
|
||||
checkBan: mockCheckBan,
|
||||
configMiddleware: mockConfigMiddleware,
|
||||
createFileLimiters: mockCreateFileLimiters,
|
||||
uaParser: mockUaParser,
|
||||
}));
|
||||
jest.mock('~/server/routes/files/multer', () => ({
|
||||
createMulterInstance: mockCreateMulterInstance,
|
||||
}));
|
||||
jest.mock('~/server/routes/files/files', () => ({ handleFileUpload: jest.fn() }));
|
||||
jest.mock('~/server/services/Config', () => ({
|
||||
checkCapability: mockCheckCapability,
|
||||
getEndpointsConfig: mockGetEndpointsConfig,
|
||||
}));
|
||||
jest.mock('~/server/controllers/agents/v1', () => ({
|
||||
createAgent: mockBrowserCreate,
|
||||
updateAgent: mockBrowserUpdate,
|
||||
|
|
@ -73,6 +126,8 @@ jest.mock('~/models', () => ({
|
|||
getRoleByName: jest.fn(),
|
||||
getAgentWithVersionCount: jest.fn(),
|
||||
getAgentManagementListByAccess: jest.fn(),
|
||||
getFiles: jest.fn(),
|
||||
removeAgentResourceFiles: jest.fn(),
|
||||
deleteAgent: jest.fn(),
|
||||
}));
|
||||
|
||||
|
|
@ -88,6 +143,11 @@ describe('Agent Management route boundary', () => {
|
|||
mockConfigMiddleware.mockClear();
|
||||
mockUaParser.mockClear();
|
||||
mockMapAgentManagementError.mockClear();
|
||||
mockFileAuthorizeUpload.mockClear();
|
||||
mockFileUpload.mockClear();
|
||||
mockUploadMiddleware.mockClear();
|
||||
mockRunUploadExclusive.mockClear();
|
||||
mockRateLimitIp = false;
|
||||
});
|
||||
|
||||
it('rejects a request before reaching management routes without machine authentication', async () => {
|
||||
|
|
@ -174,4 +234,130 @@ describe('Agent Management route boundary', () => {
|
|||
expect(mockDeleteDeps.hasCapability).toEqual(expect.any(Function));
|
||||
expect(mockDeleteDeps.deleteAgent).toEqual(expect.any(Function));
|
||||
});
|
||||
|
||||
it('dispatches authenticated Agent file list and unlink requests', async () => {
|
||||
const listResponse = await request(app)
|
||||
.get('/api/agents/v1/agents/agent-one/files')
|
||||
.set('Authorization', 'Bearer valid-token');
|
||||
const deleteResponse = await request(app)
|
||||
.delete('/api/agents/v1/agents/agent-one/files/file-one')
|
||||
.set('Authorization', 'Bearer valid-token');
|
||||
|
||||
expect(listResponse.status).toBe(200);
|
||||
expect(deleteResponse.status).toBe(200);
|
||||
expect(mockFileList).toHaveBeenCalledTimes(1);
|
||||
expect(mockFileRemove).toHaveBeenCalledTimes(1);
|
||||
expect(mockFileDeps.getAgentWithVersionCount).toEqual(expect.any(Function));
|
||||
expect(mockFileDeps.getFiles).toEqual(expect.any(Function));
|
||||
expect(mockFileDeps.removeAgentResourceFiles).toEqual(expect.any(Function));
|
||||
});
|
||||
|
||||
it('retries multipart initialization after a transient failure', async () => {
|
||||
mockCreateMulterInstance.mockRejectedValueOnce(new Error('temporary config failure'));
|
||||
mockMapAgentManagementError.mockReturnValueOnce({
|
||||
status: 500,
|
||||
body: { error: { code: 'internal_error', message: 'Internal server error' } },
|
||||
});
|
||||
|
||||
const failed = await request(app)
|
||||
.post('/api/agents/v1/agents/agent-one/files')
|
||||
.set('Authorization', 'Bearer valid-token')
|
||||
.send({ purpose: 'context' });
|
||||
const retried = await request(app)
|
||||
.post('/api/agents/v1/agents/agent-one/files')
|
||||
.set('Authorization', 'Bearer valid-token')
|
||||
.send({ purpose: 'context' });
|
||||
|
||||
expect(failed.status).toBe(500);
|
||||
expect(retried.status).toBe(200);
|
||||
expect(mockCreateMulterInstance).toHaveBeenCalledTimes(2);
|
||||
expect(mockCreateMulterInstance).toHaveBeenLastCalledWith({
|
||||
fileConfig: mockRequestFileConfig,
|
||||
resolveEndpoint: mockGetFileUploadConfig,
|
||||
uniqueTempPath: true,
|
||||
});
|
||||
expect(mockFileUpload).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('loads file configuration and dispatches authenticated multipart uploads', async () => {
|
||||
const response = await request(app)
|
||||
.post('/api/agents/v1/agents/agent-one/files')
|
||||
.set('Authorization', 'Bearer valid-token')
|
||||
.send({ purpose: 'context' });
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(mockConfigMiddleware).toHaveBeenCalledTimes(1);
|
||||
expect(mockFileAuthorizeUpload).toHaveBeenCalledTimes(1);
|
||||
expect(mockFileUpload).toHaveBeenCalledTimes(1);
|
||||
expect(mockFileAuthorizeUpload.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
mockUploadMiddleware.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(mockFileDeps.processUpload).toEqual(expect.any(Function));
|
||||
expect(mockFileDeps.deleteTempFile).toEqual(expect.any(Function));
|
||||
expect(mockFileDeps.getUploadConfig).toEqual(expect.any(Function));
|
||||
expect(mockFileDeps.isUploadPurposeEnabled).toEqual(expect.any(Function));
|
||||
});
|
||||
|
||||
it('resolves upload limits from the target Agent provider', async () => {
|
||||
const config = await mockFileDeps.getUploadConfig(
|
||||
{
|
||||
config: {
|
||||
fileConfig: {
|
||||
endpoints: {
|
||||
Moonshot: { fileLimit: 3, totalSizeLimit: 20 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{ provider: 'Moonshot' },
|
||||
);
|
||||
|
||||
expect(config).toMatchObject({
|
||||
endpoint: 'Moonshot',
|
||||
endpointType: 'custom',
|
||||
fileLimit: 3,
|
||||
totalSizeLimit: 20 * 1024 * 1024,
|
||||
});
|
||||
});
|
||||
|
||||
it('maps upload quota failures into the management error contract', async () => {
|
||||
mockRateLimitIp = true;
|
||||
|
||||
const response = await request(app)
|
||||
.post('/api/agents/v1/agents/agent-one/files')
|
||||
.set('Authorization', 'Bearer valid-token')
|
||||
.send({ purpose: 'context' });
|
||||
|
||||
expect(response.status).toBe(429);
|
||||
expect(response.body).toEqual({
|
||||
error: {
|
||||
code: 'invalid_request',
|
||||
message: 'Too many file upload requests. Try again later',
|
||||
},
|
||||
});
|
||||
expect(mockFileAuthorizeUpload).not.toHaveBeenCalled();
|
||||
expect(mockFileUpload).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('maps multipart failures into the management error contract', async () => {
|
||||
mockMapAgentManagementError.mockReturnValueOnce({
|
||||
status: 400,
|
||||
body: { error: { code: 'invalid_request', message: 'Invalid request' } },
|
||||
});
|
||||
mockUploadMiddleware.mockImplementationOnce((_req, _res, next) => {
|
||||
next(Object.assign(new Error('File too large'), { code: 'LIMIT_FILE_SIZE' }));
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.post('/api/agents/v1/agents/agent-one/files')
|
||||
.set('Authorization', 'Bearer valid-token')
|
||||
.send({ purpose: 'context' });
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(response.body).toEqual({
|
||||
error: { code: 'invalid_request', message: 'Invalid request' },
|
||||
});
|
||||
expect(mockFileUpload).not.toHaveBeenCalled();
|
||||
expect(mockMapAgentManagementError).toHaveBeenCalledWith('invalid_request');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,14 +1,31 @@
|
|||
const express = require('express');
|
||||
const fs = require('fs').promises;
|
||||
const {
|
||||
EModelEndpoint,
|
||||
EToolResources,
|
||||
AgentCapabilities,
|
||||
getEndpointFileConfig,
|
||||
mergeFileConfig,
|
||||
resolveEndpointType,
|
||||
} = require('librechat-data-provider');
|
||||
const {
|
||||
createAgentUploadLock,
|
||||
createAgentManagementCreateHandler,
|
||||
createAgentManagementDeleteHandler,
|
||||
createAgentManagementFileHandlers,
|
||||
createAgentManagementUploadResponse,
|
||||
createAgentManagementReadHandlers,
|
||||
createAgentManagementUpdateHandler,
|
||||
ioredisClient,
|
||||
mapAgentManagementError,
|
||||
restoreTenantContextFromReq,
|
||||
} = require('@librechat/api');
|
||||
const { checkBan, configMiddleware } = require('~/server/middleware');
|
||||
const { checkBan, configMiddleware, createFileLimiters } = require('~/server/middleware');
|
||||
const { hasCapability } = require('~/server/middleware/roles/capabilities');
|
||||
const { checkPermission, findAccessibleResources } = require('~/server/services/PermissionService');
|
||||
const { createMulterInstance } = require('~/server/routes/files/multer');
|
||||
const { handleFileUpload } = require('~/server/routes/files/files');
|
||||
const { checkCapability, getEndpointsConfig } = require('~/server/services/Config');
|
||||
const v1 = require('~/server/controllers/agents/v1');
|
||||
const db = require('~/models');
|
||||
const { requireAgentManagementAuth } = require('./middleware');
|
||||
|
|
@ -40,12 +57,105 @@ const deleteHandler = createAgentManagementDeleteHandler({
|
|||
hasCapability,
|
||||
deleteAgent: db.deleteAgent,
|
||||
});
|
||||
const fileHandlers = createAgentManagementFileHandlers({
|
||||
getRoleByName: db.getRoleByName,
|
||||
getAgentWithVersionCount: db.getAgentWithVersionCount,
|
||||
getFiles: db.getFiles,
|
||||
checkPermission,
|
||||
hasCapability,
|
||||
removeAgentResourceFiles: db.removeAgentResourceFiles,
|
||||
processUpload: (req, res) =>
|
||||
handleFileUpload(
|
||||
req,
|
||||
createAgentManagementUploadResponse(res, req.file, req.body.tool_resource),
|
||||
),
|
||||
deleteTempFile: fs.unlink,
|
||||
getUploadConfig: async (req, agent) => {
|
||||
const endpoint = agent.provider || EModelEndpoint.agents;
|
||||
const endpointType = resolveEndpointType(
|
||||
await getEndpointsConfig(req),
|
||||
EModelEndpoint.agents,
|
||||
endpoint,
|
||||
);
|
||||
const endpointConfig = getEndpointFileConfig({
|
||||
fileConfig: mergeFileConfig(req.config?.fileConfig),
|
||||
endpoint,
|
||||
endpointType,
|
||||
});
|
||||
return {
|
||||
endpoint,
|
||||
endpointType,
|
||||
disabled: endpointConfig.disabled,
|
||||
fileSizeLimit: endpointConfig.fileSizeLimit,
|
||||
fileLimit: endpointConfig.fileLimit,
|
||||
totalSizeLimit: endpointConfig.totalSizeLimit,
|
||||
};
|
||||
},
|
||||
isUploadPurposeEnabled: async (req, purpose) => {
|
||||
if (purpose === EToolResources.context) {
|
||||
return await checkCapability(req, AgentCapabilities.context);
|
||||
}
|
||||
if (purpose === EToolResources.execute_code) {
|
||||
return await checkCapability(req, AgentCapabilities.execute_code);
|
||||
}
|
||||
if (purpose === EToolResources.file_search) {
|
||||
return await checkCapability(req, AgentCapabilities.file_search);
|
||||
}
|
||||
return true;
|
||||
},
|
||||
runUploadExclusive: createAgentUploadLock({ redisClient: ioredisClient }),
|
||||
});
|
||||
const sendUploadRateLimit = (_req, res) =>
|
||||
res.status(429).json({
|
||||
error: {
|
||||
code: 'invalid_request',
|
||||
message: 'Too many file upload requests. Try again later',
|
||||
},
|
||||
});
|
||||
const { fileUploadIpLimiter, fileUploadUserLimiter } = createFileLimiters({
|
||||
onLimit: sendUploadRateLimit,
|
||||
});
|
||||
const uploadSingleFile = async (req, res, next) => {
|
||||
try {
|
||||
const upload = await createMulterInstance({
|
||||
fileConfig: req.config?.fileConfig ?? null,
|
||||
resolveEndpoint: fileHandlers.getUploadConfig,
|
||||
uniqueTempPath: true,
|
||||
});
|
||||
return upload.single('file')(req, res, next);
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
};
|
||||
const handleUploadError = (error, _req, res, _next) => {
|
||||
const status = Number(error?.statusCode);
|
||||
const isMultipartRequestError =
|
||||
error?.name === 'MulterError' ||
|
||||
error?.code?.startsWith?.('LIMIT_') ||
|
||||
(Number.isInteger(status) && status >= 400 && status < 500);
|
||||
const code = isMultipartRequestError ? 'invalid_request' : 'internal_error';
|
||||
const mapped = mapAgentManagementError(code);
|
||||
return res.status(mapped.status).json(mapped.body);
|
||||
};
|
||||
|
||||
router.use(requireAgentManagementAuth);
|
||||
router.use(checkBan);
|
||||
|
||||
router.post('/', configMiddleware, createHandler);
|
||||
router.get('/', readHandlers.list);
|
||||
router.post(
|
||||
'/:id/files',
|
||||
configMiddleware,
|
||||
fileUploadIpLimiter,
|
||||
fileUploadUserLimiter,
|
||||
fileHandlers.authorizeUpload,
|
||||
uploadSingleFile,
|
||||
restoreTenantContextFromReq,
|
||||
fileHandlers.upload,
|
||||
handleUploadError,
|
||||
);
|
||||
router.get('/:id/files', fileHandlers.list);
|
||||
router.delete('/:id/files/:fileId', fileHandlers.remove);
|
||||
router.get('/:id', readHandlers.get);
|
||||
router.patch('/:id', configMiddleware, updateHandler);
|
||||
router.delete('/:id', deleteHandler);
|
||||
|
|
|
|||
|
|
@ -725,7 +725,7 @@ router.get('/download/:userId/:file_id', fileAccess, async (req, res) => {
|
|||
}
|
||||
});
|
||||
|
||||
router.post('/', async (req, res) => {
|
||||
const handleFileUpload = async (req, res) => {
|
||||
const metadata = req.body;
|
||||
let cleanup = true;
|
||||
|
||||
|
|
@ -845,6 +845,9 @@ router.post('/', async (req, res) => {
|
|||
sseStream.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
router.post('/', handleFileUpload);
|
||||
|
||||
module.exports = router;
|
||||
module.exports.handleFileUpload = handleFileUpload;
|
||||
|
|
|
|||
|
|
@ -11,22 +11,28 @@ const {
|
|||
} = require('librechat-data-provider');
|
||||
const { getAppConfig } = require('~/server/services/Config');
|
||||
|
||||
const storage = multer.diskStorage({
|
||||
destination: function (req, file, cb) {
|
||||
const appConfig = req.config;
|
||||
const outputPath = path.join(appConfig.paths.uploads, 'temp', req.user.id);
|
||||
if (!fs.existsSync(outputPath)) {
|
||||
fs.mkdirSync(outputPath, { recursive: true });
|
||||
}
|
||||
cb(null, outputPath);
|
||||
},
|
||||
filename: function (req, file, cb) {
|
||||
req.file_id = crypto.randomUUID();
|
||||
file.originalname = decodeURIComponent(file.originalname);
|
||||
const sanitizedFilename = sanitizeFilename(file.originalname);
|
||||
cb(null, sanitizedFilename);
|
||||
},
|
||||
});
|
||||
const createStorage = ({ uniqueTempPath = false } = {}) =>
|
||||
multer.diskStorage({
|
||||
destination: function (req, file, cb) {
|
||||
const appConfig = req.config;
|
||||
const outputPath = path.join(appConfig.paths.uploads, 'temp', req.user.id);
|
||||
if (!fs.existsSync(outputPath)) {
|
||||
fs.mkdirSync(outputPath, { recursive: true });
|
||||
}
|
||||
cb(null, outputPath);
|
||||
},
|
||||
filename: function (req, file, cb) {
|
||||
req.file_id = crypto.randomUUID();
|
||||
file.originalname = decodeURIComponent(file.originalname);
|
||||
const sanitizedFilename = sanitizeFilename(file.originalname);
|
||||
const stagedFilename = uniqueTempPath
|
||||
? sanitizeFilename(`${req.file_id}-${sanitizedFilename}`)
|
||||
: sanitizedFilename;
|
||||
cb(null, stagedFilename);
|
||||
},
|
||||
});
|
||||
|
||||
const storage = createStorage();
|
||||
|
||||
const importFileFilter = (req, file, cb) => {
|
||||
if (file.mimetype === 'application/json') {
|
||||
|
|
@ -50,7 +56,7 @@ const normalizeUploadMimeType = (file) => {
|
|||
*
|
||||
* @param {import('librechat-data-provider').FileConfig | undefined} customFileConfig
|
||||
*/
|
||||
const createFileFilter = (customFileConfig) => {
|
||||
const createFileFilter = (customFileConfig, resolveEndpoint) => {
|
||||
/**
|
||||
* @param {ServerRequest} req
|
||||
* @param {Express.Multer.File}
|
||||
|
|
@ -67,8 +73,9 @@ const createFileFilter = (customFileConfig) => {
|
|||
return cb(null, true);
|
||||
}
|
||||
|
||||
const endpoint = req.body.endpoint;
|
||||
const endpointType = req.body.endpointType;
|
||||
const resolved = resolveEndpoint?.(req);
|
||||
const endpoint = resolved?.endpoint ?? req.body.endpoint;
|
||||
const endpointType = resolved?.endpointType ?? req.body.endpointType;
|
||||
const endpointFileConfig = getEndpointFileConfig({
|
||||
fileConfig: customFileConfig,
|
||||
endpoint,
|
||||
|
|
@ -88,15 +95,24 @@ const createFileFilter = (customFileConfig) => {
|
|||
return fileFilter;
|
||||
};
|
||||
|
||||
const createMulterInstance = async () => {
|
||||
const appConfig = await getAppConfig();
|
||||
const fileConfig = mergeFileConfig(appConfig?.fileConfig);
|
||||
const fileFilter = createFileFilter(fileConfig);
|
||||
const createMulterInstance = async (options = {}) => {
|
||||
const { resolveEndpoint, uniqueTempPath = false } = options;
|
||||
const appConfig = Object.prototype.hasOwnProperty.call(options, 'fileConfig')
|
||||
? null
|
||||
: await getAppConfig();
|
||||
const fileConfig = mergeFileConfig(options.fileConfig ?? appConfig?.fileConfig);
|
||||
const fileFilter = createFileFilter(fileConfig, resolveEndpoint);
|
||||
return multer({
|
||||
storage,
|
||||
storage: uniqueTempPath ? createStorage({ uniqueTempPath: true }) : storage,
|
||||
fileFilter,
|
||||
limits: { fileSize: fileConfig.serverFileSizeLimit },
|
||||
});
|
||||
};
|
||||
|
||||
module.exports = { createMulterInstance, storage, importFileFilter, createFileFilter };
|
||||
module.exports = {
|
||||
createMulterInstance,
|
||||
createStorage,
|
||||
storage,
|
||||
importFileFilter,
|
||||
createFileFilter,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -4,7 +4,13 @@ const fs = require('fs');
|
|||
const os = require('os');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { createMulterInstance, storage, importFileFilter, createFileFilter } = require('./multer');
|
||||
const {
|
||||
createMulterInstance,
|
||||
createStorage,
|
||||
storage,
|
||||
importFileFilter,
|
||||
createFileFilter,
|
||||
} = require('./multer');
|
||||
|
||||
// Mock only the config service that requires external dependencies
|
||||
jest.mock('~/server/services/Config', () => ({
|
||||
|
|
@ -77,6 +83,16 @@ describe('Multer Configuration', () => {
|
|||
});
|
||||
|
||||
describe('filename function', () => {
|
||||
it('uses the request file ID to make management staging paths unique', (done) => {
|
||||
const uniqueStorage = createStorage({ uniqueTempPath: true });
|
||||
|
||||
uniqueStorage.getFilename(mockReq, mockFile, (err, filename) => {
|
||||
expect(err).toBeNull();
|
||||
expect(filename).toBe(`${mockReq.file_id}-test-file.jpg`);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should generate a UUID for req.file_id', (done) => {
|
||||
const cb = jest.fn((err, filename) => {
|
||||
expect(err).toBeNull();
|
||||
|
|
@ -302,6 +318,30 @@ describe('Multer Configuration', () => {
|
|||
fileFilter(mockReq, zipFile, cb);
|
||||
});
|
||||
|
||||
it('uses a server-selected endpoint before multipart fields are parsed', (done) => {
|
||||
const { mergeFileConfig } = require('librechat-data-provider');
|
||||
const fileFilter = createFileFilter(
|
||||
mergeFileConfig({
|
||||
endpoints: {
|
||||
agents: { supportedMimeTypes: ['text/plain'] },
|
||||
default: { supportedMimeTypes: ['application/pdf'] },
|
||||
},
|
||||
}),
|
||||
() => ({ endpoint: 'agents' }),
|
||||
);
|
||||
const textFile = {
|
||||
...mockFile,
|
||||
originalname: 'notes.txt',
|
||||
mimetype: 'text/plain',
|
||||
};
|
||||
|
||||
fileFilter({ ...mockReq, body: {} }, textFile, (err, result) => {
|
||||
expect(err).toBeNull();
|
||||
expect(result).toBe(true);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it.each(['application/x-shellscript', 'text/x-shellscript'])(
|
||||
'should normalize %s to application/x-sh and accept the upload',
|
||||
(reportedType) => {
|
||||
|
|
|
|||
648
packages/api/src/agents/files.spec.ts
Normal file
648
packages/api/src/agents/files.spec.ts
Normal file
|
|
@ -0,0 +1,648 @@
|
|||
import { Types } from 'mongoose';
|
||||
import { SystemCapabilities } from '@librechat/data-schemas';
|
||||
import {
|
||||
EToolResources,
|
||||
PermissionBits,
|
||||
Permissions,
|
||||
PermissionTypes,
|
||||
ResourceType,
|
||||
} from 'librechat-data-provider';
|
||||
import type { IRole, IUser } from '@librechat/data-schemas';
|
||||
import type { Request, Response } from 'express';
|
||||
import type { AgentManagementFileDeps } from './files';
|
||||
import {
|
||||
createAgentManagementFileHandlers,
|
||||
createAgentManagementUploadResponse,
|
||||
createAgentUploadLock,
|
||||
} from './files';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => {
|
||||
return {
|
||||
ResourceCapabilityMap: { agent: 'MANAGE_AGENTS' },
|
||||
SystemCapabilities: { MANAGE_AGENTS: 'MANAGE_AGENTS' },
|
||||
logger: { warn: jest.fn(), error: jest.fn() },
|
||||
};
|
||||
});
|
||||
|
||||
const tenantId = 'tenant-a';
|
||||
const user = {
|
||||
id: new Types.ObjectId().toString(),
|
||||
tenantId,
|
||||
role: 'USER',
|
||||
} as IUser;
|
||||
const objectId = new Types.ObjectId();
|
||||
const agent = {
|
||||
_id: objectId,
|
||||
id: 'agent-one',
|
||||
provider: 'Moonshot',
|
||||
tool_resources: {
|
||||
context: { file_ids: ['file-context', 'file-shared'] },
|
||||
file_search: { file_ids: ['file-search', 'file-shared'] },
|
||||
execute_code: { file_ids: ['file-code'] },
|
||||
image_edit: { file_ids: ['file-image'] },
|
||||
ocr: { file_ids: ['file-ocr'] },
|
||||
},
|
||||
};
|
||||
|
||||
function makeRequest(params: Record<string, string>): Request {
|
||||
return { user, params, headers: {} } as unknown as Request;
|
||||
}
|
||||
|
||||
function makeResponse(): Response {
|
||||
const response = {
|
||||
status: jest.fn(),
|
||||
json: jest.fn(),
|
||||
};
|
||||
response.status.mockReturnValue(response);
|
||||
response.json.mockReturnValue(response);
|
||||
return response as unknown as Response;
|
||||
}
|
||||
|
||||
function makeDeps(overrides: Partial<AgentManagementFileDeps> = {}): AgentManagementFileDeps {
|
||||
return {
|
||||
getRoleByName: jest.fn().mockResolvedValue({
|
||||
permissions: {
|
||||
[PermissionTypes.AGENTS]: {
|
||||
[Permissions.USE]: true,
|
||||
[Permissions.CREATE]: true,
|
||||
},
|
||||
},
|
||||
} as unknown as IRole),
|
||||
getAgentWithVersionCount: jest.fn().mockResolvedValue(agent),
|
||||
getFiles: jest.fn().mockResolvedValue([
|
||||
{
|
||||
file_id: 'file-shared',
|
||||
filename: 'shared.txt',
|
||||
bytes: 12,
|
||||
type: 'text/plain',
|
||||
createdAt: new Date('2026-09-01T10:00:00.000Z'),
|
||||
},
|
||||
]),
|
||||
checkPermission: jest.fn().mockResolvedValue(true),
|
||||
hasCapability: jest.fn().mockResolvedValue(false),
|
||||
removeAgentResourceFiles: jest.fn().mockResolvedValue(agent),
|
||||
processUpload: jest.fn().mockResolvedValue(undefined),
|
||||
deleteTempFile: jest.fn().mockResolvedValue(undefined),
|
||||
getUploadConfig: jest.fn().mockResolvedValue({
|
||||
endpoint: 'Moonshot',
|
||||
endpointType: 'custom',
|
||||
fileLimit: 100,
|
||||
totalSizeLimit: 1_000_000,
|
||||
}),
|
||||
isUploadPurposeEnabled: jest.fn().mockResolvedValue(true),
|
||||
runUploadExclusive: async (_key, task) => await task(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function authorizeUpload(
|
||||
handlers: ReturnType<typeof createAgentManagementFileHandlers>,
|
||||
request: Request,
|
||||
response: Response,
|
||||
) {
|
||||
const next = jest.fn();
|
||||
await handlers.authorizeUpload(request, response, next);
|
||||
expect(next).toHaveBeenCalledTimes(1);
|
||||
}
|
||||
|
||||
describe('Agent Management file handlers', () => {
|
||||
it('holds and releases the shared upload lock around processing', async () => {
|
||||
const redisClient = {
|
||||
set: jest.fn().mockResolvedValue('OK'),
|
||||
eval: jest.fn().mockResolvedValue(1),
|
||||
};
|
||||
const task = jest.fn().mockResolvedValue('uploaded');
|
||||
|
||||
await expect(
|
||||
createAgentUploadLock({ redisClient })('tenant-a:agent-one:context', task),
|
||||
).resolves.toBe('uploaded');
|
||||
|
||||
expect(redisClient.set).toHaveBeenCalledWith(
|
||||
'agent-management:file-upload:tenant-a:agent-one:context',
|
||||
expect.any(String),
|
||||
'PX',
|
||||
10 * 60 * 1000,
|
||||
'NX',
|
||||
);
|
||||
expect(task).toHaveBeenCalledTimes(1);
|
||||
expect(redisClient.eval).toHaveBeenCalledWith(
|
||||
expect.stringContaining("redis.call('DEL', KEYS[1])"),
|
||||
1,
|
||||
'agent-management:file-upload:tenant-a:agent-one:context',
|
||||
expect.any(String),
|
||||
);
|
||||
});
|
||||
|
||||
it('renews the shared upload lock while processing is still running', async () => {
|
||||
jest.useFakeTimers();
|
||||
const redisClient = {
|
||||
set: jest.fn().mockResolvedValue('OK'),
|
||||
eval: jest.fn().mockResolvedValue(1),
|
||||
};
|
||||
let completeUpload: (value: string) => void = () => {};
|
||||
const task = jest.fn(
|
||||
() =>
|
||||
new Promise<string>((resolve) => {
|
||||
completeUpload = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
const pendingUpload = createAgentUploadLock({ redisClient })(
|
||||
'tenant-a:agent-one:context',
|
||||
task,
|
||||
);
|
||||
await Promise.resolve();
|
||||
jest.advanceTimersByTime((10 * 60 * 1000) / 3);
|
||||
await Promise.resolve();
|
||||
|
||||
expect(redisClient.eval).toHaveBeenCalledWith(
|
||||
expect.stringContaining("redis.call('PEXPIRE', KEYS[1], ARGV[2])"),
|
||||
1,
|
||||
'agent-management:file-upload:tenant-a:agent-one:context',
|
||||
expect.any(String),
|
||||
10 * 60 * 1000,
|
||||
);
|
||||
|
||||
completeUpload('uploaded');
|
||||
await expect(pendingUpload).resolves.toBe('uploaded');
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('projects upload success through the management metadata allowlist', () => {
|
||||
const response = makeResponse();
|
||||
const file = {
|
||||
originalname: 'input.txt',
|
||||
size: 12,
|
||||
mimetype: 'text/plain',
|
||||
} as Express.Multer.File;
|
||||
|
||||
createAgentManagementUploadResponse(response, file, EToolResources.context).status(200).json({
|
||||
file_id: 'file-one',
|
||||
filename: 'stored.txt',
|
||||
filepath: '/private/storage/path',
|
||||
storageKey: 'private-key',
|
||||
tenantId: 'tenant-a',
|
||||
bytes: 10,
|
||||
type: 'text/plain',
|
||||
});
|
||||
|
||||
expect(response.json).toHaveBeenCalledWith({
|
||||
id: 'file-one',
|
||||
object: 'agent.file',
|
||||
filename: 'stored.txt',
|
||||
bytes: 10,
|
||||
mime_type: 'text/plain',
|
||||
purposes: [EToolResources.context],
|
||||
created_at: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects an incomplete shared-uploader success response', () => {
|
||||
const response = makeResponse();
|
||||
const file = {
|
||||
originalname: 'input.txt',
|
||||
size: 12,
|
||||
mimetype: 'text/plain',
|
||||
} as Express.Multer.File;
|
||||
|
||||
createAgentManagementUploadResponse(response, file, EToolResources.context).status(200).json({
|
||||
filename: 'stored.txt',
|
||||
});
|
||||
|
||||
expect(response.status).toHaveBeenCalledWith(500);
|
||||
expect(response.json).toHaveBeenCalledWith({
|
||||
error: { code: 'internal_error', message: 'Internal server error' },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([EToolResources.context, EToolResources.file_search, EToolResources.execute_code])(
|
||||
'routes a %s upload through the shared browser pipeline',
|
||||
async (purpose) => {
|
||||
const deps = makeDeps();
|
||||
const response = makeResponse();
|
||||
const request = makeRequest({ id: 'agent-one' });
|
||||
const handlers = createAgentManagementFileHandlers(deps);
|
||||
await authorizeUpload(handlers, request, response);
|
||||
request.body = { purpose };
|
||||
request.file = {
|
||||
path: '/tmp/upload',
|
||||
originalname: 'input.txt',
|
||||
size: 12,
|
||||
} as Express.Multer.File;
|
||||
|
||||
await handlers.upload(request, response);
|
||||
|
||||
expect(deps.processUpload).toHaveBeenCalledWith(request, response);
|
||||
expect(request.body).toEqual({
|
||||
endpoint: 'Moonshot',
|
||||
endpointType: 'custom',
|
||||
agent_id: 'agent-one',
|
||||
tool_resource: purpose,
|
||||
});
|
||||
expect(request.headers.accept).toBe('application/json');
|
||||
expect(deps.deleteTempFile).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('rejects unsupported purposes and removes the temporary upload', async () => {
|
||||
const deps = makeDeps();
|
||||
const response = makeResponse();
|
||||
const request = makeRequest({ id: 'agent-one' });
|
||||
request.body = { purpose: 'provider_storage' };
|
||||
request.file = { path: '/tmp/rejected', originalname: 'input.txt' } as Express.Multer.File;
|
||||
|
||||
await createAgentManagementFileHandlers(deps).upload(request, response);
|
||||
|
||||
expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/rejected');
|
||||
expect(deps.processUpload).not.toHaveBeenCalled();
|
||||
expect(response.status).toHaveBeenCalledWith(400);
|
||||
});
|
||||
|
||||
it('rejects a cross-tenant Agent before staging an upload', async () => {
|
||||
const deps = makeDeps({ getAgentWithVersionCount: jest.fn().mockResolvedValue(null) });
|
||||
const response = makeResponse();
|
||||
const request = makeRequest({ id: 'agent-other-tenant' });
|
||||
const next = jest.fn();
|
||||
|
||||
await createAgentManagementFileHandlers(deps).authorizeUpload(request, response, next);
|
||||
|
||||
expect(next).not.toHaveBeenCalled();
|
||||
expect(deps.deleteTempFile).not.toHaveBeenCalled();
|
||||
expect(deps.processUpload).not.toHaveBeenCalled();
|
||||
expect(response.status).toHaveBeenCalledWith(404);
|
||||
});
|
||||
|
||||
it('reports cleanup failures with the management error contract', async () => {
|
||||
const deps = makeDeps({
|
||||
deleteTempFile: jest.fn().mockRejectedValue(new Error('cleanup failed')),
|
||||
getFiles: jest.fn().mockResolvedValue([
|
||||
{ file_id: 'file-context', bytes: 8 },
|
||||
{ file_id: 'file-shared', bytes: 12 },
|
||||
]),
|
||||
getUploadConfig: jest.fn().mockResolvedValue({
|
||||
endpoint: 'Moonshot',
|
||||
endpointType: 'custom',
|
||||
fileLimit: 2,
|
||||
}),
|
||||
});
|
||||
const response = makeResponse();
|
||||
const request = makeRequest({ id: 'agent-one' });
|
||||
const handlers = createAgentManagementFileHandlers(deps);
|
||||
await authorizeUpload(handlers, request, response);
|
||||
request.body = { purpose: EToolResources.context };
|
||||
request.file = {
|
||||
path: '/tmp/rejected',
|
||||
originalname: 'input.txt',
|
||||
size: 12,
|
||||
} as Express.Multer.File;
|
||||
|
||||
await handlers.upload(request, response);
|
||||
|
||||
expect(response.status).toHaveBeenCalledWith(500);
|
||||
expect(response.json).toHaveBeenCalledWith({
|
||||
error: { code: 'internal_error', message: 'Internal server error' },
|
||||
});
|
||||
});
|
||||
|
||||
it('serializes uploads when enforcing the per-purpose aggregate file limit', async () => {
|
||||
let attachedCount = 1;
|
||||
const getAgentWithVersionCount = jest.fn().mockImplementation(async () => ({
|
||||
...agent,
|
||||
tool_resources: {
|
||||
...agent.tool_resources,
|
||||
context: {
|
||||
file_ids: Array.from({ length: attachedCount }, (_, index) => `file-${index}`),
|
||||
},
|
||||
},
|
||||
}));
|
||||
const processUpload = jest.fn().mockImplementation(async () => {
|
||||
attachedCount += 1;
|
||||
});
|
||||
const deps = makeDeps({
|
||||
getAgentWithVersionCount,
|
||||
getFiles: jest.fn().mockImplementation(async () =>
|
||||
Array.from({ length: attachedCount }, (_, index) => ({
|
||||
file_id: `file-${index}`,
|
||||
bytes: 1,
|
||||
})),
|
||||
),
|
||||
processUpload,
|
||||
getUploadConfig: jest.fn().mockResolvedValue({ endpoint: 'Moonshot', fileLimit: 2 }),
|
||||
});
|
||||
const handlers = createAgentManagementFileHandlers(deps);
|
||||
const firstResponse = makeResponse();
|
||||
const secondResponse = makeResponse();
|
||||
const firstRequest = makeRequest({ id: 'agent-one' });
|
||||
const secondRequest = makeRequest({ id: 'agent-one' });
|
||||
await authorizeUpload(handlers, firstRequest, firstResponse);
|
||||
await authorizeUpload(handlers, secondRequest, secondResponse);
|
||||
firstRequest.body = { purpose: EToolResources.context };
|
||||
secondRequest.body = { purpose: EToolResources.context };
|
||||
firstRequest.file = {
|
||||
path: '/tmp/first',
|
||||
originalname: 'first.txt',
|
||||
size: 1,
|
||||
} as Express.Multer.File;
|
||||
secondRequest.file = {
|
||||
path: '/tmp/second',
|
||||
originalname: 'second.txt',
|
||||
size: 1,
|
||||
} as Express.Multer.File;
|
||||
|
||||
await Promise.all([
|
||||
handlers.upload(firstRequest, firstResponse),
|
||||
handlers.upload(secondRequest, secondResponse),
|
||||
]);
|
||||
|
||||
expect(processUpload).toHaveBeenCalledTimes(1);
|
||||
expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/second');
|
||||
expect(secondResponse.status).toHaveBeenCalledWith(400);
|
||||
});
|
||||
|
||||
it('rejects an upload that would exceed the per-purpose aggregate byte limit', async () => {
|
||||
const deps = makeDeps({
|
||||
getFiles: jest.fn().mockResolvedValue([
|
||||
{ file_id: 'file-context', bytes: 8 },
|
||||
{ file_id: 'file-shared', bytes: 12 },
|
||||
]),
|
||||
getUploadConfig: jest.fn().mockResolvedValue({
|
||||
endpoint: 'Moonshot',
|
||||
totalSizeLimit: 24,
|
||||
}),
|
||||
});
|
||||
const response = makeResponse();
|
||||
const request = makeRequest({ id: 'agent-one' });
|
||||
const handlers = createAgentManagementFileHandlers(deps);
|
||||
await authorizeUpload(handlers, request, response);
|
||||
request.body = { purpose: EToolResources.context };
|
||||
request.file = {
|
||||
path: '/tmp/too-large-in-aggregate',
|
||||
originalname: 'input.txt',
|
||||
size: 5,
|
||||
} as Express.Multer.File;
|
||||
|
||||
await handlers.upload(request, response);
|
||||
|
||||
expect(deps.processUpload).not.toHaveBeenCalled();
|
||||
expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/too-large-in-aggregate');
|
||||
expect(response.status).toHaveBeenCalledWith(400);
|
||||
});
|
||||
|
||||
it('does not count dangling legacy file references toward the aggregate file limit', async () => {
|
||||
const deps = makeDeps({
|
||||
getAgentWithVersionCount: jest.fn().mockResolvedValue({
|
||||
...agent,
|
||||
tool_resources: { context: { file_ids: ['file-live', 'file-missing'] } },
|
||||
}),
|
||||
getFiles: jest.fn().mockResolvedValue([{ file_id: 'file-live', bytes: 8 }]),
|
||||
getUploadConfig: jest.fn().mockResolvedValue({ endpoint: 'Moonshot', fileLimit: 2 }),
|
||||
});
|
||||
const response = makeResponse();
|
||||
const request = makeRequest({ id: 'agent-one' });
|
||||
const handlers = createAgentManagementFileHandlers(deps);
|
||||
await authorizeUpload(handlers, request, response);
|
||||
request.body = { purpose: EToolResources.context };
|
||||
request.file = {
|
||||
path: '/tmp/accepted-with-dangling-reference',
|
||||
originalname: 'input.txt',
|
||||
size: 5,
|
||||
} as Express.Multer.File;
|
||||
|
||||
await handlers.upload(request, response);
|
||||
|
||||
expect(deps.processUpload).toHaveBeenCalledTimes(1);
|
||||
expect(deps.deleteTempFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects provider file-size violations before shared upload processing', async () => {
|
||||
const deps = makeDeps({
|
||||
getUploadConfig: jest.fn().mockResolvedValue({
|
||||
endpoint: 'Moonshot',
|
||||
fileSizeLimit: 4,
|
||||
}),
|
||||
});
|
||||
const response = makeResponse();
|
||||
const request = makeRequest({ id: 'agent-one' });
|
||||
const handlers = createAgentManagementFileHandlers(deps);
|
||||
await authorizeUpload(handlers, request, response);
|
||||
request.body = { purpose: EToolResources.context };
|
||||
request.file = {
|
||||
path: '/tmp/provider-file-too-large',
|
||||
originalname: 'input.txt',
|
||||
size: 5,
|
||||
} as Express.Multer.File;
|
||||
|
||||
await handlers.upload(request, response);
|
||||
|
||||
expect(deps.processUpload).not.toHaveBeenCalled();
|
||||
expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/provider-file-too-large');
|
||||
expect(response.status).toHaveBeenCalledWith(400);
|
||||
});
|
||||
|
||||
it('rejects empty files before shared upload processing', async () => {
|
||||
const deps = makeDeps();
|
||||
const response = makeResponse();
|
||||
const request = makeRequest({ id: 'agent-one' });
|
||||
const handlers = createAgentManagementFileHandlers(deps);
|
||||
await authorizeUpload(handlers, request, response);
|
||||
request.body = { purpose: EToolResources.context };
|
||||
request.file = {
|
||||
path: '/tmp/empty-file',
|
||||
originalname: 'empty.txt',
|
||||
size: 0,
|
||||
} as Express.Multer.File;
|
||||
|
||||
await handlers.upload(request, response);
|
||||
|
||||
expect(deps.processUpload).not.toHaveBeenCalled();
|
||||
expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/empty-file');
|
||||
expect(response.status).toHaveBeenCalledWith(400);
|
||||
});
|
||||
|
||||
it.each([EToolResources.context, EToolResources.execute_code])(
|
||||
'rejects disabled %s Agent uploads before shared processing',
|
||||
async (purpose) => {
|
||||
const deps = makeDeps({ isUploadPurposeEnabled: jest.fn().mockResolvedValue(false) });
|
||||
const response = makeResponse();
|
||||
const request = makeRequest({ id: 'agent-one' });
|
||||
const handlers = createAgentManagementFileHandlers(deps);
|
||||
await authorizeUpload(handlers, request, response);
|
||||
request.body = { purpose };
|
||||
request.file = {
|
||||
path: '/tmp/disabled-purpose',
|
||||
originalname: 'input.txt',
|
||||
size: 5,
|
||||
} as Express.Multer.File;
|
||||
|
||||
await handlers.upload(request, response);
|
||||
|
||||
expect(deps.processUpload).not.toHaveBeenCalled();
|
||||
expect(deps.deleteTempFile).toHaveBeenCalledWith('/tmp/disabled-purpose');
|
||||
expect(response.status).toHaveBeenCalledWith(400);
|
||||
},
|
||||
);
|
||||
|
||||
it('lists safe file metadata with every attached purpose in the authenticated tenant', async () => {
|
||||
const deps = makeDeps();
|
||||
const response = makeResponse();
|
||||
|
||||
await createAgentManagementFileHandlers(deps).list(makeRequest({ id: 'agent-one' }), response);
|
||||
|
||||
expect(deps.getAgentWithVersionCount).toHaveBeenCalledWith({ id: 'agent-one', tenantId });
|
||||
expect(deps.checkPermission).toHaveBeenCalledWith({
|
||||
userId: user.id,
|
||||
role: user.role,
|
||||
resourceType: ResourceType.AGENT,
|
||||
resourceId: objectId,
|
||||
requiredPermission: PermissionBits.EDIT,
|
||||
});
|
||||
expect(deps.getFiles).toHaveBeenCalledWith(
|
||||
{
|
||||
file_id: {
|
||||
$in: [
|
||||
'file-context',
|
||||
'file-shared',
|
||||
'file-search',
|
||||
'file-code',
|
||||
'file-image',
|
||||
'file-ocr',
|
||||
],
|
||||
},
|
||||
tenantId,
|
||||
},
|
||||
null,
|
||||
{ text: 0 },
|
||||
);
|
||||
expect(response.json).toHaveBeenCalledWith({
|
||||
object: 'list',
|
||||
data: [
|
||||
{
|
||||
id: 'file-shared',
|
||||
object: 'agent.file',
|
||||
filename: 'shared.txt',
|
||||
bytes: 12,
|
||||
mime_type: 'text/plain',
|
||||
purposes: [EToolResources.context, EToolResources.file_search],
|
||||
created_at: '2026-09-01T10:00:00.000Z',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('unlinks a file from every purpose without deleting shared storage', async () => {
|
||||
const deps = makeDeps();
|
||||
const response = makeResponse();
|
||||
|
||||
await createAgentManagementFileHandlers(deps).remove(
|
||||
makeRequest({ id: 'agent-one', fileId: 'file-shared' }),
|
||||
response,
|
||||
);
|
||||
|
||||
expect(deps.removeAgentResourceFiles).toHaveBeenCalledWith({
|
||||
agent_id: 'agent-one',
|
||||
files: [
|
||||
{ tool_resource: EToolResources.context, file_id: 'file-shared' },
|
||||
{ tool_resource: EToolResources.file_search, file_id: 'file-shared' },
|
||||
],
|
||||
});
|
||||
expect(response.status).toHaveBeenCalledWith(200);
|
||||
expect(response.json).toHaveBeenCalledWith({ id: 'file-shared', deleted: true });
|
||||
});
|
||||
|
||||
it.each([
|
||||
[EToolResources.image_edit, 'file-image'],
|
||||
[EToolResources.ocr, 'file-ocr'],
|
||||
])('unlinks legacy %s attachments', async (purpose, fileId) => {
|
||||
const deps = makeDeps();
|
||||
const response = makeResponse();
|
||||
|
||||
await createAgentManagementFileHandlers(deps).remove(
|
||||
makeRequest({ id: 'agent-one', fileId }),
|
||||
response,
|
||||
);
|
||||
|
||||
expect(deps.removeAgentResourceFiles).toHaveBeenCalledWith({
|
||||
agent_id: 'agent-one',
|
||||
files: [{ tool_resource: purpose, file_id: fileId }],
|
||||
});
|
||||
expect(response.status).toHaveBeenCalledWith(200);
|
||||
});
|
||||
|
||||
it('checks mutation capability before looking up the agent', async () => {
|
||||
const getAgentWithVersionCount = jest.fn().mockResolvedValue(agent);
|
||||
const deps = makeDeps({
|
||||
getAgentWithVersionCount,
|
||||
getRoleByName: jest.fn().mockResolvedValue({
|
||||
permissions: {
|
||||
[PermissionTypes.AGENTS]: {
|
||||
[Permissions.USE]: true,
|
||||
[Permissions.CREATE]: false,
|
||||
},
|
||||
},
|
||||
} as unknown as IRole),
|
||||
});
|
||||
const response = makeResponse();
|
||||
|
||||
await createAgentManagementFileHandlers(deps).remove(
|
||||
makeRequest({ id: 'agent-one', fileId: 'file-shared' }),
|
||||
response,
|
||||
);
|
||||
|
||||
expect(response.status).toHaveBeenCalledWith(403);
|
||||
expect(getAgentWithVersionCount).not.toHaveBeenCalled();
|
||||
expect(deps.removeAgentResourceFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('fails closed when the agent is outside the authenticated tenant', async () => {
|
||||
const deps = makeDeps({ getAgentWithVersionCount: jest.fn().mockResolvedValue(null) });
|
||||
const response = makeResponse();
|
||||
|
||||
await createAgentManagementFileHandlers(deps).list(
|
||||
makeRequest({ id: 'agent-other-tenant' }),
|
||||
response,
|
||||
);
|
||||
|
||||
expect(response.status).toHaveBeenCalledWith(404);
|
||||
expect(deps.getFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('requires EDIT access unless the caller has the management capability', async () => {
|
||||
const deps = makeDeps({ checkPermission: jest.fn().mockResolvedValue(false) });
|
||||
const response = makeResponse();
|
||||
|
||||
await createAgentManagementFileHandlers(deps).list(makeRequest({ id: 'agent-one' }), response);
|
||||
|
||||
expect(response.status).toHaveBeenCalledWith(403);
|
||||
expect(deps.getFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('uses the manage-agents capability as the existing ACL bypass', async () => {
|
||||
const deps = makeDeps({ hasCapability: jest.fn().mockResolvedValue(true) });
|
||||
const response = makeResponse();
|
||||
|
||||
await createAgentManagementFileHandlers(deps).list(makeRequest({ id: 'agent-one' }), response);
|
||||
|
||||
expect(deps.hasCapability).toHaveBeenCalledWith(user, SystemCapabilities.MANAGE_AGENTS);
|
||||
expect(deps.checkPermission).not.toHaveBeenCalled();
|
||||
expect(response.status).toHaveBeenCalledWith(200);
|
||||
});
|
||||
|
||||
it('returns not found without mutating when the file is not attached', async () => {
|
||||
const deps = makeDeps();
|
||||
const response = makeResponse();
|
||||
|
||||
await createAgentManagementFileHandlers(deps).remove(
|
||||
makeRequest({ id: 'agent-one', fileId: 'file-missing' }),
|
||||
response,
|
||||
);
|
||||
|
||||
expect(response.status).toHaveBeenCalledWith(404);
|
||||
expect(response.json).toHaveBeenCalledWith({
|
||||
error: { code: 'not_found', message: 'File not found' },
|
||||
});
|
||||
expect(deps.removeAgentResourceFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
592
packages/api/src/agents/files.ts
Normal file
592
packages/api/src/agents/files.ts
Normal file
|
|
@ -0,0 +1,592 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { logger, ResourceCapabilityMap } from '@librechat/data-schemas';
|
||||
import {
|
||||
EToolResources,
|
||||
PermissionBits,
|
||||
Permissions,
|
||||
PermissionTypes,
|
||||
ResourceType,
|
||||
} from 'librechat-data-provider';
|
||||
import type { IRole, IUser, SystemCapability } from '@librechat/data-schemas';
|
||||
import type { NextFunction, Request, Response } from 'express';
|
||||
import type { Types } from 'mongoose';
|
||||
import type { AgentManagementProjectionSource } from './management';
|
||||
import { checkAccessWithRequestCache } from '../middleware/access';
|
||||
import { mapAgentManagementError } from './management';
|
||||
|
||||
type AgentUploadPurpose =
|
||||
| EToolResources.context
|
||||
| EToolResources.file_search
|
||||
| EToolResources.execute_code;
|
||||
|
||||
type AgentFilePurpose = AgentUploadPurpose | EToolResources.image_edit | EToolResources.ocr;
|
||||
|
||||
const UPLOAD_PURPOSES: readonly AgentUploadPurpose[] = [
|
||||
EToolResources.context,
|
||||
EToolResources.file_search,
|
||||
EToolResources.execute_code,
|
||||
] as const;
|
||||
|
||||
const FILE_PURPOSES: readonly AgentFilePurpose[] = [
|
||||
...UPLOAD_PURPOSES,
|
||||
EToolResources.image_edit,
|
||||
EToolResources.ocr,
|
||||
] as const;
|
||||
|
||||
type AgentManagementFileRecord = {
|
||||
file_id: string;
|
||||
filename: string;
|
||||
bytes: number;
|
||||
type: string;
|
||||
createdAt?: Date;
|
||||
};
|
||||
type AgentManagementFile = {
|
||||
id: string;
|
||||
object: 'agent.file';
|
||||
filename: string;
|
||||
bytes: number;
|
||||
mime_type: string;
|
||||
purposes: AgentFilePurpose[];
|
||||
created_at: string | null;
|
||||
};
|
||||
type AgentManagementUploadBody = {
|
||||
file_id?: string;
|
||||
filename?: string;
|
||||
bytes?: number;
|
||||
type?: string;
|
||||
createdAt?: string | Date;
|
||||
};
|
||||
type AgentManagementFileAgent = AgentManagementProjectionSource & {
|
||||
_id: Types.ObjectId;
|
||||
provider?: string;
|
||||
tool_resources?: Partial<Record<AgentFilePurpose, { file_ids?: string[] }>>;
|
||||
};
|
||||
type AgentUploadConfig = {
|
||||
endpoint: string;
|
||||
endpointType?: string;
|
||||
disabled?: boolean;
|
||||
fileSizeLimit?: number;
|
||||
fileLimit?: number;
|
||||
totalSizeLimit?: number;
|
||||
};
|
||||
type AgentUploadLockRedisClient = {
|
||||
set: (
|
||||
key: string,
|
||||
value: string,
|
||||
expiryMode: 'PX',
|
||||
ttlMs: number,
|
||||
condition: 'NX',
|
||||
) => Promise<unknown>;
|
||||
eval: (
|
||||
script: string,
|
||||
numberOfKeys: number,
|
||||
key: string,
|
||||
...args: Array<string | number>
|
||||
) => Promise<unknown>;
|
||||
};
|
||||
|
||||
const AGENT_UPLOAD_LOCK_TTL_MS = 10 * 60 * 1000;
|
||||
const AGENT_UPLOAD_LOCK_WAIT_MS = 2 * 60 * 1000;
|
||||
const AGENT_UPLOAD_LOCK_RENEW_MS = Math.floor(AGENT_UPLOAD_LOCK_TTL_MS / 3);
|
||||
const releaseUploadLockScript = `
|
||||
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
||||
return redis.call('DEL', KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
const renewUploadLockScript = `
|
||||
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
||||
return redis.call('PEXPIRE', KEYS[1], ARGV[2])
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
|
||||
export interface AgentManagementFileDeps {
|
||||
getRoleByName: (roleName: string, fieldsToSelect?: string | string[]) => Promise<IRole | null>;
|
||||
getAgentWithVersionCount: (search: {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
}) => Promise<AgentManagementFileAgent | null>;
|
||||
getFiles: (
|
||||
filter: { file_id: { $in: string[] }; tenantId: string },
|
||||
sort?: null,
|
||||
projection?: Record<string, 0 | 1>,
|
||||
) => Promise<AgentManagementFileRecord[] | null>;
|
||||
checkPermission: (params: {
|
||||
userId: string;
|
||||
role?: string;
|
||||
resourceType: ResourceType;
|
||||
resourceId: Types.ObjectId;
|
||||
requiredPermission: PermissionBits;
|
||||
}) => Promise<boolean>;
|
||||
hasCapability: (user: IUser, capability: SystemCapability) => Promise<boolean>;
|
||||
removeAgentResourceFiles: (params: {
|
||||
agent_id: string;
|
||||
files: Array<{ tool_resource: AgentFilePurpose; file_id: string }>;
|
||||
}) => Promise<AgentManagementFileAgent>;
|
||||
processUpload: (req: Request, res: Response) => Promise<Response | void>;
|
||||
deleteTempFile: (path: string) => Promise<void>;
|
||||
getUploadConfig: (req: Request, agent: AgentManagementFileAgent) => Promise<AgentUploadConfig>;
|
||||
isUploadPurposeEnabled: (req: Request, purpose: AgentUploadPurpose) => Promise<boolean>;
|
||||
runUploadExclusive: <T>(key: string, task: () => Promise<T>) => Promise<T>;
|
||||
}
|
||||
|
||||
/** Serialize an Agent purpose's aggregate-limit check and upload across API replicas. */
|
||||
export function createAgentUploadLock({
|
||||
redisClient,
|
||||
}: {
|
||||
redisClient: AgentUploadLockRedisClient | null;
|
||||
}): AgentManagementFileDeps['runUploadExclusive'] {
|
||||
return async function withAgentUploadLock<T>(key: string, task: () => Promise<T>): Promise<T> {
|
||||
if (!redisClient) {
|
||||
return await task();
|
||||
}
|
||||
const lockKey = `agent-management:file-upload:${key}`;
|
||||
const token = randomUUID();
|
||||
const deadline = Date.now() + AGENT_UPLOAD_LOCK_WAIT_MS;
|
||||
while ((await redisClient.set(lockKey, token, 'PX', AGENT_UPLOAD_LOCK_TTL_MS, 'NX')) !== 'OK') {
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error('Timed out waiting for Agent file upload lock');
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
|
||||
let stopped = false;
|
||||
let renewalTimer: ReturnType<typeof setTimeout>;
|
||||
const renewLease = async () => {
|
||||
try {
|
||||
const renewed = await redisClient.eval(
|
||||
renewUploadLockScript,
|
||||
1,
|
||||
lockKey,
|
||||
token,
|
||||
AGENT_UPLOAD_LOCK_TTL_MS,
|
||||
);
|
||||
if (renewed !== 1) {
|
||||
logger.warn('[AgentManagement] Lost Agent file upload lock before processing completed');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('[AgentManagement] Failed to renew Agent file upload lock', error);
|
||||
} finally {
|
||||
if (!stopped) {
|
||||
renewalTimer = setTimeout(renewLease, AGENT_UPLOAD_LOCK_RENEW_MS);
|
||||
renewalTimer.unref?.();
|
||||
}
|
||||
}
|
||||
};
|
||||
renewalTimer = setTimeout(renewLease, AGENT_UPLOAD_LOCK_RENEW_MS);
|
||||
renewalTimer.unref?.();
|
||||
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
stopped = true;
|
||||
clearTimeout(renewalTimer);
|
||||
try {
|
||||
await redisClient.eval(releaseUploadLockScript, 1, lockKey, token);
|
||||
} catch (error) {
|
||||
logger.warn('[AgentManagement] Failed to release Agent file upload lock', error);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function sendError(res: Response, code: Parameters<typeof mapAgentManagementError>[0]) {
|
||||
const mapped = mapAgentManagementError(code);
|
||||
return res.status(mapped.status).json(mapped.body);
|
||||
}
|
||||
|
||||
function sendFileNotFound(res: Response) {
|
||||
return res.status(404).json({ error: { code: 'not_found', message: 'File not found' } });
|
||||
}
|
||||
|
||||
function getUploadErrorCode(status: number): Parameters<typeof mapAgentManagementError>[0] {
|
||||
if (status === 403) {
|
||||
return 'permission_denied';
|
||||
}
|
||||
if (status === 404) {
|
||||
return 'not_found';
|
||||
}
|
||||
if (status >= 400 && status < 500) {
|
||||
return 'invalid_request';
|
||||
}
|
||||
return 'internal_error';
|
||||
}
|
||||
|
||||
function getUploadCreatedAt(value: string | Date | undefined): string | null {
|
||||
if (value instanceof Date) {
|
||||
return value.toISOString();
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Restrict the shared browser uploader's response to the management file contract. */
|
||||
export function createAgentManagementUploadResponse(
|
||||
res: Response,
|
||||
file: Express.Multer.File,
|
||||
purpose: AgentUploadPurpose,
|
||||
): Response {
|
||||
let status = 200;
|
||||
const response = Object.create(res) as Response;
|
||||
response.status = (code: number) => {
|
||||
status = code;
|
||||
return response;
|
||||
};
|
||||
response.json = (body: AgentManagementUploadBody) => {
|
||||
if (status < 200 || status >= 300) {
|
||||
return sendError(res, getUploadErrorCode(status));
|
||||
}
|
||||
if (typeof body.file_id !== 'string' || body.file_id.length === 0) {
|
||||
return sendError(res, 'internal_error');
|
||||
}
|
||||
return res.status(status).json({
|
||||
id: body.file_id,
|
||||
object: 'agent.file',
|
||||
filename: body.filename ?? file.originalname,
|
||||
bytes: body.bytes ?? file.size,
|
||||
mime_type: body.type ?? file.mimetype,
|
||||
purposes: [purpose],
|
||||
created_at: getUploadCreatedAt(body.createdAt),
|
||||
});
|
||||
};
|
||||
return response;
|
||||
}
|
||||
|
||||
async function canUseAgents(
|
||||
req: Request,
|
||||
user: IUser,
|
||||
permissions: Permissions[],
|
||||
deps: AgentManagementFileDeps,
|
||||
) {
|
||||
return await checkAccessWithRequestCache({
|
||||
req,
|
||||
user,
|
||||
permissionType: PermissionTypes.AGENTS,
|
||||
permissions,
|
||||
getRoleByName: deps.getRoleByName,
|
||||
});
|
||||
}
|
||||
|
||||
async function hasManageAgentsCapability(user: IUser, deps: AgentManagementFileDeps) {
|
||||
const capability = ResourceCapabilityMap[ResourceType.AGENT];
|
||||
try {
|
||||
return capability != null && (await deps.hasCapability(user, capability));
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`[AgentManagement] Agent capability check failed, denying file access bypass: ${(error as Error).message}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function canEditAgentFiles(
|
||||
user: IUser,
|
||||
agent: AgentManagementFileAgent,
|
||||
deps: AgentManagementFileDeps,
|
||||
) {
|
||||
if (await hasManageAgentsCapability(user, deps)) {
|
||||
return true;
|
||||
}
|
||||
return await deps.checkPermission({
|
||||
userId: user.id,
|
||||
role: user.role,
|
||||
resourceType: ResourceType.AGENT,
|
||||
resourceId: agent._id,
|
||||
requiredPermission: PermissionBits.EDIT,
|
||||
});
|
||||
}
|
||||
|
||||
function getFilePurposes(agent: AgentManagementFileAgent): Map<string, AgentFilePurpose[]> {
|
||||
const purposes = new Map<string, AgentFilePurpose[]>();
|
||||
for (const purpose of FILE_PURPOSES) {
|
||||
for (const fileId of agent.tool_resources?.[purpose]?.file_ids ?? []) {
|
||||
purposes.set(fileId, [...(purposes.get(fileId) ?? []), purpose]);
|
||||
}
|
||||
}
|
||||
return purposes;
|
||||
}
|
||||
|
||||
/** Machine-authenticated Agent file listing and unlink handlers. */
|
||||
export function createAgentManagementFileHandlers(deps: AgentManagementFileDeps): {
|
||||
authorizeUpload: (req: Request, res: Response, next: NextFunction) => Promise<Response | void>;
|
||||
getUploadConfig: (
|
||||
req: Request,
|
||||
) => Pick<AgentUploadConfig, 'endpoint' | 'endpointType'> | undefined;
|
||||
upload: (req: Request, res: Response) => Promise<Response>;
|
||||
list: (req: Request, res: Response) => Promise<Response>;
|
||||
remove: (req: Request, res: Response) => Promise<Response>;
|
||||
} {
|
||||
const authorizedUploads = new WeakMap<
|
||||
Request,
|
||||
{
|
||||
agent: AgentManagementFileAgent;
|
||||
tenantId: string;
|
||||
uploadConfig: AgentUploadConfig;
|
||||
}
|
||||
>();
|
||||
const uploadQueues = new Map<string, Promise<void>>();
|
||||
|
||||
async function getAuthorizedAgent(req: Request, permissions: Permissions[]) {
|
||||
const user = req.user as IUser | undefined;
|
||||
if (!user?.id || !user.tenantId) {
|
||||
return { allowed: false as const, code: 'permission_denied' as const };
|
||||
}
|
||||
|
||||
if (!(await canUseAgents(req, user, permissions, deps))) {
|
||||
return { allowed: false as const, code: 'permission_denied' as const };
|
||||
}
|
||||
|
||||
const agent = await deps.getAgentWithVersionCount({
|
||||
id: req.params.id,
|
||||
tenantId: user.tenantId,
|
||||
});
|
||||
if (!agent) {
|
||||
return { allowed: false as const, code: 'not_found' as const };
|
||||
}
|
||||
if (!(await canEditAgentFiles(user, agent, deps))) {
|
||||
return { allowed: false as const, code: 'permission_denied' as const };
|
||||
}
|
||||
return { allowed: true as const, agent, user, tenantId: user.tenantId };
|
||||
}
|
||||
|
||||
async function authorizeUpload(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
): Promise<Response | void> {
|
||||
try {
|
||||
const authorized = await getAuthorizedAgent(req, [Permissions.USE, Permissions.CREATE]);
|
||||
if (!authorized.allowed) {
|
||||
return sendError(res, authorized.code);
|
||||
}
|
||||
const uploadConfig = await deps.getUploadConfig(req, authorized.agent);
|
||||
if (uploadConfig.disabled === true) {
|
||||
return sendError(res, 'invalid_request');
|
||||
}
|
||||
authorizedUploads.set(req, {
|
||||
agent: authorized.agent,
|
||||
tenantId: authorized.tenantId,
|
||||
uploadConfig,
|
||||
});
|
||||
next();
|
||||
} catch (error) {
|
||||
logger.error('[AgentManagement] Error authorizing Agent file upload', error);
|
||||
return sendError(res, 'internal_error');
|
||||
}
|
||||
}
|
||||
|
||||
function getAuthorizedUploadConfig(req: Request) {
|
||||
const config = authorizedUploads.get(req)?.uploadConfig;
|
||||
if (!config) {
|
||||
return undefined;
|
||||
}
|
||||
return { endpoint: config.endpoint, endpointType: config.endpointType };
|
||||
}
|
||||
|
||||
async function withUploadQueue<T>(key: string, task: () => Promise<T>): Promise<T> {
|
||||
const previous = uploadQueues.get(key) ?? Promise.resolve();
|
||||
let release = () => {};
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const queued = previous.then(() => gate);
|
||||
uploadQueues.set(key, queued);
|
||||
await previous;
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
release();
|
||||
if (uploadQueues.get(key) === queued) {
|
||||
uploadQueues.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function isWithinAggregateLimits(
|
||||
req: Request,
|
||||
purpose: AgentUploadPurpose,
|
||||
tenantId: string,
|
||||
config: AgentUploadConfig,
|
||||
): Promise<boolean> {
|
||||
const currentAgent = await deps.getAgentWithVersionCount({
|
||||
id: req.params.id,
|
||||
tenantId,
|
||||
});
|
||||
if (!currentAgent) {
|
||||
return false;
|
||||
}
|
||||
const fileIds = [...new Set(currentAgent.tool_resources?.[purpose]?.file_ids ?? [])];
|
||||
if (!config.fileLimit && !config.totalSizeLimit) {
|
||||
return true;
|
||||
}
|
||||
const files =
|
||||
fileIds.length === 0
|
||||
? []
|
||||
: ((await deps.getFiles({ file_id: { $in: fileIds }, tenantId }, null, { text: 0 })) ?? []);
|
||||
const persistedFileCount = new Set(files.map((file) => file.file_id)).size;
|
||||
if (config.fileLimit && persistedFileCount + 1 > config.fileLimit) {
|
||||
return false;
|
||||
}
|
||||
if (!config.totalSizeLimit) {
|
||||
return true;
|
||||
}
|
||||
const currentBytes = files.reduce((total, file) => total + file.bytes, 0);
|
||||
return currentBytes + (req.file?.size ?? 0) <= config.totalSizeLimit;
|
||||
}
|
||||
|
||||
async function isValidUpload(
|
||||
req: Request,
|
||||
purpose: AgentUploadPurpose,
|
||||
config: AgentUploadConfig,
|
||||
): Promise<boolean> {
|
||||
if (req.file?.size === 0) {
|
||||
return false;
|
||||
}
|
||||
if (config.fileSizeLimit && (req.file?.size ?? 0) > config.fileSizeLimit) {
|
||||
return false;
|
||||
}
|
||||
if (purpose === EToolResources.file_search && req.file?.mimetype?.startsWith('image')) {
|
||||
return false;
|
||||
}
|
||||
return await deps.isUploadPurposeEnabled(req, purpose);
|
||||
}
|
||||
|
||||
async function cleanupRejectedUpload(req: Request): Promise<boolean> {
|
||||
if (!req.file?.path) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
await deps.deleteTempFile(req.file.path);
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.error('[AgentManagement] Error cleaning up rejected Agent file upload', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function upload(req: Request, res: Response): Promise<Response> {
|
||||
try {
|
||||
const purpose = req.body?.purpose as string | undefined;
|
||||
if (!req.file || !UPLOAD_PURPOSES.includes(purpose as AgentUploadPurpose)) {
|
||||
const cleaned = await cleanupRejectedUpload(req);
|
||||
return sendError(res, cleaned ? 'invalid_request' : 'internal_error');
|
||||
}
|
||||
|
||||
const authorized = authorizedUploads.get(req);
|
||||
if (!authorized) {
|
||||
const cleaned = await cleanupRejectedUpload(req);
|
||||
return sendError(res, cleaned ? 'permission_denied' : 'internal_error');
|
||||
}
|
||||
|
||||
const queueKey = `${authorized.tenantId}:${req.params.id}:${purpose}`;
|
||||
return await deps.runUploadExclusive(queueKey, async () =>
|
||||
withUploadQueue(queueKey, async () => {
|
||||
if (!(await isValidUpload(req, purpose as AgentUploadPurpose, authorized.uploadConfig))) {
|
||||
const cleaned = await cleanupRejectedUpload(req);
|
||||
return sendError(res, cleaned ? 'invalid_request' : 'internal_error');
|
||||
}
|
||||
if (
|
||||
!(await isWithinAggregateLimits(
|
||||
req,
|
||||
purpose as AgentUploadPurpose,
|
||||
authorized.tenantId,
|
||||
authorized.uploadConfig,
|
||||
))
|
||||
) {
|
||||
const cleaned = await cleanupRejectedUpload(req);
|
||||
return sendError(res, cleaned ? 'invalid_request' : 'internal_error');
|
||||
}
|
||||
|
||||
req.body = {
|
||||
endpoint: authorized.uploadConfig.endpoint,
|
||||
endpointType: authorized.uploadConfig.endpointType,
|
||||
agent_id: req.params.id,
|
||||
tool_resource: purpose,
|
||||
};
|
||||
req.headers.accept = 'application/json';
|
||||
await deps.processUpload(req, res);
|
||||
return res;
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
logger.error('[AgentManagement] Error preparing Agent file upload', error);
|
||||
await cleanupRejectedUpload(req);
|
||||
return sendError(res, 'internal_error');
|
||||
}
|
||||
}
|
||||
|
||||
async function list(req: Request, res: Response): Promise<Response> {
|
||||
try {
|
||||
const authorized = await getAuthorizedAgent(req, [Permissions.USE]);
|
||||
if (!authorized.allowed) {
|
||||
return sendError(res, authorized.code);
|
||||
}
|
||||
|
||||
const purposes = getFilePurposes(authorized.agent);
|
||||
const fileIds = [...purposes.keys()];
|
||||
const records =
|
||||
fileIds.length === 0
|
||||
? []
|
||||
: ((await deps.getFiles(
|
||||
{ file_id: { $in: fileIds }, tenantId: authorized.tenantId },
|
||||
null,
|
||||
{ text: 0 },
|
||||
)) ?? []);
|
||||
const data: AgentManagementFile[] = records.map((file) => ({
|
||||
id: file.file_id,
|
||||
object: 'agent.file',
|
||||
filename: file.filename,
|
||||
bytes: file.bytes,
|
||||
mime_type: file.type,
|
||||
purposes: purposes.get(file.file_id) ?? [],
|
||||
created_at: file.createdAt?.toISOString() ?? null,
|
||||
}));
|
||||
|
||||
return res.status(200).json({ object: 'list', data });
|
||||
} catch (error) {
|
||||
logger.error('[AgentManagement] Error listing Agent files', error);
|
||||
return sendError(res, 'internal_error');
|
||||
}
|
||||
}
|
||||
|
||||
async function remove(req: Request, res: Response): Promise<Response> {
|
||||
try {
|
||||
const fileId = req.params.fileId;
|
||||
if (!fileId) {
|
||||
return sendError(res, 'invalid_request');
|
||||
}
|
||||
const authorized = await getAuthorizedAgent(req, [Permissions.USE, Permissions.CREATE]);
|
||||
if (!authorized.allowed) {
|
||||
return sendError(res, authorized.code);
|
||||
}
|
||||
|
||||
const purposes = getFilePurposes(authorized.agent).get(fileId) ?? [];
|
||||
if (purposes.length === 0) {
|
||||
return sendFileNotFound(res);
|
||||
}
|
||||
await deps.removeAgentResourceFiles({
|
||||
agent_id: req.params.id,
|
||||
files: purposes.map((purpose) => ({
|
||||
tool_resource: purpose,
|
||||
file_id: fileId,
|
||||
})),
|
||||
});
|
||||
|
||||
return res.status(200).json({ id: fileId, deleted: true });
|
||||
} catch (error) {
|
||||
logger.error('[AgentManagement] Error unlinking Agent file', error);
|
||||
return sendError(res, 'internal_error');
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
authorizeUpload,
|
||||
getUploadConfig: getAuthorizedUploadConfig,
|
||||
upload,
|
||||
list,
|
||||
remove,
|
||||
};
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ export * from './errors';
|
|||
export * from './eventRetention';
|
||||
export * from './envelope';
|
||||
export * from './execution';
|
||||
export * from './files';
|
||||
export * from './handlers';
|
||||
export * from './guard';
|
||||
export * from './harvest';
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue