mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-10-21 01:22:54 +00:00
🪣 refactor: Rate-Limit Token Routes and Cap Remote File Downloads (#13978)
* harden token and remote file handling * sort s3 storage imports * split token submission rate limits
This commit is contained in:
parent
3790bdcff2
commit
b15d40e3e4
16 changed files with 518 additions and 21 deletions
|
|
@ -12,6 +12,8 @@ const promptUsageLimiter = require('./promptUsageLimiter');
|
|||
const verifyEmailLimiter = require('./verifyEmailLimiter');
|
||||
const resetPasswordLimiter = require('./resetPasswordLimiter');
|
||||
const twoFactorTempLimiter = require('./twoFactorTempLimiter');
|
||||
const verifyEmailSubmissionLimiter = require('./verifyEmailSubmissionLimiter');
|
||||
const resetPasswordSubmissionLimiter = require('./resetPasswordSubmissionLimiter');
|
||||
|
||||
module.exports = {
|
||||
...uploadLimiters,
|
||||
|
|
@ -26,5 +28,7 @@ module.exports = {
|
|||
createSTTLimiters,
|
||||
verifyEmailLimiter,
|
||||
resetPasswordLimiter,
|
||||
verifyEmailSubmissionLimiter,
|
||||
resetPasswordSubmissionLimiter,
|
||||
twoFactorTempLimiter,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
const rateLimit = require('express-rate-limit');
|
||||
const { ViolationTypes } = require('librechat-data-provider');
|
||||
const { limiterCache, removePorts } = require('@librechat/api');
|
||||
const { logViolation } = require('~/cache');
|
||||
|
||||
const {
|
||||
RESET_PASSWORD_SUBMISSION_WINDOW = process.env.RESET_PASSWORD_WINDOW ?? 2,
|
||||
RESET_PASSWORD_SUBMISSION_MAX = process.env.RESET_PASSWORD_MAX ?? 2,
|
||||
RESET_PASSWORD_SUBMISSION_VIOLATION_SCORE: score,
|
||||
} = process.env;
|
||||
const windowMs = RESET_PASSWORD_SUBMISSION_WINDOW * 60 * 1000;
|
||||
const max = RESET_PASSWORD_SUBMISSION_MAX;
|
||||
const windowInMinutes = windowMs / 60000;
|
||||
const message = `Too many attempts, please try again after ${windowInMinutes} minute(s)`;
|
||||
|
||||
const handler = async (req, res) => {
|
||||
const type = ViolationTypes.RESET_PASSWORD_LIMIT;
|
||||
const errorMessage = {
|
||||
type,
|
||||
max,
|
||||
windowInMinutes,
|
||||
limiter: 'submission',
|
||||
};
|
||||
|
||||
await logViolation(req, res, type, errorMessage, score);
|
||||
return res.status(429).json({ message });
|
||||
};
|
||||
|
||||
const limiterOptions = {
|
||||
windowMs,
|
||||
max,
|
||||
handler,
|
||||
keyGenerator: removePorts,
|
||||
store: limiterCache('reset_password_submission_limiter'),
|
||||
};
|
||||
|
||||
const resetPasswordSubmissionLimiter = rateLimit(limiterOptions);
|
||||
|
||||
module.exports = resetPasswordSubmissionLimiter;
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
const rateLimit = require('express-rate-limit');
|
||||
const { ViolationTypes } = require('librechat-data-provider');
|
||||
const { limiterCache, removePorts } = require('@librechat/api');
|
||||
const { logViolation } = require('~/cache');
|
||||
|
||||
const {
|
||||
VERIFY_EMAIL_SUBMISSION_WINDOW = process.env.VERIFY_EMAIL_WINDOW ?? 2,
|
||||
VERIFY_EMAIL_SUBMISSION_MAX = process.env.VERIFY_EMAIL_MAX ?? 2,
|
||||
VERIFY_EMAIL_SUBMISSION_VIOLATION_SCORE: score,
|
||||
} = process.env;
|
||||
const windowMs = VERIFY_EMAIL_SUBMISSION_WINDOW * 60 * 1000;
|
||||
const max = VERIFY_EMAIL_SUBMISSION_MAX;
|
||||
const windowInMinutes = windowMs / 60000;
|
||||
const message = `Too many attempts, please try again after ${windowInMinutes} minute(s)`;
|
||||
|
||||
const handler = async (req, res) => {
|
||||
const type = ViolationTypes.VERIFY_EMAIL_LIMIT;
|
||||
const errorMessage = {
|
||||
type,
|
||||
max,
|
||||
windowInMinutes,
|
||||
limiter: 'submission',
|
||||
};
|
||||
|
||||
await logViolation(req, res, type, errorMessage, score);
|
||||
return res.status(429).json({ message });
|
||||
};
|
||||
|
||||
const limiterOptions = {
|
||||
windowMs,
|
||||
max,
|
||||
handler,
|
||||
keyGenerator: removePorts,
|
||||
store: limiterCache('verify_email_submission_limiter'),
|
||||
};
|
||||
|
||||
const verifyEmailSubmissionLimiter = rateLimit(limiterOptions);
|
||||
|
||||
module.exports = verifyEmailSubmissionLimiter;
|
||||
|
|
@ -62,6 +62,7 @@ jest.mock('~/server/middleware', () => {
|
|||
checkInviteUser: pass,
|
||||
validateRegistration: pass,
|
||||
resetPasswordLimiter: pass,
|
||||
resetPasswordSubmissionLimiter: pass,
|
||||
validatePasswordReset: pass,
|
||||
requireJwtAuth: pass,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ jest.mock('~/server/middleware', () => {
|
|||
checkInviteUser: pass,
|
||||
validateRegistration: pass,
|
||||
resetPasswordLimiter: pass,
|
||||
resetPasswordSubmissionLimiter: pass,
|
||||
validatePasswordReset: pass,
|
||||
requireJwtAuth: jest.fn((req, res, next) => {
|
||||
if (req.headers.authorization !== 'Bearer ok') {
|
||||
|
|
|
|||
|
|
@ -80,6 +80,7 @@ router.post(
|
|||
);
|
||||
router.post(
|
||||
'/resetPassword',
|
||||
middleware.resetPasswordSubmissionLimiter,
|
||||
middleware.checkBan,
|
||||
middleware.validatePasswordReset,
|
||||
resetPasswordController,
|
||||
|
|
|
|||
121
api/server/routes/auth.reset-password-ratelimit.test.js
Normal file
121
api/server/routes/auth.reset-password-ratelimit.test.js
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
const express = require('express');
|
||||
const request = require('supertest');
|
||||
|
||||
const mockCheckBan = jest.fn((req, res, next) => next());
|
||||
const mockResetPasswordSubmissionLimiter = jest.fn((req, res, next) => next());
|
||||
const mockValidatePasswordReset = jest.fn((req, res, next) => next());
|
||||
const mockResetPasswordController = jest.fn((req, res) => res.status(204).end());
|
||||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
createSetBalanceConfig: jest.fn(() => (req, res, next) => next()),
|
||||
forceRefreshCloudFrontAuthCookies: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/controllers/AuthController', () => ({
|
||||
refreshController: jest.fn((req, res) => res.status(204).end()),
|
||||
registrationController: jest.fn((req, res) => res.status(204).end()),
|
||||
resetPasswordController: (...args) => mockResetPasswordController(...args),
|
||||
resetPasswordRequestController: jest.fn((req, res) => res.status(204).end()),
|
||||
graphTokenController: jest.fn((req, res) => res.status(204).end()),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/controllers/TwoFactorController', () => ({
|
||||
enable2FA: jest.fn((req, res) => res.status(204).end()),
|
||||
verify2FA: jest.fn((req, res) => res.status(204).end()),
|
||||
confirm2FA: jest.fn((req, res) => res.status(204).end()),
|
||||
disable2FA: jest.fn((req, res) => res.status(204).end()),
|
||||
regenerateBackupCodes: jest.fn((req, res) => res.status(204).end()),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/controllers/auth/TwoFactorAuthController', () => ({
|
||||
verify2FAWithTempToken: jest.fn((req, res) => res.status(204).end()),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/controllers/auth/LogoutController', () => ({
|
||||
logoutController: jest.fn((req, res) => res.status(204).end()),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/controllers/auth/LoginController', () => ({
|
||||
loginController: jest.fn((req, res) => res.status(204).end()),
|
||||
}));
|
||||
|
||||
jest.mock('~/models', () => ({
|
||||
findBalanceByUser: jest.fn(),
|
||||
upsertBalanceFields: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/services/Config', () => ({
|
||||
getAppConfig: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/middleware', () => {
|
||||
const pass = (req, res, next) => next();
|
||||
return {
|
||||
logHeaders: pass,
|
||||
loginLimiter: pass,
|
||||
setTwoFactorTempUser: pass,
|
||||
twoFactorTempLimiter: pass,
|
||||
checkBan: (...args) => mockCheckBan(...args),
|
||||
requireLocalAuth: pass,
|
||||
requireLdapAuth: pass,
|
||||
registerLimiter: pass,
|
||||
checkInviteUser: pass,
|
||||
validateRegistration: pass,
|
||||
resetPasswordLimiter: pass,
|
||||
resetPasswordSubmissionLimiter: (...args) => mockResetPasswordSubmissionLimiter(...args),
|
||||
validatePasswordReset: (...args) => mockValidatePasswordReset(...args),
|
||||
requireJwtAuth: pass,
|
||||
};
|
||||
});
|
||||
|
||||
const authRouter = require('./auth');
|
||||
|
||||
describe('POST /api/auth/resetPassword rate limiting', () => {
|
||||
let app;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
mockCheckBan.mockImplementation((req, res, next) => next());
|
||||
mockResetPasswordSubmissionLimiter.mockImplementation((req, res, next) => next());
|
||||
mockValidatePasswordReset.mockImplementation((req, res, next) => next());
|
||||
mockResetPasswordController.mockImplementation((req, res) => res.status(204).end());
|
||||
|
||||
app = express();
|
||||
app.use(express.json());
|
||||
app.use('/api/auth', authRouter);
|
||||
});
|
||||
|
||||
it('limits before ban checks, validation, and password reset work', async () => {
|
||||
await request(app).post('/api/auth/resetPassword').send({ token: 'token' }).expect(204);
|
||||
|
||||
expect(mockResetPasswordSubmissionLimiter).toHaveBeenCalledTimes(1);
|
||||
expect(mockCheckBan).toHaveBeenCalledTimes(1);
|
||||
expect(mockValidatePasswordReset).toHaveBeenCalledTimes(1);
|
||||
expect(mockResetPasswordController).toHaveBeenCalledTimes(1);
|
||||
expect(mockResetPasswordSubmissionLimiter.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
mockCheckBan.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(mockCheckBan.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
mockValidatePasswordReset.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(mockValidatePasswordReset.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
mockResetPasswordController.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
it('does not validate or reset passwords after the limiter rejects the request', async () => {
|
||||
mockResetPasswordSubmissionLimiter.mockImplementation((req, res) =>
|
||||
res.status(429).json({ message: 'Too many password reset attempts' }),
|
||||
);
|
||||
|
||||
const response = await request(app)
|
||||
.post('/api/auth/resetPassword')
|
||||
.send({ token: 'token' })
|
||||
.expect(429);
|
||||
|
||||
expect(response.body).toEqual({ message: 'Too many password reset attempts' });
|
||||
expect(mockCheckBan).not.toHaveBeenCalled();
|
||||
expect(mockValidatePasswordReset).not.toHaveBeenCalled();
|
||||
expect(mockResetPasswordController).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -10,6 +10,7 @@ const {
|
|||
} = require('~/server/controllers/UserController');
|
||||
const {
|
||||
verifyEmailLimiter,
|
||||
verifyEmailSubmissionLimiter,
|
||||
configMiddleware,
|
||||
canDeleteAccount,
|
||||
requireJwtAuth,
|
||||
|
|
@ -25,7 +26,7 @@ router.get('/terms', requireJwtAuth, getTermsStatusController);
|
|||
router.post('/terms/accept', requireJwtAuth, acceptTermsController);
|
||||
router.post('/plugins', requireJwtAuth, updateUserPluginsController);
|
||||
router.delete('/delete', requireJwtAuth, canDeleteAccount, configMiddleware, deleteUserController);
|
||||
router.post('/verify', verifyEmailController);
|
||||
router.post('/verify', verifyEmailSubmissionLimiter, verifyEmailController);
|
||||
router.post('/verify/resend', verifyEmailLimiter, resendVerificationController);
|
||||
|
||||
module.exports = router;
|
||||
|
|
|
|||
71
api/server/routes/user.verify-ratelimit.test.js
Normal file
71
api/server/routes/user.verify-ratelimit.test.js
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
const express = require('express');
|
||||
const request = require('supertest');
|
||||
|
||||
const mockVerifyEmailSubmissionLimiter = jest.fn((req, res, next) => next());
|
||||
const mockVerifyEmailController = jest.fn((req, res) => res.status(204).end());
|
||||
|
||||
jest.mock('~/server/controllers/UserController', () => ({
|
||||
getUserController: jest.fn((req, res) => res.status(204).end()),
|
||||
deleteUserController: jest.fn((req, res) => res.status(204).end()),
|
||||
acceptTermsController: jest.fn((req, res) => res.status(204).end()),
|
||||
verifyEmailController: (...args) => mockVerifyEmailController(...args),
|
||||
getTermsStatusController: jest.fn((req, res) => res.status(204).end()),
|
||||
updateUserPluginsController: jest.fn((req, res) => res.status(204).end()),
|
||||
resendVerificationController: jest.fn((req, res) => res.status(204).end()),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/middleware', () => {
|
||||
const pass = (req, res, next) => next();
|
||||
return {
|
||||
requireJwtAuth: pass,
|
||||
canDeleteAccount: pass,
|
||||
configMiddleware: pass,
|
||||
verifyEmailLimiter: pass,
|
||||
verifyEmailSubmissionLimiter: (...args) => mockVerifyEmailSubmissionLimiter(...args),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock('./settings', () => {
|
||||
const express = require('express');
|
||||
return express.Router();
|
||||
});
|
||||
|
||||
const userRouter = require('./user');
|
||||
|
||||
describe('POST /api/user/verify rate limiting', () => {
|
||||
let app;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
mockVerifyEmailSubmissionLimiter.mockImplementation((req, res, next) => next());
|
||||
mockVerifyEmailController.mockImplementation((req, res) => res.status(204).end());
|
||||
|
||||
app = express();
|
||||
app.use(express.json());
|
||||
app.use('/api/user', userRouter);
|
||||
});
|
||||
|
||||
it('limits email verification before checking the token', async () => {
|
||||
await request(app).post('/api/user/verify').send({ token: 'token' }).expect(204);
|
||||
|
||||
expect(mockVerifyEmailSubmissionLimiter).toHaveBeenCalledTimes(1);
|
||||
expect(mockVerifyEmailController).toHaveBeenCalledTimes(1);
|
||||
expect(mockVerifyEmailSubmissionLimiter.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
mockVerifyEmailController.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
it('does not check the verification token after the limiter rejects the request', async () => {
|
||||
mockVerifyEmailSubmissionLimiter.mockImplementation((req, res) =>
|
||||
res.status(429).json({ message: 'Too many verification attempts' }),
|
||||
);
|
||||
|
||||
const response = await request(app)
|
||||
.post('/api/user/verify')
|
||||
.send({ token: 'token' })
|
||||
.expect(429);
|
||||
|
||||
expect(response.body).toEqual({ message: 'Too many verification attempts' });
|
||||
expect(mockVerifyEmailController).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -4,7 +4,14 @@ const mime = require('mime');
|
|||
const axios = require('axios');
|
||||
const fetch = require('node-fetch');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { getAzureContainerClient, deleteRagFile } = require('@librechat/api');
|
||||
const {
|
||||
deleteRagFile,
|
||||
assertRemoteFileURL,
|
||||
getAzureContainerClient,
|
||||
getRemoteFileFetchMaxBytes,
|
||||
getRemoteFileFetchTimeoutMs,
|
||||
assertRemoteFileContentLength,
|
||||
} = require('@librechat/api');
|
||||
|
||||
const defaultBasePath = 'images';
|
||||
const { AZURE_STORAGE_PUBLIC_ACCESS = 'true', AZURE_CONTAINER_NAME = 'files' } = process.env;
|
||||
|
|
@ -63,8 +70,20 @@ async function saveURLToAzure({
|
|||
containerName,
|
||||
}) {
|
||||
try {
|
||||
const response = await fetch(URL);
|
||||
const maxBytes = getRemoteFileFetchMaxBytes();
|
||||
const response = await fetch(assertRemoteFileURL(URL), {
|
||||
timeout: getRemoteFileFetchTimeoutMs(),
|
||||
size: maxBytes,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
assertRemoteFileContentLength(response.headers, maxBytes);
|
||||
const buffer = await response.buffer();
|
||||
if (buffer.length > maxBytes) {
|
||||
throw new Error(`Remote file response too large: ${buffer.length} bytes`);
|
||||
}
|
||||
|
||||
return await saveBufferToAzure({ userId, buffer, fileName, basePath, containerName });
|
||||
} catch (error) {
|
||||
logger.error('[saveURLToAzure] Error uploading file from URL:', error);
|
||||
|
|
|
|||
|
|
@ -3,7 +3,14 @@ const path = require('path');
|
|||
const axios = require('axios');
|
||||
const fetch = require('node-fetch');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { getFirebaseStorage, deleteRagFile } = require('@librechat/api');
|
||||
const {
|
||||
deleteRagFile,
|
||||
getFirebaseStorage,
|
||||
assertRemoteFileURL,
|
||||
getRemoteFileFetchMaxBytes,
|
||||
getRemoteFileFetchTimeoutMs,
|
||||
assertRemoteFileContentLength,
|
||||
} = require('@librechat/api');
|
||||
const { ref, uploadBytes, getDownloadURL, deleteObject } = require('firebase/storage');
|
||||
const { getBufferMetadata } = require('~/server/utils');
|
||||
|
||||
|
|
@ -57,8 +64,19 @@ async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' })
|
|||
}
|
||||
|
||||
const storageRef = ref(storage, `${basePath}/${userId.toString()}/${fileName}`);
|
||||
const response = await fetch(URL);
|
||||
const maxBytes = getRemoteFileFetchMaxBytes();
|
||||
const response = await fetch(assertRemoteFileURL(URL), {
|
||||
timeout: getRemoteFileFetchTimeoutMs(),
|
||||
size: maxBytes,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
assertRemoteFileContentLength(response.headers, maxBytes);
|
||||
const buffer = await response.buffer();
|
||||
if (buffer.length > maxBytes) {
|
||||
throw new Error(`Remote file response too large: ${buffer.length} bytes`);
|
||||
}
|
||||
|
||||
try {
|
||||
await uploadBytes(storageRef, buffer);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,13 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const axios = require('axios');
|
||||
const { deleteRagFile } = require('@librechat/api');
|
||||
const {
|
||||
deleteRagFile,
|
||||
assertRemoteFileURL,
|
||||
getRemoteFileFetchMaxBytes,
|
||||
getRemoteFileFetchTimeoutMs,
|
||||
assertRemoteFileContentLength,
|
||||
} = require('@librechat/api');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { EModelEndpoint } = require('librechat-data-provider');
|
||||
const { resizeImageBuffer } = require('~/server/services/Files/images/resize');
|
||||
|
|
@ -115,12 +121,21 @@ async function saveLocalBuffer({ userId, buffer, fileName, basePath = 'images' }
|
|||
*/
|
||||
async function saveFileFromURL({ userId, URL, fileName, basePath = 'images' }) {
|
||||
try {
|
||||
const maxBytes = getRemoteFileFetchMaxBytes();
|
||||
const response = await axios({
|
||||
url: URL,
|
||||
url: assertRemoteFileURL(URL),
|
||||
responseType: 'arraybuffer',
|
||||
timeout: getRemoteFileFetchTimeoutMs(),
|
||||
maxContentLength: maxBytes,
|
||||
maxBodyLength: maxBytes,
|
||||
});
|
||||
assertRemoteFileContentLength(response.headers, maxBytes);
|
||||
|
||||
const buffer = Buffer.from(response.data, 'binary');
|
||||
if (buffer.length > maxBytes) {
|
||||
throw new Error(`Remote file response too large: ${buffer.length} bytes`);
|
||||
}
|
||||
|
||||
const { bytes, type, dimensions, extension } = await getBufferMetadata(buffer);
|
||||
|
||||
// Construct the outputPath based on the basePath and userId
|
||||
|
|
|
|||
|
|
@ -4,3 +4,4 @@ export * from './types';
|
|||
export * from './images';
|
||||
export * from './avatar';
|
||||
export * from './metadata';
|
||||
export * from './url';
|
||||
|
|
|
|||
|
|
@ -580,7 +580,10 @@ describe('S3 CRUD', () => {
|
|||
fileName: 'downloaded.jpg',
|
||||
});
|
||||
|
||||
expect(global.fetch).toHaveBeenCalledWith('https://example.com/image.jpg');
|
||||
expect(global.fetch).toHaveBeenCalledWith(
|
||||
'https://example.com/image.jpg',
|
||||
expect.objectContaining({ signal: expect.any(Object) }),
|
||||
);
|
||||
expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(1);
|
||||
expect(result).toBe('https://bucket.s3.amazonaws.com/test-key?signed=true');
|
||||
});
|
||||
|
|
@ -593,7 +596,10 @@ describe('S3 CRUD', () => {
|
|||
fileName: 'downloaded.jpg',
|
||||
});
|
||||
|
||||
expect(global.fetch).toHaveBeenCalledWith('https://example.com/image.jpg');
|
||||
expect(global.fetch).toHaveBeenCalledWith(
|
||||
'https://example.com/image.jpg',
|
||||
expect.objectContaining({ signal: expect.any(Object) }),
|
||||
);
|
||||
expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(1);
|
||||
expect(result).toEqual({
|
||||
filepath: 'https://bucket.s3.amazonaws.com/test-key?signed=true',
|
||||
|
|
@ -779,6 +785,47 @@ describe('S3 CRUD', () => {
|
|||
).rejects.toThrow('Failed to fetch URL');
|
||||
});
|
||||
|
||||
it('rejects non-http remote file URLs before fetching', async () => {
|
||||
const { saveURLToS3 } = await import('../crud');
|
||||
await expect(
|
||||
saveURLToS3({
|
||||
userId: 'user123',
|
||||
URL: 'file:///etc/passwd',
|
||||
fileName: 'passwd',
|
||||
}),
|
||||
).rejects.toThrow('Refusing to fetch remote file over file:');
|
||||
|
||||
expect(global.fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects remote responses larger than the configured cap', async () => {
|
||||
process.env.REMOTE_FILE_FETCH_MAX_BYTES = '4';
|
||||
try {
|
||||
(global.fetch as unknown as jest.Mock).mockResolvedValueOnce({
|
||||
ok: true,
|
||||
headers: {
|
||||
get: (name: string) =>
|
||||
({
|
||||
'content-length': '8',
|
||||
'content-type': 'image/jpeg',
|
||||
})[name.toLowerCase()] ?? null,
|
||||
},
|
||||
arrayBuffer: jest.fn().mockResolvedValue(new ArrayBuffer(8)),
|
||||
});
|
||||
|
||||
const { saveURLToS3 } = await import('../crud');
|
||||
await expect(
|
||||
saveURLToS3({
|
||||
userId: 'user123',
|
||||
URL: 'https://example.com/image.jpg',
|
||||
fileName: 'downloaded.jpg',
|
||||
}),
|
||||
).rejects.toThrow('Remote file response too large: 8 bytes');
|
||||
} finally {
|
||||
delete process.env.REMOTE_FILE_FETCH_MAX_BYTES;
|
||||
}
|
||||
});
|
||||
|
||||
it('handles fetch errors', async () => {
|
||||
(global.fetch as unknown as jest.Mock).mockRejectedValueOnce(new Error('Network error'));
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import fs from 'fs';
|
||||
import { Readable } from 'stream';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import { FileSources } from 'librechat-data-provider';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import {
|
||||
UploadPartCommand,
|
||||
PutObjectCommand,
|
||||
|
|
@ -10,16 +13,12 @@ import {
|
|||
HeadObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import { FileSources } from 'librechat-data-provider';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import type {
|
||||
CompletedPart,
|
||||
GetObjectCommandInput,
|
||||
PutObjectCommandInput,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import type { TFile } from 'librechat-data-provider';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import type {
|
||||
UploadFileParams,
|
||||
SaveBufferParams,
|
||||
|
|
@ -32,6 +31,20 @@ import type {
|
|||
UrlBuilder,
|
||||
S3FileRef,
|
||||
} from '~/storage/types';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import {
|
||||
assertRemoteFileURL,
|
||||
getRemoteFileFetchMaxBytes,
|
||||
getRemoteFileFetchTimeoutMs,
|
||||
assertRemoteFileContentLength,
|
||||
createRemoteFileByteLimitTransform,
|
||||
} from '~/storage/url';
|
||||
import {
|
||||
AVATAR_BASE_PATH,
|
||||
DEFAULT_BASE_PATH as defaultBasePath,
|
||||
INLINE_AVATAR_PATH_PREFIX,
|
||||
INLINE_IMAGE_PATH_PREFIX,
|
||||
} from '~/storage/constants';
|
||||
import {
|
||||
assertS3FileName,
|
||||
assertPathSegment,
|
||||
|
|
@ -39,12 +52,6 @@ import {
|
|||
} from '~/storage/validation';
|
||||
import { initializeS3 } from '~/cdn/s3';
|
||||
import { deleteRagFile } from '~/files';
|
||||
import {
|
||||
AVATAR_BASE_PATH,
|
||||
DEFAULT_BASE_PATH as defaultBasePath,
|
||||
INLINE_AVATAR_PATH_PREFIX,
|
||||
INLINE_IMAGE_PATH_PREFIX,
|
||||
} from '~/storage/constants';
|
||||
import { s3Config } from './s3Config';
|
||||
|
||||
const {
|
||||
|
|
@ -551,15 +558,20 @@ export async function saveURLToS3WithMetadata({
|
|||
urlBuilder,
|
||||
}: SaveURLParams & { urlBuilder?: UrlBuilder }): Promise<SaveURLResult> {
|
||||
try {
|
||||
const response = await fetch(URL);
|
||||
const maxBytes = getRemoteFileFetchMaxBytes();
|
||||
const response = await fetch(assertRemoteFileURL(URL), {
|
||||
signal: AbortSignal.timeout(getRemoteFileFetchTimeoutMs()),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
assertRemoteFileContentLength(response.headers, maxBytes);
|
||||
|
||||
const contentType = response.headers.get('content-type') ?? '';
|
||||
if (response.body) {
|
||||
const source = Readable.fromWeb(
|
||||
response.body as unknown as Parameters<typeof Readable.fromWeb>[0],
|
||||
);
|
||||
).pipe(createRemoteFileByteLimitTransform(maxBytes));
|
||||
const result = await saveReadableToS3({
|
||||
userId,
|
||||
body: source,
|
||||
|
|
@ -582,6 +594,10 @@ export async function saveURLToS3WithMetadata({
|
|||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
if (buffer.length > maxBytes) {
|
||||
throw new Error(`Remote file response too large: ${buffer.length} bytes`);
|
||||
}
|
||||
|
||||
const filepath = await saveBufferToS3({
|
||||
userId,
|
||||
buffer,
|
||||
|
|
|
|||
103
packages/api/src/storage/url.ts
Normal file
103
packages/api/src/storage/url.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import { Transform } from 'stream';
|
||||
import type { TransformCallback } from 'stream';
|
||||
|
||||
export const DEFAULT_REMOTE_FILE_FETCH_TIMEOUT_MS: number = 15000;
|
||||
export const DEFAULT_REMOTE_FILE_FETCH_MAX_BYTES: number = 512 * 1024 * 1024;
|
||||
|
||||
const REMOTE_FILE_PROTOCOLS = new Set(['http:', 'https:']);
|
||||
|
||||
type HeaderGetter = {
|
||||
get: (name: string) => string | null;
|
||||
};
|
||||
|
||||
type HeaderRecord = Record<string, string | string[] | number | undefined>;
|
||||
|
||||
export type RemoteFileHeaders = HeaderGetter | HeaderRecord;
|
||||
|
||||
function readPositiveIntEnv(name: string, fallback: number): number {
|
||||
const value = process.env[name];
|
||||
if (!value) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function isHeaderGetter(headers: RemoteFileHeaders): headers is HeaderGetter {
|
||||
return typeof (headers as HeaderGetter).get === 'function';
|
||||
}
|
||||
|
||||
function getHeader(headers: RemoteFileHeaders, name: string): string | null {
|
||||
if (isHeaderGetter(headers)) {
|
||||
return headers.get(name);
|
||||
}
|
||||
|
||||
const value = headers[name] ?? headers[name.toLowerCase()] ?? headers[name.toUpperCase()];
|
||||
if (Array.isArray(value)) {
|
||||
return value[0] ?? null;
|
||||
}
|
||||
|
||||
return value == null ? null : String(value);
|
||||
}
|
||||
|
||||
export function getRemoteFileFetchTimeoutMs(): number {
|
||||
return readPositiveIntEnv('REMOTE_FILE_FETCH_TIMEOUT_MS', DEFAULT_REMOTE_FILE_FETCH_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
export function getRemoteFileFetchMaxBytes(): number {
|
||||
return readPositiveIntEnv('REMOTE_FILE_FETCH_MAX_BYTES', DEFAULT_REMOTE_FILE_FETCH_MAX_BYTES);
|
||||
}
|
||||
|
||||
export function assertRemoteFileURL(input: string): string {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(input);
|
||||
} catch {
|
||||
throw new Error('Invalid remote file URL');
|
||||
}
|
||||
|
||||
if (!REMOTE_FILE_PROTOCOLS.has(parsed.protocol)) {
|
||||
throw new Error(`Refusing to fetch remote file over ${parsed.protocol}`);
|
||||
}
|
||||
|
||||
return parsed.href;
|
||||
}
|
||||
|
||||
export function assertRemoteFileContentLength(
|
||||
headers: RemoteFileHeaders,
|
||||
maxBytes: number = getRemoteFileFetchMaxBytes(),
|
||||
): void {
|
||||
const contentLength = Number.parseInt(getHeader(headers, 'content-length') ?? '0', 10);
|
||||
if (contentLength > maxBytes) {
|
||||
throw new Error(`Remote file response too large: ${contentLength} bytes`);
|
||||
}
|
||||
}
|
||||
|
||||
class RemoteFileByteLimitTransform extends Transform {
|
||||
private bytes = 0;
|
||||
|
||||
constructor(private readonly maxBytes: number) {
|
||||
super();
|
||||
}
|
||||
|
||||
override _transform(
|
||||
chunk: Buffer | string,
|
||||
encoding: BufferEncoding,
|
||||
callback: TransformCallback,
|
||||
): void {
|
||||
this.bytes += typeof chunk === 'string' ? Buffer.byteLength(chunk, encoding) : chunk.byteLength;
|
||||
if (this.bytes > this.maxBytes) {
|
||||
callback(new Error(`Remote file response too large: ${this.bytes} bytes`));
|
||||
return;
|
||||
}
|
||||
|
||||
callback(null, chunk);
|
||||
}
|
||||
}
|
||||
|
||||
export function createRemoteFileByteLimitTransform(
|
||||
maxBytes: number = getRemoteFileFetchMaxBytes(),
|
||||
): Transform {
|
||||
return new RemoteFileByteLimitTransform(maxBytes);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue