diff --git a/api/app/clients/tools/structured/DALLE3.js b/api/app/clients/tools/structured/DALLE3.js index 8ce46323af..0f885ed636 100644 --- a/api/app/clients/tools/structured/DALLE3.js +++ b/api/app/clients/tools/structured/DALLE3.js @@ -4,7 +4,11 @@ const { v4: uuidv4 } = require('uuid'); const { ProxyAgent, fetch } = require('undici'); const { logger } = require('@librechat/data-schemas'); const { Tool } = require('@librechat/agents/langchain/tools'); -const { getImageBasename, extractBaseURL } = require('@librechat/api'); +const { + getImageBasename, + extractBaseURL, + createMinimalRetentionRequest, +} = require('@librechat/api'); const { FileContext, ContentTypes } = require('librechat-data-provider'); const dalle3JsonSchema = { @@ -49,6 +53,7 @@ class DALLE3 extends Tool { this.userId = fields.userId; this.tenantId = fields.req?.user?.tenantId; + this.retentionRequest = createMinimalRetentionRequest(fields.req); this.fileStrategy = fields.fileStrategy; /** @type {boolean} */ this.isAgent = fields.isAgent; @@ -230,6 +235,7 @@ Error Message: ${error.message}`); fileStrategy: this.fileStrategy, context: FileContext.image_generation, tenantId: this.tenantId, + req: this.retentionRequest, }); if (this.returnMetadata) { diff --git a/api/app/clients/tools/structured/FluxAPI.js b/api/app/clients/tools/structured/FluxAPI.js index dc94a25e82..e251b2da65 100644 --- a/api/app/clients/tools/structured/FluxAPI.js +++ b/api/app/clients/tools/structured/FluxAPI.js @@ -4,6 +4,7 @@ const { v4: uuidv4 } = require('uuid'); const { logger } = require('@librechat/data-schemas'); const { HttpsProxyAgent } = require('https-proxy-agent'); const { Tool } = require('@librechat/agents/langchain/tools'); +const { createMinimalRetentionRequest } = require('@librechat/api'); const { FileContext, ContentTypes } = require('librechat-data-provider'); const fluxApiJsonSchema = { @@ -110,6 +111,7 @@ class FluxAPI extends Tool { this.userId = fields.userId; this.tenantId = fields.req?.user?.tenantId; + this.retentionRequest = createMinimalRetentionRequest(fields.req); this.fileStrategy = fields.fileStrategy; /** @type {boolean} **/ @@ -343,6 +345,7 @@ class FluxAPI extends Tool { basePath: 'images', context: FileContext.image_generation, tenantId: this.tenantId, + req: this.retentionRequest, }); logger.debug('[FluxAPI] Image saved to path:', result.filepath); @@ -574,6 +577,7 @@ class FluxAPI extends Tool { basePath: 'images', context: FileContext.image_generation, tenantId: this.tenantId, + req: this.retentionRequest, }); logger.debug('[FluxAPI] Finetuned image saved to path:', result.filepath); diff --git a/api/app/clients/tools/structured/specs/imageTools-agent.spec.js b/api/app/clients/tools/structured/specs/imageTools-agent.spec.js index f88b76a116..2d36ad4b7f 100644 --- a/api/app/clients/tools/structured/specs/imageTools-agent.spec.js +++ b/api/app/clients/tools/structured/specs/imageTools-agent.spec.js @@ -100,11 +100,21 @@ describe('image tools - agent mode ToolMessage format', () => { }); it('keeps tenant context without retaining the request object', () => { - const req = { user: { tenantId: 'tenant-a' }, socket: {} }; + const req = { + user: { id: 'user-1', tenantId: 'tenant-a' }, + body: { conversationId: 'convo-1', isTemporary: 'true' }, + config: { interfaceConfig: { retentionMode: 'all' } }, + socket: {}, + }; const dalle = new DALLE3({ isAgent: false, processFileURL: jest.fn(), req }); expect(dalle.tenantId).toBe('tenant-a'); expect(dalle.req).toBeUndefined(); + expect(dalle.retentionRequest).toEqual({ + user: { id: 'user-1', tenantId: 'tenant-a' }, + body: { conversationId: 'convo-1', isTemporary: 'true' }, + config: { interfaceConfig: { retentionMode: 'all' } }, + }); }); it('invoke() returns ToolMessage with base64 in artifact, not serialized in content', async () => { @@ -181,11 +191,90 @@ describe('image tools - agent mode ToolMessage format', () => { }); it('keeps tenant context without retaining the request object', () => { - const req = { user: { tenantId: 'tenant-a' }, socket: {} }; + const req = { + user: { id: 'user-1', tenantId: 'tenant-a' }, + body: { conversationId: 'convo-1', isTemporary: 'true' }, + config: { interfaceConfig: { retentionMode: 'all' } }, + socket: {}, + }; const flux = new FluxAPI({ isAgent: false, processFileURL: jest.fn(), req }); expect(flux.tenantId).toBe('tenant-a'); expect(flux.req).toBeUndefined(); + expect(flux.retentionRequest).toEqual({ + user: { id: 'user-1', tenantId: 'tenant-a' }, + body: { conversationId: 'convo-1', isTemporary: 'true' }, + config: { interfaceConfig: { retentionMode: 'all' } }, + }); + }); + + it('passes minimal retention context when saving generated images', async () => { + const processFileURL = jest.fn().mockResolvedValue({ filepath: '/images/generated.png' }); + const req = { + user: { id: 'user-1', tenantId: 'tenant-a' }, + body: { conversationId: 'convo-1', isTemporary: 'true' }, + config: { interfaceConfig: { retentionMode: 'all' } }, + socket: {}, + }; + const flux = new FluxAPI({ + isAgent: false, + processFileURL, + req, + userId: 'user-1', + fileStrategy: 'local', + }); + const invokePromise = flux.invoke( + makeToolCall('flux', { prompt: 'a box', endpoint: '/v1/flux-dev' }), + ); + await jest.runAllTimersAsync(); + await invokePromise; + + expect(processFileURL).toHaveBeenCalledWith( + expect.objectContaining({ + req: { + user: { id: 'user-1', tenantId: 'tenant-a' }, + body: { conversationId: 'convo-1', isTemporary: 'true' }, + config: { interfaceConfig: { retentionMode: 'all' } }, + }, + }), + ); + }); + + it('passes minimal retention context when saving finetuned generated images', async () => { + const processFileURL = jest.fn().mockResolvedValue({ filepath: '/images/generated.png' }); + const req = { + user: { id: 'user-1', tenantId: 'tenant-a' }, + body: { conversationId: 'convo-1', isTemporary: 'true' }, + config: { interfaceConfig: { retentionMode: 'all' } }, + socket: {}, + }; + const flux = new FluxAPI({ + isAgent: false, + processFileURL, + req, + userId: 'user-1', + fileStrategy: 'local', + }); + const invokePromise = flux.invoke( + makeToolCall('flux', { + action: 'generate_finetuned', + prompt: 'a box', + finetune_id: 'ft-abc123', + endpoint: '/v1/flux-pro-finetuned', + }), + ); + await jest.runAllTimersAsync(); + await invokePromise; + + expect(processFileURL).toHaveBeenCalledWith( + expect.objectContaining({ + req: { + user: { id: 'user-1', tenantId: 'tenant-a' }, + body: { conversationId: 'convo-1', isTemporary: 'true' }, + config: { interfaceConfig: { retentionMode: 'all' } }, + }, + }), + ); }); it('invoke() returns ToolMessage with base64 in artifact, not serialized in content', async () => { diff --git a/api/db/utils.js b/api/db/utils.js index 32051be78d..f3302c92da 100644 --- a/api/db/utils.js +++ b/api/db/utils.js @@ -1,4 +1,4 @@ -const { logger } = require('@librechat/data-schemas'); +const { logger, buildRetentionVisibilityFilter } = require('@librechat/data-schemas'); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -26,7 +26,10 @@ async function batchResetMeiliFlags(collection) { try { while (hasMore) { const docs = await collection - .find({ expiredAt: null, _meiliIndex: { $ne: false } }, { projection: { _id: 1 } }) + .find( + { ...buildRetentionVisibilityFilter(), _meiliIndex: { $ne: false } }, + { projection: { _id: 1 } }, + ) .limit(BATCH_SIZE) .toArray(); diff --git a/api/db/utils.spec.js b/api/db/utils.spec.js index adf4f6cd86..477bd70050 100644 --- a/api/db/utils.spec.js +++ b/api/db/utils.spec.js @@ -83,6 +83,60 @@ describe('batchResetMeiliFlags', () => { expect(expiredDoc._meiliIndex).toBe(true); }); + it('should reset active non-temporary documents with expiredAt set for all-data retention', async () => { + const retentionDate = new Date(Date.now() + 60 * 60 * 1000); + await testCollection.insertMany([ + { + _id: new mongoose.Types.ObjectId(), + isTemporary: false, + expiredAt: retentionDate, + _meiliIndex: true, + }, + { + _id: new mongoose.Types.ObjectId(), + isTemporary: true, + expiredAt: retentionDate, + _meiliIndex: true, + }, + ]); + + const result = await batchResetMeiliFlags(testCollection); + + expect(result).toBe(1); + + const retainedDoc = await testCollection.findOne({ isTemporary: false }); + const temporaryDoc = await testCollection.findOne({ isTemporary: true }); + expect(retainedDoc._meiliIndex).toBe(false); + expect(temporaryDoc._meiliIndex).toBe(true); + }); + + it('should not reset expired non-temporary documents with expiredAt set for all-data retention', async () => { + const retentionDate = new Date(Date.now() - 60 * 60 * 1000); + await testCollection.insertMany([ + { + _id: new mongoose.Types.ObjectId(), + isTemporary: false, + expiredAt: retentionDate, + _meiliIndex: true, + }, + { + _id: new mongoose.Types.ObjectId(), + isTemporary: false, + expiredAt: null, + _meiliIndex: true, + }, + ]); + + const result = await batchResetMeiliFlags(testCollection); + + expect(result).toBe(1); + + const expiredDoc = await testCollection.findOne({ expiredAt: retentionDate }); + const permanentDoc = await testCollection.findOne({ expiredAt: null }); + expect(expiredDoc._meiliIndex).toBe(true); + expect(permanentDoc._meiliIndex).toBe(false); + }); + it('should not modify documents with _meiliIndex: false', async () => { await testCollection.insertMany([ { _id: new mongoose.Types.ObjectId(), expiredAt: null, _meiliIndex: false }, diff --git a/api/server/controllers/UserController.spec.js b/api/server/controllers/UserController.spec.js index 30e6190e28..5e1419bde3 100644 --- a/api/server/controllers/UserController.spec.js +++ b/api/server/controllers/UserController.spec.js @@ -75,7 +75,7 @@ jest.mock('@librechat/api', () => ({ })); jest.mock('~/server/services/Files/process', () => ({ - processDeleteRequest: jest.fn().mockResolvedValue(undefined), + processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }), })); jest.mock('~/server/services/Config', () => ({ diff --git a/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js b/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js index 2d23b4b02c..ef605beaab 100644 --- a/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js +++ b/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js @@ -67,7 +67,7 @@ jest.mock('~/server/services/Config/getCachedTools', () => ({ })); jest.mock('~/server/services/Files/process', () => ({ - processDeleteRequest: jest.fn(), + processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }), })); jest.mock('~/server/services/Config', () => ({ diff --git a/api/server/controllers/__tests__/deleteUser.spec.js b/api/server/controllers/__tests__/deleteUser.spec.js index bc6acde53d..1d7c852153 100644 --- a/api/server/controllers/__tests__/deleteUser.spec.js +++ b/api/server/controllers/__tests__/deleteUser.spec.js @@ -127,7 +127,7 @@ function stubDeletionMocks() { mockDeleteUserById.mockResolvedValue(); mockDeleteAllSharedLinks.mockResolvedValue(); mockGetFiles.mockResolvedValue([]); - mockProcessDeleteRequest.mockResolvedValue(); + mockProcessDeleteRequest.mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }); mockDeleteFiles.mockResolvedValue(); mockDeleteToolCalls.mockResolvedValue(); mockDeleteUserAgents.mockResolvedValue(); diff --git a/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js b/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js index 65b9cf75b2..fd591aa440 100644 --- a/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js +++ b/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js @@ -81,7 +81,7 @@ jest.mock('~/server/services/Config/getCachedTools', () => ({ })); jest.mock('~/server/services/Files/process', () => ({ - processDeleteRequest: jest.fn(), + processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }), })); jest.mock('~/server/services/Config', () => ({ diff --git a/api/server/controllers/tools.js b/api/server/controllers/tools.js index 07be1210c1..4551adf617 100644 --- a/api/server/controllers/tools.js +++ b/api/server/controllers/tools.js @@ -10,6 +10,7 @@ const { } = require('librechat-data-provider'); const { getRoleByName, createToolCall, getToolCallsByConvo, getMessage } = require('~/models'); const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process'); +const { getRetentionExpiry } = require('~/server/services/Files/retention'); const { processCodeOutput, runPreviewFinalize } = require('~/server/services/Files/Code/process'); const { loadAuthValues } = require('~/server/services/Tools/credentials'); const { loadTools } = require('~/app/clients/tools/util'); @@ -167,6 +168,7 @@ const callTool = async (req, res) => { conversationId, result: content, user: req.user.id, + ...(await getRetentionExpiry(req)), }; if (!artifact || !artifact.files || toolId !== Tools.execute_code) { diff --git a/api/server/experimental.js b/api/server/experimental.js index b12b9deffe..8fd94c6723 100644 --- a/api/server/experimental.js +++ b/api/server/experimental.js @@ -24,6 +24,7 @@ const { const { connectDb, indexSync } = require('~/db'); const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager'); const createValidateImageRequest = require('./middleware/validateImageRequest'); +const { startExpiredFileSweep } = require('./services/Files/process'); const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies'); const { updateInterfacePermissions: updateInterfacePerms } = require('@librechat/api'); const { @@ -139,8 +140,32 @@ if (cluster.isMaster) { logger.info(`Spawning ${workers} workers to simulate multi-pod environment`); let activeWorkers = 0; + const listeningWorkers = new Set(); + let retentionSweepWorkerId = null; const startTime = Date.now(); + const assignRetentionSweepWorker = () => { + if (retentionSweepWorkerId && cluster.workers[retentionSweepWorkerId]) { + return; + } + + const connectedWorkers = Object.values(cluster.workers).filter( + (worker) => worker && worker.isConnected(), + ); + const availableWorkers = connectedWorkers.filter((worker) => listeningWorkers.has(worker.id)); + const workerPool = availableWorkers.length > 0 ? availableWorkers : connectedWorkers; + const retentionSweepWorker = workerPool[workerPool.length - 1]; + if (!retentionSweepWorker) { + return; + } + + retentionSweepWorkerId = retentionSweepWorker.id; + logger.info( + wrapLogMessage(`Worker ${retentionSweepWorker.process.pid} assigned to file-retention sweep`), + ); + retentionSweepWorker.send({ type: 'file-retention-sweep-worker' }); + }; + /** Flush Redis cache before starting workers */ flushRedisCache() .then(() => { @@ -162,19 +187,29 @@ if (cluster.isMaster) { `Worker ${worker.process.pid} is online (${activeWorkers}/${workers}) after ${uptime}s`, ); - /** Notify the last worker to perform one-time initialization tasks */ + /** Assign one worker for process-wide background jobs */ if (activeWorkers === workers) { - const allWorkers = Object.values(cluster.workers); - const lastWorker = allWorkers[allWorkers.length - 1]; - if (lastWorker) { - logger.info(wrapLogMessage(`All ${workers} workers are online`)); - lastWorker.send({ type: 'last-worker' }); - } + logger.info(wrapLogMessage(`All ${workers} workers are online`)); + } + }); + + cluster.on('listening', (worker) => { + listeningWorkers.add(worker.id); + if ( + listeningWorkers.size === workers || + (!retentionSweepWorkerId && activeWorkers >= workers) + ) { + assignRetentionSweepWorker(); } }); cluster.on('exit', (worker, code, signal) => { activeWorkers--; + listeningWorkers.delete(worker.id); + if (worker.id === retentionSweepWorkerId) { + retentionSweepWorkerId = null; + assignRetentionSweepWorker(); + } logger.error( `Worker ${worker.process.pid} died (${activeWorkers}/${workers}). Code: ${code}, Signal: ${signal}`, ); @@ -202,6 +237,32 @@ if (cluster.isMaster) { * Each worker runs a full Express server instance */ const app = express(); + /** + * The master may assign the sweep worker before or after this worker has + * loaded app config. These flags join the IPC assignment with config + * availability and ensure the background sweep starts only once. + */ + let shouldStartExpiredFileSweep = false; + let expiredFileSweepOptions = null; + let expiredFileSweepStarted = false; + + const startExpiredFileSweepOnce = () => { + if (!shouldStartExpiredFileSweep || expiredFileSweepStarted || !expiredFileSweepOptions) { + return; + } + + expiredFileSweepStarted = true; + startExpiredFileSweep(expiredFileSweepOptions); + }; + + /** Handle inter-process messages from master */ + process.on('message', (msg) => { + if (msg.type === 'file-retention-sweep-worker') { + shouldStartExpiredFileSweep = true; + logger.info(wrapLogMessage(`Worker ${process.pid} is assigned file-retention sweep`)); + startExpiredFileSweepOnce(); + } + }); const startServer = async () => { logger.info(`Worker ${process.pid} initializing...`); @@ -233,6 +294,8 @@ if (cluster.isMaster) { /** Initialize app configuration */ const appConfig = await getAppConfig(); initializeFileStorage(appConfig); + expiredFileSweepOptions = { appConfig, loadAppConfig: getAppConfig }; + startExpiredFileSweepOnce(); await performStartupChecks(appConfig); await updateInterfacePerms({ appConfig, getRoleByName, updateAccessPermissions }); @@ -390,19 +453,6 @@ if (cluster.isMaster) { process.exit(1); } }); - - /** Handle inter-process messages from master */ - process.on('message', async (msg) => { - if (msg.type === 'last-worker') { - logger.info( - wrapLogMessage( - `Worker ${process.pid} is the last worker and can perform special initialization tasks`, - ), - ); - /** Add any one-time initialization tasks here */ - /** For example: scheduled jobs, cleanup tasks, etc. */ - } - }); }; startServer().catch((err) => { diff --git a/api/server/index.js b/api/server/index.js index 9e094724f7..60b1a96b3d 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -34,6 +34,7 @@ const { const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager'); const { capabilityContextMiddleware } = require('./middleware/roles/capabilities'); const createValidateImageRequest = require('./middleware/validateImageRequest'); +const { startExpiredFileSweep } = require('./services/Files/process'); const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies'); const { checkMigrations } = require('./services/start/migration'); const optionalJwtAuth = require('./middleware/optionalJwtAuth'); @@ -89,6 +90,7 @@ const startServer = async () => { }); const appConfig = await getAppConfig({ baseOnly: true }); initializeFileStorage(appConfig); + startExpiredFileSweep({ appConfig, loadAppConfig: getAppConfig }); await runAsSystem(async () => { await performStartupChecks(appConfig); await updateInterfacePermissions({ appConfig, getRoleByName, updateAccessPermissions }); diff --git a/api/server/routes/__tests__/share.spec.js b/api/server/routes/__tests__/share.spec.js new file mode 100644 index 0000000000..541ae451c6 --- /dev/null +++ b/api/server/routes/__tests__/share.spec.js @@ -0,0 +1,262 @@ +const express = require('express'); +const request = require('supertest'); +const mongoose = require('mongoose'); + +const mockGetSharedLinkExpiration = jest.fn(); + +jest.mock('@librechat/api', () => ({ + isEnabled: jest.fn(() => true), + getSharedLinkExpiration: (...args) => mockGetSharedLinkExpiration(...args), + isActiveExpirationDate: jest.fn((expiredAt) => expiredAt > new Date()), +})); + +jest.mock('@librechat/data-schemas', () => ({ + logger: { error: jest.fn() }, + createTempChatExpirationDate: jest.fn(() => new Date('2030-01-01T00:00:00.000Z')), +})); + +jest.mock('librechat-data-provider', () => ({ + RetentionMode: { + ALL: 'all', + TEMPORARY: 'temporary', + }, +})); + +jest.mock('mongoose', () => ({ + models: { + Conversation: { + findOne: jest.fn(), + }, + SharedLink: { + findOne: jest.fn(), + }, + }, +})); + +jest.mock('~/models', () => ({ + getSharedMessages: jest.fn(), + createSharedLink: jest.fn(), + updateSharedLink: jest.fn(), + deleteSharedLink: jest.fn(), + getSharedLinks: jest.fn(), + getSharedLink: jest.fn(), +})); + +jest.mock('~/server/middleware/requireJwtAuth', () => (req, res, next) => next()); + +const { RetentionMode } = require('librechat-data-provider'); +const { createTempChatExpirationDate, logger } = require('@librechat/data-schemas'); +const { createSharedLink, updateSharedLink } = require('~/models'); +const shareRouter = require('../share'); + +const activeExpiration = new Date('2030-01-01T00:00:00.000Z'); +const expiredExpiration = new Date('2020-01-01T00:00:00.000Z'); + +const lean = (value) => ({ + lean: jest.fn().mockResolvedValue(value), +}); + +const buildApp = ({ retentionMode = RetentionMode.TEMPORARY } = {}) => { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.user = { id: 'user-123' }; + req.config = { interfaceConfig: { retentionMode } }; + next(); + }); + app.use('/api/share', shareRouter); + return app; +}; + +describe('share routes retention', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('expires new shares for retained non-temporary conversations', async () => { + mockGetSharedLinkExpiration.mockResolvedValue(activeExpiration); + createSharedLink.mockResolvedValue({ shareId: 'share-123' }); + + const response = await request(buildApp()) + .post('/api/share/convo-123') + .send({ targetMessageId: 'msg-123' }); + + expect(response.status).toBe(200); + expect(mockGetSharedLinkExpiration).toHaveBeenCalledWith( + expect.objectContaining({ + conversationId: 'convo-123', + req: expect.objectContaining({ user: { id: 'user-123' } }), + }), + expect.objectContaining({ + getConvo: expect.any(Function), + createExpirationDate: createTempChatExpirationDate, + logger, + }), + ); + const [, dependencies] = mockGetSharedLinkExpiration.mock.calls[0]; + mongoose.models.Conversation.findOne.mockReturnValue(lean({ expiredAt: activeExpiration })); + await dependencies.getConvo('user-123', 'convo-123'); + expect(mongoose.models.Conversation.findOne).toHaveBeenCalledWith( + { conversationId: 'convo-123', user: 'user-123' }, + 'isTemporary expiredAt', + ); + expect(createSharedLink).toHaveBeenCalledWith( + 'user-123', + 'convo-123', + 'msg-123', + new Date('2030-01-01T00:00:00.000Z'), + ); + }); + + it('rejects new shares when the retained conversation expired', async () => { + mockGetSharedLinkExpiration.mockResolvedValue(expiredExpiration); + createSharedLink.mockResolvedValue({ shareId: 'share-123' }); + + const response = await request(buildApp()) + .post('/api/share/convo-123') + .send({ targetMessageId: 'msg-123' }); + + expect(response.status).toBe(404); + expect(createSharedLink).not.toHaveBeenCalled(); + }); + + it('rejects new shares for expired conversations in all retention mode', async () => { + mockGetSharedLinkExpiration.mockResolvedValue(expiredExpiration); + createSharedLink.mockResolvedValue({ shareId: 'share-123' }); + + const response = await request(buildApp({ retentionMode: RetentionMode.ALL })) + .post('/api/share/convo-123') + .send({ targetMessageId: 'msg-123' }); + + expect(response.status).toBe(404); + expect(createSharedLink).not.toHaveBeenCalled(); + }); + + it('expires updated shares for retained non-temporary conversations', async () => { + mongoose.models.SharedLink.findOne.mockReturnValue(lean({ conversationId: 'convo-123' })); + mockGetSharedLinkExpiration.mockResolvedValue(activeExpiration); + updateSharedLink.mockResolvedValue({ shareId: 'share-456' }); + + const response = await request(buildApp()).patch('/api/share/share-123'); + + expect(response.status).toBe(200); + expect(mongoose.models.SharedLink.findOne).toHaveBeenCalledWith( + { shareId: 'share-123', user: 'user-123' }, + 'conversationId', + ); + expect(mockGetSharedLinkExpiration).toHaveBeenCalledTimes(1); + expect(mockGetSharedLinkExpiration).toHaveBeenCalledWith( + expect.objectContaining({ + conversationId: 'convo-123', + req: expect.objectContaining({ user: { id: 'user-123' } }), + }), + expect.objectContaining({ + getConvo: expect.any(Function), + createExpirationDate: createTempChatExpirationDate, + logger, + }), + ); + expect(updateSharedLink).toHaveBeenCalledWith( + 'user-123', + 'share-123', + undefined, + new Date('2030-01-01T00:00:00.000Z'), + ); + }); + + it('rejects updated shares when the retained conversation expired', async () => { + mongoose.models.SharedLink.findOne.mockReturnValue(lean({ conversationId: 'convo-123' })); + mockGetSharedLinkExpiration.mockResolvedValue(expiredExpiration); + updateSharedLink.mockResolvedValue({ shareId: 'share-456' }); + + const response = await request(buildApp()).patch('/api/share/share-123'); + + expect(response.status).toBe(404); + expect(updateSharedLink).not.toHaveBeenCalled(); + }); + + it('rejects updated shares for expired conversations in all retention mode', async () => { + mongoose.models.SharedLink.findOne.mockReturnValue(lean({ conversationId: 'convo-123' })); + mockGetSharedLinkExpiration.mockResolvedValue(expiredExpiration); + updateSharedLink.mockResolvedValue({ shareId: 'share-456' }); + + const response = await request(buildApp({ retentionMode: RetentionMode.ALL })).patch( + '/api/share/share-123', + ); + + expect(response.status).toBe(404); + expect(mongoose.models.SharedLink.findOne).toHaveBeenCalledWith( + { shareId: 'share-123', user: 'user-123' }, + 'conversationId', + ); + expect(updateSharedLink).not.toHaveBeenCalled(); + }); + + it('clears updated share expiration when the conversation is no longer retained', async () => { + mongoose.models.SharedLink.findOne.mockReturnValue(lean({ conversationId: 'convo-123' })); + mockGetSharedLinkExpiration.mockResolvedValue(null); + updateSharedLink.mockResolvedValue({ shareId: 'share-456' }); + + const response = await request(buildApp()).patch('/api/share/share-123'); + + expect(response.status).toBe(200); + expect(updateSharedLink).toHaveBeenCalledWith('user-123', 'share-123', undefined, null); + }); + + it('preserves updated share expiration when the conversation cannot be found', async () => { + mongoose.models.SharedLink.findOne.mockReturnValue(lean({ conversationId: 'convo-123' })); + mockGetSharedLinkExpiration.mockResolvedValue(undefined); + updateSharedLink.mockResolvedValue({ shareId: 'share-456' }); + + const response = await request(buildApp()).patch('/api/share/share-123'); + + expect(response.status).toBe(200); + expect(updateSharedLink).toHaveBeenCalledWith('user-123', 'share-123', undefined, undefined); + }); + + it('clears updated share expiration when creating a new expiration throws', async () => { + const error = new Error('bad config'); + mongoose.models.SharedLink.findOne.mockReturnValue(lean({ conversationId: 'convo-123' })); + mockGetSharedLinkExpiration.mockImplementationOnce(async (_input, dependencies) => { + dependencies.logger.error('[getSharedLinkExpiration] Error creating expiration date:', error); + return null; + }); + updateSharedLink.mockResolvedValue({ shareId: 'share-456' }); + + const response = await request(buildApp()).patch('/api/share/share-123'); + + expect(response.status).toBe(200); + expect(logger.error).toHaveBeenCalledWith( + '[getSharedLinkExpiration] Error creating expiration date:', + error, + ); + expect(updateSharedLink).toHaveBeenCalledWith('user-123', 'share-123', undefined, null); + }); + + it('updates share target message while applying retention expiration', async () => { + mongoose.models.SharedLink.findOne.mockReturnValue(lean({ conversationId: 'convo-123' })); + mockGetSharedLinkExpiration.mockResolvedValue(activeExpiration); + updateSharedLink.mockResolvedValue({ shareId: 'share-456', targetMessageId: 'msg-456' }); + + const response = await request(buildApp()) + .patch('/api/share/share-123') + .send({ targetMessageId: 'msg-456' }); + + expect(response.status).toBe(200); + expect(updateSharedLink).toHaveBeenCalledWith( + 'user-123', + 'share-123', + 'msg-456', + new Date('2030-01-01T00:00:00.000Z'), + ); + }); + + it('rejects non-string target message updates', async () => { + const response = await request(buildApp()) + .patch('/api/share/share-123') + .send({ targetMessageId: 123 }); + + expect(response.status).toBe(400); + expect(updateSharedLink).not.toHaveBeenCalled(); + }); +}); diff --git a/api/server/routes/convos.js b/api/server/routes/convos.js index 3d65343648..dc59482afa 100644 --- a/api/server/routes/convos.js +++ b/api/server/routes/convos.js @@ -276,6 +276,7 @@ router.post( filepath: req.file.path, requestUserId: req.user.id, userRole: req.user.role, + interfaceConfig: req.config?.interfaceConfig, }); res.status(201).json({ message: 'Conversation(s) imported successfully' }); } catch (error) { diff --git a/api/server/routes/files/files.agents.test.js b/api/server/routes/files/files.agents.test.js index d2c76ea139..664721f35b 100644 --- a/api/server/routes/files/files.agents.test.js +++ b/api/server/routes/files/files.agents.test.js @@ -14,7 +14,7 @@ const { createAgent, createFile } = require('~/models'); // Only mock the external dependencies that we don't want to test jest.mock('~/server/services/Files/process', () => ({ - processDeleteRequest: jest.fn().mockResolvedValue({}), + processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }), filterFile: jest.fn(), processFileUpload: jest.fn(), processAgentFileUpload: jest.fn().mockImplementation(async ({ res }) => { diff --git a/api/server/routes/files/files.test.js b/api/server/routes/files/files.test.js index 5758b77387..473731de01 100644 --- a/api/server/routes/files/files.test.js +++ b/api/server/routes/files/files.test.js @@ -16,7 +16,7 @@ const { createAgent, createFile } = require('~/models'); // Only mock the external dependencies that we don't want to test jest.mock('~/server/services/Files/process', () => ({ - processDeleteRequest: jest.fn().mockResolvedValue({}), + processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }), filterFile: jest.fn(), processFileUpload: jest.fn(), processAgentFileUpload: jest.fn(), diff --git a/api/server/routes/files/preview.spec.js b/api/server/routes/files/preview.spec.js index 426b0697c6..36de49c223 100644 --- a/api/server/routes/files/preview.spec.js +++ b/api/server/routes/files/preview.spec.js @@ -36,7 +36,7 @@ jest.mock('~/models', () => ({ jest.mock('~/server/services/Files/process', () => ({ filterFile: jest.fn(), processFileUpload: jest.fn(), - processDeleteRequest: jest.fn(), + processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }), processAgentFileUpload: jest.fn(), })); diff --git a/api/server/routes/share.js b/api/server/routes/share.js index 4c0427f197..ce4dee1a1f 100644 --- a/api/server/routes/share.js +++ b/api/server/routes/share.js @@ -1,6 +1,7 @@ +const mongoose = require('mongoose'); const express = require('express'); -const { isEnabled } = require('@librechat/api'); -const { logger } = require('@librechat/data-schemas'); +const { isEnabled, isActiveExpirationDate, getSharedLinkExpiration } = require('@librechat/api'); +const { logger, createTempChatExpirationDate } = require('@librechat/data-schemas'); const { getSharedMessages, createSharedLink, @@ -12,6 +13,22 @@ const { const requireJwtAuth = require('~/server/middleware/requireJwtAuth'); const router = express.Router(); +const resolveSharedLinkExpiration = (req, conversationId) => + getSharedLinkExpiration( + { req, conversationId }, + { + getConvo: async (userId, sourceConversationId) => { + const Conversation = mongoose.models.Conversation; + return Conversation.findOne( + { conversationId: sourceConversationId, user: userId }, + 'isTemporary expiredAt', + ).lean(); + }, + createExpirationDate: createTempChatExpirationDate, + logger, + }, + ); + /** * Shared messages */ @@ -99,7 +116,17 @@ router.get('/link/:conversationId', requireJwtAuth, async (req, res) => { router.post('/:conversationId', requireJwtAuth, async (req, res) => { try { const { targetMessageId } = req.body; - const created = await createSharedLink(req.user.id, req.params.conversationId, targetMessageId); + const expiredAt = await resolveSharedLinkExpiration(req, req.params.conversationId); + if (expiredAt != null && !isActiveExpirationDate(expiredAt)) { + return res.status(404).end(); + } + + const created = await createSharedLink( + req.user.id, + req.params.conversationId, + targetMessageId, + expiredAt, + ); if (created) { res.status(200).json(created); } else { @@ -118,7 +145,25 @@ router.patch('/:shareId', requireJwtAuth, async (req, res) => { return res.status(400).json({ message: 'targetMessageId must be a string' }); } - const updatedShare = await updateSharedLink(req.user.id, req.params.shareId, targetMessageId); + let expiredAt; + const SharedLink = mongoose.models.SharedLink; + const existing = await SharedLink.findOne( + { shareId: req.params.shareId, user: req.user.id }, + 'conversationId', + ).lean(); + if (existing?.conversationId) { + expiredAt = await resolveSharedLinkExpiration(req, existing.conversationId); + } + if (expiredAt != null && !isActiveExpirationDate(expiredAt)) { + return res.status(404).end(); + } + + const updatedShare = await updateSharedLink( + req.user.id, + req.params.shareId, + targetMessageId, + expiredAt, + ); if (updatedShare) { res.status(200).json(updatedShare); } else { diff --git a/api/server/services/Files/Code/__tests__/process-traversal.spec.js b/api/server/services/Files/Code/__tests__/process-traversal.spec.js index 791d6d258c..57609c545a 100644 --- a/api/server/services/Files/Code/__tests__/process-traversal.spec.js +++ b/api/server/services/Files/Code/__tests__/process-traversal.spec.js @@ -92,6 +92,11 @@ jest.mock('~/server/utils', () => ({ determineFileType: jest.fn().mockResolvedValue({ mime: 'text/csv' }), })); +jest.mock('~/server/services/Files/retention', () => ({ + getRetentionExpiry: jest.fn(() => ({})), +})); + +const { getRetentionExpiry } = require('~/server/services/Files/retention'); const { createFile } = require('~/models'); const { processCodeOutput } = require('../process'); @@ -143,6 +148,12 @@ describe('processCodeOutput path traversal protection', () => { expect(fileArg.tenantId).toBe('tenantA'); }); + test('getRetentionExpiry is called with the request object', async () => { + mockSanitizeArtifactPath.mockReturnValueOnce('output.csv'); + await processCodeOutput({ ...baseParams, name: 'output.csv' }); + expect(getRetentionExpiry).toHaveBeenCalledWith(baseParams.req); + }); + test('sanitized name is used for image file records', async () => { const { convertImage } = require('~/server/services/Files/images/convert'); convertImage.mockResolvedValueOnce({ diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js index a04c6329c6..d9940da05b 100644 --- a/api/server/services/Files/Code/process.js +++ b/api/server/services/Files/Code/process.js @@ -36,6 +36,7 @@ const { filterFilesByAgentAccess } = require('~/server/services/Files/permission const { createFile, getFiles, updateFile, claimCodeFile } = require('~/models'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { convertImage } = require('~/server/services/Files/images/convert'); +const { getRetentionExpiry } = require('~/server/services/Files/retention'); const { determineFileType } = require('~/server/utils'); const axios = createAxiosInstance(); @@ -463,6 +464,7 @@ const processCodeOutput = async ({ source: appConfig.fileStrategy, context: FileContext.execute_code, metadata: { codeEnvRef }, + ...(await getRetentionExpiry(req)), }; await createFile(file, true); return { file: Object.assign(file, { messageId, toolCallId }) }; @@ -565,6 +567,7 @@ const processCodeOutput = async ({ context: FileContext.execute_code, usage: isUpdate ? (claimed.usage ?? 0) + 1 : 1, createdAt: isUpdate ? claimed.createdAt : formattedDate, + ...(await getRetentionExpiry(req)), }; if (expectsPreview) { diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js index 6ade00b1f7..b1a11a44a4 100644 --- a/api/server/services/Files/Code/process.spec.js +++ b/api/server/services/Files/Code/process.spec.js @@ -137,6 +137,10 @@ jest.mock('~/server/services/Files/images/convert', () => ({ convertImage: jest.fn(), })); +jest.mock('~/server/services/Files/retention', () => ({ + getRetentionExpiry: jest.fn(() => ({})), +})); + // Mock determineFileType jest.mock('~/server/utils', () => ({ determineFileType: jest.fn(), @@ -145,6 +149,7 @@ jest.mock('~/server/utils', () => ({ const http = require('http'); const https = require('https'); const { createFile, getFiles } = require('~/models'); +const { getRetentionExpiry } = require('~/server/services/Files/retention'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { convertImage } = require('~/server/services/Files/images/convert'); const { determineFileType } = require('~/server/utils'); @@ -233,6 +238,7 @@ describe('Code Process', () => { expect(result.file_id).toBe('mock-uuid-1234'); expect(result.usage).toBe(1); + expect(getRetentionExpiry).toHaveBeenCalledWith(baseParams.req); }); }); diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index ea8ee14840..6f2ee2b619 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -18,12 +18,14 @@ const { getEndpointFileConfig, documentParserMimeTypes, } = require('librechat-data-provider'); -const { logger } = require('@librechat/data-schemas'); +const { logger, runAsSystem } = require('@librechat/data-schemas'); const { sanitizeFilename, parseText, processAudioFile, getStorageMetadata, + sweepExpiredFiles: sweepExpiredFilesWithDeps, + startExpiredFileSweep: startExpiredFileSweepWithDeps, } = require('@librechat/api'); const { convertImage, @@ -36,6 +38,7 @@ const { loadAuthValues } = require('~/server/services/Tools/credentials'); const { getFileStrategy } = require('~/server/utils/getFileStrategy'); const { checkCapability } = require('~/server/services/Config'); const { LB_QueueAsyncCall } = require('~/server/utils/queue'); +const { getRetentionExpiry } = require('./retention'); const { getStrategyFunctions } = require('./strategies'); const { determineFileType } = require('~/server/utils'); const { STTService } = require('./Audio/STTService'); @@ -64,6 +67,17 @@ const createSanitizedUploadWrapper = (uploadFunction) => { }; }; +const isMissingStorageError = (err) => { + const code = err?.code ?? err?.status ?? err?.statusCode ?? err?.response?.status; + if ([404, '404', 'ENOENT', 'NoSuchKey', 'NotFound', 'ResourceNotFound'].includes(code)) { + return true; + } + + return /(?:file|object|blob|key|resource) (?:not found|does not exist)|no such (?:file|key)/i.test( + String(err?.message ?? ''), + ); +}; + /** * Enqueues the delete operation to the leaky bucket queue if necessary, or adds it directly to promises. * @@ -72,10 +86,19 @@ const createSanitizedUploadWrapper = (uploadFunction) => { * @param {MongoFile} params.file - The file object to delete. * @param {Function} params.deleteFile - The delete file function. * @param {Promise[]} params.promises - The array of promises to await. - * @param {string[]} params.resolvedFileIds - The array of promises to await. + * @param {Set} params.resolvedFileIds - File IDs whose storage delete succeeded. + * @param {Set} params.failedFileIds - File IDs whose storage delete failed. * @param {OpenAI | undefined} [params.openai] - If an OpenAI file, the initialized OpenAI client. */ -function enqueueDeleteOperation({ req, file, deleteFile, promises, resolvedFileIds, openai }) { +function enqueueDeleteOperation({ + req, + file, + deleteFile, + promises, + resolvedFileIds, + failedFileIds, + openai, +}) { if (checkOpenAIStorage(file.source)) { // Enqueue to leaky bucket promises.push( @@ -85,10 +108,17 @@ function enqueueDeleteOperation({ req, file, deleteFile, promises, resolvedFileI [], (err, result) => { if (err) { + if (isMissingStorageError(err)) { + resolvedFileIds.add(file.file_id); + logger.warn('File storage was already missing during delete', err); + resolve(result); + return; + } + failedFileIds.add(file.file_id); logger.error('Error deleting file from OpenAI source', err); reject(err); } else { - resolvedFileIds.push(file.file_id); + resolvedFileIds.add(file.file_id); resolve(result); } }, @@ -99,8 +129,14 @@ function enqueueDeleteOperation({ req, file, deleteFile, promises, resolvedFileI // Add directly to promises promises.push( deleteFile(req, file) - .then(() => resolvedFileIds.push(file.file_id)) + .then(() => resolvedFileIds.add(file.file_id)) .catch((err) => { + if (isMissingStorageError(err)) { + resolvedFileIds.add(file.file_id); + logger.warn('File storage was already missing during delete', err); + return; + } + failedFileIds.add(file.file_id); logger.error('Error deleting file', err); return Promise.reject(err); }), @@ -121,11 +157,13 @@ function enqueueDeleteOperation({ req, file, deleteFile, promises, resolvedFileI * @param {string} [params.req.body.assistant_id] - The assistant ID if file uploaded is associated to an assistant. * @param {string} [params.req.body.tool_resource] - The tool resource if assistant file uploaded is associated to a tool resource. * - * @returns {Promise} + * @returns {Promise<{ deletedFileIds: string[], failedFileIds: string[] }>} + * @throws {Error} When storage deletion cannot be scheduled or file metadata cleanup fails. */ const processDeleteRequest = async ({ req, files }) => { const appConfig = req.config; - const resolvedFileIds = []; + const resolvedFileIds = new Set(); + const failedFileIds = new Set(); const deletionMethods = {}; const promises = []; @@ -167,7 +205,7 @@ const processDeleteRequest = async ({ req, files }) => { } if (source === FileSources.text) { - resolvedFileIds.push(file.file_id); + resolvedFileIds.add(file.file_id); continue; } @@ -198,6 +236,7 @@ const processDeleteRequest = async ({ req, files }) => { deleteFile: deletionMethods[source], promises, resolvedFileIds, + failedFileIds, openai, }); continue; @@ -209,7 +248,15 @@ const processDeleteRequest = async ({ req, files }) => { } deletionMethods[source] = deleteFile; - enqueueDeleteOperation({ req, file, deleteFile, promises, resolvedFileIds, openai }); + enqueueDeleteOperation({ + req, + file, + deleteFile, + promises, + resolvedFileIds, + failedFileIds, + openai, + }); } if (agentFiles.length > 0) { @@ -222,17 +269,60 @@ const processDeleteRequest = async ({ req, files }) => { } await Promise.allSettled(promises); - await db.deleteFiles(resolvedFileIds); - - if (resolvedFileIds.length > 0) { + const deletedFileIds = [...resolvedFileIds]; + let metadataDeletedFileIds = deletedFileIds; + if (deletedFileIds.length > 0) { try { - await db.removeAgentResourceFilesFromAllAgents({ file_ids: resolvedFileIds }); + await db.deleteFiles(deletedFileIds); } catch (error) { - logger.error('Error cleaning up orphaned agent file references', error); + logger.error('Error deleting file metadata after storage deletion', error); + deletedFileIds.forEach((fileId) => failedFileIds.add(fileId)); + metadataDeletedFileIds = []; + throw error; + } + if (metadataDeletedFileIds.length > 0) { + try { + await db.removeAgentResourceFilesFromAllAgents({ file_ids: metadataDeletedFileIds }); + } catch (error) { + logger.error('Error cleaning up orphaned agent file references', error); + } } } + + return { + deletedFileIds: metadataDeletedFileIds, + failedFileIds: [...failedFileIds], + }; }; +/** + * Deletes expired file storage before removing the corresponding File records. + * + * Mongo TTL indexes delete only the metadata document, so file retention uses + * this application sweep for records with `expiredAt` instead. + * + * @param {object} params + * @param {AppConfig} params.appConfig + * @param {number} [params.limit] + * @param {() => Promise} [params.loadAppConfig] + * @returns {Promise<{ scanned: number, deleted: number, failed: number }>} + */ +async function sweepExpiredFiles(options = {}) { + return sweepExpiredFilesWithDeps(options, { + getExpiredFiles: db.getExpiredFiles, + processDeleteRequest, + logger, + }); +} + +function startExpiredFileSweep(options = {}) { + return startExpiredFileSweepWithDeps(options, { + sweepExpiredFiles, + runAsSystem, + logger, + }); +} + /** * Processes a file URL using a specified file handling strategy. This function accepts a strategy name, * fetches the corresponding file processing functions (for saving and retrieving file URLs), and then @@ -251,6 +341,7 @@ const processDeleteRequest = async ({ req, files }) => { * @param {string} params.basePath - The base path or directory where the file will be saved or retrieved from. * @param {FileContext} params.context - The context of the file (e.g., 'avatar', 'image_generation', etc.) * @param {string} [params.tenantId] - Optional tenant identifier for tenant-prefixed storage paths. + * @param {ServerRequest} [params.req] - Request context used to apply data retention metadata. * @returns {Promise} A promise that resolves to the DB representation (MongoFile) * of the processed file. It throws an error if the file processing fails at any stage. */ @@ -262,6 +353,7 @@ const processFileURL = async ({ basePath, context, tenantId, + req, }) => { const { saveURL, getFileURL } = getStrategyFunctions(fileStrategy); try { @@ -305,6 +397,7 @@ const processFileURL = async ({ source: fileStrategy, type, context, + ...(await getRetentionExpiry(req)), tenantId, width: dimensions.width, height: dimensions.height, @@ -355,6 +448,7 @@ const processImageFile = async ({ req, res, metadata, returnFile = false }) => { context: FileContext.message_attachment, source, type: `image/${appConfig.imageOutputType}`, + ...(await getRetentionExpiry(req)), width, height, tenantId: req.user.tenantId, @@ -415,6 +509,7 @@ const uploadImageBuffer = async ({ req, context, metadata = {}, resize = true }) source, type, width, + ...(await getRetentionExpiry(req)), height, tenantId: req.user.tenantId, }, @@ -517,6 +612,7 @@ const processFileUpload = async ({ req, res, metadata }) => { context: isAssistantUpload ? FileContext.assistants : FileContext.message_attachment, model: isAssistantUpload ? req.body.model : undefined, type: file.mimetype, + ...(await getRetentionExpiry(req)), embedded, source, height, @@ -631,20 +727,24 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { `Extracted text from "${file.originalname}" exceeds the 15MB storage limit (${Math.round(textBytes / megabyte)}MB). Try a shorter document.`, ); } - const fileInfo = removeNullishValues({ - text, - bytes, - file_id, - temp_file_id, - user: req.user.id, - type, - filepath: filepath ?? file.path, - source: FileSources.text, - filename: file.originalname, - model: messageAttachment ? undefined : req.body.model, - context: messageAttachment ? FileContext.message_attachment : FileContext.agents, - tenantId: req.user.tenantId, - }); + const retentionExpiry = await getRetentionExpiry(req); + const fileInfo = { + ...removeNullishValues({ + text, + bytes, + file_id, + temp_file_id, + user: req.user.id, + type, + filepath: filepath ?? file.path, + source: FileSources.text, + filename: file.originalname, + model: messageAttachment ? undefined : req.body.model, + context: messageAttachment ? FileContext.message_attachment : FileContext.agents, + tenantId: req.user.tenantId, + }), + ...retentionExpiry, + }; if (!messageAttachment && tool_resource) { await db.addAgentResourceFile({ @@ -825,24 +925,28 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { }); } - const fileInfo = removeNullishValues({ - user: req.user.id, - file_id, - temp_file_id, - bytes, - filepath, - ...storageMetadata, - filename: filename ?? sanitizeFilename(file.originalname), - context: messageAttachment ? FileContext.message_attachment : FileContext.agents, - model: messageAttachment ? undefined : req.body.model, - metadata: fileInfoMetadata, - type: file.mimetype, - embedded, - source, - height, - width, - tenantId: req.user.tenantId, - }); + const retentionExpiry = await getRetentionExpiry(req); + const fileInfo = { + ...removeNullishValues({ + user: req.user.id, + file_id, + temp_file_id, + bytes, + filepath, + ...storageMetadata, + filename: filename ?? sanitizeFilename(file.originalname), + context: messageAttachment ? FileContext.message_attachment : FileContext.agents, + model: messageAttachment ? undefined : req.body.model, + metadata: fileInfoMetadata, + type: file.mimetype, + embedded, + source, + height, + width, + tenantId: req.user.tenantId, + }), + ...retentionExpiry, + }; const result = await db.createFile(fileInfo, true); @@ -887,6 +991,7 @@ const processOpenAIFile = async ({ source, model: openai.req.body.model, filename: originalName ?? file_id, + ...(await getRetentionExpiry(openai.req)), tenantId: openai.req?.user?.tenantId, }; @@ -931,9 +1036,14 @@ const processOpenAIImageOutput = async ({ req, buffer, file_id, filename, fileEx context: FileContext.assistants_output, file_id, filename, + ...(await getRetentionExpiry(req)), tenantId: req.user.tenantId, }; - db.createFile(file, true); + try { + await db.createFile(file, true); + } catch (error) { + logger.warn('Error saving OpenAI image output file metadata', error); + } return file; }; @@ -1091,6 +1201,7 @@ async function saveBase64Image( user: req.user.id, bytes: image.bytes, width: image.width, + ...(await getRetentionExpiry(req)), height: image.height, tenantId: req.user.tenantId, }, @@ -1182,6 +1293,8 @@ module.exports = { saveBase64Image, processImageFile, uploadImageBuffer, + sweepExpiredFiles, + startExpiredFileSweep, processFileUpload, processDeleteRequest, processAgentFileUpload, diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index 99457522d4..01cfb2f1e5 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -1,22 +1,41 @@ jest.mock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') })); jest.mock('@librechat/data-schemas', () => ({ - logger: { warn: jest.fn(), debug: jest.fn(), error: jest.fn() }, + logger: { warn: jest.fn(), debug: jest.fn(), error: jest.fn(), info: jest.fn() }, + runAsSystem: jest.fn((fn) => fn()), + createTempChatExpirationDate: jest.fn(() => new Date('2030-01-01T00:00:00.000Z')), })); -jest.mock('@librechat/agents', () => ({})); - -jest.mock('@librechat/api', () => ({ - sanitizeFilename: jest.fn((n) => n), - parseText: jest.fn().mockResolvedValue({ text: '', bytes: 0 }), - processAudioFile: jest.fn(), - getStorageMetadata: jest.fn(() => ({})), +jest.mock('@librechat/agents', () => ({ + Providers: { + XAI: 'xai', + DEEPSEEK: 'deepseek', + MOONSHOT: 'moonshot', + OPENROUTER: 'openrouter', + VERTEXAI: 'vertexai', + }, })); -jest.mock('librechat-data-provider', () => ({ - ...jest.requireActual('librechat-data-provider'), - mergeFileConfig: jest.fn(), -})); +jest.mock('librechat-data-provider', () => { + const actual = jest.requireActual('librechat-data-provider'); + return { + ...actual, + Providers: actual.Providers, + mergeFileConfig: jest.fn(), + }; +}); + +jest.mock('@librechat/api', () => { + return { + sanitizeFilename: jest.fn((n) => n), + parseText: jest.fn().mockResolvedValue({ text: '', bytes: 0 }), + processAudioFile: jest.fn(), + getStorageMetadata: jest.fn(() => ({})), + getRetentionExpiry: jest.fn(() => ({})), + sweepExpiredFiles: jest.fn().mockResolvedValue({ scanned: 0, deleted: 0, failed: 0 }), + startExpiredFileSweep: jest.fn().mockReturnValue('sweep-interval'), + }; +}); jest.mock('~/server/services/Files/images', () => ({ convertImage: jest.fn(), @@ -41,8 +60,12 @@ jest.mock('~/models', () => ({ createFile: jest.fn().mockResolvedValue({ file_id: 'created-file-id' }), updateFileUsage: jest.fn(), deleteFiles: jest.fn(), + findFileById: jest.fn(), + getConvo: jest.fn(), + getExpiredFiles: jest.fn(), addAgentResourceFile: jest.fn().mockResolvedValue({}), removeAgentResourceFiles: jest.fn(), + removeAgentResourceFilesFromAllAgents: jest.fn(), })); jest.mock('~/server/utils/getFileStrategy', () => ({ @@ -69,17 +92,29 @@ jest.mock('~/server/services/Files/Audio/STTService', () => ({ STTService: { getInstance: jest.fn() }, })); +const { + getRetentionExpiry, + sweepExpiredFiles: sweepExpiredFilesWithDeps, + startExpiredFileSweep: startExpiredFileSweepWithDeps, +} = require('@librechat/api'); const { EToolResources, FileSources, FileContext, + RetentionMode, AgentCapabilities, } = require('librechat-data-provider'); const { mergeFileConfig } = require('librechat-data-provider'); const { checkCapability } = require('~/server/services/Config'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const db = require('~/models'); -const { processAgentFileUpload, processFileURL } = require('./process'); +const { + processAgentFileUpload, + processDeleteRequest, + processFileURL, + sweepExpiredFiles, + startExpiredFileSweep, +} = require('./process'); const PDF_MIME = 'application/pdf'; const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; @@ -534,6 +569,110 @@ describe('processFileURL', () => { ); }); + it('applies retention metadata for generated images when retention mode is all', async () => { + getRetentionExpiry.mockResolvedValueOnce({ + expiredAt: new Date('2030-01-01T00:00:00.000Z'), + }); + const saveURL = jest.fn().mockResolvedValue({ + filepath: 'https://cdn.example.com/t/tenant-a/images/user-123/image.png', + bytes: 512, + type: 'image/png', + }); + const getFileURL = jest.fn(); + getStrategyFunctions.mockReturnValue({ saveURL, getFileURL }); + + await processFileURL({ + fileStrategy: FileSources.cloudfront, + userId: 'user-123', + URL: 'https://example.com/image.png', + fileName: 'image.png', + basePath: 'images', + context: FileContext.image_generation, + tenantId: 'tenant-a', + req: { + user: { id: 'user-123', tenantId: 'tenant-a' }, + body: {}, + config: { interfaceConfig: { retentionMode: 'all' } }, + }, + }); + + expect(db.createFile).toHaveBeenCalledWith( + expect.objectContaining({ + expiredAt: new Date('2030-01-01T00:00:00.000Z'), + }), + true, + ); + }); + + it('applies retention metadata for retained non-temporary conversations', async () => { + const saveURL = jest.fn().mockResolvedValue({ + filepath: 'https://cdn.example.com/t/tenant-a/images/user-123/image.png', + bytes: 512, + type: 'image/png', + }); + const getFileURL = jest.fn(); + getStrategyFunctions.mockReturnValue({ saveURL, getFileURL }); + getRetentionExpiry.mockResolvedValueOnce({ + expiredAt: new Date('2030-01-01T00:00:00.000Z'), + }); + + await processFileURL({ + fileStrategy: FileSources.cloudfront, + userId: 'user-123', + URL: 'https://example.com/image.png', + fileName: 'image.png', + basePath: 'images', + context: FileContext.image_generation, + tenantId: 'tenant-a', + req: { + user: { id: 'user-123', tenantId: 'tenant-a' }, + body: { conversationId: 'convo-123' }, + config: { interfaceConfig: { retentionMode: RetentionMode.TEMPORARY } }, + }, + }); + + expect(db.createFile).toHaveBeenCalledWith( + expect.objectContaining({ + expiredAt: new Date('2030-01-01T00:00:00.000Z'), + }), + true, + ); + }); + + it('keeps expired retained conversation files on the parent expiration', async () => { + const parentExpiredAt = new Date('2020-01-01T00:00:00.000Z'); + const saveURL = jest.fn().mockResolvedValue({ + filepath: 'https://cdn.example.com/t/tenant-a/images/user-123/image.png', + bytes: 512, + type: 'image/png', + }); + const getFileURL = jest.fn(); + getStrategyFunctions.mockReturnValue({ saveURL, getFileURL }); + getRetentionExpiry.mockResolvedValueOnce({ expiredAt: parentExpiredAt }); + + await processFileURL({ + fileStrategy: FileSources.cloudfront, + userId: 'user-123', + URL: 'https://example.com/image.png', + fileName: 'image.png', + basePath: 'images', + context: FileContext.image_generation, + tenantId: 'tenant-a', + req: { + user: { id: 'user-123', tenantId: 'tenant-a' }, + body: { conversationId: 'convo-123' }, + config: { interfaceConfig: { retentionMode: RetentionMode.TEMPORARY } }, + }, + }); + + expect(db.createFile).toHaveBeenCalledWith( + expect.objectContaining({ + expiredAt: parentExpiredAt, + }), + true, + ); + }); + it('falls back to getFileURL with user and tenant context when metadata lacks filepath', async () => { const saveURL = jest.fn().mockResolvedValue({ bytes: 256, @@ -602,3 +741,142 @@ describe('processFileURL', () => { ); }); }); + +describe('processDeleteRequest', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('removes metadata when backing storage is already missing', async () => { + const missingError = Object.assign(new Error('no such file'), { code: 'ENOENT' }); + const deleteFile = jest.fn().mockRejectedValue(missingError); + getStrategyFunctions.mockReturnValue({ deleteFile }); + db.deleteFiles.mockResolvedValue({ deletedCount: 1 }); + + const result = await processDeleteRequest({ + req: { + body: {}, + config: {}, + user: { id: 'user-123', tenantId: 'tenant-a' }, + }, + files: [ + { + file_id: 'expired-file', + filepath: '/images/user-123/expired.png', + source: FileSources.local, + }, + ], + }); + + expect(db.deleteFiles).toHaveBeenCalledWith(['expired-file']); + expect(result).toEqual({ deletedFileIds: ['expired-file'], failedFileIds: [] }); + }); + + it('does not treat unrelated not found messages as missing storage', async () => { + const deleteFile = jest.fn().mockRejectedValue(new Error('Configuration not found')); + getStrategyFunctions.mockReturnValue({ deleteFile }); + + const result = await processDeleteRequest({ + req: { + body: {}, + config: {}, + user: { id: 'user-123', tenantId: 'tenant-a' }, + }, + files: [ + { + file_id: 'expired-file', + filepath: '/images/user-123/expired.png', + source: FileSources.local, + }, + ], + }); + + expect(db.deleteFiles).not.toHaveBeenCalled(); + expect(result).toEqual({ deletedFileIds: [], failedFileIds: ['expired-file'] }); + }); + + it('throws metadata delete failures after storage deletion succeeds', async () => { + const deleteFile = jest.fn().mockResolvedValue(undefined); + const metadataError = new Error('mongo unavailable'); + getStrategyFunctions.mockReturnValue({ deleteFile }); + db.deleteFiles.mockRejectedValue(metadataError); + + await expect( + processDeleteRequest({ + req: { + body: {}, + config: {}, + user: { id: 'user-123', tenantId: 'tenant-a' }, + }, + files: [ + { + file_id: 'expired-file', + filepath: '/images/user-123/expired.png', + source: FileSources.local, + }, + ], + }), + ).rejects.toThrow('mongo unavailable'); + + expect(db.deleteFiles).toHaveBeenCalledWith(['expired-file']); + expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled(); + }); +}); + +describe('sweepExpiredFiles', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('delegates expired file sweeping to the shared package with backend dependencies', async () => { + const options = { + appConfig: { paths: { publicPath: '/tmp/public', uploads: '/tmp/uploads' } }, + limit: 1, + }; + sweepExpiredFilesWithDeps.mockResolvedValue({ scanned: 1, deleted: 1, failed: 0 }); + + const result = await sweepExpiredFiles(options); + + expect(sweepExpiredFilesWithDeps).toHaveBeenCalledWith( + options, + expect.objectContaining({ + getExpiredFiles: db.getExpiredFiles, + processDeleteRequest: expect.any(Function), + logger: expect.objectContaining({ + error: expect.any(Function), + info: expect.any(Function), + warn: expect.any(Function), + }), + }), + ); + expect(result).toEqual({ scanned: 1, deleted: 1, failed: 0 }); + }); +}); + +describe('startExpiredFileSweep', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('delegates background sweep startup to the shared package with system context', () => { + const options = { + appConfig: { paths: { publicPath: '/tmp/public', uploads: '/tmp/uploads' } }, + }; + + const interval = startExpiredFileSweep(options); + + expect(startExpiredFileSweepWithDeps).toHaveBeenCalledWith( + options, + expect.objectContaining({ + sweepExpiredFiles: expect.any(Function), + runAsSystem: expect.any(Function), + logger: expect.objectContaining({ + error: expect.any(Function), + info: expect.any(Function), + warn: expect.any(Function), + }), + }), + ); + expect(interval).toBe('sweep-interval'); + }); +}); diff --git a/api/server/services/Files/retention.js b/api/server/services/Files/retention.js new file mode 100644 index 0000000000..2895d5764a --- /dev/null +++ b/api/server/services/Files/retention.js @@ -0,0 +1,21 @@ +const { getRetentionExpiry: getRetentionExpiryWithDeps } = require('@librechat/api'); +const { logger, createTempChatExpirationDate } = require('@librechat/data-schemas'); +const db = require('~/models'); + +/** + * Returns `{ expiredAt }` when the request indicates data retention applies, otherwise `{}`. + * Spread into file data objects before calling createFile. + * @param {ServerRequest} req + * @returns {Promise<{ expiredAt?: Date | null }>} + */ +async function getRetentionExpiry(req) { + return getRetentionExpiryWithDeps(req, { + getConvo: db.getConvoRetention ?? db.getConvo, + createExpirationDate: createTempChatExpirationDate, + logger, + }); +} + +module.exports = { + getRetentionExpiry, +}; diff --git a/api/server/utils/import/importBatchBuilder.js b/api/server/utils/import/importBatchBuilder.js index be47cd3692..b1856737cd 100644 --- a/api/server/utils/import/importBatchBuilder.js +++ b/api/server/utils/import/importBatchBuilder.js @@ -1,16 +1,26 @@ const { v4: uuidv4 } = require('uuid'); -const { logger } = require('@librechat/data-schemas'); -const { EModelEndpoint, Constants, openAISettings } = require('librechat-data-provider'); +const { + logger, + createFallbackRetentionDate, + createTempChatExpirationDate, +} = require('@librechat/data-schemas'); +const { + EModelEndpoint, + Constants, + RetentionMode, + openAISettings, +} = require('librechat-data-provider'); const { bulkIncrementTagCounts, bulkSaveConvos, bulkSaveMessages } = require('~/models'); const { FALLBACK_MODEL_BY_ENDPOINT } = require('./defaults'); /** * Factory function for creating an instance of ImportBatchBuilder. * @param {string} requestUserId - The ID of the user making the request. + * @param {object} [interfaceConfig] - Runtime interface config for import retention. * @returns {ImportBatchBuilder} - The newly created ImportBatchBuilder instance. */ -function createImportBatchBuilder(requestUserId) { - return new ImportBatchBuilder(requestUserId); +function createImportBatchBuilder(requestUserId, interfaceConfig) { + return new ImportBatchBuilder(requestUserId, interfaceConfig); } /** @@ -20,11 +30,36 @@ class ImportBatchBuilder { /** * Creates an instance of ImportBatchBuilder. * @param {string} requestUserId - The ID of the user making the import request. + * @param {object} [interfaceConfig] - Runtime interface config for import retention. */ - constructor(requestUserId) { + constructor(requestUserId, interfaceConfig) { this.requestUserId = requestUserId; + this.interfaceConfig = interfaceConfig; this.conversations = []; this.messages = []; + this.retentionFields = undefined; + } + + getRetentionFields() { + if (this.retentionFields !== undefined) { + return this.retentionFields; + } + + if (this.interfaceConfig?.retentionMode !== RetentionMode.ALL) { + this.retentionFields = {}; + return this.retentionFields; + } + + try { + this.retentionFields = { + isTemporary: false, + expiredAt: createTempChatExpirationDate(this.interfaceConfig), + }; + } catch (error) { + logger.error('[ImportBatchBuilder] Error creating import expiration date:', error); + this.retentionFields = { isTemporary: false, expiredAt: createFallbackRetentionDate() }; + } + return this.retentionFields; } /** @@ -89,6 +124,7 @@ class ImportBatchBuilder { overrideTimestamp: true, endpoint: this.endpoint, model: originalConvo.model ?? fallbackModel, + ...this.getRetentionFields(), }; convo._id && delete convo._id; this.conversations.push(convo); @@ -161,6 +197,7 @@ class ImportBatchBuilder { error: false, sender, text, + ...this.getRetentionFields(), }; message._id && delete message._id; this.lastMessageId = newMessageId; diff --git a/api/server/utils/import/importConversations.js b/api/server/utils/import/importConversations.js index ad2d743f01..21bba86e3a 100644 --- a/api/server/utils/import/importConversations.js +++ b/api/server/utils/import/importConversations.js @@ -2,15 +2,16 @@ const fs = require('fs').promises; const { resolveImportMaxFileSize } = require('@librechat/api'); const { logger } = require('@librechat/data-schemas'); const { getImporter } = require('./importers'); +const { createImportBatchBuilder } = require('./importBatchBuilder'); const maxFileSize = resolveImportMaxFileSize(); /** * Job definition for importing a conversation. - * @param {{ filepath: string, requestUserId: string, userRole?: string }} job + * @param {{ filepath: string, requestUserId: string, userRole?: string, interfaceConfig?: object }} job */ const importConversations = async (job) => { - const { filepath, requestUserId, userRole } = job; + const { filepath, requestUserId, userRole, interfaceConfig } = job; try { logger.debug(`user: ${requestUserId} | Importing conversation(s) from file...`); @@ -24,7 +25,12 @@ const importConversations = async (job) => { const fileData = await fs.readFile(filepath, 'utf8'); const jsonData = JSON.parse(fileData); const importer = getImporter(jsonData); - await importer(jsonData, requestUserId, undefined, userRole); + await importer( + jsonData, + requestUserId, + (userId) => createImportBatchBuilder(userId, interfaceConfig), + userRole, + ); logger.debug(`user: ${requestUserId} | Finished importing conversations`); } catch (error) { logger.error(`user: ${requestUserId} | Failed to import conversation: `, error); diff --git a/api/server/utils/import/importers.spec.js b/api/server/utils/import/importers.spec.js index cbd39afb34..6ccd2f3728 100644 --- a/api/server/utils/import/importers.spec.js +++ b/api/server/utils/import/importers.spec.js @@ -3,6 +3,7 @@ const path = require('path'); const { EModelEndpoint, Constants, + RetentionMode, openAISettings, anthropicSettings, } = require('librechat-data-provider'); @@ -28,6 +29,7 @@ jest.mock('~/server/controllers/ModelController', () => ({ jest.mock('~/models', () => ({ bulkSaveConvos: jest.fn(), bulkSaveMessages: jest.fn(), + bulkIncrementTagCounts: jest.fn(), })); afterEach(() => { @@ -1046,6 +1048,23 @@ describe('importLibreChatConvo', () => { expect(result.conversation.endpoint).toBe(EModelEndpoint.openAI); expect(result.conversation.model).toBe(openAISettings.model.default); }); + + it('applies all-data retention to imported conversations and messages', () => { + const requestUserId = 'user-123'; + const builder = new ImportBatchBuilder(requestUserId, { + retentionMode: RetentionMode.ALL, + temporaryChatRetention: 24, + }); + builder.startConversation(EModelEndpoint.openAI); + const message = builder.addUserMessage('Retained import'); + const result = builder.finishConversation('Imported retained chat'); + + expect(message.isTemporary).toBe(false); + expect(message.expiredAt).toBeInstanceOf(Date); + expect(result.conversation.isTemporary).toBe(false); + expect(result.conversation.expiredAt).toBeInstanceOf(Date); + expect(result.conversation.expiredAt).toBe(message.expiredAt); + }); }); }); diff --git a/client/src/components/Chat/Menus/BookmarkMenu.tsx b/client/src/components/Chat/Menus/BookmarkMenu.tsx index d66fccd24b..a42a917fbd 100644 --- a/client/src/components/Chat/Menus/BookmarkMenu.tsx +++ b/client/src/components/Chat/Menus/BookmarkMenu.tsx @@ -14,7 +14,7 @@ import { BookmarkContext } from '~/Providers/BookmarkContext'; import { BookmarkEditDialog } from '~/components/Bookmarks'; import { useBookmarkSuccess, useLocalize } from '~/hooks'; import { NotificationSeverity } from '~/common'; -import { cn, logger } from '~/utils'; +import { cn, isTemporaryConversation, logger } from '~/utils'; import store from '~/store'; const BookmarkMenu: FC = () => { @@ -26,8 +26,7 @@ const BookmarkMenu: FC = () => { const conversationId = conversation?.conversationId ?? ''; const updateConvoTags = useBookmarkSuccess(conversationId); const tags = conversation?.tags; - const isTemporary = conversation?.expiredAt != null; - + const isTemporary = isTemporaryConversation(conversation); const menuId = useId(); const [isMenuOpen, setIsMenuOpen] = useState(false); const [isDialogOpen, setIsDialogOpen] = useState(false); diff --git a/client/src/components/Messages/Content/RunCode.tsx b/client/src/components/Messages/Content/RunCode.tsx index da7db2739a..15c31bb85b 100644 --- a/client/src/components/Messages/Content/RunCode.tsx +++ b/client/src/components/Messages/Content/RunCode.tsx @@ -1,5 +1,6 @@ import React, { useState, useMemo, useCallback, useEffect, useRef } from 'react'; import debounce from 'lodash/debounce'; +import { useRecoilCallback } from 'recoil'; import { Tools } from 'librechat-data-provider'; import { TerminalSquareIcon, Check, X } from 'lucide-react'; import { Spinner, TooltipAnchor, useToastContext } from '@librechat/client'; @@ -8,6 +9,7 @@ import { useToolCallMutation } from '~/data-provider'; import { useLocalize } from '~/hooks'; import { cn, normalizeLanguage } from '~/utils'; import { useMessageContext } from '~/Providers'; +import store from '~/store'; type RunState = 'idle' | 'loading' | 'success' | 'error'; @@ -23,6 +25,13 @@ const RunCode: React.FC = React.memo( const { messageId, conversationId, partIndex } = useMessageContext(); const normalizedLang = useMemo(() => normalizeLanguage(lang), [lang]); + // Read at click time so retention context is current without re-rendering every code block. + const getIsTemporary = useRecoilCallback( + ({ snapshot }) => + () => + snapshot.getPromise(store.isTemporary), + [], + ); const handleExecute = useCallback(async () => { const codeString: string = codeRef.current?.textContent ?? ''; @@ -42,8 +51,18 @@ const RunCode: React.FC = React.memo( conversationId: conversationId ?? '', lang: normalizedLang, code: codeString, + isTemporary: await getIsTemporary(), }); - }, [codeRef, execute, partIndex, messageId, blockIndex, conversationId, normalizedLang]); + }, [ + codeRef, + execute, + partIndex, + messageId, + blockIndex, + conversationId, + normalizedLang, + getIsTemporary, + ]); const debouncedExecute = useMemo( () => debounce(handleExecute, 1000, { leading: true }), diff --git a/client/src/hooks/Files/__tests__/useFileHandling.test.ts b/client/src/hooks/Files/__tests__/useFileHandling.test.ts index 0a07c5f2b4..fdb098e2d5 100644 --- a/client/src/hooks/Files/__tests__/useFileHandling.test.ts +++ b/client/src/hooks/Files/__tests__/useFileHandling.test.ts @@ -11,6 +11,7 @@ const mockSetFilesLoading = jest.fn(); const mockMutate = jest.fn(); let mockConversation: Record = {}; +let mockIsTemporary = false; jest.mock('~/Providers/ChatContext', () => ({ useChatContext: jest.fn(() => ({ @@ -30,9 +31,12 @@ jest.mock('@librechat/client', () => ({ jest.mock('recoil', () => ({ ...jest.requireActual('recoil'), useSetRecoilState: jest.fn(() => jest.fn()), + useRecoilValue: jest.fn(() => mockIsTemporary), })); jest.mock('~/store', () => ({ + __esModule: true, + default: { isTemporary: { key: 'isTemporary' } }, ephemeralAgentByConvoId: jest.fn(() => ({ key: 'mock' })), })); @@ -99,6 +103,7 @@ describe('useFileHandling', () => { beforeEach(() => { jest.clearAllMocks(); mockConversation = {}; + mockIsTemporary = false; }); const loadHook = async () => (await import('../useFileHandling')).default; @@ -209,6 +214,7 @@ describe('useFileHandling', () => { const formData: FormData = mockMutate.mock.calls[0][0]; expect(formData.get('endpoint')).toBe(EModelEndpoint.agents); expect(formData.get('endpointType')).toBe(EModelEndpoint.agents); + expect(formData.get('conversationId')).toBeNull(); }); it('does not enter assistants upload path when override is agents', async () => { @@ -284,6 +290,87 @@ describe('useFileHandling', () => { expect(mockMutate).toHaveBeenCalledTimes(1); const formData: FormData = mockMutate.mock.calls[0][0]; expect(formData.get('endpoint')).toBe('default'); + expect(formData.get('conversationId')).toBeNull(); + }); + + it('sends temporary flag for temporary chat uploads', async () => { + mockIsTemporary = true; + mockConversation = { + conversationId: Constants.NEW_CONVO as string, + endpoint: 'openAI', + endpointType: 'custom', + }; + + const useFileHandling = await loadHook(); + const { result } = renderHook(() => useFileHandling()); + + const textFile = new File(['hello'], 'test.txt', { type: 'text/plain' }); + + await act(async () => { + await result.current.handleFiles([textFile]); + }); + + expect(mockMutate).toHaveBeenCalledTimes(1); + const formData: FormData = mockMutate.mock.calls[0][0]; + expect(formData.get('conversationId')).toBeNull(); + expect(formData.get('isTemporary')).toBe('true'); + }); + + it('does not send temporary flag for assistant builder uploads', async () => { + mockIsTemporary = true; + mockConversation = { + conversationId: 'temporary-convo', + endpoint: 'openAI', + endpointType: 'custom', + }; + + const useFileHandling = await loadHook(); + const { result } = renderHook(() => + useFileHandling({ + additionalMetadata: { assistant_id: 'asst-123' }, + }), + ); + + const textFile = new File(['hello'], 'test.txt', { type: 'text/plain' }); + + await act(async () => { + await result.current.handleFiles([textFile]); + }); + + expect(mockMutate).toHaveBeenCalledTimes(1); + const formData: FormData = mockMutate.mock.calls[0][0]; + expect(formData.get('assistant_id')).toBe('asst-123'); + expect(formData.get('conversationId')).toBeNull(); + expect(formData.get('isTemporary')).toBeNull(); + }); + + it('does not send temporary flag for agent builder uploads', async () => { + mockIsTemporary = true; + mockConversation = { + conversationId: 'temporary-convo', + endpoint: 'openAI', + endpointType: 'custom', + }; + + const useFileHandling = await loadHook(); + const { result } = renderHook(() => + useFileHandling({ + endpointOverride: EModelEndpoint.agents, + additionalMetadata: { agent_id: 'agent-123' }, + }), + ); + + const textFile = new File(['hello'], 'test.txt', { type: 'text/plain' }); + + await act(async () => { + await result.current.handleFiles([textFile]); + }); + + expect(mockMutate).toHaveBeenCalledTimes(1); + const formData: FormData = mockMutate.mock.calls[0][0]; + expect(formData.get('agent_id')).toBe('agent-123'); + expect(formData.get('conversationId')).toBeNull(); + expect(formData.get('isTemporary')).toBeNull(); }); }); }); diff --git a/client/src/hooks/Files/useFileHandling.ts b/client/src/hooks/Files/useFileHandling.ts index 635937a6fa..94f26039b6 100644 --- a/client/src/hooks/Files/useFileHandling.ts +++ b/client/src/hooks/Files/useFileHandling.ts @@ -1,6 +1,6 @@ import React, { useCallback, useEffect, useRef, useMemo, useState } from 'react'; import { v4 } from 'uuid'; -import { useSetRecoilState } from 'recoil'; +import { useRecoilValue, useSetRecoilState } from 'recoil'; import { useToastContext } from '@librechat/client'; import { useQueryClient } from '@tanstack/react-query'; import { @@ -22,7 +22,7 @@ import useLocalize, { TranslationKeys } from '~/hooks/useLocalize'; import { useDelayedUploadToast } from './useDelayedUploadToast'; import { processFileForUpload } from '~/utils/heicConverter'; import { useChatContext } from '~/Providers/ChatContext'; -import { ephemeralAgentByConvoId } from '~/store'; +import store, { ephemeralAgentByConvoId } from '~/store'; import useClientResize from './useClientResize'; import useUpdateFiles from './useUpdateFiles'; @@ -57,6 +57,7 @@ const useFileHandlingCore = (params: UseFileHandling | undefined, fileState: Fil const setEphemeralAgent = useSetRecoilState( ephemeralAgentByConvoId(conversation?.conversationId ?? Constants.NEW_CONVO), ); + const isTemporary = useRecoilValue(store.isTemporary); const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]); const { addFile, replaceFile, updateFileById, deleteFileById } = useUpdateFiles( params?.fileSetter ?? setFiles, @@ -65,6 +66,7 @@ const useFileHandlingCore = (params: UseFileHandling | undefined, fileState: Fil const agent_id = params?.additionalMetadata?.agent_id ?? ''; const assistant_id = params?.additionalMetadata?.assistant_id ?? ''; + const isConversationUpload = !agent_id && !assistant_id; const endpointOverride = params?.endpointOverride; const endpointTypeOverride = params?.endpointTypeOverride; const endpointType = useMemo( @@ -192,6 +194,16 @@ const useFileHandlingCore = (params: UseFileHandling | undefined, fileState: Fil formData.append('endpointType', endpointType ?? ''); formData.append('file', extendedFile.file as File, encodeURIComponent(filename)); formData.append('file_id', extendedFile.file_id); + if ( + isConversationUpload && + conversation?.conversationId && + conversation.conversationId !== Constants.NEW_CONVO + ) { + formData.append('conversationId', conversation.conversationId); + } + if (isTemporary && isConversationUpload) { + formData.append('isTemporary', 'true'); + } const width = extendedFile.width ?? 0; const height = extendedFile.height ?? 0; diff --git a/client/src/routes/ChatRoute.tsx b/client/src/routes/ChatRoute.tsx index a17d349037..374e2330cd 100644 --- a/client/src/routes/ChatRoute.tsx +++ b/client/src/routes/ChatRoute.tsx @@ -11,6 +11,7 @@ import { getDefaultModelSpec, getModelSpecPreset, isNotFoundError, + isTemporaryConversation, logger, } from '~/utils'; import { @@ -62,7 +63,7 @@ export default function ChatRoute() { const endpointsQuery = useGetEndpointsQuery({ enabled: isAuthenticated }); const assistantListMap = useAssistantListMap(); - const isTemporaryChat = conversation && conversation.expiredAt ? true : false; + const isTemporaryChat = isTemporaryConversation(conversation); useEffect(() => { if (conversationId === Constants.NEW_CONVO) { diff --git a/client/src/utils/conversation.ts b/client/src/utils/conversation.ts new file mode 100644 index 0000000000..31cdc39ebe --- /dev/null +++ b/client/src/utils/conversation.ts @@ -0,0 +1,5 @@ +import type { TConversation } from 'librechat-data-provider'; + +export const isTemporaryConversation = (conversation?: Partial | null): boolean => + conversation?.isTemporary === true || + (conversation?.isTemporary === undefined && conversation?.expiredAt != null); diff --git a/client/src/utils/index.ts b/client/src/utils/index.ts index 643cb76183..befcef81a1 100644 --- a/client/src/utils/index.ts +++ b/client/src/utils/index.ts @@ -22,6 +22,7 @@ export * from './textarea'; export * from './messages'; export * from './redirect'; export * from './languages'; +export * from './conversation'; export * from './endpoints'; export * from './resources'; export * from './downloadFile'; diff --git a/librechat.example.yaml b/librechat.example.yaml index b88fff9a60..d3e0d006cc 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -167,6 +167,15 @@ interface: # Temporary chat retention period in hours (default: 720, min: 1, max: 8760) # temporaryChatRetention: 1 + # Retention mode: "all" applies expiry to all data types, "temporary" (default) only to temporary chats + # Before switching from "all" back to "temporary", remove retention deadlines from non-temporary data + # that should stop expiring: + # db.conversations.updateMany({ isTemporary: false, expiredAt: { $ne: null } }, { $unset: { expiredAt: 1 } }) + # db.messages.updateMany({ isTemporary: false, expiredAt: { $ne: null } }, { $unset: { expiredAt: 1 } }) + # MongoDB does not drop superseded indexes automatically. After upgrading, old Meili indexes + # such as "_meiliIndex_1_expiredAt_1" can be dropped from conversations/messages once the new + # "_meiliIndex_1_isTemporary_1_expiredAt_1" indexes exist. + # retentionMode: "temporary" # Example Cloudflare turnstile (optional) #turnstile: diff --git a/packages/api/src/files/index.ts b/packages/api/src/files/index.ts index 8200f21195..0e9a23ff59 100644 --- a/packages/api/src/files/index.ts +++ b/packages/api/src/files/index.ts @@ -9,5 +9,7 @@ export * from './mistral/crud'; export * from './ocr'; export * from './parse'; export * from './rag'; +export * from './retention'; +export * from './sweep'; export * from './validation'; export * from './text'; diff --git a/packages/api/src/files/retention.spec.ts b/packages/api/src/files/retention.spec.ts new file mode 100644 index 0000000000..65ede6a5b1 --- /dev/null +++ b/packages/api/src/files/retention.spec.ts @@ -0,0 +1,273 @@ +import { RetentionMode } from 'librechat-data-provider'; +import { + createMinimalRetentionRequest, + getConversationExpirationDate, + getRetentionExpiry, + getSharedLinkExpiration, + isActiveExpirationDate, + isBooleanOrStringTrue, + type RetentionDependencies, + type RetentionRequest, +} from './retention'; + +describe('retention helpers', () => { + const expirationDate = new Date('2030-01-01T00:00:00.000Z'); + let dependencies: jest.Mocked; + + beforeEach(() => { + dependencies = { + getConvo: jest.fn(), + createExpirationDate: jest.fn().mockReturnValue(expirationDate), + logger: { + error: jest.fn(), + }, + }; + }); + + const request = (overrides: RetentionRequest = {}): RetentionRequest => ({ + user: { + id: 'user-1', + tenantId: 'tenant-1', + ...overrides.user, + }, + body: { + conversationId: 'convo-1', + ...overrides.body, + }, + config: { + interfaceConfig: { + ...overrides.config?.interfaceConfig, + }, + }, + }); + + it('returns expiry when retentionMode is ALL', async () => { + const result = await getRetentionExpiry( + request({ config: { interfaceConfig: { retentionMode: RetentionMode.ALL } } }), + dependencies, + ); + + expect(result).toEqual({ expiredAt: expirationDate }); + expect(dependencies.getConvo).not.toHaveBeenCalled(); + }); + + it('returns a fresh expiry when the conversation has an active expiration', async () => { + dependencies.getConvo.mockResolvedValue({ + expiredAt: new Date(Date.now() + 60 * 60 * 1000), + }); + + const result = await getRetentionExpiry(request(), dependencies); + + expect(result).toEqual({ expiredAt: expirationDate }); + }); + + it('returns the conversation expiration when the conversation is already expired', async () => { + const expiredAt = new Date(Date.now() - 60 * 60 * 1000); + dependencies.getConvo.mockResolvedValue({ expiredAt }); + + const result = await getRetentionExpiry(request(), dependencies); + + expect(result).toEqual({ expiredAt }); + expect(dependencies.createExpirationDate).not.toHaveBeenCalled(); + }); + + it('returns no retention fields when the conversation has no expiration', async () => { + dependencies.getConvo.mockResolvedValue({ expiredAt: null }); + + await expect(getRetentionExpiry(request(), dependencies)).resolves.toEqual({}); + }); + + it('returns expiry when the conversation has no expiration but explicit temporary intent is present', async () => { + dependencies.getConvo.mockResolvedValue({ expiredAt: null }); + + const result = await getRetentionExpiry( + request({ body: { conversationId: 'convo-1', isTemporary: true } }), + dependencies, + ); + + expect(result).toEqual({ expiredAt: expirationDate }); + }); + + it('returns no retention fields when conversation is missing and isTemporary is false', async () => { + dependencies.getConvo.mockResolvedValue(null); + + const result = await getRetentionExpiry( + request({ body: { conversationId: 'convo-1', isTemporary: false } }), + dependencies, + ); + + expect(result).toEqual({}); + }); + + it('returns expiry when isTemporary is true', async () => { + dependencies.getConvo.mockResolvedValue(null); + + const result = await getRetentionExpiry( + request({ body: { conversationId: 'convo-1', isTemporary: true } }), + dependencies, + ); + + expect(result).toEqual({ expiredAt: expirationDate }); + }); + + it('returns expiry when isTemporary is the string "true"', async () => { + dependencies.getConvo.mockResolvedValue(null); + + const result = await getRetentionExpiry( + request({ body: { conversationId: 'convo-1', isTemporary: 'true' } }), + dependencies, + ); + + expect(result).toEqual({ expiredAt: expirationDate }); + }); + + it('returns no retention fields when conversation lookup throws without explicit temporary intent', async () => { + const error = new Error('database unavailable'); + dependencies.getConvo.mockRejectedValue(error); + + const result = await getRetentionExpiry(request(), dependencies); + + expect(result).toEqual({}); + expect(dependencies.logger?.error).toHaveBeenCalledWith( + '[getRetentionExpiry] Error checking conversation retention:', + error, + ); + }); + + it('applies retention when explicit temporary intent is present and conversation lookup throws', async () => { + const error = new Error('database unavailable'); + dependencies.getConvo.mockRejectedValue(error); + + const result = await getRetentionExpiry( + request({ body: { conversationId: 'convo-1', isTemporary: true } }), + dependencies, + ); + + expect(result).toEqual({ expiredAt: expirationDate }); + expect(dependencies.logger?.error).toHaveBeenCalledWith( + '[getRetentionExpiry] Error checking conversation retention:', + error, + ); + }); + + it('returns a fallback expiration when expiration creation throws', async () => { + const error = new Error('bad config'); + const nowSpy = jest + .spyOn(Date, 'now') + .mockReturnValue(new Date('2026-01-01T00:00:00.000Z').getTime()); + dependencies.createExpirationDate.mockImplementation(() => { + throw error; + }); + + const result = await getRetentionExpiry( + request({ body: { conversationId: undefined, isTemporary: true } }), + dependencies, + ); + + expect(result).toEqual({ expiredAt: new Date('2026-01-31T00:00:00.000Z') }); + expect(dependencies.logger?.error).toHaveBeenCalledWith( + '[getRetentionExpiry] Error creating file expiration date:', + error, + ); + nowSpy.mockRestore(); + }); + + it('memoizes retention lookup per request object', async () => { + dependencies.getConvo.mockResolvedValue({ + expiredAt: new Date(Date.now() + 60 * 60 * 1000), + }); + const req = request(); + + const first = await getRetentionExpiry(req, dependencies); + const second = await getRetentionExpiry(req, dependencies); + + expect(first).toEqual({ expiredAt: expirationDate }); + expect(second).toEqual({ expiredAt: expirationDate }); + expect(dependencies.getConvo).toHaveBeenCalledTimes(1); + }); + + it('returns no retention fields when req is null or undefined', async () => { + await expect(getRetentionExpiry(null, dependencies)).resolves.toEqual({}); + await expect(getRetentionExpiry(undefined, dependencies)).resolves.toEqual({}); + }); + + it('parses valid conversation expiration dates and ignores invalid ones', () => { + expect(getConversationExpirationDate({ expiredAt: expirationDate })).toBe(expirationDate); + expect(getConversationExpirationDate({ expiredAt: expirationDate.toISOString() })).toEqual( + expirationDate, + ); + expect(getConversationExpirationDate({ expiredAt: 'not-a-date' })).toBeNull(); + expect(getConversationExpirationDate({ expiredAt: null })).toBeNull(); + }); + + it('compares active expiration dates against the provided clock', () => { + const now = new Date('2026-01-01T00:00:00.000Z'); + + expect(isActiveExpirationDate(new Date('2026-01-01T00:00:01.000Z'), now)).toBe(true); + expect(isActiveExpirationDate(new Date('2025-12-31T23:59:59.000Z'), now)).toBe(false); + }); + + it('uses strict temporary truthiness semantics', () => { + expect(isBooleanOrStringTrue(true)).toBe(true); + expect(isBooleanOrStringTrue('true')).toBe(true); + expect(isBooleanOrStringTrue(1)).toBe(false); + expect(isBooleanOrStringTrue('1')).toBe(false); + }); + + it('creates minimal retention requests for tool calls', () => { + expect( + createMinimalRetentionRequest({ + user: { id: 'user-1', tenantId: 'tenant-1' }, + body: { conversationId: 'convo-1', isTemporary: 'true' }, + config: { interfaceConfig: { retentionMode: RetentionMode.TEMPORARY } }, + }), + ).toEqual({ + user: { id: 'user-1', tenantId: 'tenant-1' }, + body: { conversationId: 'convo-1', isTemporary: 'true' }, + config: { interfaceConfig: { retentionMode: RetentionMode.TEMPORARY } }, + }); + + expect(createMinimalRetentionRequest()).toBeUndefined(); + }); + + describe('getSharedLinkExpiration', () => { + it('returns undefined when the conversation id is missing', async () => { + await expect( + getSharedLinkExpiration({ req: request() }, dependencies), + ).resolves.toBeUndefined(); + expect(dependencies.getConvo).not.toHaveBeenCalled(); + }); + + it('returns null for non-retained conversations in temporary retention mode', async () => { + dependencies.getConvo.mockResolvedValue({ expiredAt: null }); + + await expect( + getSharedLinkExpiration({ req: request(), conversationId: 'convo-1' }, dependencies), + ).resolves.toBeNull(); + }); + + it('returns a fresh expiry for retentionMode ALL conversations without an expiration', async () => { + dependencies.getConvo.mockResolvedValue({ expiredAt: null }); + + await expect( + getSharedLinkExpiration( + { + req: request({ config: { interfaceConfig: { retentionMode: RetentionMode.ALL } } }), + conversationId: 'convo-1', + }, + dependencies, + ), + ).resolves.toBe(expirationDate); + }); + + it('returns an expired source conversation date so callers can reject the share', async () => { + const expiredAt = new Date(Date.now() - 60 * 60 * 1000); + dependencies.getConvo.mockResolvedValue({ expiredAt }); + + await expect( + getSharedLinkExpiration({ req: request(), conversationId: 'convo-1' }, dependencies), + ).resolves.toBe(expiredAt); + expect(dependencies.createExpirationDate).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/api/src/files/retention.ts b/packages/api/src/files/retention.ts new file mode 100644 index 0000000000..edbebadd90 --- /dev/null +++ b/packages/api/src/files/retention.ts @@ -0,0 +1,231 @@ +import { RetentionMode } from 'librechat-data-provider'; +import { createFallbackRetentionDate } from '@librechat/data-schemas'; +import type { AppConfig } from '@librechat/data-schemas'; + +type InterfaceConfig = AppConfig['interfaceConfig']; + +const retentionExpiryCache = new WeakMap< + RetentionRequest, + { + key: string; + promise: Promise; + } +>(); + +export type RetentionConversation = { + expiredAt?: Date | string | number | null; +}; + +export type RetentionRequest = { + user?: { + id?: string; + tenantId?: string; + }; + body?: { + conversationId?: string; + isTemporary?: boolean | string | null; + }; + config?: { + interfaceConfig?: InterfaceConfig; + }; +}; + +export type RetentionExpiry = { + expiredAt?: Date | null; +}; + +export type RetentionLogger = { + error: (message: string, error?: unknown) => void; +}; + +export type RetentionDependencies = { + getConvo: ( + userId: string, + conversationId: string, + ) => Promise; + createExpirationDate: (interfaceConfig?: InterfaceConfig) => Date; + logger?: RetentionLogger; +}; + +export type SharedLinkRetentionDependencies = { + getConvo: ( + userId: string, + conversationId: string, + ) => Promise; + createExpirationDate: (interfaceConfig?: InterfaceConfig) => Date; + logger?: RetentionLogger; +}; + +export const isBooleanOrStringTrue = (value: unknown): boolean => + value === true || value === 'true'; + +export const getConversationExpirationDate = ( + convo?: RetentionConversation | null, +): Date | null => { + if (convo?.expiredAt == null) { + return null; + } + + const expiredAt = convo.expiredAt instanceof Date ? convo.expiredAt : new Date(convo.expiredAt); + return Number.isNaN(expiredAt.getTime()) ? null : expiredAt; +}; + +export const isActiveExpirationDate = (expiredAt: Date, now = new Date()): boolean => + expiredAt > now; + +const createRetentionExpiry = ( + req: RetentionRequest | null | undefined, + { createExpirationDate, logger }: RetentionDependencies, +): RetentionExpiry => { + try { + return { expiredAt: createExpirationDate(req?.config?.interfaceConfig) }; + } catch (err) { + logger?.error('[getRetentionExpiry] Error creating file expiration date:', err); + return { expiredAt: createFallbackRetentionDate() }; + } +}; + +const getRetentionCacheKey = (req: RetentionRequest): string => + [ + req.config?.interfaceConfig?.retentionMode ?? '', + req.user?.id ?? '', + req.body?.conversationId ?? '', + String(req.body?.isTemporary ?? ''), + ].join('|'); + +async function computeRetentionExpiry( + req: RetentionRequest | null | undefined, + dependencies: RetentionDependencies, +): Promise { + if (req?.config?.interfaceConfig?.retentionMode === RetentionMode.ALL) { + return createRetentionExpiry(req, dependencies); + } + + const conversationId = req?.body?.conversationId; + const userId = req?.user?.id; + + if (conversationId && userId) { + try { + const convo = await dependencies.getConvo(userId, conversationId); + if (convo) { + const expiredAt = getConversationExpirationDate(convo); + if (expiredAt == null) { + if (isBooleanOrStringTrue(req?.body?.isTemporary)) { + return createRetentionExpiry(req, dependencies); + } + return {}; + } + + if (!isActiveExpirationDate(expiredAt)) { + return { expiredAt }; + } + + return createRetentionExpiry(req, dependencies); + } + } catch (err) { + dependencies.logger?.error( + '[getRetentionExpiry] Error checking conversation retention:', + err, + ); + if (isBooleanOrStringTrue(req?.body?.isTemporary)) { + return createRetentionExpiry(req, dependencies); + } + return {}; + } + } + + if (!isBooleanOrStringTrue(req?.body?.isTemporary)) { + return {}; + } + + return createRetentionExpiry(req, dependencies); +} + +export async function getRetentionExpiry( + req: RetentionRequest | null | undefined, + dependencies: RetentionDependencies, +): Promise { + if (!req) { + return {}; + } + + const key = getRetentionCacheKey(req); + const cached = retentionExpiryCache.get(req); + if (cached?.key === key) { + return cached.promise; + } + + const promise = computeRetentionExpiry(req, dependencies); + retentionExpiryCache.set(req, { key, promise }); + return promise; +} + +/** + * Resolves the retention deadline for a shared link derived from a conversation. + * + * Return values are intentionally tri-state: + * - `undefined`: no decision can be made because the conversation id or row is missing. + * - `null`: the share should be stored without an expiration. + * - `Date`: the share should expire at that date; callers reject already-expired dates. + */ +export async function getSharedLinkExpiration( + { + req, + conversationId, + }: { + req: RetentionRequest | null | undefined; + conversationId?: string | null; + }, + dependencies: SharedLinkRetentionDependencies, +): Promise { + const userId = req?.user?.id; + if (!conversationId || !userId) { + return undefined; + } + + const isRetentionAll = req?.config?.interfaceConfig?.retentionMode === RetentionMode.ALL; + const convo = await dependencies.getConvo(userId, conversationId); + if (!convo) { + return undefined; + } + + const conversationExpiredAt = getConversationExpirationDate(convo); + if (conversationExpiredAt == null) { + if (!isRetentionAll) { + return null; + } + } else if (!isActiveExpirationDate(conversationExpiredAt)) { + return conversationExpiredAt; + } + + try { + return dependencies.createExpirationDate(req?.config?.interfaceConfig); + } catch (err) { + dependencies.logger?.error('[getSharedLinkExpiration] Error creating expiration date:', err); + return null; + } +} + +export const createMinimalRetentionRequest = ( + req?: RetentionRequest | null, +): RetentionRequest | undefined => { + if (!req) { + return undefined; + } + + return { + user: req.user + ? { + id: req.user.id, + tenantId: req.user.tenantId, + } + : undefined, + body: { + conversationId: req.body?.conversationId, + isTemporary: req.body?.isTemporary, + }, + config: { + interfaceConfig: req.config?.interfaceConfig, + }, + }; +}; diff --git a/packages/api/src/files/sweep.spec.ts b/packages/api/src/files/sweep.spec.ts new file mode 100644 index 0000000000..4c5465f97e --- /dev/null +++ b/packages/api/src/files/sweep.spec.ts @@ -0,0 +1,96 @@ +import { EModelEndpoint, FileSources } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import { getFileRetentionSweepInterval, startExpiredFileSweep, sweepExpiredFiles } from './sweep'; + +describe('expired file sweep helpers', () => { + const logger = { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + delete process.env.FILE_RETENTION_SWEEP_INTERVAL_MS; + }); + + afterEach(() => { + jest.useRealTimers(); + delete process.env.FILE_RETENTION_SWEEP_INTERVAL_MS; + }); + + it('loads endpoint config and deletes expired OpenAI storage files', async () => { + const getExpiredFiles = jest.fn().mockResolvedValue([ + { + file_id: 'expired-openai-file', + source: FileSources.openai, + user: { toString: () => 'user-123' }, + tenantId: 'tenant-a', + }, + ]); + const processDeleteRequest = jest.fn().mockResolvedValue({ + deletedFileIds: ['expired-openai-file'], + failedFileIds: [], + }); + const loadAppConfig = jest.fn().mockResolvedValue({ + endpoints: { + [EModelEndpoint.assistants]: { version: 'v3' }, + }, + } as AppConfig); + + const result = await sweepExpiredFiles( + { appConfig: {} as AppConfig, loadAppConfig, limit: 1 }, + { getExpiredFiles, processDeleteRequest, logger }, + ); + + expect(loadAppConfig).toHaveBeenCalledTimes(1); + expect(processDeleteRequest).toHaveBeenCalledWith({ + req: expect.objectContaining({ + baseUrl: '/api/assistants/v3', + originalUrl: '/api/assistants/v3/files', + body: { endpoint: EModelEndpoint.assistants, version: '3' }, + user: { id: 'user-123', tenantId: 'tenant-a' }, + }), + files: [expect.objectContaining({ file_id: 'expired-openai-file' })], + }); + expect(result).toEqual({ scanned: 1, deleted: 1, failed: 0 }); + }); + + it('counts files without owners as failed without deleting them', async () => { + const getExpiredFiles = jest.fn().mockResolvedValue([{ file_id: 'orphaned-file' }]); + const processDeleteRequest = jest.fn(); + + const result = await sweepExpiredFiles( + { appConfig: {} as AppConfig, limit: 1 }, + { getExpiredFiles, processDeleteRequest, logger }, + ); + + expect(processDeleteRequest).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith( + '[sweepExpiredFiles] Skipping expired file without user: orphaned-file', + ); + expect(result).toEqual({ scanned: 1, deleted: 0, failed: 1 }); + }); + + it('falls back to the default interval for sub-millisecond values', () => { + expect(getFileRetentionSweepInterval('0.5')).toBe(60 * 60 * 1000); + }); + + it('does not start the interval when the sweep is disabled', () => { + process.env.FILE_RETENTION_SWEEP_INTERVAL_MS = '0'; + + const interval = startExpiredFileSweep( + { appConfig: {} as AppConfig }, + { + sweepExpiredFiles: jest.fn(), + runAsSystem: jest.fn((fn) => fn()), + logger, + }, + ); + + expect(interval).toBeNull(); + expect(logger.info).toHaveBeenCalledWith( + '[sweepExpiredFiles] Disabled by FILE_RETENTION_SWEEP_INTERVAL_MS=0', + ); + }); +}); diff --git a/packages/api/src/files/sweep.ts b/packages/api/src/files/sweep.ts new file mode 100644 index 0000000000..29562ffac5 --- /dev/null +++ b/packages/api/src/files/sweep.ts @@ -0,0 +1,286 @@ +import { + FileSources, + EModelEndpoint, + checkOpenAIStorage, + defaultAssistantsVersion, +} from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; + +const DEFAULT_FILE_RETENTION_SWEEP_INTERVAL_MS = 60 * 60 * 1000; + +type ExpiredFile = { + file_id: string; + source?: string; + user?: string | { toString?: () => string }; + tenantId?: string; +}; + +type SweepRequest = { + baseUrl: string; + originalUrl: string; + path: string; + method: string; + headers: Record; + query: Record; + params: Record; + config?: AppConfig; + body: { + endpoint: string; + version: string; + }; + user: { + id: string; + tenantId?: string; + }; +}; + +type SweepLogger = { + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string, error?: unknown) => void; +}; + +type VersionedEndpointConfig = { + version?: unknown; + assistants?: { version?: unknown } | boolean; +}; + +type SweepDependencies = { + getExpiredFiles: (limit: number) => Promise; + processDeleteRequest: (params: { + req: SweepRequest; + files: ExpiredFile[]; + }) => Promise<{ deletedFileIds: string[]; failedFileIds: string[] }>; + logger: SweepLogger; +}; + +type StartSweepDependencies = { + sweepExpiredFiles: (options?: ExpiredFileSweepOptions) => Promise; + runAsSystem: (fn: () => Promise) => Promise; + logger: SweepLogger; +}; + +export type ExpiredFileSweepOptions = { + appConfig?: AppConfig; + limit?: number; + loadAppConfig?: () => Promise; +}; + +export type ExpiredFileSweepResult = { + scanned: number; + deleted: number; + failed: number; +}; + +export function getFileRetentionSweepInterval( + interval = process.env.FILE_RETENTION_SWEEP_INTERVAL_MS, +): number { + if (interval == null || interval.trim() === '') { + return DEFAULT_FILE_RETENTION_SWEEP_INTERVAL_MS; + } + + const value = Number(interval); + if (!Number.isFinite(value) || value < 0 || (value > 0 && value < 1)) { + return DEFAULT_FILE_RETENTION_SWEEP_INTERVAL_MS; + } + return value; +} + +export function getExpiredFileEndpoint(source?: string): string { + return source === FileSources.azure ? EModelEndpoint.azureAssistants : EModelEndpoint.assistants; +} + +export function hasExpiredFileEndpointConfig(appConfig: AppConfig | undefined, source?: string) { + if (source === FileSources.azure) { + return Boolean(appConfig?.endpoints?.[EModelEndpoint.azureOpenAI]?.assistants); + } + + return Boolean(appConfig?.endpoints?.[EModelEndpoint.assistants]); +} + +export function getConfiguredExpiredFileAssistantVersion({ + appConfig, + source, + endpoint, +}: { + appConfig?: AppConfig; + source?: string; + endpoint: string; +}): unknown { + const endpoints = appConfig?.endpoints as + | Record + | undefined; + const endpointVersion = endpoints?.[endpoint]?.version; + if (endpointVersion != null) { + return endpointVersion; + } + + if (source === FileSources.azure) { + const azureAssistantsConfig = endpoints?.[EModelEndpoint.azureOpenAI]?.assistants; + if (typeof azureAssistantsConfig === 'object' && azureAssistantsConfig?.version != null) { + return azureAssistantsConfig.version; + } + } + + return undefined; +} + +export function getExpiredFileAssistantVersion({ + appConfig, + source, + endpoint, +}: { + appConfig?: AppConfig; + source?: string; + endpoint: string; +}): string { + const configuredVersion = getConfiguredExpiredFileAssistantVersion({ + appConfig, + source, + endpoint, + }); + const assistantVersions = defaultAssistantsVersion as Record; + const fallbackVersion = assistantVersions[endpoint] ?? defaultAssistantsVersion.assistants ?? 2; + + return String(configuredVersion ?? fallbackVersion).replace(/^v/, ''); +} + +export function createExpiredFileSweepRequest({ + appConfig, + file, + userId, +}: { + appConfig?: AppConfig; + file: ExpiredFile; + userId: string; +}): SweepRequest { + const source = file.source ?? FileSources.local; + const endpoint = getExpiredFileEndpoint(source); + const version = getExpiredFileAssistantVersion({ appConfig, source, endpoint }); + const baseUrl = `/api/assistants/v${version}`; + + return { + baseUrl, + originalUrl: `${baseUrl}/files`, + path: '/files', + method: 'DELETE', + headers: {}, + query: {}, + params: {}, + config: appConfig, + body: { + endpoint, + version, + }, + user: { + id: userId, + tenantId: file.tenantId, + }, + }; +} + +export async function resolveExpiredFileSweepConfig({ + appConfig, + file, + loadAppConfig, +}: { + appConfig?: AppConfig; + file: ExpiredFile; + loadAppConfig?: () => Promise; +}): Promise { + const source = file.source ?? FileSources.local; + if ( + !checkOpenAIStorage(source) || + hasExpiredFileEndpointConfig(appConfig, source) || + typeof loadAppConfig !== 'function' + ) { + return appConfig; + } + + return (await loadAppConfig()) ?? appConfig; +} + +export async function sweepExpiredFiles( + { appConfig, limit = 100, loadAppConfig }: ExpiredFileSweepOptions = {}, + { getExpiredFiles, processDeleteRequest, logger }: SweepDependencies, +): Promise { + const files = (await getExpiredFiles(limit)) ?? []; + let resolvedAppConfig = appConfig; + let deleted = 0; + let failed = 0; + + for (const file of files) { + const userId = typeof file.user === 'string' ? file.user : file.user?.toString?.(); + if (!userId) { + logger.warn(`[sweepExpiredFiles] Skipping expired file without user: ${file.file_id}`); + failed++; + continue; + } + + try { + resolvedAppConfig = await resolveExpiredFileSweepConfig({ + appConfig: resolvedAppConfig, + file, + loadAppConfig, + }); + const req = createExpiredFileSweepRequest({ appConfig: resolvedAppConfig, file, userId }); + const { deletedFileIds, failedFileIds } = await processDeleteRequest({ req, files: [file] }); + if (failedFileIds.includes(file.file_id)) { + failed++; + continue; + } + + if (deletedFileIds.includes(file.file_id)) { + deleted++; + } else { + failed++; + logger.error( + `[sweepExpiredFiles] Delete request finished without resolving expired file ${file.file_id}`, + ); + } + } catch (error) { + failed++; + logger.error(`[sweepExpiredFiles] Error deleting expired file ${file.file_id}:`, error); + } + } + + if (deleted > 0 || failed > 0) { + logger.info( + `[sweepExpiredFiles] Processed ${files.length} expired files: ${deleted} deleted, ${failed} failed`, + ); + } + + return { scanned: files.length, deleted, failed }; +} + +export function startExpiredFileSweep( + options: ExpiredFileSweepOptions = {}, + { sweepExpiredFiles, runAsSystem, logger }: StartSweepDependencies, +): NodeJS.Timeout | null { + const intervalMs = getFileRetentionSweepInterval(); + if (intervalMs === 0) { + logger.info('[sweepExpiredFiles] Disabled by FILE_RETENTION_SWEEP_INTERVAL_MS=0'); + return null; + } + + let isSweeping = false; + const runSweep = async () => { + if (isSweeping) { + return; + } + + isSweeping = true; + try { + await runAsSystem(() => sweepExpiredFiles(options)); + } catch (error) { + logger.error('[sweepExpiredFiles] Background sweep failed:', error); + } finally { + isSweeping = false; + } + }; + + runSweep(); + const interval = setInterval(runSweep, intervalMs); + interval.unref?.(); + return interval; +} diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index ebc86122d1..f5a2be1fe9 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -45,6 +45,7 @@ export const excludedKeys = new Set([ 'createdAt', 'updatedAt', 'expiredAt', + 'isTemporary', 'messages', 'isArchived', 'tags', @@ -895,6 +896,11 @@ const mcpServersSchema = z export type TMcpServersConfig = z.infer; +export enum RetentionMode { + ALL = 'all', + TEMPORARY = 'temporary', +} + export const interfaceSchema = z .object({ privacyPolicy: z @@ -937,6 +943,7 @@ export const interfaceSchema = z temporaryChat: z.boolean().optional(), temporaryChatRetention: z.number().min(1).max(8760).optional(), autoSubmitFromUrl: z.boolean().optional(), + retentionMode: z.nativeEnum(RetentionMode).default(RetentionMode.TEMPORARY), runCode: z.boolean().optional(), webSearch: z.boolean().optional(), peoplePicker: z diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts index 3d72940872..7a8d7e47b7 100644 --- a/packages/data-provider/src/schemas.ts +++ b/packages/data-provider/src/schemas.ts @@ -648,6 +648,8 @@ export const tMessageSchema = z.object({ /** @deprecated */ generation: z.string().nullable().optional(), isCreatedByUser: z.boolean(), + isTemporary: z.boolean().optional(), + expiredAt: z.string().nullable().optional(), error: z.boolean().optional(), clientTimestamp: z.string().optional(), createdAt: z @@ -855,6 +857,7 @@ export const tConversationSchema = z.object({ iconURL: z.string().nullable().optional(), /* temporary chat */ expiredAt: z.string().nullable().optional(), + isTemporary: z.boolean().optional(), /* file token limits */ fileTokenLimit: coerceNumber.optional(), /** @deprecated */ diff --git a/packages/data-provider/src/types/mutations.ts b/packages/data-provider/src/types/mutations.ts index 4f40794527..d0f69cac18 100644 --- a/packages/data-provider/src/types/mutations.ts +++ b/packages/data-provider/src/types/mutations.ts @@ -446,6 +446,7 @@ export type ToolParams = ToolParamsMap[T] & { partIndex?: number; blockIndex?: number; conversationId: string; + isTemporary?: boolean; }; export type ToolCallResponse = { result: unknown; attachments?: types.TAttachment[] }; export type ToolCallMutationOptions = MutationOptions< diff --git a/packages/data-schemas/src/app/assistants.ts b/packages/data-schemas/src/app/assistants.ts index c41a8d603e..ddc56cf495 100644 --- a/packages/data-schemas/src/app/assistants.ts +++ b/packages/data-schemas/src/app/assistants.ts @@ -52,6 +52,10 @@ export function assistantsConfigSetup( return { ...prevConfig, + version: + assistantsConfig?.version != null + ? parsedConfig.version + : (prevConfig.version ?? parsedConfig.version), retrievalModels: parsedConfig.retrievalModels, disableBuilder: parsedConfig.disableBuilder, pollIntervalMs: parsedConfig.pollIntervalMs, diff --git a/packages/data-schemas/src/app/interface.ts b/packages/data-schemas/src/app/interface.ts index 4e719e8ac5..2dbe387ccf 100644 --- a/packages/data-schemas/src/app/interface.ts +++ b/packages/data-schemas/src/app/interface.ts @@ -48,6 +48,7 @@ export async function loadDefaultInterface({ agents: interfaceConfig?.agents, temporaryChat: interfaceConfig?.temporaryChat, temporaryChatRetention: interfaceConfig?.temporaryChatRetention, + retentionMode: interfaceConfig?.retentionMode, runCode: interfaceConfig?.runCode, webSearch: interfaceConfig?.webSearch, fileSearch: interfaceConfig?.fileSearch, diff --git a/packages/data-schemas/src/app/service.spec.ts b/packages/data-schemas/src/app/service.spec.ts index 80298b3e18..2a2a8ffd1d 100644 --- a/packages/data-schemas/src/app/service.spec.ts +++ b/packages/data-schemas/src/app/service.spec.ts @@ -1,5 +1,6 @@ import type { DeepPartial, TCustomConfig } from 'librechat-data-provider'; -import { loadSummarizationConfig } from './service'; +import { EModelEndpoint, defaultAssistantsVersion } from 'librechat-data-provider'; +import { AppService, loadSummarizationConfig } from './service'; import logger from '~/config/winston'; jest.mock('~/config/winston', () => ({ @@ -78,3 +79,70 @@ describe('loadSummarizationConfig', () => { expect(String(warnSpy.mock.calls[0][0])).toContain('Invalid summarization config'); }); }); + +describe('AppService assistants config', () => { + it('preserves configured Assistants API versions', async () => { + const config = { + endpoints: { + [EModelEndpoint.assistants]: { + version: 'v3', + }, + [EModelEndpoint.azureOpenAI]: { + assistants: true, + groups: [ + { + group: 'azure-assistants-test', + apiKey: 'test-key', + instanceName: 'azure-assistants-test', + assistants: true, + version: '2024-02-15-preview', + models: { + 'gpt-4': { + deploymentName: 'gpt-4', + }, + }, + }, + ], + }, + [EModelEndpoint.azureAssistants]: { + version: 4, + }, + }, + } as DeepPartial; + + const result = await AppService({ config }); + + expect(result.endpoints?.[EModelEndpoint.assistants]?.version).toBe('v3'); + expect(result.endpoints?.[EModelEndpoint.azureAssistants]?.version).toBe(4); + }); + + it('keeps Azure Assistants default version when only Azure OpenAI enables assistants', async () => { + const config = { + endpoints: { + [EModelEndpoint.azureOpenAI]: { + assistants: true, + groups: [ + { + group: 'azure-assistants-test', + apiKey: 'test-key', + instanceName: 'azure-assistants-test', + assistants: true, + version: '2024-02-15-preview', + models: { + 'gpt-4': { + deploymentName: 'gpt-4', + }, + }, + }, + ], + }, + }, + } as DeepPartial; + + const result = await AppService({ config }); + + expect(result.endpoints?.[EModelEndpoint.azureAssistants]?.version).toBe( + defaultAssistantsVersion.azureAssistants, + ); + }); +}); diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index 166e5b51c8..5646d7dfff 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -1,6 +1,6 @@ import mongoose from 'mongoose'; import { v4 as uuidv4 } from 'uuid'; -import { EModelEndpoint } from 'librechat-data-provider'; +import { EModelEndpoint, RetentionMode } from 'librechat-data-provider'; import type { IConversation } from '../types'; import { MongoMemoryServer } from 'mongodb-memory-server'; import { ConversationMethods, createConversationMethods } from './conversation'; @@ -58,6 +58,8 @@ const saveConvo = (...args: Parameters) => methods.saveConvo(...args) as Promise; const getConvo = (...args: Parameters) => methods.getConvo(...args); +const getConvoRetention = (...args: Parameters) => + methods.getConvoRetention(...args); const getConvoTitle = (...args: Parameters) => methods.getConvoTitle(...args); const getConvoFiles = (...args: Parameters) => @@ -78,7 +80,7 @@ describe('Conversation Operations', () => { let mockCtx: { userId: string; isTemporary?: boolean; - interfaceConfig?: { temporaryChatRetention?: number }; + interfaceConfig?: { temporaryChatRetention?: number; retentionMode?: RetentionMode }; }; let mockConversationData: { conversationId: string; @@ -263,7 +265,7 @@ describe('Conversation Operations', () => { const result = await saveConvo(mockCtx, mockConversationData); expect(result?.conversationId).toBe(mockConversationData.conversationId); - expect(result?.expiredAt).toBeNull(); + expect(result?.expiredAt).toBeUndefined(); }); it('should use custom retention period from config', async () => { @@ -401,6 +403,21 @@ describe('Conversation Operations', () => { ); }); + it('should preserve temporary retention when saving without isTemporary', async () => { + mockCtx.interfaceConfig = { temporaryChatRetention: 24 }; + mockCtx.isTemporary = true; + const firstSave = await saveConvo(mockCtx, mockConversationData); + const originalExpiredAt = firstSave?.expiredAt; + + mockCtx.isTemporary = undefined; + const updatedData = { ...mockConversationData, title: 'Updated Title' }; + const secondSave = await saveConvo(mockCtx, updatedData); + + expect(secondSave?.title).toBe('Updated Title'); + expect(secondSave?.isTemporary).toBe(true); + expect(secondSave?.expiredAt).toEqual(originalExpiredAt); + }); + it('should not set expiredAt when updating non-temporary conversation', async () => { // First save a non-temporary conversation mockCtx.isTemporary = false; @@ -416,23 +433,124 @@ describe('Conversation Operations', () => { expect(secondSave?.expiredAt).toBeNull(); }); - it('should filter out expired conversations in getConvosByCursor', async () => { + it('should set expiredAt for non-temporary conversation when retentionMode is ALL', async () => { + mockCtx.isTemporary = false; + mockCtx.interfaceConfig = { + temporaryChatRetention: 24, + retentionMode: RetentionMode.ALL, + }; + const result = await saveConvo(mockCtx, mockConversationData); + expect(result?.expiredAt).toBeDefined(); + expect(result?.isTemporary).toBe(false); + }); + + it('should mark retained conversation non-temporary when retentionMode is ALL and isTemporary is omitted', async () => { + mockCtx.isTemporary = undefined; + mockCtx.interfaceConfig = { + temporaryChatRetention: 24, + retentionMode: RetentionMode.ALL, + }; + + const result = await saveConvo(mockCtx, mockConversationData); + + expect(result?.expiredAt).toBeDefined(); + expect(result?.isTemporary).toBe(false); + }); + + it('should preserve existing temporary flag when retentionMode is ALL and isTemporary is omitted', async () => { + mockCtx.isTemporary = true; + mockCtx.interfaceConfig = { + temporaryChatRetention: 24, + retentionMode: RetentionMode.ALL, + }; + + const firstSave = await saveConvo(mockCtx, mockConversationData); + + mockCtx.isTemporary = undefined; + const secondSave = await saveConvo(mockCtx, { + ...mockConversationData, + title: 'Updated Title', + }); + + expect(firstSave?.isTemporary).toBe(true); + expect(secondSave?.title).toBe('Updated Title'); + expect(secondSave?.isTemporary).toBe(true); + expect(secondSave?.expiredAt).toBeDefined(); + }); + + it('should not set expiredAt when retentionMode is temporary and not isTemporary', async () => { + mockCtx.isTemporary = false; + mockCtx.interfaceConfig = { + temporaryChatRetention: 24, + retentionMode: RetentionMode.TEMPORARY, + }; + const result = await saveConvo(mockCtx, mockConversationData); + expect(result?.expiredAt).toBeNull(); + expect(result?.isTemporary).toBe(false); + }); + + it('should filter out temporary conversations in getConvosByCursor', async () => { // Create some test conversations - const nonExpiredConvo = await Conversation.create({ + const newNonTemporaryConvo = await Conversation.create({ conversationId: uuidv4(), user: 'user123', - title: 'Non-expired', + title: 'New Non-temporary Conversation', endpoint: EModelEndpoint.openAI, + isTemporary: false, + expiredAt: new Date(Date.now() + 24 * 60 * 60 * 1000), + updatedAt: new Date(), + }); + + const oldNonTemporaryConvo = await Conversation.create({ + conversationId: uuidv4(), + user: 'user123', + title: 'Old Non-Temporary Conversation', + endpoint: EModelEndpoint.openAI, + isTemporary: undefined, expiredAt: null, updatedAt: new Date(), }); + const legacyNullNonTemporaryConvoId = uuidv4(); + await Conversation.collection.insertOne({ + conversationId: legacyNullNonTemporaryConvoId, + user: 'user123', + title: 'Legacy Null Non-Temporary Conversation', + endpoint: EModelEndpoint.openAI, + isTemporary: null, + expiredAt: null, + updatedAt: new Date(), + createdAt: new Date(), + }); + + const legacyTemporaryConvoId = uuidv4(); + await Conversation.collection.insertOne({ + conversationId: legacyTemporaryConvoId, + user: 'user123', + title: 'Legacy Temporary Conversation', + endpoint: EModelEndpoint.openAI, + expiredAt: new Date(Date.now() + 24 * 60 * 60 * 1000), + updatedAt: new Date(), + createdAt: new Date(), + }); + + const expiredRetainedConvo = await Conversation.create({ + conversationId: uuidv4(), + user: 'user123', + title: 'Expired Retained Conversation', + endpoint: EModelEndpoint.openAI, + isTemporary: false, + expiredAt: new Date(Date.now() - 60 * 60 * 1000), + updatedAt: new Date(), + }); + await Conversation.create({ conversationId: uuidv4(), user: 'user123', - title: 'Future expired', + title: 'Temporary conversation', endpoint: EModelEndpoint.openAI, - expiredAt: new Date(Date.now() + 24 * 60 * 60 * 1000), // 24 hours from now + isTemporary: true, + expiredAt: new Date(Date.now() + 24 * 60 * 60 * 1000), updatedAt: new Date(), }); @@ -441,41 +559,104 @@ describe('Conversation Operations', () => { const result = await getConvosByCursor('user123'); - // Should only return conversations with null or non-existent expiredAt - expect(result?.conversations).toHaveLength(1); - expect(result?.conversations[0]?.conversationId).toBe(nonExpiredConvo.conversationId); + // Should return both non-temporary conversations, not the temporary one + expect(result?.conversations).toHaveLength(3); + const convoIds = result?.conversations.map((c) => c.conversationId); + expect(convoIds).toContain(newNonTemporaryConvo.conversationId); + expect(convoIds).toContain(oldNonTemporaryConvo.conversationId); + expect(convoIds).toContain(legacyNullNonTemporaryConvoId); + expect(convoIds).not.toContain(legacyTemporaryConvoId); + expect(convoIds).not.toContain(expiredRetainedConvo.conversationId); }); - it('should filter out expired conversations in getConvosQueried', async () => { - // Create test conversations - const nonExpiredConvo = await Conversation.create({ + it('should filter out temporary conversations in getConvosQueried', async () => { + const newNonTemporaryConvo = await Conversation.create({ conversationId: uuidv4(), user: 'user123', - title: 'Non-expired', + title: 'New Non-temporary Conversation', endpoint: EModelEndpoint.openAI, - expiredAt: null, + isTemporary: false, + expiredAt: new Date(Date.now() + 24 * 60 * 60 * 1000), + updatedAt: new Date(), }); - const expiredConvo = await Conversation.create({ + const oldNonTemporaryConvo = await Conversation.create({ conversationId: uuidv4(), user: 'user123', - title: 'Expired', + title: 'Old Non-Temporary Conversation', + endpoint: EModelEndpoint.openAI, + isTemporary: undefined, + expiredAt: null, + updatedAt: new Date(), + }); + + const legacyNullNonTemporaryConvoId = uuidv4(); + await Conversation.collection.insertOne({ + conversationId: legacyNullNonTemporaryConvoId, + user: 'user123', + title: 'Legacy Null Non-Temporary Conversation', + endpoint: EModelEndpoint.openAI, + isTemporary: null, + expiredAt: null, + updatedAt: new Date(), + createdAt: new Date(), + }); + + const legacyTemporaryConvoId = uuidv4(); + await Conversation.collection.insertOne({ + conversationId: legacyTemporaryConvoId, + user: 'user123', + title: 'Legacy Temporary Conversation', endpoint: EModelEndpoint.openAI, expiredAt: new Date(Date.now() + 24 * 60 * 60 * 1000), + updatedAt: new Date(), + createdAt: new Date(), + }); + + const expiredRetainedConvo = await Conversation.create({ + conversationId: uuidv4(), + user: 'user123', + title: 'Expired Retained Conversation', + endpoint: EModelEndpoint.openAI, + isTemporary: false, + expiredAt: new Date(Date.now() - 60 * 60 * 1000), + updatedAt: new Date(), + }); + + const tempConvo = await Conversation.create({ + conversationId: uuidv4(), + user: 'user123', + title: 'Temporary conversation', + endpoint: EModelEndpoint.openAI, + isTemporary: true, + expiredAt: new Date(Date.now() + 24 * 60 * 60 * 1000), + updatedAt: new Date(), }); const convoIds = [ - { conversationId: nonExpiredConvo.conversationId }, - { conversationId: expiredConvo.conversationId }, + { conversationId: newNonTemporaryConvo.conversationId }, + { conversationId: oldNonTemporaryConvo.conversationId }, + { conversationId: legacyNullNonTemporaryConvoId }, + { conversationId: legacyTemporaryConvoId }, + { conversationId: expiredRetainedConvo.conversationId }, + { conversationId: tempConvo.conversationId }, ]; const result = await getConvosQueried('user123', convoIds); - // Should only return the non-expired conversation - expect(result?.conversations).toHaveLength(1); - expect(result?.conversations[0].conversationId).toBe(nonExpiredConvo.conversationId); - expect(result?.convoMap[nonExpiredConvo.conversationId]).toBeDefined(); - expect(result?.convoMap[expiredConvo.conversationId]).toBeUndefined(); + // Should only return the non-temporary conversations + expect(result?.conversations).toHaveLength(3); + + const resultIds = result?.conversations.map((c) => c.conversationId); + expect(resultIds).toContain(newNonTemporaryConvo.conversationId); + expect(resultIds).toContain(oldNonTemporaryConvo.conversationId); + expect(resultIds).toContain(legacyNullNonTemporaryConvoId); + expect(result?.convoMap[newNonTemporaryConvo.conversationId]).toBeDefined(); + expect(result?.convoMap[oldNonTemporaryConvo.conversationId]).toBeDefined(); + expect(result?.convoMap[legacyNullNonTemporaryConvoId]).toBeDefined(); + expect(result?.convoMap[legacyTemporaryConvoId]).toBeUndefined(); + expect(result?.convoMap[expiredRetainedConvo.conversationId]).toBeUndefined(); + expect(result?.convoMap[tempConvo.conversationId]).toBeUndefined(); }); }); @@ -524,6 +705,24 @@ describe('Conversation Operations', () => { }); }); + describe('getConvoRetention', () => { + it('should retrieve only retention fields for a user conversation', async () => { + await Conversation.create({ + conversationId: mockConversationData.conversationId, + user: 'user123', + title: 'Test Conversation', + endpoint: EModelEndpoint.openAI, + expiredAt: new Date('2030-01-01T00:00:00.000Z'), + }); + + const result = await getConvoRetention('user123', mockConversationData.conversationId); + + expect(result?.expiredAt).toEqual(new Date('2030-01-01T00:00:00.000Z')); + expect(result).not.toHaveProperty('title'); + expect(result).not.toHaveProperty('messages'); + }); + }); + describe('getConvoTitle', () => { it('should return the conversation title', async () => { await Conversation.create({ diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index 82c22f3947..c46487e519 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -1,5 +1,7 @@ import type { FilterQuery, Model, SortOrder } from 'mongoose'; +import { RetentionMode } from 'librechat-data-provider'; import { createTempChatExpirationDate } from '~/utils/tempChatRetention'; +import { buildRetentionVisibilityFilter, createFallbackRetentionDate } from '~/utils/retention'; import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite'; import logger from '~/config/winston'; import type { AppConfig, IConversation } from '~/types'; @@ -47,6 +49,10 @@ export interface ConversationMethods { convoMap: Record; }>; getConvo(user: string, conversationId: string): Promise; + getConvoRetention( + user: string, + conversationId: string, + ): Promise | null>; getConvoTitle(user: string, conversationId: string): Promise; deleteConvos( user: string, @@ -65,6 +71,10 @@ export function createConversationMethods( return messageMethods; } + function getVisibleConversationRetentionFilter(): FilterQuery { + return buildRetentionVisibilityFilter(); + } + /** * Searches for a conversation by conversationId and returns a lean document with only conversationId and user. */ @@ -94,6 +104,24 @@ export function createConversationMethods( } } + /** + * Retrieves only the retention deadline for a conversation. + */ + async function getConvoRetention( + user: string, + conversationId: string, + ): Promise | null> { + try { + const Conversation = mongoose.models.Conversation as Model; + return await Conversation.findOne({ user, conversationId }, 'expiredAt').lean< + Pick + >(); + } catch (error) { + logger.error('[getConvoRetention] Error getting conversation retention fields', error); + throw new Error('Error getting conversation retention fields'); + } + } + /** * Deletes conversations and messages with null or empty IDs. */ @@ -185,15 +213,28 @@ export function createConversationMethods( update.conversationId = newConversationId; } - if (isTemporary) { + if (interfaceConfig?.retentionMode === RetentionMode.ALL) { + if (typeof isTemporary === 'boolean') { + update.isTemporary = isTemporary; + } try { update.expiredAt = createTempChatExpirationDate(interfaceConfig); } catch (err) { logger.error('Error creating temporary chat expiration date:', err); logger.info(`---\`saveConvo\` context: ${metadata?.context}`); - update.expiredAt = null; + update.expiredAt = createFallbackRetentionDate(); } - } else { + } else if (isTemporary === true) { + update.isTemporary = true; + try { + update.expiredAt = createTempChatExpirationDate(interfaceConfig); + } catch (err) { + logger.error('Error creating temporary chat expiration date:', err); + logger.info(`---\`saveConvo\` context: ${metadata?.context}`); + update.expiredAt = createFallbackRetentionDate(); + } + } else if (isTemporary === false) { + update.isTemporary = false; update.expiredAt = null; } @@ -229,6 +270,19 @@ export function createConversationMethods( return null; } + if ( + interfaceConfig?.retentionMode === RetentionMode.ALL && + typeof isTemporary !== 'boolean' && + (conversation.isTemporary == null || + (conversation.isTemporary === false && conversation.$isDefault('isTemporary'))) + ) { + await Conversation.updateOne( + { _id: conversation._id, isTemporary: { $ne: false } }, + { $set: { isTemporary: false } }, + ); + conversation.isTemporary = false; + } + return conversation.toObject(); } catch (error) { logger.error('[saveConvo] Error saving conversation', error); @@ -302,9 +356,7 @@ export function createConversationMethods( filters.push({ tags: { $in: tags } } as FilterQuery); } - filters.push({ - $or: [{ expiredAt: null }, { expiredAt: { $exists: false } }], - } as FilterQuery); + filters.push(getVisibleConversationRetentionFilter()); if (search) { try { @@ -429,7 +481,7 @@ export function createConversationMethods( const results = await Conversation.find({ user, conversationId: { $in: conversationIds }, - $or: [{ expiredAt: { $exists: false } }, { expiredAt: null }], + ...getVisibleConversationRetentionFilter(), }).lean(); results.sort( @@ -516,6 +568,7 @@ export function createConversationMethods( getConvosByCursor, getConvosQueried, getConvo, + getConvoRetention, getConvoTitle, deleteConvos, }; diff --git a/packages/data-schemas/src/methods/file.spec.ts b/packages/data-schemas/src/methods/file.spec.ts index 9bb3895d27..08dd02326c 100644 --- a/packages/data-schemas/src/methods/file.spec.ts +++ b/packages/data-schemas/src/methods/file.spec.ts @@ -233,6 +233,69 @@ describe('File Methods', () => { }); }); + describe('getExpiredFiles', () => { + it('returns only files whose expiredAt date has passed', async () => { + const userId = new mongoose.Types.ObjectId(); + const now = new Date('2030-01-01T00:00:00.000Z'); + const expiredFileId = uuidv4(); + const futureFileId = uuidv4(); + const permanentFileId = uuidv4(); + const missingExpiryFileId = uuidv4(); + + await fileMethods.createFile( + { + file_id: expiredFileId, + user: userId, + filename: 'expired.txt', + filepath: '/uploads/expired.txt', + type: 'text/plain', + bytes: 100, + expiredAt: new Date('2029-12-31T23:59:59.000Z'), + }, + true, + ); + await fileMethods.createFile( + { + file_id: futureFileId, + user: userId, + filename: 'future.txt', + filepath: '/uploads/future.txt', + type: 'text/plain', + bytes: 100, + expiredAt: new Date('2030-01-01T00:00:01.000Z'), + }, + true, + ); + await fileMethods.createFile( + { + file_id: permanentFileId, + user: userId, + filename: 'permanent.txt', + filepath: '/uploads/permanent.txt', + type: 'text/plain', + bytes: 100, + expiredAt: null, + }, + true, + ); + await fileMethods.createFile( + { + file_id: missingExpiryFileId, + user: userId, + filename: 'missing-expiry.txt', + filepath: '/uploads/missing-expiry.txt', + type: 'text/plain', + bytes: 100, + }, + true, + ); + + const files = await fileMethods.getExpiredFiles(100, now); + + expect(files.map((file) => file.file_id)).toEqual([expiredFileId]); + }); + }); + describe('getToolFilesByIds', () => { it('should retrieve files for file_search tool (embedded files)', async () => { const userId = new mongoose.Types.ObjectId(); diff --git a/packages/data-schemas/src/methods/file.ts b/packages/data-schemas/src/methods/file.ts index e1b49b2c67..a76cbc1315 100644 --- a/packages/data-schemas/src/methods/file.ts +++ b/packages/data-schemas/src/methods/file.ts @@ -47,6 +47,14 @@ export function createFileMethods(mongoose: typeof import('mongoose')) { return await query.sort(sortOptions).lean(); } + async function getExpiredFiles(limit = 100, now = new Date()): Promise { + const File = mongoose.models.File as Model; + return await File.find({ expiredAt: { $ne: null, $lte: now } }) + .sort({ expiredAt: 1 }) + .limit(limit) + .lean(); + } + /** * Retrieves tool files (files that are embedded or have a fileIdentifier) from an array of file IDs. * Note: execute_code files are handled separately by getCodeGeneratedFiles. @@ -457,6 +465,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')) { return { findFileById, getFiles, + getExpiredFiles, getToolFilesByIds, getCodeGeneratedFiles, getUserCodeFiles, diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.ts index 14fe28596d..98b02dd19d 100644 --- a/packages/data-schemas/src/methods/message.spec.ts +++ b/packages/data-schemas/src/methods/message.spec.ts @@ -1,5 +1,6 @@ import mongoose from 'mongoose'; import { v4 as uuidv4 } from 'uuid'; +import { RetentionMode } from 'librechat-data-provider'; import { MongoMemoryServer } from 'mongodb-memory-server'; import type { IMessage } from '..'; import { createMessageMethods } from './message'; @@ -54,7 +55,7 @@ describe('Message Operations', () => { let mockCtx: { userId: string; isTemporary?: boolean; - interfaceConfig?: { temporaryChatRetention?: number }; + interfaceConfig?: { temporaryChatRetention?: number; retentionMode?: RetentionMode }; }; let mockMessageData: Partial = { messageId: 'msg123', @@ -403,7 +404,7 @@ describe('Message Operations', () => { const result = await saveMessage(mockCtx, mockMessageData); expect(result?.messageId).toBe('msg123'); - expect(result?.expiredAt).toBeNull(); + expect(result?.expiredAt).toBeUndefined(); }); it('should use custom retention period from config', async () => { @@ -475,6 +476,61 @@ describe('Message Operations', () => { ); }); + it('should set expiredAt for non-temporary message when retentionMode is ALL', async () => { + mockCtx.isTemporary = false; + mockCtx.interfaceConfig = { + temporaryChatRetention: 24, + retentionMode: RetentionMode.ALL, + }; + const result = await saveMessage(mockCtx, mockMessageData); + expect(result?.expiredAt).toBeDefined(); + expect(result?.expiredAt).toBeInstanceOf(Date); + }); + + it('should mark retained message non-temporary when retentionMode is ALL and isTemporary is omitted', async () => { + mockCtx.isTemporary = undefined; + mockCtx.interfaceConfig = { + temporaryChatRetention: 24, + retentionMode: RetentionMode.ALL, + }; + + const result = await saveMessage(mockCtx, mockMessageData); + + expect(result?.expiredAt).toBeDefined(); + expect(result?.isTemporary).toBe(false); + }); + + it('should preserve existing temporary flag when retentionMode is ALL and isTemporary is omitted', async () => { + mockCtx.isTemporary = true; + mockCtx.interfaceConfig = { + temporaryChatRetention: 24, + retentionMode: RetentionMode.ALL, + }; + + const firstSave = await saveMessage(mockCtx, mockMessageData); + + mockCtx.isTemporary = undefined; + const secondSave = await saveMessage(mockCtx, { + ...mockMessageData, + text: 'Updated text', + }); + + expect(firstSave?.isTemporary).toBe(true); + expect(secondSave?.text).toBe('Updated text'); + expect(secondSave?.isTemporary).toBe(true); + expect(secondSave?.expiredAt).toBeDefined(); + }); + + it('should not set expiredAt when retentionMode is temporary and not isTemporary', async () => { + mockCtx.isTemporary = false; + mockCtx.interfaceConfig = { + temporaryChatRetention: 24, + retentionMode: RetentionMode.TEMPORARY, + }; + const result = await saveMessage(mockCtx, mockMessageData); + expect(result?.expiredAt).toBeNull(); + }); + it('should handle missing config gracefully', async () => { // Simulate missing config - should use default retention period delete mockCtx.interfaceConfig; @@ -571,6 +627,22 @@ describe('Message Operations', () => { ); }); + it('should preserve temporary retention when saving without isTemporary', async () => { + mockCtx.interfaceConfig = { temporaryChatRetention: 24 }; + + mockCtx.isTemporary = true; + const firstSave = await saveMessage(mockCtx, mockMessageData); + const originalExpiredAt = firstSave?.expiredAt; + + mockCtx.isTemporary = undefined; + const updatedData = { ...mockMessageData, text: 'Updated text' }; + const secondSave = await saveMessage(mockCtx, updatedData); + + expect(secondSave?.text).toBe('Updated text'); + expect(secondSave?.isTemporary).toBe(true); + expect(secondSave?.expiredAt).toEqual(originalExpiredAt); + }); + it('should handle bulk operations with temporary messages', async () => { // This test verifies bulkSaveMessages doesn't interfere with expiredAt const messages = [ diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts index b4b5038ca8..acaff7c778 100644 --- a/packages/data-schemas/src/methods/message.ts +++ b/packages/data-schemas/src/methods/message.ts @@ -1,6 +1,8 @@ import type { DeleteResult, FilterQuery, Model } from 'mongoose'; +import { RetentionMode } from 'librechat-data-provider'; import logger from '~/config/winston'; import { createTempChatExpirationDate } from '~/utils/tempChatRetention'; +import { createFallbackRetentionDate } from '~/utils/retention'; import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite'; import type { AppConfig, IMessage } from '~/types'; @@ -91,15 +93,28 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa messageId: params.newMessageId || params.messageId, }; - if (isTemporary) { + if (interfaceConfig?.retentionMode === RetentionMode.ALL) { + if (typeof isTemporary === 'boolean') { + update.isTemporary = isTemporary; + } try { update.expiredAt = createTempChatExpirationDate(interfaceConfig); } catch (err) { logger.error('Error creating temporary chat expiration date:', err); logger.info(`---\`saveMessage\` context: ${metadata?.context}`); - update.expiredAt = null; + update.expiredAt = createFallbackRetentionDate(); } - } else { + } else if (isTemporary === true) { + update.isTemporary = true; + try { + update.expiredAt = createTempChatExpirationDate(interfaceConfig); + } catch (err) { + logger.error('Error creating temporary chat expiration date:', err); + logger.info(`---\`saveMessage\` context: ${metadata?.context}`); + update.expiredAt = createFallbackRetentionDate(); + } + } else if (isTemporary === false) { + update.isTemporary = false; update.expiredAt = null; } @@ -116,6 +131,19 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa { upsert: true, new: true }, ); + if ( + interfaceConfig?.retentionMode === RetentionMode.ALL && + typeof isTemporary !== 'boolean' && + (message.isTemporary == null || + (message.isTemporary === false && message.$isDefault('isTemporary'))) + ) { + await Message.updateOne( + { _id: message._id, isTemporary: { $ne: false } }, + { $set: { isTemporary: false } }, + ); + message.isTemporary = false; + } + return message.toObject(); } catch (err: unknown) { logger.error('Error saving message:', err); diff --git a/packages/data-schemas/src/methods/share.test.ts b/packages/data-schemas/src/methods/share.test.ts index 4f045d5b97..60b789c3dd 100644 --- a/packages/data-schemas/src/methods/share.test.ts +++ b/packages/data-schemas/src/methods/share.test.ts @@ -28,6 +28,7 @@ describe('Share Methods', () => { shareId: { type: String, index: true }, targetMessageId: { type: String, required: false, index: true }, isPublic: { type: Boolean, default: true }, + expiredAt: { type: Date }, }, { timestamps: true }, ); @@ -154,6 +155,41 @@ describe('Share Methods', () => { ); }); + test('should ignore expired public shares when checking for duplicates', async () => { + const userId = new mongoose.Types.ObjectId().toString(); + const conversationId = `conv_${nanoid()}`; + const expiredShareId = `share_${nanoid()}`; + + await Conversation.create({ + conversationId, + title: 'Test Conversation', + user: userId, + }); + + const message = await Message.create({ + messageId: `msg_${nanoid()}`, + conversationId, + user: userId, + text: 'Test message', + isCreatedByUser: true, + }); + + await SharedLink.create({ + shareId: expiredShareId, + conversationId, + user: userId, + messages: [message._id], + isPublic: true, + expiredAt: new Date(Date.now() - 60 * 60 * 1000), + }); + + const result = await shareMethods.createSharedLink(userId, conversationId); + + expect(result.shareId).toBeDefined(); + expect(result.shareId).not.toBe(expiredShareId); + expect(result.conversationId).toBe(conversationId); + }); + test('should throw error with missing parameters', async () => { await expect(shareMethods.createSharedLink('', 'conv123')).rejects.toThrow( 'Missing required parameters', @@ -329,6 +365,21 @@ describe('Share Methods', () => { expect(result).toBeNull(); }); + test('should return null for expired share', async () => { + const shareId = `share_${nanoid()}`; + + await SharedLink.create({ + shareId, + conversationId: 'conv123', + user: 'user123', + isPublic: true, + expiredAt: new Date(Date.now() - 60 * 60 * 1000), + }); + + const result = await shareMethods.getSharedMessages(shareId); + expect(result).toBeNull(); + }); + test('should handle messages with attachments', async () => { const userId = new mongoose.Types.ObjectId().toString(); const conversationId = `conv_${nanoid()}`; @@ -428,6 +479,34 @@ describe('Share Methods', () => { expect(privateResults.links[0].title).toBe('Private Share'); }); + test('should exclude expired shares', async () => { + const userId = new mongoose.Types.ObjectId().toString(); + + await SharedLink.create([ + { + shareId: 'active_share', + conversationId: 'conv1', + user: userId, + title: 'Active Share', + isPublic: true, + expiredAt: new Date(Date.now() + 60 * 60 * 1000), + }, + { + shareId: 'expired_share', + conversationId: 'conv2', + user: userId, + title: 'Expired Share', + isPublic: true, + expiredAt: new Date(Date.now() - 60 * 60 * 1000), + }, + ]); + + const result = await shareMethods.getSharedLinks(userId, undefined, 10, true); + + expect(result.links).toHaveLength(1); + expect(result.links[0].shareId).toBe('active_share'); + }); + test('should handle search with mocked meiliSearch and user filter', async () => { const userId = new mongoose.Types.ObjectId().toString(); @@ -659,6 +738,62 @@ describe('Share Methods', () => { expect(updatedShare?.messages).toHaveLength(2); }); + test('should preserve stale expiration when updating without an expiration decision', async () => { + const userId = new mongoose.Types.ObjectId().toString(); + const conversationId = `conv_${nanoid()}`; + const shareId = `share_${nanoid()}`; + const expiresAt = new Date('2030-01-01T00:00:00.000Z'); + + await SharedLink.create({ + shareId, + conversationId, + user: userId, + messages: [], + isPublic: true, + expiredAt: expiresAt, + }); + await Message.create({ + messageId: `msg_${nanoid()}`, + conversationId, + user: userId, + text: 'Retained no longer applies', + isCreatedByUser: true, + }); + + const result = await shareMethods.updateSharedLink(userId, shareId); + const updatedShare = await SharedLink.findOne({ shareId: result.shareId }).lean(); + + expect(updatedShare?.expiredAt?.toISOString()).toBe(expiresAt.toISOString()); + }); + + test('should clear stale expiration when updating with null expiration', async () => { + const userId = new mongoose.Types.ObjectId().toString(); + const conversationId = `conv_${nanoid()}`; + const shareId = `share_${nanoid()}`; + const expiresAt = new Date('2030-01-01T00:00:00.000Z'); + + await SharedLink.create({ + shareId, + conversationId, + user: userId, + messages: [], + isPublic: true, + expiredAt: expiresAt, + }); + await Message.create({ + messageId: `msg_${nanoid()}`, + conversationId, + user: userId, + text: 'Retained no longer applies', + isCreatedByUser: true, + }); + + const result = await shareMethods.updateSharedLink(userId, shareId, undefined, null); + const updatedShare = await SharedLink.findOne({ shareId: result.shareId }).lean(); + + expect(updatedShare?.expiredAt).toBeUndefined(); + }); + test('should throw error if share not found', async () => { await expect(shareMethods.updateSharedLink('user123', 'non_existent')).rejects.toThrow( 'Share not found', @@ -929,6 +1064,24 @@ describe('Share Methods', () => { expect(result.shareId).toBeNull(); }); + test('should return null shareId for expired shares', async () => { + const userId = new mongoose.Types.ObjectId().toString(); + const conversationId = `conv_${nanoid()}`; + + await SharedLink.create({ + shareId: 'share123', + conversationId, + user: userId, + isPublic: true, + expiredAt: new Date(Date.now() - 60 * 60 * 1000), + }); + + const result = await shareMethods.getSharedLink(userId, conversationId); + + expect(result.success).toBe(false); + expect(result.shareId).toBeNull(); + }); + test('should not return share from different user', async () => { const userId1 = new mongoose.Types.ObjectId().toString(); const userId2 = new mongoose.Types.ObjectId().toString(); diff --git a/packages/data-schemas/src/methods/share.ts b/packages/data-schemas/src/methods/share.ts index c6b6400e64..0551b349c4 100644 --- a/packages/data-schemas/src/methods/share.ts +++ b/packages/data-schemas/src/methods/share.ts @@ -4,6 +4,7 @@ import type { FilterQuery, Model } from 'mongoose'; import type { SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili'; import type * as t from '~/types'; import logger from '~/config/winston'; +import { activeExpirationFilter } from '~/utils/retention'; class ShareServiceError extends Error { code: string; @@ -161,7 +162,11 @@ export function createShareMethods(mongoose: typeof import('mongoose')) { async function getSharedMessages(shareId: string): Promise { try { const SharedLink = mongoose.models.SharedLink as Model; - const share = (await SharedLink.findOne({ shareId, isPublic: true }) + const share = (await SharedLink.findOne({ + shareId, + isPublic: true, + ...activeExpirationFilter(), + }) .populate({ path: 'messages', select: '-_id -__v -user', @@ -215,7 +220,11 @@ export function createShareMethods(mongoose: typeof import('mongoose')) { try { const SharedLink = mongoose.models.SharedLink as Model; const Conversation = mongoose.models.Conversation as SchemaWithMeiliMethods; - const query: FilterQuery = { user, isPublic }; + const query: FilterQuery = { + user, + isPublic, + ...activeExpirationFilter(), + }; if (pageParam) { if (sortDirection === 'desc') { @@ -345,6 +354,7 @@ export function createShareMethods(mongoose: typeof import('mongoose')) { user: string, conversationId: string, targetMessageId?: string, + expiredAt?: Date, ): Promise { if (!user || !conversationId) { throw new ShareServiceError('Missing required parameters', 'INVALID_PARAMS'); @@ -359,6 +369,7 @@ export function createShareMethods(mongoose: typeof import('mongoose')) { conversationId, user, isPublic: true, + ...activeExpirationFilter(), ...(targetMessageId && { targetMessageId }), }) .select('-_id -__v -user') @@ -408,6 +419,7 @@ export function createShareMethods(mongoose: typeof import('mongoose')) { title, user, ...(targetMessageId && { targetMessageId }), + ...(expiredAt && { expiredAt }), }); return { shareId, conversationId, targetMessageId }; @@ -438,7 +450,12 @@ export function createShareMethods(mongoose: typeof import('mongoose')) { try { const SharedLink = mongoose.models.SharedLink as Model; - const share = (await SharedLink.findOne({ conversationId, user, isPublic: true }) + const share = (await SharedLink.findOne({ + conversationId, + user, + isPublic: true, + ...activeExpirationFilter(), + }) .select('shareId targetMessageId -_id') .sort({ updatedAt: -1 }) .lean()) as { shareId?: string; targetMessageId?: string } | null; @@ -469,6 +486,7 @@ export function createShareMethods(mongoose: typeof import('mongoose')) { user: string, shareId: string, targetMessageId?: string, + expiredAt?: Date | null, ): Promise { if (!user || !shareId) { throw new ShareServiceError('Missing required parameters', 'INVALID_PARAMS'); @@ -490,12 +508,17 @@ export function createShareMethods(mongoose: typeof import('mongoose')) { .lean(); const newShareId = nanoid(); + const hasNewExpiration = expiredAt instanceof Date; const resolvedTargetMessageId = targetMessageId ?? share.targetMessageId; const update = { - messages: updatedMessages, - user, - shareId: newShareId, - ...(resolvedTargetMessageId && { targetMessageId: resolvedTargetMessageId }), + $set: { + messages: updatedMessages, + user, + shareId: newShareId, + ...(resolvedTargetMessageId && { targetMessageId: resolvedTargetMessageId }), + ...(hasNewExpiration && { expiredAt }), + }, + ...(expiredAt === null ? { $unset: { expiredAt: 1 } } : {}), }; const updatedShare = (await SharedLink.findOneAndUpdate({ shareId, user }, update, { diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts index 1d341a7939..b4084fa936 100644 --- a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts +++ b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts @@ -3,7 +3,55 @@ import mongoose from 'mongoose'; import { EModelEndpoint } from 'librechat-data-provider'; import { createConversationModel } from '~/models/convo'; import { createMessageModel } from '~/models/message'; -import { SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili'; +import mongoMeili, { type SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili'; + +interface DynamicMeiliDocument extends mongoose.Document { + docId: string; + user: string; + title: string; + isTemporary?: boolean; + expiredAt?: Date | null; + _meiliIndex?: boolean; +} + +type DynamicMeiliModel = mongoose.Model & SchemaWithMeiliMethods; + +const createDynamicMeiliModel = (modelName: string): DynamicMeiliModel => { + const schema = new mongoose.Schema({ + docId: { + type: String, + required: true, + meiliIndex: true, + }, + title: { + type: String, + meiliIndex: true, + }, + user: { + type: String, + meiliIndex: true, + }, + isTemporary: { + type: Boolean, + default: false, + }, + expiredAt: { + type: Date, + }, + }); + + schema.plugin(mongoMeili, { + mongoose, + host: 'foo', + apiKey: 'bar', + indexName: modelName.toLowerCase(), + primaryKey: 'docId', + }); + + return mongoose.model(modelName, schema) as unknown as DynamicMeiliModel; +}; + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const mockAddDocuments = jest.fn(); const mockAddDocumentsInBatches = jest.fn(); @@ -90,12 +138,37 @@ describe('Meilisearch Mongoose plugin', () => { expect(mockAddDocuments).toHaveBeenCalled(); }); - test('saving TTL conversation does NOT index w/ meilisearch', async () => { + test('saving retained non-temporary conversation indexes w/ meilisearch', async () => { await createConversationModel(mongoose).create({ conversationId: new mongoose.Types.ObjectId(), user: new mongoose.Types.ObjectId(), title: 'Test Conversation', endpoint: EModelEndpoint.openAI, + isTemporary: false, + expiredAt: new Date(Date.now() + 60 * 60 * 1000), + }); + expect(mockAddDocuments).toHaveBeenCalled(); + }); + + test('saving expired retained non-temporary conversation does NOT index w/ meilisearch', async () => { + await createConversationModel(mongoose).create({ + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + title: 'Test Conversation', + endpoint: EModelEndpoint.openAI, + isTemporary: false, + expiredAt: new Date(Date.now() - 60 * 60 * 1000), + }); + expect(mockAddDocuments).not.toHaveBeenCalled(); + }); + + test('saving temporary conversation does NOT index w/ meilisearch', async () => { + await createConversationModel(mongoose).create({ + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + title: 'Test Conversation', + endpoint: EModelEndpoint.openAI, + isTemporary: true, expiredAt: new Date(), }); expect(mockAddDocuments).not.toHaveBeenCalled(); @@ -125,12 +198,37 @@ describe('Meilisearch Mongoose plugin', () => { expect(mockAddDocuments).toHaveBeenCalled(); }); - test('saving TTL messages does NOT index w/ meilisearch', async () => { + test('saving retained non-temporary messages indexes w/ meilisearch', async () => { await createMessageModel(mongoose).create({ messageId: new mongoose.Types.ObjectId(), conversationId: new mongoose.Types.ObjectId(), user: new mongoose.Types.ObjectId(), isCreatedByUser: true, + isTemporary: false, + expiredAt: new Date(Date.now() + 60 * 60 * 1000), + }); + expect(mockAddDocuments).toHaveBeenCalled(); + }); + + test('saving expired retained non-temporary message does NOT index w/ meilisearch', async () => { + await createMessageModel(mongoose).create({ + messageId: new mongoose.Types.ObjectId(), + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + isCreatedByUser: true, + isTemporary: false, + expiredAt: new Date(Date.now() - 60 * 60 * 1000), + }); + expect(mockAddDocuments).not.toHaveBeenCalled(); + }); + + test('saving temporary messages does NOT index w/ meilisearch', async () => { + await createMessageModel(mongoose).create({ + messageId: new mongoose.Types.ObjectId(), + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + isCreatedByUser: true, + isTemporary: true, expiredAt: new Date(), }); expect(mockAddDocuments).not.toHaveBeenCalled(); @@ -224,6 +322,7 @@ describe('Meilisearch Mongoose plugin', () => { user: new mongoose.Types.ObjectId(), title: 'Test Conversation', endpoint: EModelEndpoint.openAI, + isTemporary: true, expiredAt: new Date(), }); @@ -232,6 +331,173 @@ describe('Meilisearch Mongoose plugin', () => { expect(mockAddDocuments).not.toHaveBeenCalled(); }); + test('sync w/ meili excludes legacy temporary conversations without isTemporary', async () => { + const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods; + await conversationModel.deleteMany({}); + mockAddDocumentsInBatches.mockClear(); + const conversationId = new mongoose.Types.ObjectId().toString(); + + await conversationModel.collection.insertOne({ + conversationId, + user: new mongoose.Types.ObjectId().toString(), + title: 'Legacy Temporary Conversation', + endpoint: EModelEndpoint.openAI, + expiredAt: new Date(Date.now() + 60 * 60 * 1000), + _meiliIndex: false, + createdAt: new Date(), + updatedAt: new Date(), + }); + + await conversationModel.syncWithMeili(); + const storedDoc = await conversationModel.collection.findOne({ conversationId }); + + expect(mockAddDocumentsInBatches).not.toHaveBeenCalled(); + expect(storedDoc?._meiliIndex).toBe(false); + }); + + test('saving hydrated legacy temporary conversations without isTemporary does NOT index', async () => { + const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods; + await conversationModel.deleteMany({}); + mockAddDocuments.mockClear(); + mockUpdateDocuments.mockClear(); + const conversationId = new mongoose.Types.ObjectId().toString(); + + await conversationModel.collection.insertOne({ + conversationId, + user: new mongoose.Types.ObjectId().toString(), + title: 'Legacy Temporary Conversation', + endpoint: EModelEndpoint.openAI, + expiredAt: new Date(Date.now() + 60 * 60 * 1000), + _meiliIndex: false, + createdAt: new Date(), + updatedAt: new Date(), + }); + + const legacyConvo = await conversationModel.findOne({ conversationId }); + expect(legacyConvo).toBeTruthy(); + + legacyConvo!.title = 'Updated Legacy Temporary Conversation'; + await legacyConvo!.save(); + const storedDoc = await conversationModel.collection.findOne({ conversationId }); + + expect(mockAddDocuments).not.toHaveBeenCalled(); + expect(mockUpdateDocuments).not.toHaveBeenCalled(); + expect(storedDoc?._meiliIndex).toBe(false); + }); + + test('findOneAndUpdate on legacy temporary conversations without isTemporary does NOT index', async () => { + const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods; + await conversationModel.deleteMany({}); + mockAddDocuments.mockClear(); + mockUpdateDocuments.mockClear(); + const conversationId = new mongoose.Types.ObjectId().toString(); + + await conversationModel.collection.insertOne({ + conversationId, + user: new mongoose.Types.ObjectId().toString(), + title: 'Legacy Temporary Conversation', + endpoint: EModelEndpoint.openAI, + expiredAt: new Date(Date.now() + 60 * 60 * 1000), + _meiliIndex: false, + createdAt: new Date(), + updatedAt: new Date(), + }); + + await conversationModel.findOneAndUpdate( + { conversationId }, + { $set: { title: 'Updated via findOneAndUpdate' } }, + { new: true }, + ); + const storedDoc = await conversationModel.collection.findOne({ conversationId }); + + expect(mockAddDocuments).not.toHaveBeenCalled(); + expect(mockUpdateDocuments).not.toHaveBeenCalled(); + expect(storedDoc?._meiliIndex).toBe(false); + }); + + test('sync w/ meili excludes legacy temporary messages without isTemporary', async () => { + const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods; + await messageModel.deleteMany({}); + mockAddDocumentsInBatches.mockClear(); + const messageId = new mongoose.Types.ObjectId().toString(); + + await messageModel.collection.insertOne({ + messageId, + conversationId: new mongoose.Types.ObjectId().toString(), + user: new mongoose.Types.ObjectId().toString(), + isCreatedByUser: true, + text: 'Legacy temporary message', + expiredAt: new Date(Date.now() + 60 * 60 * 1000), + _meiliIndex: false, + createdAt: new Date(), + updatedAt: new Date(), + }); + + await messageModel.syncWithMeili(); + const storedDoc = await messageModel.collection.findOne({ messageId }); + + expect(mockAddDocumentsInBatches).not.toHaveBeenCalled(); + expect(storedDoc?._meiliIndex).toBe(false); + }); + + test('sync w/ meili treats null isTemporary with no expiration like missing legacy fields', async () => { + const modelName = `DynamicMeiliNullTemporary${new mongoose.Types.ObjectId().toString()}`; + const dynamicModel = createDynamicMeiliModel(modelName); + mockAddDocumentsInBatches.mockClear(); + + try { + await dynamicModel.collection.insertOne({ + docId: 'legacy-null-temporary', + user: 'user-123', + title: 'Legacy Null Temporary', + isTemporary: null as unknown as boolean, + expiredAt: null, + _meiliIndex: false, + }); + + const progress = await dynamicModel.getSyncProgress(); + await dynamicModel.syncWithMeili(); + const storedDoc = await dynamicModel.collection.findOne({ docId: 'legacy-null-temporary' }); + + expect(progress.totalDocuments).toBe(1); + expect(mockAddDocumentsInBatches).toHaveBeenCalled(); + expect(storedDoc?._meiliIndex).toBe(true); + } finally { + await mongoose.connection.dropCollection(modelName.toLowerCase()).catch(() => undefined); + delete mongoose.models[modelName]; + } + }); + + test('sync queries use a fresh expiration cutoff after plugin initialization', async () => { + const modelName = `DynamicMeiliCutoff${new mongoose.Types.ObjectId().toString()}`; + const dynamicModel = createDynamicMeiliModel(modelName); + mockAddDocumentsInBatches.mockClear(); + + try { + await dynamicModel.collection.insertOne({ + docId: 'expires-soon', + user: 'user-123', + title: 'Expires Soon', + isTemporary: false, + expiredAt: new Date(Date.now() + 25), + _meiliIndex: false, + }); + + await wait(100); + + const progress = await dynamicModel.getSyncProgress(); + await dynamicModel.syncWithMeili(); + const storedDoc = await dynamicModel.collection.findOne({ docId: 'expires-soon' }); + + expect(progress.totalDocuments).toBe(0); + expect(mockAddDocumentsInBatches).not.toHaveBeenCalled(); + expect(storedDoc?._meiliIndex).toBe(false); + } finally { + await dynamicModel.deleteMany({}); + mongoose.deleteModel(modelName); + } + }); + describe('estimatedDocumentCount usage in syncWithMeili', () => { test('syncWithMeili completes successfully with estimatedDocumentCount', async () => { // Clear any previous documents @@ -335,6 +601,7 @@ describe('Meilisearch Mongoose plugin', () => { conversationId: new mongoose.Types.ObjectId(), user: new mongoose.Types.ObjectId(), isCreatedByUser: true, + isTemporary: true, expiredAt: new Date(), }); @@ -343,6 +610,7 @@ describe('Meilisearch Mongoose plugin', () => { conversationId: new mongoose.Types.ObjectId(), user: new mongoose.Types.ObjectId(), isCreatedByUser: false, + isTemporary: true, expiredAt: new Date(), }); diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.ts b/packages/data-schemas/src/models/plugins/mongoMeili.ts index 125e7bab71..0594c272b1 100644 --- a/packages/data-schemas/src/models/plugins/mongoMeili.ts +++ b/packages/data-schemas/src/models/plugins/mongoMeili.ts @@ -13,6 +13,7 @@ import type { } from 'mongoose'; import type { IConversation, IMessage } from '~/types'; import logger from '~/config/meiliLogger'; +import { buildRetentionVisibilityFilter, legacyPermanentExpirationFilter } from '~/utils/retention'; interface MongoMeiliOptions { host: string; @@ -38,6 +39,8 @@ interface SyncProgress { interface _DocumentWithMeiliIndex extends Document { _meiliIndex?: boolean; + isTemporary?: boolean; + expiredAt?: Date | null; preprocessObjectForIndex?: () => Record; addObjectToMeili?: (next: CallbackWithoutResultAndOptionalError) => Promise; updateObjectToMeili?: (next: CallbackWithoutResultAndOptionalError) => Promise; @@ -90,6 +93,49 @@ const getSyncConfig = () => ({ delayMs: parseInt(process.env.MEILI_SYNC_DELAY_MS || '100', 10), }); +const hasSchemaPath = (schema: Schema, path: string): boolean => + Object.prototype.hasOwnProperty.call(schema.obj, path); + +const explicitTemporaryFlagKey = 'meiliExplicitTemporaryFlag'; + +const buildIndexableQuery = (schema: Schema): FilterQuery => { + if (!hasSchemaPath(schema, 'isTemporary')) { + return hasSchemaPath(schema, 'expiredAt') ? legacyPermanentExpirationFilter() : {}; + } + + return buildRetentionVisibilityFilter(); +}; + +const hasActiveExpiration = (expiredAt?: Date | null): boolean => + _.isNil(expiredAt) || new Date(expiredAt).getTime() > Date.now(); + +/** + * `isTemporary` defaults to `false` on the schema, so hydrated legacy documents + * can appear non-temporary even when the field is absent from MongoDB. `$isDefault` + * lets us distinguish that schema default from an explicit stored flag, and + * `$locals` carries the pre-save answer into post hooks after Mongoose mutates + * document state. + */ +const hasExplicitTemporaryFlag = (doc: DocumentWithMeiliIndex): boolean => + typeof doc.$locals?.[explicitTemporaryFlagKey] === 'boolean' + ? (doc.$locals[explicitTemporaryFlagKey] as boolean) + : doc.isTemporary != null && !doc.$isDefault('isTemporary'); + +const captureExplicitTemporaryFlag = (doc: DocumentWithMeiliIndex): void => { + doc.$locals[explicitTemporaryFlagKey] = doc.isTemporary != null && !doc.$isDefault('isTemporary'); +}; + +/** + * Index only retained non-temporary records whose flag was explicitly stored, + * plus legacy permanent records that have no retention deadline. Legacy records + * with an expiration are treated as temporary and stay out of search. + */ +const isIndexableDocument = (doc: DocumentWithMeiliIndex): boolean => + (doc.isTemporary === false && + hasExplicitTemporaryFlag(doc) && + hasActiveExpiration(doc.expiredAt)) || + (!hasExplicitTemporaryFlag(doc) && _.isNil(doc.expiredAt)); + /** * Validates the required options for configuring the mongoMeili plugin. */ @@ -136,11 +182,13 @@ const processBatch = async ( */ const createMeiliMongooseModel = ({ index, + getIndexableQuery, attributesToIndex, primaryKey, syncOptions, }: { index: Index; + getIndexableQuery: () => FilterQuery; attributesToIndex: string[]; primaryKey: string; syncOptions: { batchSize: number; delayMs: number }; @@ -152,8 +200,12 @@ const createMeiliMongooseModel = ({ * Get the current sync progress */ static async getSyncProgress(this: SchemaWithMeiliMethods): Promise { - const totalDocuments = await this.countDocuments({ expiredAt: null }); - const indexedDocuments = await this.countDocuments({ expiredAt: null, _meiliIndex: true }); + const indexableQuery = getIndexableQuery(); + const totalDocuments = await this.countDocuments(indexableQuery); + const indexedDocuments = await this.countDocuments({ + ...indexableQuery, + _meiliIndex: true, + }); return { totalProcessed: indexedDocuments, @@ -164,8 +216,7 @@ const createMeiliMongooseModel = ({ /** * Synchronizes data between the MongoDB collection and the MeiliSearch index by - * incrementally indexing only documents where `expiredAt` is `null` and `_meiliIndex` is not `true` - * (i.e., non-expired documents that have not yet been indexed, including those with missing or null `_meiliIndex`). + * incrementally indexing only non-temporary documents where `_meiliIndex` is not `true`. * */ static async syncWithMeili(this: SchemaWithMeiliMethods): Promise { const startTime = Date.now(); @@ -196,8 +247,9 @@ const createMeiliMongooseModel = ({ let hasMore = true; while (hasMore) { + const indexableQuery = getIndexableQuery(); const query: FilterQuery = { - expiredAt: null, + ...indexableQuery, _meiliIndex: { $ne: true }, }; @@ -299,8 +351,9 @@ const createMeiliMongooseModel = ({ const query: Record = {}; query[primaryKey] = { $in: meiliIds }; - // Find which documents exist in MongoDB - const existingDocs = await this.find(query).select(primaryKey).lean(); + const existingDocs = await this.find({ ...query, ...getIndexableQuery() }) + .select(primaryKey) + .lean(); const existingIds = new Set( existingDocs.map((doc: Record) => doc[primaryKey]), @@ -413,8 +466,7 @@ const createMeiliMongooseModel = ({ this: DocumentWithMeiliIndex, next: CallbackWithoutResultAndOptionalError, ): Promise { - // If this conversation or message has a TTL, don't index it - if (!_.isNil(this.expiredAt)) { + if (!isIndexableDocument(this)) { return next(); } @@ -459,6 +511,16 @@ const createMeiliMongooseModel = ({ next: CallbackWithoutResultAndOptionalError, ): Promise { try { + if (!isIndexableDocument(this)) { + await index.deleteDocument(String(this[primaryKey as keyof DocumentWithMeiliIndex])); + const model = this.constructor as Model; + await model.updateOne( + { _id: this._id as Types.ObjectId }, + { $set: { _meiliIndex: false } }, + ); + return next(); + } + const object = this.preprocessObjectForIndex!(); await index.updateDocuments([object], { primaryKey }); next(); @@ -644,9 +706,24 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions): logger.debug(`[mongoMeili] Added 'user' field to ${indexName} index attributes`); } - schema.loadClass(createMeiliMongooseModel({ index, attributesToIndex, primaryKey, syncOptions })); + schema.loadClass( + createMeiliMongooseModel({ + index, + getIndexableQuery: () => buildIndexableQuery(schema), + attributesToIndex, + primaryKey, + syncOptions, + }), + ); // Register Mongoose hooks + schema.pre('save', function (this: DocumentWithMeiliIndex, next) { + if (hasSchemaPath(schema, 'isTemporary')) { + captureExplicitTemporaryFlag(this); + } + next(); + }); + schema.post('save', function (doc: DocumentWithMeiliIndex, next) { doc.postSaveHook?.(next); }); diff --git a/packages/data-schemas/src/schema/convo.ts b/packages/data-schemas/src/schema/convo.ts index c8f394935a..24915af8a5 100644 --- a/packages/data-schemas/src/schema/convo.ts +++ b/packages/data-schemas/src/schema/convo.ts @@ -21,6 +21,10 @@ const convoSchema: Schema = new Schema( meiliIndex: true, }, messages: [{ type: Schema.Types.ObjectId, ref: 'Message' }], + isTemporary: { + type: Boolean, + default: false, + }, ...conversationPreset, agent_id: { type: String, @@ -48,7 +52,8 @@ convoSchema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 }); convoSchema.index({ createdAt: 1, updatedAt: 1 }); convoSchema.index({ conversationId: 1, user: 1, tenantId: 1 }, { unique: true }); +convoSchema.index({ user: 1, isTemporary: 1, expiredAt: 1 }); // index for MeiliSearch sync operations -convoSchema.index({ _meiliIndex: 1, expiredAt: 1 }); +convoSchema.index({ _meiliIndex: 1, isTemporary: 1, expiredAt: 1 }); export default convoSchema; diff --git a/packages/data-schemas/src/schema/file.ts b/packages/data-schemas/src/schema/file.ts index c8e7c72f52..b1b6eb3169 100644 --- a/packages/data-schemas/src/schema/file.ts +++ b/packages/data-schemas/src/schema/file.ts @@ -137,6 +137,9 @@ const file: Schema = new Schema( }, }, expiresAt: { + /* Short-lived upload TTL managed by MongoDB. This is separate from + * retention-scoped `expiredAt`, which is swept by application code + * after storage cleanup succeeds. */ type: Date, expires: 3600, // 1 hour in seconds }, @@ -144,12 +147,18 @@ const file: Schema = new Schema( type: String, index: true, }, + expiredAt: { + /* Retention deadline for persisted files. The file sweep deletes the + * backing storage first, then removes this metadata record. */ + type: Date, + }, }, { timestamps: true, }, ); +file.index({ expiredAt: 1 }); file.index({ createdAt: 1, updatedAt: 1 }); file.index( { filename: 1, conversationId: 1, context: 1, tenantId: 1 }, diff --git a/packages/data-schemas/src/schema/message.ts b/packages/data-schemas/src/schema/message.ts index d68edd0df3..f4002d076c 100644 --- a/packages/data-schemas/src/schema/message.ts +++ b/packages/data-schemas/src/schema/message.ts @@ -63,6 +63,10 @@ const messageSchema: Schema = new Schema( required: true, default: false, }, + isTemporary: { + type: Boolean, + default: false, + }, unfinished: { type: Boolean, default: false, @@ -180,6 +184,6 @@ messageSchema.index({ createdAt: 1 }); messageSchema.index({ messageId: 1, user: 1, tenantId: 1 }, { unique: true }); // index for MeiliSearch sync operations -messageSchema.index({ _meiliIndex: 1, expiredAt: 1 }); +messageSchema.index({ _meiliIndex: 1, isTemporary: 1, expiredAt: 1 }); export default messageSchema; diff --git a/packages/data-schemas/src/schema/share.ts b/packages/data-schemas/src/schema/share.ts index 3238084889..616536fe52 100644 --- a/packages/data-schemas/src/schema/share.ts +++ b/packages/data-schemas/src/schema/share.ts @@ -8,6 +8,7 @@ export interface ISharedLink extends Document { shareId?: string; targetMessageId?: string; isPublic: boolean; + expiredAt?: Date; createdAt?: Date; updatedAt?: Date; tenantId?: string; @@ -45,10 +46,14 @@ const shareSchema: Schema = new Schema( type: String, index: true, }, + expiredAt: { + type: Date, + }, }, { timestamps: true }, ); +shareSchema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 }); shareSchema.index({ conversationId: 1, user: 1, targetMessageId: 1, tenantId: 1 }); export default shareSchema; diff --git a/packages/data-schemas/src/schema/toolCall.ts b/packages/data-schemas/src/schema/toolCall.ts index d36d6b758a..cde2163ca3 100644 --- a/packages/data-schemas/src/schema/toolCall.ts +++ b/packages/data-schemas/src/schema/toolCall.ts @@ -10,6 +10,7 @@ export interface IToolCallData extends Document { attachments?: TAttachment[]; blockIndex?: number; partIndex?: number; + expiredAt?: Date; createdAt?: Date; updatedAt?: Date; tenantId?: string; @@ -50,10 +51,14 @@ const toolCallSchema: Schema = new Schema( type: String, index: true, }, + expiredAt: { + type: Date, + }, }, { timestamps: true }, ); +toolCallSchema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 }); toolCallSchema.index({ messageId: 1, user: 1, tenantId: 1 }); toolCallSchema.index({ conversationId: 1, user: 1, tenantId: 1 }); diff --git a/packages/data-schemas/src/types/convo.ts b/packages/data-schemas/src/types/convo.ts index c7888efba2..5cd828e9d6 100644 --- a/packages/data-schemas/src/types/convo.ts +++ b/packages/data-schemas/src/types/convo.ts @@ -6,6 +6,7 @@ export interface IConversation extends Document { title?: string; user?: string; messages?: Types.ObjectId[]; + isTemporary?: boolean; // Fields provided by conversationPreset (adjust types as needed) endpoint?: string; endpointType?: string; diff --git a/packages/data-schemas/src/types/file.ts b/packages/data-schemas/src/types/file.ts index b47a18abb6..9ccdf16034 100644 --- a/packages/data-schemas/src/types/file.ts +++ b/packages/data-schemas/src/types/file.ts @@ -72,6 +72,7 @@ export interface IMongoFile extends Omit { codeEnvRef?: CodeEnvRef; }; expiresAt?: Date; + expiredAt?: Date | null; createdAt?: Date; updatedAt?: Date; tenantId?: string; diff --git a/packages/data-schemas/src/types/message.ts b/packages/data-schemas/src/types/message.ts index cf2213d88b..16ec2a5ffb 100644 --- a/packages/data-schemas/src/types/message.ts +++ b/packages/data-schemas/src/types/message.ts @@ -18,6 +18,7 @@ export interface IMessage extends Document { text?: string; summary?: string; isCreatedByUser: boolean; + isTemporary?: boolean; unfinished?: boolean; error?: boolean; finish_reason?: string; diff --git a/packages/data-schemas/src/types/share.ts b/packages/data-schemas/src/types/share.ts index 6ef16ad87a..285818c35a 100644 --- a/packages/data-schemas/src/types/share.ts +++ b/packages/data-schemas/src/types/share.ts @@ -10,6 +10,7 @@ export interface ISharedLink { shareId?: string; targetMessageId?: string; isPublic: boolean; + expiredAt?: Date; createdAt?: Date; updatedAt?: Date; } diff --git a/packages/data-schemas/src/utils/index.ts b/packages/data-schemas/src/utils/index.ts index cdc69ce6b3..1cdf337be0 100644 --- a/packages/data-schemas/src/utils/index.ts +++ b/packages/data-schemas/src/utils/index.ts @@ -1,6 +1,7 @@ export * from './principal'; export * from './string'; export * from './tempChatRetention'; +export * from './retention'; export { tenantSafeBulkWrite } from './tenantBulkWrite'; export * from './transactions'; export * from './objectId'; diff --git a/packages/data-schemas/src/utils/retention.ts b/packages/data-schemas/src/utils/retention.ts new file mode 100644 index 0000000000..c25896fbf9 --- /dev/null +++ b/packages/data-schemas/src/utils/retention.ts @@ -0,0 +1,33 @@ +import type { FilterQuery } from 'mongoose'; +import { DEFAULT_RETENTION_HOURS } from './tempChatRetention'; + +export type RetentionFilterDocument = { + isTemporary?: boolean | null; + expiredAt?: Date | null; +}; + +export const activeExpirationFilter = < + T extends RetentionFilterDocument = RetentionFilterDocument, +>(): FilterQuery => + ({ + $or: [{ expiredAt: null }, { expiredAt: { $gt: new Date() } }], + }) as FilterQuery; + +export const legacyPermanentExpirationFilter = < + T extends RetentionFilterDocument = RetentionFilterDocument, +>(): FilterQuery => + ({ expiredAt: null }) as FilterQuery; + +export const buildRetentionVisibilityFilter = < + T extends RetentionFilterDocument = RetentionFilterDocument, +>(): FilterQuery => + ({ + $or: [ + { isTemporary: false, expiredAt: null }, + { isTemporary: false, expiredAt: { $gt: new Date() } }, + { isTemporary: null, expiredAt: null }, + ], + }) as FilterQuery; + +export const createFallbackRetentionDate = (now = Date.now()): Date => + new Date(now + DEFAULT_RETENTION_HOURS * 60 * 60 * 1000);