diff --git a/config/migrate-ephemeral-retention.js b/config/migrate-ephemeral-retention.js index fb86fda2ca..ab9f0e33af 100644 --- a/config/migrate-ephemeral-retention.js +++ b/config/migrate-ephemeral-retention.js @@ -12,16 +12,16 @@ require('module-alias')({ base: path.resolve(__dirname, '..', 'api') }); const connect = require('./connect'); const { getAppConfig } = require('~/server/services/Config'); -const { Conversation, Message, SharedLink } = require('~/db/models'); +const { Conversation, Message, SharedLink, File } = require('~/db/models'); /** * Backfills forced (ephemeral) retention over conversations that predate the mode. * * Convert-on-touch only converts chats that are subsequently written, so enabling ephemeral * retention on a deployment with existing data leaves untouched permanent chats visible and - * non-expiring. This sweep converts every non-conforming conversation, its messages, and its - * shares to the forced window (capping rather than extending sooner deadlines). It is - * idempotent and safe to re-run. + * non-expiring. This sweep converts every non-conforming conversation, its messages, its + * shares, and its uploaded files to the forced window (capping rather than extending sooner + * deadlines). It is idempotent and safe to re-run. */ async function migrateEphemeralRetention({ dryRun = true, force = false } = {}) { await connect(); @@ -55,7 +55,13 @@ async function migrateEphemeralRetention({ dryRun = true, force = false } = {}) }; } - const result = await sweepForcedRetention(Conversation, Message, SharedLink, forcedExpiredAt); + const result = await sweepForcedRetention( + Conversation, + Message, + SharedLink, + File, + forcedExpiredAt, + ); logger.info('Ephemeral Retention Migration completed', result); return { dryRun: false, forcedExpiredAt, ...result }; }); diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.ts index f54d625ac6..cd67cd8159 100644 --- a/packages/data-schemas/src/methods/message.spec.ts +++ b/packages/data-schemas/src/methods/message.spec.ts @@ -2,7 +2,7 @@ import mongoose from 'mongoose'; import { v4 as uuidv4 } from 'uuid'; import { RetentionMode } from 'librechat-data-provider'; import { MongoMemoryServer } from 'mongodb-memory-server'; -import type { IConversation, IMessage, ISharedLink } from '..'; +import type { IConversation, IMessage, IMongoFile, ISharedLink } from '..'; import { tenantStorage, runAsSystem } from '~/config/tenantContext'; import { sweepForcedRetention } from '../utils/retention'; import { createMessageMethods } from './message'; @@ -1412,10 +1412,12 @@ describe('Message Operations', () => { describe('sweepForcedRetention', () => { const Conversation = () => mongoose.models.Conversation as mongoose.Model; const SharedLink = () => mongoose.models.SharedLink as mongoose.Model; + const File = () => mongoose.models.File as mongoose.Model; beforeEach(async () => { await Conversation().deleteMany({}); await SharedLink().deleteMany({}); + await File().deleteMany({}); }); it('converts untouched permanent conversations, messages, and shares and skips conforming ones', async () => { @@ -1455,6 +1457,7 @@ describe('Message Operations', () => { Conversation(), Message, SharedLink(), + File(), forcedExpiredAt, ); expect(result).toEqual({ conversations: 1, errors: 0 }); @@ -1476,6 +1479,54 @@ describe('Message Operations', () => { expect(conforming?.expiredAt?.getTime()).toBe(soonerExpiry.getTime()); }); + it("caps a converted conversation's permanent files while preserving sooner and unrelated ones", async () => { + const forcedExpiredAt = new Date(Date.now() + 24 * 60 * 60 * 1000); + const soonerExpiry = new Date(Date.now() + 30 * 60 * 1000); + const conversationId = uuidv4(); + const unrelatedConversationId = uuidv4(); + const permanentFileId = uuidv4(); + const soonerFileId = uuidv4(); + const unrelatedFileId = uuidv4(); + + await Conversation().create({ + conversationId, + user: 'user123', + endpoint: 'openAI', + isTemporary: false, + }); + await File().collection.insertMany([ + { + file_id: permanentFileId, + conversationId, + user: new mongoose.Types.ObjectId(), + expiredAt: null, + }, + { + file_id: soonerFileId, + conversationId, + user: new mongoose.Types.ObjectId(), + expiredAt: soonerExpiry, + }, + { + file_id: unrelatedFileId, + conversationId: unrelatedConversationId, + user: new mongoose.Types.ObjectId(), + expiredAt: null, + }, + ]); + + await sweepForcedRetention(Conversation(), Message, SharedLink(), File(), forcedExpiredAt); + + const permanentFile = await File().findOne({ file_id: permanentFileId }).lean(); + expect(permanentFile?.expiredAt?.getTime()).toBe(forcedExpiredAt.getTime()); + + const soonerFile = await File().findOne({ file_id: soonerFileId }).lean(); + expect(soonerFile?.expiredAt?.getTime()).toBe(soonerExpiry.getTime()); + + const unrelatedFile = await File().findOne({ file_id: unrelatedFileId }).lean(); + expect(unrelatedFile?.expiredAt ?? null).toBeNull(); + }); + it('aligns a permanent message to a sooner parent deadline instead of the forced window', async () => { const forcedExpiredAt = new Date(Date.now() + 24 * 60 * 60 * 1000); const conversationId = uuidv4(); @@ -1497,7 +1548,7 @@ describe('Message Operations', () => { isTemporary: false, }); - await sweepForcedRetention(Conversation(), Message, SharedLink(), forcedExpiredAt); + await sweepForcedRetention(Conversation(), Message, SharedLink(), File(), forcedExpiredAt); const convo = await Conversation().findOne({ conversationId }).lean(); expect(convo?.expiredAt?.getTime()).toBe(soonerExpiry.getTime()); @@ -1525,7 +1576,7 @@ describe('Message Operations', () => { expiredAt: null, }); - await sweepForcedRetention(Conversation(), Message, SharedLink(), forcedExpiredAt); + await sweepForcedRetention(Conversation(), Message, SharedLink(), File(), forcedExpiredAt); const message = await Message.findOne({ messageId: legacyMessageId }).lean(); expect(message?.isTemporary).toBe(true); @@ -1550,6 +1601,7 @@ describe('Message Operations', () => { Conversation(), Message, throwingSharedLink, + File(), forcedExpiredAt, ); expect(failed).toEqual({ conversations: 0, errors: 1 }); @@ -1562,6 +1614,7 @@ describe('Message Operations', () => { Conversation(), Message, SharedLink(), + File(), forcedExpiredAt, ); expect(retried).toEqual({ conversations: 1, errors: 0 }); diff --git a/packages/data-schemas/src/utils/retention.ts b/packages/data-schemas/src/utils/retention.ts index 723667bcdc..946d6244f6 100644 --- a/packages/data-schemas/src/utils/retention.ts +++ b/packages/data-schemas/src/utils/retention.ts @@ -1,5 +1,5 @@ import type { FilterQuery, Model } from 'mongoose'; -import type { AppConfig, IConversation, IMessage, ISharedLink } from '~/types'; +import type { AppConfig, IConversation, IMessage, IMongoFile, ISharedLink } from '~/types'; import { createTempChatExpirationDate, DEFAULT_RETENTION_HOURS } from './tempChatRetention'; import logger from '~/config/winston'; @@ -159,6 +159,33 @@ export const capConversationSharedLinks = async ( ); }; +/** + * Caps a conversation's uploaded files to the forced deadline. Files use a retention-scoped + * `expiredAt` swept by application code (`getExpiredFiles` only sweeps files whose own `expiredAt` + * is set), so a permanent file (`expiredAt: null`) uploaded before forced retention would linger + * in storage after the conversation and messages TTL out. Only files with no expiration or a later + * one are touched, so it is a no-op once a conversation conforms and never extends a file that + * already expires sooner. Under ephemeral retention every conversation-scoped file is meant to + * expire (persistent agent files are not retained), so no agent-file exclusion is needed here. + * + * Scoped by `conversationId` alone: it is a globally unique per-conversation id, and unlike the + * message/share caps the `File.user` field is an `ObjectId` rather than the string user id, so + * filtering by user would require a cast the callers cannot guarantee. + */ +export const capConversationFiles = async ( + File: Model, + conversationId: string, + forcedExpiredAt: Date, +): Promise => { + await File.updateMany( + { + conversationId, + $or: [{ expiredAt: null }, { expiredAt: { $gt: forcedExpiredAt } }], + }, + { $set: { expiredAt: forcedExpiredAt } }, + ); +}; + /** * Caps a message-only forced save to a parent that already expires sooner than the freshly * computed window. Returns the parent's earlier deadline (so the message cannot outlive it) @@ -347,16 +374,17 @@ export const cascadeForcedRetentionByProject = ( * deployment with existing chats leaves untouched permanent rows visible and non-expiring. * * Streams every conversation that does not yet conform to the forced window and converts it, - * its messages, and its shares one conversation at a time. Each conversation is capped to the - * earlier of its own deadline and the forced window, and its messages and shares are capped to - * that same per-conversation deadline, so the sweep never extends data that already expires - * sooner and never lets a message outlive its conversation. It is idempotent: re-running skips - * conversations that already conform. + * its messages, its shares, and its uploaded files one conversation at a time. Each conversation + * is capped to the earlier of its own deadline and the forced window, and its messages, shares, + * and files are capped to that same per-conversation deadline, so the sweep never extends data + * that already expires sooner and never lets a dependent record outlive its conversation. It is + * idempotent: re-running skips conversations that already conform. */ export const sweepForcedRetention = async ( Conversation: Model, Message: Model, SharedLink: Model, + File: Model, forcedExpiredAt: Date, ): Promise<{ conversations: number; errors: number }> => { const result = { conversations: 0, errors: 0 }; @@ -373,12 +401,13 @@ export const sweepForcedRetention = async ( try { const expiredAt = capForcedRetentionExpiry(convo.expiredAt, forcedExpiredAt); /** - * Convert the dependent messages and shares before marking the conversation itself + * Convert the dependent messages, shares, and files before marking the conversation itself * conforming. If a child backfill throws, the conversation stays non-conforming so the * gap-filtered query picks it up again on a re-run, keeping the sweep safe to repeat. */ await forceConversationMessagesTemporary(Message, user, conversationId, expiredAt); await capConversationSharedLinks(SharedLink, user, conversationId, expiredAt); + await capConversationFiles(File, conversationId, expiredAt); await Conversation.updateOne({ _id: convo._id }, { $set: { isTemporary: true, expiredAt } }); result.conversations += 1; } catch {