feat: add migration to force retention on pre-existing chats

Forced ephemeral retention only converts conversations that are
subsequently written (convert-on-touch), so enabling the mode on a
deployment with existing data left untouched permanent chats
(isTemporary: false/expiredAt: null) satisfying the visibility filter
and never expiring.

Add a sweepForcedRetention helper that streams every non-conforming
conversation and converts it, its messages, and its shares to the
forced window, capping each to the earlier of its own deadline and the
window so it never extends data scheduled to expire sooner and never
lets a message outlive its conversation. Expose it through a
config/migrate-ephemeral-retention.js script (and npm run
migrate:ephemeral-retention) that loads the app config, refuses to run
unless ephemeral mode is enabled (or --force), and supports --dry-run.
The sweep is idempotent and safe to re-run.
This commit is contained in:
Marco Beretta 2026-06-26 22:20:29 +02:00
parent c6deea47eb
commit 516ef2f2dd
No known key found for this signature in database
GPG key ID: D918033D8E74CC11
4 changed files with 241 additions and 1 deletions

View file

@ -0,0 +1,96 @@
const path = require('path');
const {
logger,
runAsSystem,
createTempChatExpirationDate,
forcedRetentionGapFilter,
sweepForcedRetention,
} = require('@librechat/data-schemas');
const { RetentionMode } = require('librechat-data-provider');
require('module-alias')({ base: path.resolve(__dirname, '..', 'api') });
const connect = require('./connect');
const { getAppConfig } = require('~/server/services/Config');
const { Conversation, Message, SharedLink } = require('~/db/models');
/**
* Backfills forced (ephemeral) retention over conversations that predate the mode.
*
* Convert-on-touch only converts chats that are subsequently written, so enabling ephemeral
* retention on a deployment with existing data leaves untouched permanent chats visible and
* non-expiring. This sweep converts every non-conforming conversation, its messages, and its
* shares to the forced window (capping rather than extending sooner deadlines). It is
* idempotent and safe to re-run.
*/
async function migrateEphemeralRetention({ dryRun = true, force = false } = {}) {
await connect();
return runAsSystem(async () => {
const appConfig = await getAppConfig();
const interfaceConfig = appConfig?.interfaceConfig;
const retentionMode = interfaceConfig?.retentionMode;
logger.info('Starting Ephemeral Retention Migration', { dryRun, force, retentionMode });
if (retentionMode !== RetentionMode.EPHEMERAL && !force) {
logger.error(
`retentionMode is "${retentionMode ?? 'unset'}", not "ephemeral". This migration ` +
'converts every conversation into a temporary, expiring chat. Enable ephemeral ' +
'retention first, or pass --force to run anyway.',
);
return { aborted: true, reason: 'retentionMode is not ephemeral', retentionMode };
}
const forcedExpiredAt = createTempChatExpirationDate(interfaceConfig);
const nonConforming = await Conversation.countDocuments(
forcedRetentionGapFilter(forcedExpiredAt),
);
logger.info(`Found ${nonConforming} non-conforming conversation(s)`, { forcedExpiredAt });
if (dryRun) {
return {
dryRun: true,
summary: { nonConformingConversations: nonConforming, forcedExpiredAt },
};
}
const result = await sweepForcedRetention(Conversation, Message, SharedLink, forcedExpiredAt);
logger.info('Ephemeral Retention Migration completed', result);
return { dryRun: false, forcedExpiredAt, ...result };
});
}
if (require.main === module) {
const dryRun = process.argv.includes('--dry-run');
const force = process.argv.includes('--force');
migrateEphemeralRetention({ dryRun, force })
.then((result) => {
if (result.aborted) {
console.log('\n=== MIGRATION ABORTED ===');
console.log(`Reason: ${result.reason}`);
console.log(`Current retentionMode: ${result.retentionMode ?? 'unset'}`);
console.log('\nEnable ephemeral retention, or pass --force to run anyway.');
process.exit(1);
}
if (result.dryRun) {
console.log('\n=== DRY RUN RESULTS ===');
console.log(`Non-conforming conversations: ${result.summary.nonConformingConversations}`);
const expiry = result.summary.forcedExpiredAt;
console.log(`Forced expiry: ${expiry?.toISOString?.() ?? expiry}`);
console.log('\nTo run the actual migration, remove the --dry-run flag.');
} else {
console.log('\n=== MIGRATION RESULTS ===');
console.log(JSON.stringify(result, null, 2));
}
process.exit(0);
})
.catch((error) => {
console.error('Ephemeral retention migration failed:', error);
process.exit(1);
});
}
module.exports = { migrateEphemeralRetention };

View file

@ -109,7 +109,9 @@
"migrate:shared-link-permissions:batch": "node config/migrate-shared-link-permissions.js --batch-size=50",
"migrate:orphaned-agent-files:dry-run": "node config/migrate-orphaned-agent-files.js --dry-run",
"migrate:orphaned-agent-files": "node config/migrate-orphaned-agent-files.js",
"migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50"
"migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50",
"migrate:ephemeral-retention:dry-run": "node config/migrate-ephemeral-retention.js --dry-run",
"migrate:ephemeral-retention": "node config/migrate-ephemeral-retention.js"
},
"repository": {
"type": "git",

View file

@ -4,6 +4,7 @@ import { RetentionMode } from 'librechat-data-provider';
import { MongoMemoryServer } from 'mongodb-memory-server';
import type { IConversation, IMessage, ISharedLink } from '..';
import { tenantStorage, runAsSystem } from '~/config/tenantContext';
import { sweepForcedRetention } from '../utils/retention';
import { createMessageMethods } from './message';
import { createModels } from '../models';
import logger from '~/config/winston';
@ -1372,6 +1373,104 @@ describe('Message Operations', () => {
});
});
describe('sweepForcedRetention', () => {
const Conversation = () => mongoose.models.Conversation as mongoose.Model<IConversation>;
const SharedLink = () => mongoose.models.SharedLink as mongoose.Model<ISharedLink>;
beforeEach(async () => {
await Conversation().deleteMany({});
await SharedLink().deleteMany({});
});
it('converts untouched permanent conversations, messages, and shares and skips conforming ones', async () => {
const forcedExpiredAt = new Date(Date.now() + 24 * 60 * 60 * 1000);
const permanentId = uuidv4();
const conformingId = uuidv4();
const soonerExpiry = new Date(Date.now() + 30 * 60 * 1000);
await Conversation().create([
{ conversationId: permanentId, user: 'user123', endpoint: 'openAI', isTemporary: false },
{
conversationId: conformingId,
user: 'user123',
endpoint: 'openAI',
isTemporary: true,
expiredAt: soonerExpiry,
},
]);
await Message.create([
{ messageId: uuidv4(), conversationId: permanentId, user: 'user123', text: 'permanent' },
{
messageId: uuidv4(),
conversationId: conformingId,
user: 'user123',
text: 'conforming',
isTemporary: true,
expiredAt: soonerExpiry,
},
]);
await SharedLink().create({
conversationId: permanentId,
user: 'user123',
shareId: uuidv4(),
});
const result = await sweepForcedRetention(
Conversation(),
Message,
SharedLink(),
forcedExpiredAt,
);
expect(result).toEqual({ conversations: 1, errors: 0 });
const permanent = await Conversation().findOne({ conversationId: permanentId }).lean();
expect(permanent?.isTemporary).toBe(true);
expect(permanent?.expiredAt?.getTime()).toBe(forcedExpiredAt.getTime());
const permanentMessages = await getMessages({ conversationId: permanentId, user: 'user123' });
for (const message of permanentMessages) {
expect(message.isTemporary).toBe(true);
expect(message.expiredAt?.getTime()).toBe(forcedExpiredAt.getTime());
}
const share = await SharedLink().findOne({ conversationId: permanentId }).lean();
expect(share?.expiredAt?.getTime()).toBe(forcedExpiredAt.getTime());
const conforming = await Conversation().findOne({ conversationId: conformingId }).lean();
expect(conforming?.expiredAt?.getTime()).toBe(soonerExpiry.getTime());
});
it('aligns a permanent message to a sooner parent deadline instead of the forced window', async () => {
const forcedExpiredAt = new Date(Date.now() + 24 * 60 * 60 * 1000);
const conversationId = uuidv4();
const soonerExpiry = new Date(Date.now() + 30 * 60 * 1000);
await Conversation().create({
conversationId,
user: 'user123',
endpoint: 'openAI',
isTemporary: false,
expiredAt: soonerExpiry,
});
const permanentMessageId = uuidv4();
await Message.create({
messageId: permanentMessageId,
conversationId,
user: 'user123',
text: 'permanent',
isTemporary: false,
});
await sweepForcedRetention(Conversation(), Message, SharedLink(), forcedExpiredAt);
const convo = await Conversation().findOne({ conversationId }).lean();
expect(convo?.expiredAt?.getTime()).toBe(soonerExpiry.getTime());
const message = await Message.findOne({ messageId: permanentMessageId }).lean();
expect(message?.isTemporary).toBe(true);
expect(message?.expiredAt?.getTime()).toBe(soonerExpiry.getTime());
});
});
describe('Message cursor pagination', () => {
/**
* Helper to create messages with specific timestamps

View file

@ -262,3 +262,46 @@ export const cascadeForcedRetentionByTag = async (
);
}
};
/**
* One-time backfill of forced (ephemeral) retention over pre-existing data. Convert-on-touch
* only converts conversations that are subsequently written, so enabling ephemeral mode on a
* deployment with existing chats leaves untouched permanent rows visible and non-expiring.
*
* Streams every conversation that does not yet conform to the forced window and converts it,
* its messages, and its shares one conversation at a time. Each conversation is capped to the
* earlier of its own deadline and the forced window, and its messages and shares are capped to
* that same per-conversation deadline, so the sweep never extends data that already expires
* sooner and never lets a message outlive its conversation. It is idempotent: re-running skips
* conversations that already conform.
*/
export const sweepForcedRetention = async (
Conversation: Model<IConversation>,
Message: Model<IMessage>,
SharedLink: Model<ISharedLink>,
forcedExpiredAt: Date,
): Promise<{ conversations: number; errors: number }> => {
const result = { conversations: 0, errors: 0 };
const cursor = Conversation.find(forcedRetentionGapFilter<IConversation>(forcedExpiredAt))
.select('_id conversationId user expiredAt')
.lean()
.cursor();
for await (const convo of cursor) {
const { conversationId, user } = convo;
if (typeof conversationId !== 'string' || !conversationId || !user) {
continue;
}
try {
const expiredAt = capForcedRetentionExpiry(convo.expiredAt, forcedExpiredAt);
await Conversation.updateOne({ _id: convo._id }, { $set: { isTemporary: true, expiredAt } });
await forceConversationMessagesTemporary(Message, user, conversationId, expiredAt);
await capConversationSharedLinks(SharedLink, user, conversationId, expiredAt);
result.conversations += 1;
} catch {
result.errors += 1;
}
}
return result;
};