🧹 chore: Migrate Legacy Duplicate Code Files Blocking Dedupe Index (#14593)

Atomic file claiming (#11675) added a unique partial index on
(filename, conversationId, context, tenantId) for execute_code outputs.
Records written before it inserted a new document per regeneration, so
any deployment that re-ran a cell producing the same filename carries
duplicates the index cannot span: Mongo aborts the build with E11000 and
the constraint is silently absent — the claim path still works, but
without its database-level guard against concurrent inserts.

Adds config/migrate-code-file-duplicates.js to normalize that legacy
data, following the existing migration conventions (dry-run default,
--batch-size, runAsSystem for cross-tenant scans).

Renames rather than deletes: each duplicate is a distinct stored object,
typically still referenced by a message attachment, so removing one
would strip a real artifact from a user's history. The newest record
keeps the canonical name — matching the claim path's latest-write-wins
behavior — and older copies gain a ' (n)' suffix that skips names
already taken in the conversation. Attachments embed their own filename,
so rendered history is unchanged.

After a successful apply the script builds the index directly (targeted
createIndex, not syncIndexes) so the operator learns immediately whether
the constraint is now in place.
This commit is contained in:
Danny Avila 2026-08-02 06:41:21 -04:00 committed by GitHub
parent 928b14f5bc
commit 2d606a9783
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 543 additions and 1 deletions

View file

@ -0,0 +1,303 @@
const mongoose = require('mongoose');
const { v4: uuidv4 } = require('uuid');
const { FileContext } = require('librechat-data-provider');
const { logger } = require('@librechat/data-schemas');
const { MongoMemoryServer } = require('mongodb-memory-server');
// Mock the config/connect module to prevent connection attempts during tests
jest.mock('../connect', () => jest.fn().mockResolvedValue(true));
// Disable console for tests
logger.silent = true;
describe('Code File Duplicate Migration Script', () => {
let mongoServer;
let File;
let migrateCodeFileDuplicates;
/** The unique partial index this migration exists to unblock. */
const INDEX_KEYS = { filename: 1, conversationId: 1, context: 1, tenantId: 1 };
const INDEX_NAME = 'filename_1_conversationId_1_context_1_tenantId_1';
const INDEX_OPTIONS = {
unique: true,
partialFilterExpression: { context: FileContext.execute_code },
};
/**
* Reproduces the state this migration is written for: legacy duplicates
* present and the unique index absent because its build failed. Mongoose
* builds schema indexes in the background at startup, so without dropping it
* here the fixtures would race an index the affected deployments don't have.
*/
async function dropUniqueIndex() {
await File.init().catch(() => {
/* the background build may itself fail — that IS the scenario */
});
await File.collection.dropIndex(INDEX_NAME).catch(() => {
/* already absent */
});
}
async function createCodeFile({ filename, conversationId, createdAt, context }) {
return File.create({
user: new mongoose.Types.ObjectId(),
file_id: uuidv4(),
filename,
filepath: `/images/user/${uuidv4()}.png`,
object: 'file',
type: 'image/png',
bytes: 1024,
conversationId,
context: context ?? FileContext.execute_code,
createdAt,
updatedAt: createdAt,
});
}
const namesFor = async (conversationId) => {
const files = await File.find({ conversationId }).lean();
return files.map((file) => file.filename).sort();
};
beforeAll(async () => {
mongoServer = await MongoMemoryServer.create();
await mongoose.connect(mongoServer.getUri());
const dbModels = require('~/db/models');
File = dbModels.File;
({ migrateCodeFileDuplicates } = require('../migrate-code-file-duplicates'));
await dropUniqueIndex();
});
afterAll(async () => {
await mongoose.disconnect();
await mongoServer.stop();
});
afterEach(async () => {
await File.deleteMany({});
await File.collection.dropIndex(INDEX_NAME).catch(() => {
/* the test never built it */
});
});
it('renames older duplicates and leaves the newest record canonical', async () => {
const conversationId = uuidv4();
await createCodeFile({
filename: 'bar_chart.png',
conversationId,
createdAt: new Date('2025-03-29T03:54:53Z'),
});
await createCodeFile({
filename: 'bar_chart.png',
conversationId,
createdAt: new Date('2025-03-29T03:56:32Z'),
});
const result = await migrateCodeFileDuplicates({ dryRun: false });
expect(result.duplicateGroups).toBe(1);
expect(result.filesRenamed).toBe(1);
/* Newest keeps the canonical name — the claim path's "latest write wins". */
expect(await namesFor(conversationId)).toEqual(['bar_chart (1).png', 'bar_chart.png']);
const newest = await File.findOne({ filename: 'bar_chart.png' }).lean();
expect(newest.createdAt).toEqual(new Date('2025-03-29T03:56:32Z'));
});
it('never deletes: every original record survives the rename', async () => {
const conversationId = uuidv4();
const older = await createCodeFile({
filename: 'plot.png',
conversationId,
createdAt: new Date('2025-01-01T00:00:00Z'),
});
const newer = await createCodeFile({
filename: 'plot.png',
conversationId,
createdAt: new Date('2025-01-01T00:05:00Z'),
});
await migrateCodeFileDuplicates({ dryRun: false });
/* Both file_ids still resolve a deleted record would strip a real
* artifact out of the message attachment that references it. */
expect(await File.countDocuments({})).toBe(2);
const kept = await File.findOne({ file_id: older.file_id }).lean();
expect(kept).not.toBeNull();
expect(kept.filepath).toBe(older.filepath);
expect((await File.findOne({ file_id: newer.file_id }).lean()).filename).toBe('plot.png');
});
it('unblocks the unique partial index that could not build before', async () => {
const conversationId = uuidv4();
await createCodeFile({
filename: 'report.png',
conversationId,
createdAt: new Date('2025-02-01T00:00:00Z'),
});
await createCodeFile({
filename: 'report.png',
conversationId,
createdAt: new Date('2025-02-01T00:01:00Z'),
});
/* Precondition: the duplicates genuinely block the build (E11000). */
await expect(File.collection.createIndex(INDEX_KEYS, INDEX_OPTIONS)).rejects.toThrow();
const result = await migrateCodeFileDuplicates({ dryRun: false });
expect(result.indexBuilt).toBe(true);
const indexes = await File.collection.indexes();
expect(
indexes.some((index) => index.name === 'filename_1_conversationId_1_context_1_tenantId_1'),
).toBe(true);
});
it('reports without writing in dry-run mode', async () => {
const conversationId = uuidv4();
await createCodeFile({
filename: 'chart.png',
conversationId,
createdAt: new Date('2025-04-01T00:00:00Z'),
});
await createCodeFile({
filename: 'chart.png',
conversationId,
createdAt: new Date('2025-04-01T00:02:00Z'),
});
const result = await migrateCodeFileDuplicates({ dryRun: true });
expect(result.filesRenamed).toBe(1);
expect(result.indexBuilt).toBe(false);
expect(await namesFor(conversationId)).toEqual(['chart.png', 'chart.png']);
});
it('skips names already taken in the conversation', async () => {
const conversationId = uuidv4();
await createCodeFile({
filename: 'out.png',
conversationId,
createdAt: new Date('2025-05-01T00:00:00Z'),
});
await createCodeFile({
filename: 'out.png',
conversationId,
createdAt: new Date('2025-05-01T00:01:00Z'),
});
/* An unrelated record already occupies the first replacement name. */
await createCodeFile({
filename: 'out (1).png',
conversationId,
createdAt: new Date('2025-05-01T00:03:00Z'),
});
await migrateCodeFileDuplicates({ dryRun: false });
expect(await namesFor(conversationId)).toEqual(['out (1).png', 'out (2).png', 'out.png']);
});
it('resolves three copies into distinct names in one pass', async () => {
const conversationId = uuidv4();
for (const minute of [0, 1, 2]) {
await createCodeFile({
filename: 'fig.png',
conversationId,
createdAt: new Date(`2025-06-01T00:0${minute}:00Z`),
});
}
const result = await migrateCodeFileDuplicates({ dryRun: false });
expect(result.filesRenamed).toBe(2);
expect(await namesFor(conversationId)).toEqual(['fig (1).png', 'fig (2).png', 'fig.png']);
expect(result.indexBuilt).toBe(true);
});
it('leaves same-named files in DIFFERENT conversations alone', async () => {
const first = uuidv4();
const second = uuidv4();
await createCodeFile({
filename: 'shared.png',
conversationId: first,
createdAt: new Date('2025-07-01T00:00:00Z'),
});
await createCodeFile({
filename: 'shared.png',
conversationId: second,
createdAt: new Date('2025-07-01T00:01:00Z'),
});
const result = await migrateCodeFileDuplicates({ dryRun: false });
expect(result.duplicateGroups).toBe(0);
expect(result.filesRenamed).toBe(0);
expect(await namesFor(first)).toEqual(['shared.png']);
expect(await namesFor(second)).toEqual(['shared.png']);
});
it('ignores duplicates outside the execute_code context', async () => {
const conversationId = uuidv4();
await createCodeFile({
filename: 'upload.png',
conversationId,
createdAt: new Date('2025-08-01T00:00:00Z'),
context: FileContext.message_attachment,
});
await createCodeFile({
filename: 'upload.png',
conversationId,
createdAt: new Date('2025-08-01T00:01:00Z'),
context: FileContext.message_attachment,
});
const result = await migrateCodeFileDuplicates({ dryRun: false });
/* The index is partial — only code outputs must be unique. */
expect(result.duplicateGroups).toBe(0);
expect(await namesFor(conversationId)).toEqual(['upload.png', 'upload.png']);
});
it('is safe to re-run once each conversation is unique', async () => {
const conversationId = uuidv4();
await createCodeFile({
filename: 'idempotent.png',
conversationId,
createdAt: new Date('2025-09-01T00:00:00Z'),
});
await createCodeFile({
filename: 'idempotent.png',
conversationId,
createdAt: new Date('2025-09-01T00:01:00Z'),
});
await migrateCodeFileDuplicates({ dryRun: false });
const afterFirst = await namesFor(conversationId);
const second = await migrateCodeFileDuplicates({ dryRun: false });
expect(second.duplicateGroups).toBe(0);
expect(second.filesRenamed).toBe(0);
expect(await namesFor(conversationId)).toEqual(afterFirst);
});
it('handles filenames without an extension', async () => {
const conversationId = uuidv4();
await createCodeFile({
filename: 'Makefile',
conversationId,
createdAt: new Date('2025-10-01T00:00:00Z'),
});
await createCodeFile({
filename: 'Makefile',
conversationId,
createdAt: new Date('2025-10-01T00:01:00Z'),
});
await migrateCodeFileDuplicates({ dryRun: false });
expect(await namesFor(conversationId)).toEqual(['Makefile', 'Makefile (1)']);
});
});

View file

@ -0,0 +1,236 @@
const path = require('path');
const { FileContext } = require('librechat-data-provider');
const { logger, runAsSystem } = require('@librechat/data-schemas');
require('module-alias')({ base: path.resolve(__dirname, '..', 'api') });
const connect = require('./connect');
const { File } = require('~/db/models');
/**
* Cap on the number of per-group entries retained in `results.details`. Larger
* runs still rename every affected record and still report accurate aggregate
* counts we just stop accumulating sample data past this threshold to keep
* memory bounded on deployments with thousands of legacy duplicates.
*/
const DETAIL_SAMPLE_LIMIT = 50;
/** Mirrors the unique partial index declared on the file schema. */
const INDEX_KEYS = { filename: 1, conversationId: 1, context: 1, tenantId: 1 };
const INDEX_OPTIONS = {
unique: true,
partialFilterExpression: { context: FileContext.execute_code },
};
/** `report.png` -> `report (2).png`; a name without an extension keeps its shape. */
function suffixFilename(filename, n) {
const extension = path.extname(filename);
const base = extension ? filename.slice(0, -extension.length) : filename;
return `${base} (${n})${extension}`;
}
/**
* Picks a name that is free within the group's uniqueness scope. `taken` holds
* both the names already in the database and the ones handed out earlier in
* this run, so a group with several duplicates can't rename two records onto
* the same replacement.
*/
function nextAvailableName(filename, taken) {
for (let n = 1; ; n++) {
const candidate = suffixFilename(filename, n);
if (!taken.has(candidate)) {
taken.add(candidate);
return candidate;
}
}
}
/**
* Normalizes code-execution output files that share a filename within one
* conversation, so the unique partial index on
* `(filename, conversationId, context, tenantId)` can finish building.
*
* That index arrived with atomic file claiming: a regenerated output now
* converges on ONE record with a cache-busted filepath. Records written before
* that change instead inserted a second document per regeneration, so any
* deployment that re-ran a code cell producing the same filename carries
* duplicates the index cannot span. Mongo then aborts the build with E11000 and
* the constraint is silently absent the claim path keeps working, but without
* its database-level guarantee against concurrent inserts.
*
* Renames rather than deletes: every duplicate is a distinct stored object,
* usually still referenced by a message attachment, so removing one would
* strip a real artifact out of a user's history. The newest record keeps the
* canonical name (matching the "latest write wins" behavior of the claim path);
* older ones gain a ` (n)` suffix. Attachments carry their own filename copy,
* so rendered history is untouched.
*
* Safe to re-run once each scope is unique, nothing is written.
*
* @param {{ dryRun?: boolean, batchSize?: number }} [options]
*/
async function migrateCodeFileDuplicates({ dryRun = true, batchSize = 100 } = {}) {
await connect();
logger.info('Starting Code File Duplicate Migration', { dryRun, batchSize });
/*
* Scan and heal across every tenant. Without this wrapper the tenant
* isolation plugin either scopes queries to a (non-existent) tenant or
* throws under TENANT_ISOLATION_STRICT=true, making the script unusable as
* the intended remediation path.
*/
return runAsSystem(async () => {
const results = {
dryRun,
scannedFiles: 0,
duplicateGroups: 0,
filesRenamed: 0,
indexBuilt: false,
errors: 0,
details: [],
};
results.scannedFiles = await File.countDocuments({ context: FileContext.execute_code });
logger.info(`Scanning ${results.scannedFiles} code-execution file(s) for duplicates`);
const groups = await File.aggregate([
{ $match: { context: FileContext.execute_code } },
{
$group: {
_id: {
filename: '$filename',
conversationId: '$conversationId',
tenantId: '$tenantId',
},
count: { $sum: 1 },
files: { $push: { _id: '$_id', file_id: '$file_id', createdAt: '$createdAt' } },
},
},
{ $match: { count: { $gt: 1 } } },
]).option({ batchSize });
results.duplicateGroups = groups.length;
for (const group of groups) {
try {
/* Newest first: it keeps the canonical name, older copies get suffixed. */
const ordered = [...group.files].sort(
(a, b) => new Date(b.createdAt ?? 0) - new Date(a.createdAt ?? 0),
);
const [, ...stale] = ordered;
/* Reserve every name already used in this uniqueness scope, so a
* suffixed replacement can't collide with an unrelated record that
* happens to be called `report (1).png` already. */
const scopeNames = await File.find(
{
context: FileContext.execute_code,
conversationId: group._id.conversationId ?? null,
tenantId: group._id.tenantId ?? null,
},
{ filename: 1, _id: 0 },
).lean();
const taken = new Set(scopeNames.map((file) => file.filename));
const renames = stale.map((file) => ({
file_id: file.file_id,
_id: file._id,
from: group._id.filename,
to: nextAvailableName(group._id.filename, taken),
}));
if (!dryRun) {
for (const rename of renames) {
await File.updateOne({ _id: rename._id }, { $set: { filename: rename.to } });
}
}
results.filesRenamed += renames.length;
if (results.details.length < DETAIL_SAMPLE_LIMIT) {
results.details.push({
filename: group._id.filename,
conversationId: group._id.conversationId,
count: group.count,
renames: renames.map(({ file_id, to }) => ({ file_id, to })),
});
}
} catch (error) {
results.errors++;
logger.error(
`Failed to normalize duplicates for "${group._id.filename}" in conversation ${group._id.conversationId}: ${error.message}`,
);
}
}
/*
* Build the index here rather than waiting for the next boot: the operator
* ran this to fix a failing build, so they should learn immediately whether
* it now succeeds. Targeted `createIndex` (not `syncIndexes`, which would
* drop indexes absent from the schema).
*/
if (!dryRun && results.errors === 0) {
try {
await File.collection.createIndex(INDEX_KEYS, { ...INDEX_OPTIONS, background: true });
results.indexBuilt = true;
} catch (error) {
results.errors++;
logger.error(
`Duplicates normalized but the unique index still failed to build: ${error.message}`,
);
}
}
logger.info('Code File Duplicate Migration completed', {
dryRun,
scannedFiles: results.scannedFiles,
duplicateGroups: results.duplicateGroups,
filesRenamed: results.filesRenamed,
indexBuilt: results.indexBuilt,
errors: results.errors,
});
return results;
});
}
if (require.main === module) {
const dryRun = process.argv.includes('--dry-run');
const batchSize =
parseInt(process.argv.find((arg) => arg.startsWith('--batch-size='))?.split('=')[1]) || 100;
migrateCodeFileDuplicates({ dryRun, batchSize })
.then((result) => {
console.log(`\n=== ${dryRun ? 'DRY RUN ' : ''}RESULTS ===`);
console.log(`Code files scanned: ${result.scannedFiles}`);
console.log(`Duplicate groups: ${result.duplicateGroups}`);
console.log(`Files ${dryRun ? 'to rename' : 'renamed'}: ${result.filesRenamed}`);
if (!dryRun && result.duplicateGroups > 0) {
console.log(`Unique index built: ${result.indexBuilt ? 'yes' : 'no'}`);
}
if (result.errors > 0) {
console.log(`Errors: ${result.errors}`);
}
if (result.details.length > 0) {
console.log('\nAffected files:');
result.details.forEach((d, i) => {
console.log(
` ${i + 1}. "${d.filename}" in ${d.conversationId}${d.count} copies, ${d.renames.length} renamed`,
);
d.renames.forEach((r) => console.log(` ${r.file_id} -> "${r.to}"`));
});
if (result.duplicateGroups > result.details.length) {
console.log(
` ... and ${result.duplicateGroups - result.details.length} more (sample capped at ${DETAIL_SAMPLE_LIMIT})`,
);
}
}
process.exit(0);
})
.catch((error) => {
console.error('Code file duplicate migration failed:', error);
process.exit(1);
});
}
module.exports = { migrateCodeFileDuplicates };

View file

@ -111,7 +111,10 @@
"migrate:shared-link-permissions:batch": "node config/migrate-shared-link-permissions.js --batch-size=50",
"migrate:orphaned-agent-files:dry-run": "node config/migrate-orphaned-agent-files.js --dry-run",
"migrate:orphaned-agent-files": "node config/migrate-orphaned-agent-files.js",
"migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50"
"migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50",
"migrate:code-file-duplicates:dry-run": "node config/migrate-code-file-duplicates.js --dry-run",
"migrate:code-file-duplicates": "node config/migrate-code-file-duplicates.js",
"migrate:code-file-duplicates:batch": "node config/migrate-code-file-duplicates.js --batch-size=50"
},
"repository": {
"type": "git",