diff --git a/config/__tests__/migrate-code-file-duplicates.spec.js b/config/__tests__/migrate-code-file-duplicates.spec.js new file mode 100644 index 0000000000..def867d6d1 --- /dev/null +++ b/config/__tests__/migrate-code-file-duplicates.spec.js @@ -0,0 +1,303 @@ +const mongoose = require('mongoose'); +const { v4: uuidv4 } = require('uuid'); +const { FileContext } = require('librechat-data-provider'); +const { logger } = require('@librechat/data-schemas'); +const { MongoMemoryServer } = require('mongodb-memory-server'); + +// Mock the config/connect module to prevent connection attempts during tests +jest.mock('../connect', () => jest.fn().mockResolvedValue(true)); + +// Disable console for tests +logger.silent = true; + +describe('Code File Duplicate Migration Script', () => { + let mongoServer; + let File; + let migrateCodeFileDuplicates; + + /** The unique partial index this migration exists to unblock. */ + const INDEX_KEYS = { filename: 1, conversationId: 1, context: 1, tenantId: 1 }; + const INDEX_NAME = 'filename_1_conversationId_1_context_1_tenantId_1'; + const INDEX_OPTIONS = { + unique: true, + partialFilterExpression: { context: FileContext.execute_code }, + }; + + /** + * Reproduces the state this migration is written for: legacy duplicates + * present and the unique index absent because its build failed. Mongoose + * builds schema indexes in the background at startup, so without dropping it + * here the fixtures would race an index the affected deployments don't have. + */ + async function dropUniqueIndex() { + await File.init().catch(() => { + /* the background build may itself fail — that IS the scenario */ + }); + await File.collection.dropIndex(INDEX_NAME).catch(() => { + /* already absent */ + }); + } + + async function createCodeFile({ filename, conversationId, createdAt, context }) { + return File.create({ + user: new mongoose.Types.ObjectId(), + file_id: uuidv4(), + filename, + filepath: `/images/user/${uuidv4()}.png`, + object: 'file', + type: 'image/png', + bytes: 1024, + conversationId, + context: context ?? FileContext.execute_code, + createdAt, + updatedAt: createdAt, + }); + } + + const namesFor = async (conversationId) => { + const files = await File.find({ conversationId }).lean(); + return files.map((file) => file.filename).sort(); + }; + + beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + await mongoose.connect(mongoServer.getUri()); + + const dbModels = require('~/db/models'); + File = dbModels.File; + + ({ migrateCodeFileDuplicates } = require('../migrate-code-file-duplicates')); + + await dropUniqueIndex(); + }); + + afterAll(async () => { + await mongoose.disconnect(); + await mongoServer.stop(); + }); + + afterEach(async () => { + await File.deleteMany({}); + await File.collection.dropIndex(INDEX_NAME).catch(() => { + /* the test never built it */ + }); + }); + + it('renames older duplicates and leaves the newest record canonical', async () => { + const conversationId = uuidv4(); + await createCodeFile({ + filename: 'bar_chart.png', + conversationId, + createdAt: new Date('2025-03-29T03:54:53Z'), + }); + await createCodeFile({ + filename: 'bar_chart.png', + conversationId, + createdAt: new Date('2025-03-29T03:56:32Z'), + }); + + const result = await migrateCodeFileDuplicates({ dryRun: false }); + + expect(result.duplicateGroups).toBe(1); + expect(result.filesRenamed).toBe(1); + /* Newest keeps the canonical name — the claim path's "latest write wins". */ + expect(await namesFor(conversationId)).toEqual(['bar_chart (1).png', 'bar_chart.png']); + const newest = await File.findOne({ filename: 'bar_chart.png' }).lean(); + expect(newest.createdAt).toEqual(new Date('2025-03-29T03:56:32Z')); + }); + + it('never deletes: every original record survives the rename', async () => { + const conversationId = uuidv4(); + const older = await createCodeFile({ + filename: 'plot.png', + conversationId, + createdAt: new Date('2025-01-01T00:00:00Z'), + }); + const newer = await createCodeFile({ + filename: 'plot.png', + conversationId, + createdAt: new Date('2025-01-01T00:05:00Z'), + }); + + await migrateCodeFileDuplicates({ dryRun: false }); + + /* Both file_ids still resolve — a deleted record would strip a real + * artifact out of the message attachment that references it. */ + expect(await File.countDocuments({})).toBe(2); + const kept = await File.findOne({ file_id: older.file_id }).lean(); + expect(kept).not.toBeNull(); + expect(kept.filepath).toBe(older.filepath); + expect((await File.findOne({ file_id: newer.file_id }).lean()).filename).toBe('plot.png'); + }); + + it('unblocks the unique partial index that could not build before', async () => { + const conversationId = uuidv4(); + await createCodeFile({ + filename: 'report.png', + conversationId, + createdAt: new Date('2025-02-01T00:00:00Z'), + }); + await createCodeFile({ + filename: 'report.png', + conversationId, + createdAt: new Date('2025-02-01T00:01:00Z'), + }); + + /* Precondition: the duplicates genuinely block the build (E11000). */ + await expect(File.collection.createIndex(INDEX_KEYS, INDEX_OPTIONS)).rejects.toThrow(); + + const result = await migrateCodeFileDuplicates({ dryRun: false }); + + expect(result.indexBuilt).toBe(true); + const indexes = await File.collection.indexes(); + expect( + indexes.some((index) => index.name === 'filename_1_conversationId_1_context_1_tenantId_1'), + ).toBe(true); + }); + + it('reports without writing in dry-run mode', async () => { + const conversationId = uuidv4(); + await createCodeFile({ + filename: 'chart.png', + conversationId, + createdAt: new Date('2025-04-01T00:00:00Z'), + }); + await createCodeFile({ + filename: 'chart.png', + conversationId, + createdAt: new Date('2025-04-01T00:02:00Z'), + }); + + const result = await migrateCodeFileDuplicates({ dryRun: true }); + + expect(result.filesRenamed).toBe(1); + expect(result.indexBuilt).toBe(false); + expect(await namesFor(conversationId)).toEqual(['chart.png', 'chart.png']); + }); + + it('skips names already taken in the conversation', async () => { + const conversationId = uuidv4(); + await createCodeFile({ + filename: 'out.png', + conversationId, + createdAt: new Date('2025-05-01T00:00:00Z'), + }); + await createCodeFile({ + filename: 'out.png', + conversationId, + createdAt: new Date('2025-05-01T00:01:00Z'), + }); + /* An unrelated record already occupies the first replacement name. */ + await createCodeFile({ + filename: 'out (1).png', + conversationId, + createdAt: new Date('2025-05-01T00:03:00Z'), + }); + + await migrateCodeFileDuplicates({ dryRun: false }); + + expect(await namesFor(conversationId)).toEqual(['out (1).png', 'out (2).png', 'out.png']); + }); + + it('resolves three copies into distinct names in one pass', async () => { + const conversationId = uuidv4(); + for (const minute of [0, 1, 2]) { + await createCodeFile({ + filename: 'fig.png', + conversationId, + createdAt: new Date(`2025-06-01T00:0${minute}:00Z`), + }); + } + + const result = await migrateCodeFileDuplicates({ dryRun: false }); + + expect(result.filesRenamed).toBe(2); + expect(await namesFor(conversationId)).toEqual(['fig (1).png', 'fig (2).png', 'fig.png']); + expect(result.indexBuilt).toBe(true); + }); + + it('leaves same-named files in DIFFERENT conversations alone', async () => { + const first = uuidv4(); + const second = uuidv4(); + await createCodeFile({ + filename: 'shared.png', + conversationId: first, + createdAt: new Date('2025-07-01T00:00:00Z'), + }); + await createCodeFile({ + filename: 'shared.png', + conversationId: second, + createdAt: new Date('2025-07-01T00:01:00Z'), + }); + + const result = await migrateCodeFileDuplicates({ dryRun: false }); + + expect(result.duplicateGroups).toBe(0); + expect(result.filesRenamed).toBe(0); + expect(await namesFor(first)).toEqual(['shared.png']); + expect(await namesFor(second)).toEqual(['shared.png']); + }); + + it('ignores duplicates outside the execute_code context', async () => { + const conversationId = uuidv4(); + await createCodeFile({ + filename: 'upload.png', + conversationId, + createdAt: new Date('2025-08-01T00:00:00Z'), + context: FileContext.message_attachment, + }); + await createCodeFile({ + filename: 'upload.png', + conversationId, + createdAt: new Date('2025-08-01T00:01:00Z'), + context: FileContext.message_attachment, + }); + + const result = await migrateCodeFileDuplicates({ dryRun: false }); + + /* The index is partial — only code outputs must be unique. */ + expect(result.duplicateGroups).toBe(0); + expect(await namesFor(conversationId)).toEqual(['upload.png', 'upload.png']); + }); + + it('is safe to re-run once each conversation is unique', async () => { + const conversationId = uuidv4(); + await createCodeFile({ + filename: 'idempotent.png', + conversationId, + createdAt: new Date('2025-09-01T00:00:00Z'), + }); + await createCodeFile({ + filename: 'idempotent.png', + conversationId, + createdAt: new Date('2025-09-01T00:01:00Z'), + }); + + await migrateCodeFileDuplicates({ dryRun: false }); + const afterFirst = await namesFor(conversationId); + + const second = await migrateCodeFileDuplicates({ dryRun: false }); + + expect(second.duplicateGroups).toBe(0); + expect(second.filesRenamed).toBe(0); + expect(await namesFor(conversationId)).toEqual(afterFirst); + }); + + it('handles filenames without an extension', async () => { + const conversationId = uuidv4(); + await createCodeFile({ + filename: 'Makefile', + conversationId, + createdAt: new Date('2025-10-01T00:00:00Z'), + }); + await createCodeFile({ + filename: 'Makefile', + conversationId, + createdAt: new Date('2025-10-01T00:01:00Z'), + }); + + await migrateCodeFileDuplicates({ dryRun: false }); + + expect(await namesFor(conversationId)).toEqual(['Makefile', 'Makefile (1)']); + }); +}); diff --git a/config/migrate-code-file-duplicates.js b/config/migrate-code-file-duplicates.js new file mode 100644 index 0000000000..4dd411e0b1 --- /dev/null +++ b/config/migrate-code-file-duplicates.js @@ -0,0 +1,236 @@ +const path = require('path'); +const { FileContext } = require('librechat-data-provider'); +const { logger, runAsSystem } = require('@librechat/data-schemas'); + +require('module-alias')({ base: path.resolve(__dirname, '..', 'api') }); +const connect = require('./connect'); + +const { File } = require('~/db/models'); + +/** + * Cap on the number of per-group entries retained in `results.details`. Larger + * runs still rename every affected record and still report accurate aggregate + * counts — we just stop accumulating sample data past this threshold to keep + * memory bounded on deployments with thousands of legacy duplicates. + */ +const DETAIL_SAMPLE_LIMIT = 50; + +/** Mirrors the unique partial index declared on the file schema. */ +const INDEX_KEYS = { filename: 1, conversationId: 1, context: 1, tenantId: 1 }; +const INDEX_OPTIONS = { + unique: true, + partialFilterExpression: { context: FileContext.execute_code }, +}; + +/** `report.png` -> `report (2).png`; a name without an extension keeps its shape. */ +function suffixFilename(filename, n) { + const extension = path.extname(filename); + const base = extension ? filename.slice(0, -extension.length) : filename; + return `${base} (${n})${extension}`; +} + +/** + * Picks a name that is free within the group's uniqueness scope. `taken` holds + * both the names already in the database and the ones handed out earlier in + * this run, so a group with several duplicates can't rename two records onto + * the same replacement. + */ +function nextAvailableName(filename, taken) { + for (let n = 1; ; n++) { + const candidate = suffixFilename(filename, n); + if (!taken.has(candidate)) { + taken.add(candidate); + return candidate; + } + } +} + +/** + * Normalizes code-execution output files that share a filename within one + * conversation, so the unique partial index on + * `(filename, conversationId, context, tenantId)` can finish building. + * + * That index arrived with atomic file claiming: a regenerated output now + * converges on ONE record with a cache-busted filepath. Records written before + * that change instead inserted a second document per regeneration, so any + * deployment that re-ran a code cell producing the same filename carries + * duplicates the index cannot span. Mongo then aborts the build with E11000 and + * the constraint is silently absent — the claim path keeps working, but without + * its database-level guarantee against concurrent inserts. + * + * Renames rather than deletes: every duplicate is a distinct stored object, + * usually still referenced by a message attachment, so removing one would + * strip a real artifact out of a user's history. The newest record keeps the + * canonical name (matching the "latest write wins" behavior of the claim path); + * older ones gain a ` (n)` suffix. Attachments carry their own filename copy, + * so rendered history is untouched. + * + * Safe to re-run — once each scope is unique, nothing is written. + * + * @param {{ dryRun?: boolean, batchSize?: number }} [options] + */ +async function migrateCodeFileDuplicates({ dryRun = true, batchSize = 100 } = {}) { + await connect(); + + logger.info('Starting Code File Duplicate Migration', { dryRun, batchSize }); + + /* + * Scan and heal across every tenant. Without this wrapper the tenant + * isolation plugin either scopes queries to a (non-existent) tenant or + * throws under TENANT_ISOLATION_STRICT=true, making the script unusable as + * the intended remediation path. + */ + return runAsSystem(async () => { + const results = { + dryRun, + scannedFiles: 0, + duplicateGroups: 0, + filesRenamed: 0, + indexBuilt: false, + errors: 0, + details: [], + }; + + results.scannedFiles = await File.countDocuments({ context: FileContext.execute_code }); + logger.info(`Scanning ${results.scannedFiles} code-execution file(s) for duplicates`); + + const groups = await File.aggregate([ + { $match: { context: FileContext.execute_code } }, + { + $group: { + _id: { + filename: '$filename', + conversationId: '$conversationId', + tenantId: '$tenantId', + }, + count: { $sum: 1 }, + files: { $push: { _id: '$_id', file_id: '$file_id', createdAt: '$createdAt' } }, + }, + }, + { $match: { count: { $gt: 1 } } }, + ]).option({ batchSize }); + + results.duplicateGroups = groups.length; + + for (const group of groups) { + try { + /* Newest first: it keeps the canonical name, older copies get suffixed. */ + const ordered = [...group.files].sort( + (a, b) => new Date(b.createdAt ?? 0) - new Date(a.createdAt ?? 0), + ); + const [, ...stale] = ordered; + + /* Reserve every name already used in this uniqueness scope, so a + * suffixed replacement can't collide with an unrelated record that + * happens to be called `report (1).png` already. */ + const scopeNames = await File.find( + { + context: FileContext.execute_code, + conversationId: group._id.conversationId ?? null, + tenantId: group._id.tenantId ?? null, + }, + { filename: 1, _id: 0 }, + ).lean(); + const taken = new Set(scopeNames.map((file) => file.filename)); + + const renames = stale.map((file) => ({ + file_id: file.file_id, + _id: file._id, + from: group._id.filename, + to: nextAvailableName(group._id.filename, taken), + })); + + if (!dryRun) { + for (const rename of renames) { + await File.updateOne({ _id: rename._id }, { $set: { filename: rename.to } }); + } + } + + results.filesRenamed += renames.length; + if (results.details.length < DETAIL_SAMPLE_LIMIT) { + results.details.push({ + filename: group._id.filename, + conversationId: group._id.conversationId, + count: group.count, + renames: renames.map(({ file_id, to }) => ({ file_id, to })), + }); + } + } catch (error) { + results.errors++; + logger.error( + `Failed to normalize duplicates for "${group._id.filename}" in conversation ${group._id.conversationId}: ${error.message}`, + ); + } + } + + /* + * Build the index here rather than waiting for the next boot: the operator + * ran this to fix a failing build, so they should learn immediately whether + * it now succeeds. Targeted `createIndex` (not `syncIndexes`, which would + * drop indexes absent from the schema). + */ + if (!dryRun && results.errors === 0) { + try { + await File.collection.createIndex(INDEX_KEYS, { ...INDEX_OPTIONS, background: true }); + results.indexBuilt = true; + } catch (error) { + results.errors++; + logger.error( + `Duplicates normalized but the unique index still failed to build: ${error.message}`, + ); + } + } + + logger.info('Code File Duplicate Migration completed', { + dryRun, + scannedFiles: results.scannedFiles, + duplicateGroups: results.duplicateGroups, + filesRenamed: results.filesRenamed, + indexBuilt: results.indexBuilt, + errors: results.errors, + }); + + return results; + }); +} + +if (require.main === module) { + const dryRun = process.argv.includes('--dry-run'); + const batchSize = + parseInt(process.argv.find((arg) => arg.startsWith('--batch-size='))?.split('=')[1]) || 100; + + migrateCodeFileDuplicates({ dryRun, batchSize }) + .then((result) => { + console.log(`\n=== ${dryRun ? 'DRY RUN ' : ''}RESULTS ===`); + console.log(`Code files scanned: ${result.scannedFiles}`); + console.log(`Duplicate groups: ${result.duplicateGroups}`); + console.log(`Files ${dryRun ? 'to rename' : 'renamed'}: ${result.filesRenamed}`); + if (!dryRun && result.duplicateGroups > 0) { + console.log(`Unique index built: ${result.indexBuilt ? 'yes' : 'no'}`); + } + if (result.errors > 0) { + console.log(`Errors: ${result.errors}`); + } + if (result.details.length > 0) { + console.log('\nAffected files:'); + result.details.forEach((d, i) => { + console.log( + ` ${i + 1}. "${d.filename}" in ${d.conversationId} — ${d.count} copies, ${d.renames.length} renamed`, + ); + d.renames.forEach((r) => console.log(` ${r.file_id} -> "${r.to}"`)); + }); + if (result.duplicateGroups > result.details.length) { + console.log( + ` ... and ${result.duplicateGroups - result.details.length} more (sample capped at ${DETAIL_SAMPLE_LIMIT})`, + ); + } + } + process.exit(0); + }) + .catch((error) => { + console.error('Code file duplicate migration failed:', error); + process.exit(1); + }); +} + +module.exports = { migrateCodeFileDuplicates }; diff --git a/package.json b/package.json index 8b836380e7..5fb6c358a3 100644 --- a/package.json +++ b/package.json @@ -111,7 +111,10 @@ "migrate:shared-link-permissions:batch": "node config/migrate-shared-link-permissions.js --batch-size=50", "migrate:orphaned-agent-files:dry-run": "node config/migrate-orphaned-agent-files.js --dry-run", "migrate:orphaned-agent-files": "node config/migrate-orphaned-agent-files.js", - "migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50" + "migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50", + "migrate:code-file-duplicates:dry-run": "node config/migrate-code-file-duplicates.js --dry-run", + "migrate:code-file-duplicates": "node config/migrate-code-file-duplicates.js", + "migrate:code-file-duplicates:batch": "node config/migrate-code-file-duplicates.js --batch-size=50" }, "repository": { "type": "git",