From 94fed5bcd532111161e5408bf3f86632b8ebf8db Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 02:59:25 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=A5=A1=20fix:=20Delete=20Storage=20and=20?= =?UTF-8?q?Embeddings=20With=20Owned=20Agent=20Files=20(#16007)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent tool-resource branch of DELETE /files removed a file's association from the agent and answered 200 without reaching processDeleteRequest, so the bytes on disk and the RAG chunks of a file the caller owned survived the delete. A replaced File Search document kept answering from its old chunks (#15930). An attached file now goes through the full delete pass, but only when nothing else holds it: a file the caller does not own, and a file any other (agent, tool_resource) pair still references, are unlinked and left whole. Nothing is unlinked until the destroy it depends on has succeeded. - packages/api/src/files/deletion.ts: deleteAgentResourceFiles owns the operation with its db methods and delete pass injected. - packages/data-schemas: getSharedResourceFileIds answers the last-reference question, excluding the pair being removed by the agent's _id. - Local/crud.js: unlinkFile propagates non-ENOENT, so a file whose bytes survive is no longer reported as deleted. Co-authored-by: Mohammed Alshyakh --- api/server/routes/files/files.js | 37 ++- api/server/routes/files/files.test.js | 239 +++++++++++++++++ .../Files/Local/__tests__/crud-delete.spec.js | 70 +++++ api/server/services/Files/Local/crud.js | 10 + api/server/services/Files/process.js | 20 +- api/server/services/Files/process.spec.js | 83 ++++++ packages/api/src/files/deletion.spec.ts | 247 +++++++++++++++++- packages/api/src/files/deletion.ts | 172 ++++++++++++ .../data-schemas/src/methods/agent.spec.ts | 142 ++++++++++ packages/data-schemas/src/methods/agent.ts | 72 +++++ 10 files changed, 1064 insertions(+), 28 deletions(-) create mode 100644 api/server/services/Files/Local/__tests__/crud-delete.spec.js diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js index de84c67c95..f6b97e587f 100644 --- a/api/server/routes/files/files.js +++ b/api/server/routes/files/files.js @@ -8,6 +8,7 @@ const { refreshS3FileUrls, handleFilesUsageRequest, buildDeleteFilesResponse, + deleteAgentResourceFiles, shouldUseUploadSse, startUploadSseStream, sendUploadPolicyError, @@ -245,20 +246,36 @@ router.delete('/', async (req, res) => { }); } - const toolResourceFiles = agent.tool_resources?.[req.body.tool_resource]?.file_ids ?? []; - const agentFiles = files - .filter((f) => toolResourceFiles.includes(f.file_id)) - .map((file) => ({ tool_resource: req.body.tool_resource, file_id: file.file_id })); - if (agentFiles.length === 0) { + const agentDeletion = await deleteAgentResourceFiles( + { + agentId: req.body.agent_id, + agentObjectId: agent._id.toString(), + toolResource: req.body.tool_resource, + requestedFileIds: fileIds, + attachedFileIds: agent.tool_resources?.[req.body.tool_resource]?.file_ids ?? [], + files: dbFiles.map((file) => ({ + file_id: file.file_id, + owner: file.user?.toString() ?? null, + file, + })), + userId: req.user.id.toString(), + }, + { + getSharedResourceFileIds: db.getSharedResourceFileIds, + removeAgentResourceFiles: db.removeAgentResourceFiles, + deleteFiles: (agentFiles) => processDeleteRequest({ req, files: agentFiles }), + }, + ); + + if (agentDeletion.outcome == null) { res.status(200).json({ message: 'File associations removed successfully from agent' }); return; } - await db.removeAgentResourceFiles({ - agent_id: req.body.agent_id, - files: agentFiles, - }); - res.status(200).json({ message: 'File associations removed successfully from agent' }); + logger.debug( + `[/files] Agent files deleted successfully: ${agentDeletion.destroyedFileIds.join(', ')}`, + ); + sendDeleteResult(agentDeletion.outcome, 'Files deleted successfully'); return; } diff --git a/api/server/routes/files/files.test.js b/api/server/routes/files/files.test.js index 2ad58d350d..9df2acb062 100644 --- a/api/server/routes/files/files.test.js +++ b/api/server/routes/files/files.test.js @@ -327,6 +327,245 @@ describe('File Routes - Delete with Agent Access', () => { expect(updatedAgent.tool_resources.file_search.file_ids).toEqual([]); }); + it('deletes storage and embeddings for an attached file the caller owns', async () => { + const ownedFileId = uuidv4(); + await createFile({ + user: otherUserId, + file_id: ownedFileId, + filename: 'owned-knowledge.txt', + filepath: '/uploads/owned-knowledge.txt', + bytes: 100, + type: 'text/plain', + source: FileSources.vectordb, + embedded: true, + }); + + const agent = await createAgent({ + id: uuidv4(), + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: otherUserId, + tool_resources: { + file_search: { + file_ids: [ownedFileId], + }, + }, + }); + + const response = await request(app) + .delete('/files') + .send({ + agent_id: agent.id, + tool_resource: 'file_search', + files: [{ file_id: ownedFileId, filepath: '/uploads/owned-knowledge.txt' }], + }); + + expect(response.status).toBe(200); + expect(response.body.message).toBe('Files deleted successfully'); + expect(processDeleteRequest).toHaveBeenCalledTimes(1); + + const [{ req, files: deletedFiles }] = processDeleteRequest.mock.calls[0]; + expect(deletedFiles.map((file) => file.file_id)).toEqual([ownedFileId]); + expect(deletedFiles[0].source).toBe(FileSources.vectordb); + expect(req.body.agent_id).toBe(agent.id); + expect(req.body.tool_resource).toBe('file_search'); + }); + + it('unlinks another user’s attached file while deleting the caller’s own', async () => { + const ownedFileId = uuidv4(); + await createFile({ + user: otherUserId, + file_id: ownedFileId, + filename: 'owned-knowledge.txt', + filepath: '/uploads/owned-knowledge.txt', + bytes: 100, + type: 'text/plain', + source: FileSources.vectordb, + embedded: true, + }); + + const agent = await createAgent({ + id: uuidv4(), + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: otherUserId, + tool_resources: { + file_search: { + file_ids: [ownedFileId, fileId], + }, + }, + }); + + const response = await request(app) + .delete('/files') + .send({ + agent_id: agent.id, + tool_resource: 'file_search', + files: [ + { file_id: ownedFileId, filepath: '/uploads/owned-knowledge.txt' }, + { file_id: fileId, filepath: '/uploads/test.txt' }, + ], + }); + + expect(response.status).toBe(200); + expect(response.body.message).toBe('Files deleted successfully'); + + const [{ files: deletedFiles }] = processDeleteRequest.mock.calls[0]; + expect(deletedFiles.map((file) => file.file_id)).toEqual([ownedFileId]); + + const updatedAgent = await Agent.findOne({ id: agent.id }).lean(); + expect(updatedAgent.tool_resources.file_search.file_ids).toEqual([ownedFileId]); + + const retainedFile = await File.findOne({ file_id: fileId }).lean(); + expect(retainedFile).toBeTruthy(); + }); + + it('keeps a file the same agent holds under another tool resource', async () => { + const sharedFileId = uuidv4(); + await createFile({ + user: otherUserId, + file_id: sharedFileId, + filename: 'dual-purpose.txt', + filepath: '/uploads/dual-purpose.txt', + bytes: 100, + type: 'text/plain', + source: FileSources.vectordb, + embedded: true, + }); + + /* One agent can hold the same file under two resources, so the reference being removed is the + `(agent, tool_resource)` pair rather than the agent. */ + const agent = await createAgent({ + id: uuidv4(), + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: otherUserId, + tool_resources: { + file_search: { file_ids: [sharedFileId] }, + context: { file_ids: [sharedFileId] }, + }, + }); + + const response = await request(app) + .delete('/files') + .send({ + agent_id: agent.id, + tool_resource: 'file_search', + files: [{ file_id: sharedFileId, filepath: '/uploads/dual-purpose.txt' }], + }); + + expect(response.status).toBe(200); + expect(response.body.message).toBe('File associations removed successfully from agent'); + expect(processDeleteRequest).not.toHaveBeenCalled(); + + const updatedAgent = await Agent.findOne({ id: agent.id }).lean(); + expect(updatedAgent.tool_resources.file_search.file_ids).toEqual([]); + expect(updatedAgent.tool_resources.context.file_ids).toEqual([sharedFileId]); + + const retainedFile = await File.findOne({ file_id: sharedFileId }).lean(); + expect(retainedFile).toBeTruthy(); + }); + + it('keeps a file a duplicated agent still references, unlinking it here only', async () => { + const sharedFileId = uuidv4(); + await createFile({ + user: otherUserId, + file_id: sharedFileId, + filename: 'shared-knowledge.txt', + filepath: '/uploads/shared-knowledge.txt', + bytes: 100, + type: 'text/plain', + source: FileSources.vectordb, + embedded: true, + }); + + const agent = await createAgent({ + id: uuidv4(), + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: otherUserId, + tool_resources: { file_search: { file_ids: [sharedFileId] } }, + }); + + /* Duplicating an agent copies file_ids rather than the files behind them, and lands them + under `context`, so the second holder is found across tool resources. */ + const duplicate = await createAgent({ + id: uuidv4(), + name: 'Test Agent (copy)', + provider: 'openai', + model: 'gpt-4', + author: otherUserId, + tool_resources: { context: { file_ids: [sharedFileId] } }, + }); + + const response = await request(app) + .delete('/files') + .send({ + agent_id: agent.id, + tool_resource: 'file_search', + files: [{ file_id: sharedFileId, filepath: '/uploads/shared-knowledge.txt' }], + }); + + expect(response.status).toBe(200); + expect(response.body.message).toBe('File associations removed successfully from agent'); + expect(processDeleteRequest).not.toHaveBeenCalled(); + + const updatedAgent = await Agent.findOne({ id: agent.id }).lean(); + expect(updatedAgent.tool_resources.file_search.file_ids).toEqual([]); + + const untouchedDuplicate = await Agent.findOne({ id: duplicate.id }).lean(); + expect(untouchedDuplicate.tool_resources.context.file_ids).toEqual([sharedFileId]); + + const retainedFile = await File.findOne({ file_id: sharedFileId }).lean(); + expect(retainedFile).toBeTruthy(); + }); + + it('leaves an owned file alone when the tool resource does not hold it', async () => { + const ownedFileId = uuidv4(); + await createFile({ + user: otherUserId, + file_id: ownedFileId, + filename: 'detached-knowledge.txt', + filepath: '/uploads/detached-knowledge.txt', + bytes: 100, + type: 'text/plain', + source: FileSources.vectordb, + embedded: true, + }); + + const agent = await createAgent({ + id: uuidv4(), + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: otherUserId, + tool_resources: { + file_search: { + file_ids: [fileId], + }, + }, + }); + + const response = await request(app) + .delete('/files') + .send({ + agent_id: agent.id, + tool_resource: 'file_search', + files: [{ file_id: ownedFileId, filepath: '/uploads/detached-knowledge.txt' }], + }); + + expect(response.status).toBe(200); + expect(response.body.message).toBe('File associations removed successfully from agent'); + expect(processDeleteRequest).not.toHaveBeenCalled(); + + const updatedAgent = await Agent.findOne({ id: agent.id }).lean(); + expect(updatedAgent.tool_resources.file_search.file_ids).toEqual([fileId]); + }); + it('rejects invalid agent tool_resource values before unlinking', async () => { const agent = await createAgent({ id: uuidv4(), diff --git a/api/server/services/Files/Local/__tests__/crud-delete.spec.js b/api/server/services/Files/Local/__tests__/crud-delete.spec.js new file mode 100644 index 0000000000..7a7a30340a --- /dev/null +++ b/api/server/services/Files/Local/__tests__/crud-delete.spec.js @@ -0,0 +1,70 @@ +/** Exactly what `deleteLocalFile` reaches for, so the double does not depend on a built package. */ +jest.mock('@librechat/api', () => ({ + deleteRagFile: jest.fn().mockResolvedValue(undefined), + stripCacheBust: (filepath) => String(filepath).split('?')[0], +})); +jest.mock('@librechat/data-schemas', () => ({ + logger: { warn: jest.fn(), error: jest.fn() }, +})); + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { deleteLocalFile } = require('../crud'); + +/* The resolved promise of a delete is what `processDeleteRequest` reads to decide that a record may + lose its metadata and its agent references, so this adapter may only resolve once the bytes are + actually gone. Storage that was already missing is the one benign case. */ +describe('deleteLocalFile failure reporting', () => { + const userId = 'user-1'; + let tmpBase; + let req; + + beforeEach(() => { + jest.restoreAllMocks(); + tmpBase = fs.mkdtempSync(path.join(os.tmpdir(), 'crud-delete-')); + fs.mkdirSync(path.join(tmpBase, 'uploads', userId), { recursive: true }); + req = { + user: { id: userId }, + config: { + paths: { + publicPath: path.join(tmpBase, 'public'), + uploads: path.join(tmpBase, 'uploads'), + }, + }, + }; + }); + + afterEach(() => { + fs.rmSync(tmpBase, { recursive: true, force: true }); + }); + + const uploadedFile = (filename) => { + const filepath = path.join(tmpBase, 'uploads', userId, filename); + fs.writeFileSync(filepath, 'contents'); + return { file_id: 'file-1', filepath: `/uploads/${userId}/${filename}` }; + }; + + it('removes the file and resolves', async () => { + const file = uploadedFile('knowledge.txt'); + + await expect(deleteLocalFile(req, file)).resolves.toBeUndefined(); + expect(fs.existsSync(path.join(tmpBase, 'uploads', userId, 'knowledge.txt'))).toBe(false); + }); + + it('resolves when the file is already gone', async () => { + await expect( + deleteLocalFile(req, { file_id: 'file-1', filepath: `/uploads/${userId}/missing.txt` }), + ).resolves.toBeUndefined(); + }); + + it('rejects when the bytes survive the delete', async () => { + const file = uploadedFile('locked.txt'); + jest + .spyOn(fs.promises, 'unlink') + .mockRejectedValue(Object.assign(new Error('permission denied'), { code: 'EACCES' })); + + await expect(deleteLocalFile(req, file)).rejects.toThrow('permission denied'); + expect(fs.existsSync(path.join(tmpBase, 'uploads', userId, 'locked.txt'))).toBe(true); + }); +}); diff --git a/api/server/services/Files/Local/crud.js b/api/server/services/Files/Local/crud.js index fe52455098..9555df49c0 100644 --- a/api/server/services/Files/Local/crud.js +++ b/api/server/services/Files/Local/crud.js @@ -208,11 +208,21 @@ const isValidPath = (req, base, subfolder, filepath) => { /** * @param {string} filepath */ +/** + * A file whose bytes are still on disk must not be reported as deleted: callers use the resolved + * promise to decide that a record may lose its metadata and its agent references. Storage that was + * already gone is the one benign case, and `processDeleteRequest` treats it as deleted by design. + */ const unlinkFile = async (filepath) => { try { await fs.promises.unlink(filepath); } catch (error) { + if (error?.code === 'ENOENT') { + logger.warn('Local file was already missing during delete:', error); + return; + } logger.error('Error deleting file:', error); + throw error; } }; diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 05b02ae6df..35117ab248 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -265,16 +265,8 @@ const processDeleteRequest = async ({ req, files }) => { await initializeClients(); } - const agentFiles = []; - for (const file of files) { const source = file.source ?? FileSources.local; - if (req.body.agent_id && req.body.tool_resource) { - agentFiles.push({ - tool_resource: req.body.tool_resource, - file_id: file.file_id, - }); - } if (source === FileSources.text) { resolvedFileIds.add(file.file_id); @@ -313,15 +305,6 @@ const processDeleteRequest = async ({ req, files }) => { }); } - if (agentFiles.length > 0) { - promises.push( - db.removeAgentResourceFiles({ - agent_id: req.body.agent_id, - files: agentFiles, - }), - ); - } - await Promise.allSettled(promises); const deletedFileIds = [...resolvedFileIds]; let metadataDeletedFileIds = deletedFileIds; @@ -334,6 +317,9 @@ const processDeleteRequest = async ({ req, files }) => { metadataDeletedFileIds = []; throw error; } + /* The only place a delete removes agent references, and it runs after the metadata delete + succeeded: a file that kept its storage, its chunks or its record keeps its references too, + so the agent it was removed from can be asked again (see issue #12776). */ if (metadataDeletedFileIds.length > 0) { try { await db.removeAgentResourceFilesFromAllAgents({ file_ids: metadataDeletedFileIds }); diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index efbb8a1b00..33ec48b786 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -2610,6 +2610,89 @@ describe('processDeleteRequest', () => { expect(result).toEqual({ deletedFileIds: [], failedFileIds: ['embedded-file'] }); }); + it('keeps a failed agent file attached so the delete can be retried', async () => { + const deleteFile = jest.fn().mockRejectedValue(new Error('rag unavailable')); + getStrategyFunctions.mockReturnValue({ deleteFile }); + const req = { + body: { agent_id: 'agent_1', tool_resource: 'file_search' }, + config: {}, + user: { id: 'user-123', tenantId: 'tenant-a' }, + }; + + const result = await processDeleteRequest({ + req, + files: [ + { + file_id: 'knowledge-file', + filepath: '/uploads/knowledge.txt', + source: FileSources.local, + }, + ], + }); + + expect(result).toEqual({ deletedFileIds: [], failedFileIds: ['knowledge-file'] }); + expect(db.deleteFiles).not.toHaveBeenCalled(); + expect(db.removeAgentResourceFiles).not.toHaveBeenCalled(); + expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled(); + }); + + it('strips agent references only for the files it deleted', async () => { + getStrategyFunctions.mockReturnValue({ + deleteFile: jest + .fn() + .mockImplementation((_req, file) => + file.file_id === 'kept-file' + ? Promise.reject(new Error('rag unavailable')) + : Promise.resolve(undefined), + ), + }); + db.deleteFiles.mockResolvedValue({ deletedCount: 1 }); + const req = { + body: { agent_id: 'agent_1', tool_resource: 'file_search' }, + config: {}, + user: { id: 'user-123', tenantId: 'tenant-a' }, + }; + + const result = await processDeleteRequest({ + req, + files: [ + { file_id: 'gone-file', filepath: '/uploads/gone.txt', source: FileSources.local }, + { file_id: 'kept-file', filepath: '/uploads/kept.txt', source: FileSources.local }, + ], + }); + + expect(result).toEqual({ deletedFileIds: ['gone-file'], failedFileIds: ['kept-file'] }); + expect(db.removeAgentResourceFilesFromAllAgents).toHaveBeenCalledWith({ + file_ids: ['gone-file'], + }); + }); + + it('keeps agent references when the metadata delete fails', async () => { + getStrategyFunctions.mockReturnValue({ deleteFile: jest.fn().mockResolvedValue(undefined) }); + db.deleteFiles.mockRejectedValue(new Error('mongo unavailable')); + const req = { + body: { agent_id: 'agent_1', tool_resource: 'file_search' }, + config: {}, + user: { id: 'user-123', tenantId: 'tenant-a' }, + }; + + await expect( + processDeleteRequest({ + req, + files: [ + { + file_id: 'knowledge-file', + filepath: '/uploads/knowledge.txt', + source: FileSources.local, + }, + ], + }), + ).rejects.toThrow('mongo unavailable'); + + expect(db.removeAgentResourceFiles).not.toHaveBeenCalled(); + expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled(); + }); + it('does not delete vector storage when primary embedded file deletion fails', async () => { const primaryDelete = jest.fn().mockRejectedValue(new Error('permission denied')); const vectorDelete = jest.fn().mockResolvedValue(undefined); diff --git a/packages/api/src/files/deletion.spec.ts b/packages/api/src/files/deletion.spec.ts index 409c1d2f82..86bbebdea0 100644 --- a/packages/api/src/files/deletion.spec.ts +++ b/packages/api/src/files/deletion.spec.ts @@ -1,4 +1,10 @@ -import { buildDeleteFilesResponse, PARTIAL_FILE_DELETION_MESSAGE } from './deletion'; +import type { AgentResourceFileInput } from './deletion'; +import { + buildDeleteFilesResponse, + deleteAgentResourceFiles, + partitionAgentResourceFiles, + PARTIAL_FILE_DELETION_MESSAGE, +} from './deletion'; describe('delete files response', () => { it('reports the caller’s success message when nothing failed', () => { @@ -32,3 +38,242 @@ describe('delete files response', () => { }); }); }); + +type TestFile = { file_id: string; filename: string }; + +const input = (file_id: string, owner: string | null): AgentResourceFileInput => ({ + file_id, + owner, + file: { file_id, filename: `${file_id}.txt` }, +}); + +describe('partition agent resource files', () => { + const userId = 'user-1'; + const owned = input('owned', userId); + const foreign = input('foreign', 'user-2'); + + it('makes an attached file the caller owns a candidate for the delete pass', () => { + expect( + partitionAgentResourceFiles({ + requestedFileIds: ['owned'], + attachedFileIds: ['owned'], + files: [owned], + toolResource: 'file_search', + userId, + }), + ).toEqual({ ownedFiles: [owned], unlinkOnlyFiles: [] }); + }); + + it('only unlinks an attached file owned by someone else', () => { + expect( + partitionAgentResourceFiles({ + requestedFileIds: ['foreign'], + attachedFileIds: ['foreign'], + files: [foreign], + toolResource: 'file_search', + userId, + }), + ).toEqual({ + ownedFiles: [], + unlinkOnlyFiles: [{ tool_resource: 'file_search', file_id: 'foreign' }], + }); + }); + + it('only unlinks an attached file whose metadata record is gone', () => { + expect( + partitionAgentResourceFiles({ + requestedFileIds: ['missing'], + attachedFileIds: ['missing'], + files: [], + toolResource: 'file_search', + userId, + }), + ).toEqual({ + ownedFiles: [], + unlinkOnlyFiles: [{ tool_resource: 'file_search', file_id: 'missing' }], + }); + }); + + it('does not treat an ownerless record as the caller’s', () => { + expect( + partitionAgentResourceFiles({ + requestedFileIds: ['orphan'], + attachedFileIds: ['orphan'], + files: [input('orphan', null)], + toolResource: 'file_search', + userId, + }), + ).toEqual({ + ownedFiles: [], + unlinkOnlyFiles: [{ tool_resource: 'file_search', file_id: 'orphan' }], + }); + }); + + it('ignores files the tool resource does not hold', () => { + expect( + partitionAgentResourceFiles({ + requestedFileIds: ['owned', 'elsewhere'], + attachedFileIds: ['owned'], + files: [owned, input('elsewhere', userId)], + toolResource: 'file_search', + userId, + }), + ).toEqual({ ownedFiles: [owned], unlinkOnlyFiles: [] }); + }); + + it('splits a mixed request and counts each file once', () => { + expect( + partitionAgentResourceFiles({ + requestedFileIds: ['owned', 'foreign', 'owned'], + attachedFileIds: ['owned', 'foreign'], + files: [owned, foreign], + toolResource: 'ocr', + userId, + }), + ).toEqual({ + ownedFiles: [owned], + unlinkOnlyFiles: [{ tool_resource: 'ocr', file_id: 'foreign' }], + }); + }); +}); + +describe('delete agent resource files', () => { + const userId = 'user-1'; + + const makeDeps = (sharedFileIds: string[] = [], calls: string[] = []) => ({ + getSharedResourceFileIds: jest.fn().mockResolvedValue(sharedFileIds), + removeAgentResourceFiles: jest.fn().mockImplementation(() => { + calls.push('unlink'); + return Promise.resolve(undefined); + }), + deleteFiles: jest.fn().mockImplementation((files: TestFile[]) => { + calls.push('destroy'); + return Promise.resolve({ + deletedFileIds: files.map((file) => file.file_id), + failedFileIds: [], + }); + }), + }); + + const run = ( + files: Array>, + deps: ReturnType, + attachedFileIds?: string[], + ) => + deleteAgentResourceFiles( + { + agentId: 'agent_1', + agentObjectId: '65f000000000000000000001', + toolResource: 'file_search', + requestedFileIds: files.map((file) => file.file_id), + attachedFileIds: attachedFileIds ?? files.map((file) => file.file_id), + files, + userId, + }, + deps, + ); + + it('destroys a file this agent was the last to reference', async () => { + const deps = makeDeps(); + const result = await run([input('owned', userId)], deps); + + expect(deps.deleteFiles).toHaveBeenCalledWith([{ file_id: 'owned', filename: 'owned.txt' }]); + expect(deps.removeAgentResourceFiles).not.toHaveBeenCalled(); + expect(result).toEqual({ + outcome: { deletedFileIds: ['owned'], failedFileIds: [] }, + unlinkedFileIds: [], + destroyedFileIds: ['owned'], + }); + }); + + it('excludes the agent by a globally unique identity, not its logical id', async () => { + const deps = makeDeps(); + await run([input('owned', userId)], deps); + + expect(deps.getSharedResourceFileIds).toHaveBeenCalledWith({ + file_ids: ['owned'], + excludeAgentObjectId: '65f000000000000000000001', + excludeToolResource: 'file_search', + }); + }); + + it('keeps a file another agent still references, unlinking it here only', async () => { + const deps = makeDeps(['shared']); + const result = await run([input('shared', userId)], deps); + + expect(deps.deleteFiles).not.toHaveBeenCalled(); + expect(deps.removeAgentResourceFiles).toHaveBeenCalledWith({ + agent_id: 'agent_1', + files: [{ tool_resource: 'file_search', file_id: 'shared' }], + }); + expect(result).toEqual({ + outcome: null, + unlinkedFileIds: ['shared'], + destroyedFileIds: [], + }); + }); + + it('destroys the last-reference file and unlinks the shared one in the same request', async () => { + const deps = makeDeps(['shared']); + const result = await run([input('shared', userId), input('owned', userId)], deps); + + expect(deps.deleteFiles).toHaveBeenCalledWith([{ file_id: 'owned', filename: 'owned.txt' }]); + expect(deps.removeAgentResourceFiles).toHaveBeenCalledWith({ + agent_id: 'agent_1', + files: [{ tool_resource: 'file_search', file_id: 'shared' }], + }); + expect(result.unlinkedFileIds).toEqual(['shared']); + expect(result.destroyedFileIds).toEqual(['owned']); + }); + + it('unlinks a file the caller does not own without asking to destroy it', async () => { + const deps = makeDeps(); + const result = await run([input('foreign', 'user-2')], deps); + + expect(deps.getSharedResourceFileIds).not.toHaveBeenCalled(); + expect(deps.deleteFiles).not.toHaveBeenCalled(); + expect(deps.removeAgentResourceFiles).toHaveBeenCalledWith({ + agent_id: 'agent_1', + files: [{ tool_resource: 'file_search', file_id: 'foreign' }], + }); + expect(result.outcome).toBeNull(); + }); + + it('touches nothing when the tool resource holds none of the requested files', async () => { + const deps = makeDeps(); + const result = await run([input('owned', userId)], deps, []); + + expect(deps.getSharedResourceFileIds).not.toHaveBeenCalled(); + expect(deps.removeAgentResourceFiles).not.toHaveBeenCalled(); + expect(deps.deleteFiles).not.toHaveBeenCalled(); + expect(result).toEqual({ outcome: null, unlinkedFileIds: [], destroyedFileIds: [] }); + }); + + it('destroys before removing any reference, so a failed destroy leaves the file attached', async () => { + const calls: string[] = []; + const deps = makeDeps(['shared'], calls); + await run([input('shared', userId), input('owned', userId)], deps); + + expect(calls).toEqual(['destroy', 'unlink']); + }); + + it('reports what the delete pass deleted rather than what it was handed', async () => { + const deps = makeDeps(); + deps.deleteFiles.mockResolvedValue({ + deletedFileIds: ['owned'], + failedFileIds: ['other-owned'], + }); + const result = await run([input('owned', userId), input('other-owned', userId)], deps); + + expect(result.destroyedFileIds).toEqual(['owned']); + }); + + it('passes a partial delete outcome back to the caller', async () => { + const deps = makeDeps(); + deps.deleteFiles.mockResolvedValue({ deletedFileIds: [], failedFileIds: ['owned'] }); + const result = await run([input('owned', userId)], deps); + + expect(result.outcome).toEqual({ deletedFileIds: [], failedFileIds: ['owned'] }); + expect(deps.removeAgentResourceFiles).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/files/deletion.ts b/packages/api/src/files/deletion.ts index 92f176a23f..c320814063 100644 --- a/packages/api/src/files/deletion.ts +++ b/packages/api/src/files/deletion.ts @@ -6,6 +6,52 @@ export type FileDeletionOutcome = { failedFileIds?: string[]; }; +/** The `{ tool_resource, file_id }` pair an agent resource unlink takes. */ +export type AgentResourceFileRef = { + tool_resource: string; + file_id: string; +}; + +/** + * One requested file, as this module needs to see it: an id, an owner already normalized to a + * plain string by the caller, and the opaque record the delete pass will be handed. Ownership + * arrives as a string so no storage type crosses into this package. + */ +export type AgentResourceFileInput = { + file_id: string; + owner: string | null; + file: TFile; +}; + +export type AgentResourceDeletion = { + /** Attached files the caller owns: candidates for storage, vector and metadata deletion. */ + ownedFiles: Array>; + /** Attached files owned by someone else, or with no metadata left: unlink the association only. */ + unlinkOnlyFiles: AgentResourceFileRef[]; +}; + +export type AgentResourceDeletionDeps = { + /** Which of these ids another agent still references; a shared file is never destroyed. */ + getSharedResourceFileIds: (params: { + file_ids: string[]; + excludeAgentObjectId: string; + excludeToolResource: string; + }) => Promise; + removeAgentResourceFiles: (params: { + agent_id: string; + files: AgentResourceFileRef[]; + }) => Promise; + /** The full delete pass, injected: storage, vectors, metadata, and the unlink of what it deleted. */ + deleteFiles: (files: TFile[]) => Promise; +}; + +export type AgentResourceDeletionResult = { + /** `null` when nothing was destroyed, so the caller answers with the unlink message. */ + outcome: FileDeletionOutcome | null; + unlinkedFileIds: string[]; + destroyedFileIds: string[]; +}; + export const PARTIAL_FILE_DELETION_MESSAGE = 'Some files could not be deleted'; /** @@ -25,3 +71,129 @@ export const buildDeleteFilesResponse = ( failedFileIds, }; }; + +/** + * Splits the files a delete request names for one agent tool resource into the ones whose storage + * and embeddings may go with the unlink, and the ones only the association can be removed for. + * + * A file the caller owns is theirs to destroy, so it becomes a candidate for the full delete pass; + * a file that belongs to another user, or that no longer has a metadata record, keeps its bytes and + * its chunks and loses only its link to the agent. A file the request names but the tool resource + * does not hold appears in neither list. + */ +export const partitionAgentResourceFiles = ({ + requestedFileIds, + attachedFileIds, + files, + toolResource, + userId, +}: { + requestedFileIds: string[]; + attachedFileIds: string[]; + files: Array>; + toolResource: string; + userId: string; +}): AgentResourceDeletion => { + const attached = new Set(attachedFileIds); + const inputsById = new Map(files.map((input) => [input.file_id, input])); + const seen = new Set(); + const ownedFiles: Array> = []; + const unlinkOnlyFiles: AgentResourceFileRef[] = []; + + for (const fileId of requestedFileIds) { + if (!attached.has(fileId) || seen.has(fileId)) { + continue; + } + seen.add(fileId); + const input = inputsById.get(fileId); + if (input != null && input.owner === userId) { + ownedFiles.push(input); + continue; + } + unlinkOnlyFiles.push({ tool_resource: toolResource, file_id: fileId }); + } + + return { ownedFiles, unlinkOnlyFiles }; +}; + +/** + * Removes files from one agent tool resource, destroying only what this agent was the last holder + * of. + * + * Three outcomes, decided per file. A file the caller does not own is unlinked and left whole. A + * file the caller owns that any other `(agent, tool_resource)` pair still references is unlinked here + * and left whole too: + * duplicating an agent copies `file_ids` rather than the files behind them, so destroying the bytes + * would empty the other agent's knowledge without touching its configuration. Only a file the + * caller owns and this agent was the last to reference goes through the delete pass, which removes + * the storage, the vector chunks and the metadata, and unlinks exactly what it managed to delete. + */ +export const deleteAgentResourceFiles = async ( + { + agentId, + agentObjectId, + toolResource, + requestedFileIds, + attachedFileIds, + files, + userId, + }: { + agentId: string; + /** The agent's globally unique `_id`; `id` alone repeats across tenants. */ + agentObjectId: string; + toolResource: string; + requestedFileIds: string[]; + attachedFileIds: string[]; + files: Array>; + userId: string; + }, + deps: AgentResourceDeletionDeps, +): Promise => { + const { ownedFiles, unlinkOnlyFiles } = partitionAgentResourceFiles({ + requestedFileIds, + attachedFileIds, + files, + toolResource, + userId, + }); + + const sharedFileIds = + ownedFiles.length === 0 + ? new Set() + : new Set( + await deps.getSharedResourceFileIds({ + file_ids: ownedFiles.map((input) => input.file_id), + excludeAgentObjectId: agentObjectId, + excludeToolResource: toolResource, + }), + ); + + const destroyable: Array> = []; + const unlinkOnly = [...unlinkOnlyFiles]; + for (const input of ownedFiles) { + if (sharedFileIds.has(input.file_id)) { + unlinkOnly.push({ tool_resource: toolResource, file_id: input.file_id }); + continue; + } + destroyable.push(input); + } + + /* The destroy runs before any reference is removed, and the delete pass strips the references of + the files it actually deleted. A reference removed ahead of a destroy that then fails is what + strands a file: it leaves the agent panel while its storage, chunks or metadata remain, and + neither this route nor the client's retry queue can name it again. */ + const outcome = + destroyable.length === 0 + ? null + : await deps.deleteFiles(destroyable.map((input) => input.file)); + + if (unlinkOnly.length > 0) { + await deps.removeAgentResourceFiles({ agent_id: agentId, files: unlinkOnly }); + } + + return { + outcome, + unlinkedFileIds: unlinkOnly.map((ref) => ref.file_id), + destroyedFileIds: outcome?.deletedFileIds ?? [], + }; +}; diff --git a/packages/data-schemas/src/methods/agent.spec.ts b/packages/data-schemas/src/methods/agent.spec.ts index fc24723e3c..57facac58d 100644 --- a/packages/data-schemas/src/methods/agent.spec.ts +++ b/packages/data-schemas/src/methods/agent.spec.ts @@ -65,6 +65,7 @@ let revertAgentVersion: AgentMethods['revertAgentVersion']; let addAgentResourceFile: AgentMethods['addAgentResourceFile']; let removeAgentResourceFiles: AgentMethods['removeAgentResourceFiles']; let removeAgentResourceFilesFromAllAgents: AgentMethods['removeAgentResourceFilesFromAllAgents']; +let getSharedResourceFileIds: AgentMethods['getSharedResourceFileIds']; let getListAgentsByAccess: AgentMethods['getListAgentsByAccess']; let getAgentManagementListByAccess: AgentMethods['getAgentManagementListByAccess']; let generateActionMetadataHash: AgentMethods['generateActionMetadataHash']; @@ -117,6 +118,7 @@ beforeAll(async () => { addAgentResourceFile = methods.addAgentResourceFile; removeAgentResourceFiles = methods.removeAgentResourceFiles; removeAgentResourceFilesFromAllAgents = methods.removeAgentResourceFilesFromAllAgents; + getSharedResourceFileIds = methods.getSharedResourceFileIds; getListAgentsByAccess = methods.getListAgentsByAccess; getAgentManagementListByAccess = methods.getAgentManagementListByAccess; generateActionMetadataHash = methods.generateActionMetadataHash; @@ -4238,6 +4240,146 @@ describe('Agent Methods', () => { ).rejects.toThrow('Agent not found for removing resource files'); }); + describe('getSharedResourceFileIds', () => { + beforeEach(async () => { + await Agent.deleteMany({}); + }); + + test('reports a file another agent still references, across tool resources', async () => { + const sharedFileId = `file_${uuidv4()}`; + const soleFileId = `file_${uuidv4()}`; + + const agent = await createBasicAgent(); + const duplicate = await createBasicAgent(); + + await addAgentResourceFile({ + agent_id: agent.id, + tool_resource: EToolResources.file_search, + file_id: sharedFileId, + }); + await addAgentResourceFile({ + agent_id: agent.id, + tool_resource: EToolResources.file_search, + file_id: soleFileId, + }); + await addAgentResourceFile({ + agent_id: duplicate.id, + tool_resource: EToolResources.context, + file_id: sharedFileId, + }); + + const shared = await getSharedResourceFileIds({ + file_ids: [sharedFileId, soleFileId], + excludeAgentObjectId: String(agent._id), + }); + + expect(shared).toEqual([sharedFileId]); + }); + + test('does not count the excluded agent’s own reference', async () => { + const fileId = `file_${uuidv4()}`; + const agent = await createBasicAgent(); + + await addAgentResourceFile({ + agent_id: agent.id, + tool_resource: EToolResources.file_search, + file_id: fileId, + }); + + expect( + await getSharedResourceFileIds({ + file_ids: [fileId], + excludeAgentObjectId: String(agent._id), + }), + ).toEqual([]); + expect(await getSharedResourceFileIds({ file_ids: [fileId] })).toEqual([fileId]); + }); + + test('reports a file the same agent still holds under another tool resource', async () => { + const fileId = `file_${uuidv4()}`; + const agent = await createBasicAgent(); + + await addAgentResourceFile({ + agent_id: agent.id, + tool_resource: EToolResources.file_search, + file_id: fileId, + }); + await addAgentResourceFile({ + agent_id: agent.id, + tool_resource: EToolResources.context, + file_id: fileId, + }); + + expect( + await getSharedResourceFileIds({ + file_ids: [fileId], + excludeAgentObjectId: String(agent._id), + excludeToolResource: EToolResources.file_search, + }), + ).toEqual([fileId]); + }); + + test('does not count the pair being removed', async () => { + const fileId = `file_${uuidv4()}`; + const agent = await createBasicAgent(); + + await addAgentResourceFile({ + agent_id: agent.id, + tool_resource: EToolResources.file_search, + file_id: fileId, + }); + + expect( + await getSharedResourceFileIds({ + file_ids: [fileId], + excludeAgentObjectId: String(agent._id), + excludeToolResource: EToolResources.file_search, + }), + ).toEqual([]); + }); + + test('does not skip another tenant’s agent that shares the logical id', async () => { + const sharedId = `agent_${uuidv4()}`; + const fileId = `file_${uuidv4()}`; + const tenantA = `tenant-${uuidv4()}`; + const tenantB = `tenant-${uuidv4()}`; + + /* `id` is unique only with `tenantId`, so two tenants can hold the same logical agent id. + Excluding by `id` would call the file unreferenced and destroy tenant B's bytes. */ + const agentA = await tenantStorage.run({ tenantId: tenantA }, () => + createBasicAgent({ id: sharedId }), + ); + await tenantStorage.run({ tenantId: tenantB }, () => createBasicAgent({ id: sharedId })); + + await tenantStorage.run({ tenantId: tenantA }, () => + addAgentResourceFile({ + agent_id: sharedId, + tool_resource: EToolResources.file_search, + file_id: fileId, + }), + ); + await tenantStorage.run({ tenantId: tenantB }, () => + addAgentResourceFile({ + agent_id: sharedId, + tool_resource: EToolResources.file_search, + file_id: fileId, + }), + ); + + expect( + await getSharedResourceFileIds({ + file_ids: [fileId], + excludeAgentObjectId: String(agentA._id), + excludeToolResource: EToolResources.file_search, + }), + ).toEqual([fileId]); + }); + + test('answers without querying when given no file_ids', async () => { + expect(await getSharedResourceFileIds({ file_ids: [] })).toEqual([]); + }); + }); + describe('removeAgentResourceFilesFromAllAgents', () => { beforeEach(async () => { await Agent.deleteMany({}); diff --git a/packages/data-schemas/src/methods/agent.ts b/packages/data-schemas/src/methods/agent.ts index ceeaa354b0..e25aa52cc9 100644 --- a/packages/data-schemas/src/methods/agent.ts +++ b/packages/data-schemas/src/methods/agent.ts @@ -650,6 +650,15 @@ export function createAgentMethods( }: { file_ids: string[]; }) => Promise<{ matchedCount: number; modifiedCount: number }>; + getSharedResourceFileIds: ({ + file_ids, + excludeAgentObjectId, + excludeToolResource, + }: { + file_ids: string[]; + excludeAgentObjectId?: string; + excludeToolResource?: string; + }) => Promise; } { const { removeAllPermissions, getActions, getSoleOwnedResourceIds, isExternalSkillId } = deps; @@ -1229,6 +1238,68 @@ export function createAgentMethods( }; } + /** + * Reports which of the given file_ids keep a reference once the caller's own is removed, so a + * caller can tell a last reference from a shared one. + * + * The unit of a reference is the `(agent, tool_resource)` pair rather than the agent: + * `addAgentResourceFile` stores `file_ids` per resource, so one agent can hold the same file under + * both `file_search` and `context`, and removing it from one leaves the other needing the bytes. + * `excludeToolResource` narrows the exclusion to the pair being removed; omitting it excludes the + * whole agent. + * + * The agent is excluded by `_id`, because `id` is unique only together with `tenantId`: matching on + * `id` alone would skip another tenant's agent of the same name and call its file unreferenced. + * + * Duplicating an agent copies `file_ids` rather than the files behind them, so one file record + * can back two agents; destroying its bytes on behalf of one agent would empty the other. This + * is deliberately not scoped by tenant: a reference is a reference, whoever holds it. + */ + async function getSharedResourceFileIds({ + file_ids, + excludeAgentObjectId, + excludeToolResource, + }: { + file_ids: string[]; + excludeAgentObjectId?: string; + excludeToolResource?: string; + }): Promise { + if (!file_ids || file_ids.length === 0) { + return []; + } + + const Agent = mongoose.models.Agent as Model; + const requested = new Set(file_ids); + const searchParameter: FilterQuery = { + $or: TOOL_RESOURCE_KEYS.map((key) => ({ + [`tool_resources.${key}.file_ids`]: { $in: file_ids }, + })), + }; + + const agents = await Agent.find(searchParameter, { _id: 1, tool_resources: 1 }).lean(); + const shared = new Set(); + for (const agent of agents) { + const isExcludedAgent = + excludeAgentObjectId != null && String(agent._id) === excludeAgentObjectId; + for (const key of TOOL_RESOURCE_KEYS) { + if (isExcludedAgent && (excludeToolResource == null || key === excludeToolResource)) { + continue; + } + const fileIds = agent.tool_resources?.[key]?.file_ids; + if (fileIds == null) { + continue; + } + for (const fileId of fileIds) { + if (requested.has(fileId)) { + shared.add(fileId); + } + } + } + } + + return [...shared]; + } + /** * Deletes an agent based on the provided search parameter. */ @@ -1698,6 +1769,7 @@ export function createAgentMethods( generateActionMetadataHash, removeAgentFromUserFavorites, removeAgentResourceFilesFromAllAgents, + getSharedResourceFileIds, }; }