mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-17 21:43:06 +00:00
🥡 fix: Delete Storage and Embeddings With Owned Agent Files (#16007)
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Votes / vote (full suite) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
The agent tool-resource branch of DELETE /files removed a file's association from the agent and answered 200 without reaching processDeleteRequest, so the bytes on disk and the RAG chunks of a file the caller owned survived the delete. A replaced File Search document kept answering from its old chunks (#15930). An attached file now goes through the full delete pass, but only when nothing else holds it: a file the caller does not own, and a file any other (agent, tool_resource) pair still references, are unlinked and left whole. Nothing is unlinked until the destroy it depends on has succeeded. - packages/api/src/files/deletion.ts: deleteAgentResourceFiles owns the operation with its db methods and delete pass injected. - packages/data-schemas: getSharedResourceFileIds answers the last-reference question, excluding the pair being removed by the agent's _id. - Local/crud.js: unlinkFile propagates non-ENOENT, so a file whose bytes survive is no longer reported as deleted. Co-authored-by: Mohammed Alshyakh <mohmedmm@users.noreply.github.com>
This commit is contained in:
parent
12d78909d3
commit
94fed5bcd5
10 changed files with 1064 additions and 28 deletions
|
|
@ -8,6 +8,7 @@ const {
|
|||
refreshS3FileUrls,
|
||||
handleFilesUsageRequest,
|
||||
buildDeleteFilesResponse,
|
||||
deleteAgentResourceFiles,
|
||||
shouldUseUploadSse,
|
||||
startUploadSseStream,
|
||||
sendUploadPolicyError,
|
||||
|
|
@ -245,20 +246,36 @@ router.delete('/', async (req, res) => {
|
|||
});
|
||||
}
|
||||
|
||||
const toolResourceFiles = agent.tool_resources?.[req.body.tool_resource]?.file_ids ?? [];
|
||||
const agentFiles = files
|
||||
.filter((f) => toolResourceFiles.includes(f.file_id))
|
||||
.map((file) => ({ tool_resource: req.body.tool_resource, file_id: file.file_id }));
|
||||
if (agentFiles.length === 0) {
|
||||
const agentDeletion = await deleteAgentResourceFiles(
|
||||
{
|
||||
agentId: req.body.agent_id,
|
||||
agentObjectId: agent._id.toString(),
|
||||
toolResource: req.body.tool_resource,
|
||||
requestedFileIds: fileIds,
|
||||
attachedFileIds: agent.tool_resources?.[req.body.tool_resource]?.file_ids ?? [],
|
||||
files: dbFiles.map((file) => ({
|
||||
file_id: file.file_id,
|
||||
owner: file.user?.toString() ?? null,
|
||||
file,
|
||||
})),
|
||||
userId: req.user.id.toString(),
|
||||
},
|
||||
{
|
||||
getSharedResourceFileIds: db.getSharedResourceFileIds,
|
||||
removeAgentResourceFiles: db.removeAgentResourceFiles,
|
||||
deleteFiles: (agentFiles) => processDeleteRequest({ req, files: agentFiles }),
|
||||
},
|
||||
);
|
||||
|
||||
if (agentDeletion.outcome == null) {
|
||||
res.status(200).json({ message: 'File associations removed successfully from agent' });
|
||||
return;
|
||||
}
|
||||
|
||||
await db.removeAgentResourceFiles({
|
||||
agent_id: req.body.agent_id,
|
||||
files: agentFiles,
|
||||
});
|
||||
res.status(200).json({ message: 'File associations removed successfully from agent' });
|
||||
logger.debug(
|
||||
`[/files] Agent files deleted successfully: ${agentDeletion.destroyedFileIds.join(', ')}`,
|
||||
);
|
||||
sendDeleteResult(agentDeletion.outcome, 'Files deleted successfully');
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -327,6 +327,245 @@ describe('File Routes - Delete with Agent Access', () => {
|
|||
expect(updatedAgent.tool_resources.file_search.file_ids).toEqual([]);
|
||||
});
|
||||
|
||||
it('deletes storage and embeddings for an attached file the caller owns', async () => {
|
||||
const ownedFileId = uuidv4();
|
||||
await createFile({
|
||||
user: otherUserId,
|
||||
file_id: ownedFileId,
|
||||
filename: 'owned-knowledge.txt',
|
||||
filepath: '/uploads/owned-knowledge.txt',
|
||||
bytes: 100,
|
||||
type: 'text/plain',
|
||||
source: FileSources.vectordb,
|
||||
embedded: true,
|
||||
});
|
||||
|
||||
const agent = await createAgent({
|
||||
id: uuidv4(),
|
||||
name: 'Test Agent',
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
author: otherUserId,
|
||||
tool_resources: {
|
||||
file_search: {
|
||||
file_ids: [ownedFileId],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/files')
|
||||
.send({
|
||||
agent_id: agent.id,
|
||||
tool_resource: 'file_search',
|
||||
files: [{ file_id: ownedFileId, filepath: '/uploads/owned-knowledge.txt' }],
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.message).toBe('Files deleted successfully');
|
||||
expect(processDeleteRequest).toHaveBeenCalledTimes(1);
|
||||
|
||||
const [{ req, files: deletedFiles }] = processDeleteRequest.mock.calls[0];
|
||||
expect(deletedFiles.map((file) => file.file_id)).toEqual([ownedFileId]);
|
||||
expect(deletedFiles[0].source).toBe(FileSources.vectordb);
|
||||
expect(req.body.agent_id).toBe(agent.id);
|
||||
expect(req.body.tool_resource).toBe('file_search');
|
||||
});
|
||||
|
||||
it('unlinks another user’s attached file while deleting the caller’s own', async () => {
|
||||
const ownedFileId = uuidv4();
|
||||
await createFile({
|
||||
user: otherUserId,
|
||||
file_id: ownedFileId,
|
||||
filename: 'owned-knowledge.txt',
|
||||
filepath: '/uploads/owned-knowledge.txt',
|
||||
bytes: 100,
|
||||
type: 'text/plain',
|
||||
source: FileSources.vectordb,
|
||||
embedded: true,
|
||||
});
|
||||
|
||||
const agent = await createAgent({
|
||||
id: uuidv4(),
|
||||
name: 'Test Agent',
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
author: otherUserId,
|
||||
tool_resources: {
|
||||
file_search: {
|
||||
file_ids: [ownedFileId, fileId],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/files')
|
||||
.send({
|
||||
agent_id: agent.id,
|
||||
tool_resource: 'file_search',
|
||||
files: [
|
||||
{ file_id: ownedFileId, filepath: '/uploads/owned-knowledge.txt' },
|
||||
{ file_id: fileId, filepath: '/uploads/test.txt' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.message).toBe('Files deleted successfully');
|
||||
|
||||
const [{ files: deletedFiles }] = processDeleteRequest.mock.calls[0];
|
||||
expect(deletedFiles.map((file) => file.file_id)).toEqual([ownedFileId]);
|
||||
|
||||
const updatedAgent = await Agent.findOne({ id: agent.id }).lean();
|
||||
expect(updatedAgent.tool_resources.file_search.file_ids).toEqual([ownedFileId]);
|
||||
|
||||
const retainedFile = await File.findOne({ file_id: fileId }).lean();
|
||||
expect(retainedFile).toBeTruthy();
|
||||
});
|
||||
|
||||
it('keeps a file the same agent holds under another tool resource', async () => {
|
||||
const sharedFileId = uuidv4();
|
||||
await createFile({
|
||||
user: otherUserId,
|
||||
file_id: sharedFileId,
|
||||
filename: 'dual-purpose.txt',
|
||||
filepath: '/uploads/dual-purpose.txt',
|
||||
bytes: 100,
|
||||
type: 'text/plain',
|
||||
source: FileSources.vectordb,
|
||||
embedded: true,
|
||||
});
|
||||
|
||||
/* One agent can hold the same file under two resources, so the reference being removed is the
|
||||
`(agent, tool_resource)` pair rather than the agent. */
|
||||
const agent = await createAgent({
|
||||
id: uuidv4(),
|
||||
name: 'Test Agent',
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
author: otherUserId,
|
||||
tool_resources: {
|
||||
file_search: { file_ids: [sharedFileId] },
|
||||
context: { file_ids: [sharedFileId] },
|
||||
},
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/files')
|
||||
.send({
|
||||
agent_id: agent.id,
|
||||
tool_resource: 'file_search',
|
||||
files: [{ file_id: sharedFileId, filepath: '/uploads/dual-purpose.txt' }],
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.message).toBe('File associations removed successfully from agent');
|
||||
expect(processDeleteRequest).not.toHaveBeenCalled();
|
||||
|
||||
const updatedAgent = await Agent.findOne({ id: agent.id }).lean();
|
||||
expect(updatedAgent.tool_resources.file_search.file_ids).toEqual([]);
|
||||
expect(updatedAgent.tool_resources.context.file_ids).toEqual([sharedFileId]);
|
||||
|
||||
const retainedFile = await File.findOne({ file_id: sharedFileId }).lean();
|
||||
expect(retainedFile).toBeTruthy();
|
||||
});
|
||||
|
||||
it('keeps a file a duplicated agent still references, unlinking it here only', async () => {
|
||||
const sharedFileId = uuidv4();
|
||||
await createFile({
|
||||
user: otherUserId,
|
||||
file_id: sharedFileId,
|
||||
filename: 'shared-knowledge.txt',
|
||||
filepath: '/uploads/shared-knowledge.txt',
|
||||
bytes: 100,
|
||||
type: 'text/plain',
|
||||
source: FileSources.vectordb,
|
||||
embedded: true,
|
||||
});
|
||||
|
||||
const agent = await createAgent({
|
||||
id: uuidv4(),
|
||||
name: 'Test Agent',
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
author: otherUserId,
|
||||
tool_resources: { file_search: { file_ids: [sharedFileId] } },
|
||||
});
|
||||
|
||||
/* Duplicating an agent copies file_ids rather than the files behind them, and lands them
|
||||
under `context`, so the second holder is found across tool resources. */
|
||||
const duplicate = await createAgent({
|
||||
id: uuidv4(),
|
||||
name: 'Test Agent (copy)',
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
author: otherUserId,
|
||||
tool_resources: { context: { file_ids: [sharedFileId] } },
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/files')
|
||||
.send({
|
||||
agent_id: agent.id,
|
||||
tool_resource: 'file_search',
|
||||
files: [{ file_id: sharedFileId, filepath: '/uploads/shared-knowledge.txt' }],
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.message).toBe('File associations removed successfully from agent');
|
||||
expect(processDeleteRequest).not.toHaveBeenCalled();
|
||||
|
||||
const updatedAgent = await Agent.findOne({ id: agent.id }).lean();
|
||||
expect(updatedAgent.tool_resources.file_search.file_ids).toEqual([]);
|
||||
|
||||
const untouchedDuplicate = await Agent.findOne({ id: duplicate.id }).lean();
|
||||
expect(untouchedDuplicate.tool_resources.context.file_ids).toEqual([sharedFileId]);
|
||||
|
||||
const retainedFile = await File.findOne({ file_id: sharedFileId }).lean();
|
||||
expect(retainedFile).toBeTruthy();
|
||||
});
|
||||
|
||||
it('leaves an owned file alone when the tool resource does not hold it', async () => {
|
||||
const ownedFileId = uuidv4();
|
||||
await createFile({
|
||||
user: otherUserId,
|
||||
file_id: ownedFileId,
|
||||
filename: 'detached-knowledge.txt',
|
||||
filepath: '/uploads/detached-knowledge.txt',
|
||||
bytes: 100,
|
||||
type: 'text/plain',
|
||||
source: FileSources.vectordb,
|
||||
embedded: true,
|
||||
});
|
||||
|
||||
const agent = await createAgent({
|
||||
id: uuidv4(),
|
||||
name: 'Test Agent',
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
author: otherUserId,
|
||||
tool_resources: {
|
||||
file_search: {
|
||||
file_ids: [fileId],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/files')
|
||||
.send({
|
||||
agent_id: agent.id,
|
||||
tool_resource: 'file_search',
|
||||
files: [{ file_id: ownedFileId, filepath: '/uploads/detached-knowledge.txt' }],
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.message).toBe('File associations removed successfully from agent');
|
||||
expect(processDeleteRequest).not.toHaveBeenCalled();
|
||||
|
||||
const updatedAgent = await Agent.findOne({ id: agent.id }).lean();
|
||||
expect(updatedAgent.tool_resources.file_search.file_ids).toEqual([fileId]);
|
||||
});
|
||||
|
||||
it('rejects invalid agent tool_resource values before unlinking', async () => {
|
||||
const agent = await createAgent({
|
||||
id: uuidv4(),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
/** Exactly what `deleteLocalFile` reaches for, so the double does not depend on a built package. */
|
||||
jest.mock('@librechat/api', () => ({
|
||||
deleteRagFile: jest.fn().mockResolvedValue(undefined),
|
||||
stripCacheBust: (filepath) => String(filepath).split('?')[0],
|
||||
}));
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: { warn: jest.fn(), error: jest.fn() },
|
||||
}));
|
||||
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const { deleteLocalFile } = require('../crud');
|
||||
|
||||
/* The resolved promise of a delete is what `processDeleteRequest` reads to decide that a record may
|
||||
lose its metadata and its agent references, so this adapter may only resolve once the bytes are
|
||||
actually gone. Storage that was already missing is the one benign case. */
|
||||
describe('deleteLocalFile failure reporting', () => {
|
||||
const userId = 'user-1';
|
||||
let tmpBase;
|
||||
let req;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
tmpBase = fs.mkdtempSync(path.join(os.tmpdir(), 'crud-delete-'));
|
||||
fs.mkdirSync(path.join(tmpBase, 'uploads', userId), { recursive: true });
|
||||
req = {
|
||||
user: { id: userId },
|
||||
config: {
|
||||
paths: {
|
||||
publicPath: path.join(tmpBase, 'public'),
|
||||
uploads: path.join(tmpBase, 'uploads'),
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpBase, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const uploadedFile = (filename) => {
|
||||
const filepath = path.join(tmpBase, 'uploads', userId, filename);
|
||||
fs.writeFileSync(filepath, 'contents');
|
||||
return { file_id: 'file-1', filepath: `/uploads/${userId}/${filename}` };
|
||||
};
|
||||
|
||||
it('removes the file and resolves', async () => {
|
||||
const file = uploadedFile('knowledge.txt');
|
||||
|
||||
await expect(deleteLocalFile(req, file)).resolves.toBeUndefined();
|
||||
expect(fs.existsSync(path.join(tmpBase, 'uploads', userId, 'knowledge.txt'))).toBe(false);
|
||||
});
|
||||
|
||||
it('resolves when the file is already gone', async () => {
|
||||
await expect(
|
||||
deleteLocalFile(req, { file_id: 'file-1', filepath: `/uploads/${userId}/missing.txt` }),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects when the bytes survive the delete', async () => {
|
||||
const file = uploadedFile('locked.txt');
|
||||
jest
|
||||
.spyOn(fs.promises, 'unlink')
|
||||
.mockRejectedValue(Object.assign(new Error('permission denied'), { code: 'EACCES' }));
|
||||
|
||||
await expect(deleteLocalFile(req, file)).rejects.toThrow('permission denied');
|
||||
expect(fs.existsSync(path.join(tmpBase, 'uploads', userId, 'locked.txt'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
|
@ -208,11 +208,21 @@ const isValidPath = (req, base, subfolder, filepath) => {
|
|||
/**
|
||||
* @param {string} filepath
|
||||
*/
|
||||
/**
|
||||
* A file whose bytes are still on disk must not be reported as deleted: callers use the resolved
|
||||
* promise to decide that a record may lose its metadata and its agent references. Storage that was
|
||||
* already gone is the one benign case, and `processDeleteRequest` treats it as deleted by design.
|
||||
*/
|
||||
const unlinkFile = async (filepath) => {
|
||||
try {
|
||||
await fs.promises.unlink(filepath);
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') {
|
||||
logger.warn('Local file was already missing during delete:', error);
|
||||
return;
|
||||
}
|
||||
logger.error('Error deleting file:', error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -265,16 +265,8 @@ const processDeleteRequest = async ({ req, files }) => {
|
|||
await initializeClients();
|
||||
}
|
||||
|
||||
const agentFiles = [];
|
||||
|
||||
for (const file of files) {
|
||||
const source = file.source ?? FileSources.local;
|
||||
if (req.body.agent_id && req.body.tool_resource) {
|
||||
agentFiles.push({
|
||||
tool_resource: req.body.tool_resource,
|
||||
file_id: file.file_id,
|
||||
});
|
||||
}
|
||||
|
||||
if (source === FileSources.text) {
|
||||
resolvedFileIds.add(file.file_id);
|
||||
|
|
@ -313,15 +305,6 @@ const processDeleteRequest = async ({ req, files }) => {
|
|||
});
|
||||
}
|
||||
|
||||
if (agentFiles.length > 0) {
|
||||
promises.push(
|
||||
db.removeAgentResourceFiles({
|
||||
agent_id: req.body.agent_id,
|
||||
files: agentFiles,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
await Promise.allSettled(promises);
|
||||
const deletedFileIds = [...resolvedFileIds];
|
||||
let metadataDeletedFileIds = deletedFileIds;
|
||||
|
|
@ -334,6 +317,9 @@ const processDeleteRequest = async ({ req, files }) => {
|
|||
metadataDeletedFileIds = [];
|
||||
throw error;
|
||||
}
|
||||
/* The only place a delete removes agent references, and it runs after the metadata delete
|
||||
succeeded: a file that kept its storage, its chunks or its record keeps its references too,
|
||||
so the agent it was removed from can be asked again (see issue #12776). */
|
||||
if (metadataDeletedFileIds.length > 0) {
|
||||
try {
|
||||
await db.removeAgentResourceFilesFromAllAgents({ file_ids: metadataDeletedFileIds });
|
||||
|
|
|
|||
|
|
@ -2610,6 +2610,89 @@ describe('processDeleteRequest', () => {
|
|||
expect(result).toEqual({ deletedFileIds: [], failedFileIds: ['embedded-file'] });
|
||||
});
|
||||
|
||||
it('keeps a failed agent file attached so the delete can be retried', async () => {
|
||||
const deleteFile = jest.fn().mockRejectedValue(new Error('rag unavailable'));
|
||||
getStrategyFunctions.mockReturnValue({ deleteFile });
|
||||
const req = {
|
||||
body: { agent_id: 'agent_1', tool_resource: 'file_search' },
|
||||
config: {},
|
||||
user: { id: 'user-123', tenantId: 'tenant-a' },
|
||||
};
|
||||
|
||||
const result = await processDeleteRequest({
|
||||
req,
|
||||
files: [
|
||||
{
|
||||
file_id: 'knowledge-file',
|
||||
filepath: '/uploads/knowledge.txt',
|
||||
source: FileSources.local,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual({ deletedFileIds: [], failedFileIds: ['knowledge-file'] });
|
||||
expect(db.deleteFiles).not.toHaveBeenCalled();
|
||||
expect(db.removeAgentResourceFiles).not.toHaveBeenCalled();
|
||||
expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('strips agent references only for the files it deleted', async () => {
|
||||
getStrategyFunctions.mockReturnValue({
|
||||
deleteFile: jest
|
||||
.fn()
|
||||
.mockImplementation((_req, file) =>
|
||||
file.file_id === 'kept-file'
|
||||
? Promise.reject(new Error('rag unavailable'))
|
||||
: Promise.resolve(undefined),
|
||||
),
|
||||
});
|
||||
db.deleteFiles.mockResolvedValue({ deletedCount: 1 });
|
||||
const req = {
|
||||
body: { agent_id: 'agent_1', tool_resource: 'file_search' },
|
||||
config: {},
|
||||
user: { id: 'user-123', tenantId: 'tenant-a' },
|
||||
};
|
||||
|
||||
const result = await processDeleteRequest({
|
||||
req,
|
||||
files: [
|
||||
{ file_id: 'gone-file', filepath: '/uploads/gone.txt', source: FileSources.local },
|
||||
{ file_id: 'kept-file', filepath: '/uploads/kept.txt', source: FileSources.local },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual({ deletedFileIds: ['gone-file'], failedFileIds: ['kept-file'] });
|
||||
expect(db.removeAgentResourceFilesFromAllAgents).toHaveBeenCalledWith({
|
||||
file_ids: ['gone-file'],
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps agent references when the metadata delete fails', async () => {
|
||||
getStrategyFunctions.mockReturnValue({ deleteFile: jest.fn().mockResolvedValue(undefined) });
|
||||
db.deleteFiles.mockRejectedValue(new Error('mongo unavailable'));
|
||||
const req = {
|
||||
body: { agent_id: 'agent_1', tool_resource: 'file_search' },
|
||||
config: {},
|
||||
user: { id: 'user-123', tenantId: 'tenant-a' },
|
||||
};
|
||||
|
||||
await expect(
|
||||
processDeleteRequest({
|
||||
req,
|
||||
files: [
|
||||
{
|
||||
file_id: 'knowledge-file',
|
||||
filepath: '/uploads/knowledge.txt',
|
||||
source: FileSources.local,
|
||||
},
|
||||
],
|
||||
}),
|
||||
).rejects.toThrow('mongo unavailable');
|
||||
|
||||
expect(db.removeAgentResourceFiles).not.toHaveBeenCalled();
|
||||
expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not delete vector storage when primary embedded file deletion fails', async () => {
|
||||
const primaryDelete = jest.fn().mockRejectedValue(new Error('permission denied'));
|
||||
const vectorDelete = jest.fn().mockResolvedValue(undefined);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
import { buildDeleteFilesResponse, PARTIAL_FILE_DELETION_MESSAGE } from './deletion';
|
||||
import type { AgentResourceFileInput } from './deletion';
|
||||
import {
|
||||
buildDeleteFilesResponse,
|
||||
deleteAgentResourceFiles,
|
||||
partitionAgentResourceFiles,
|
||||
PARTIAL_FILE_DELETION_MESSAGE,
|
||||
} from './deletion';
|
||||
|
||||
describe('delete files response', () => {
|
||||
it('reports the caller’s success message when nothing failed', () => {
|
||||
|
|
@ -32,3 +38,242 @@ describe('delete files response', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
type TestFile = { file_id: string; filename: string };
|
||||
|
||||
const input = (file_id: string, owner: string | null): AgentResourceFileInput<TestFile> => ({
|
||||
file_id,
|
||||
owner,
|
||||
file: { file_id, filename: `${file_id}.txt` },
|
||||
});
|
||||
|
||||
describe('partition agent resource files', () => {
|
||||
const userId = 'user-1';
|
||||
const owned = input('owned', userId);
|
||||
const foreign = input('foreign', 'user-2');
|
||||
|
||||
it('makes an attached file the caller owns a candidate for the delete pass', () => {
|
||||
expect(
|
||||
partitionAgentResourceFiles({
|
||||
requestedFileIds: ['owned'],
|
||||
attachedFileIds: ['owned'],
|
||||
files: [owned],
|
||||
toolResource: 'file_search',
|
||||
userId,
|
||||
}),
|
||||
).toEqual({ ownedFiles: [owned], unlinkOnlyFiles: [] });
|
||||
});
|
||||
|
||||
it('only unlinks an attached file owned by someone else', () => {
|
||||
expect(
|
||||
partitionAgentResourceFiles({
|
||||
requestedFileIds: ['foreign'],
|
||||
attachedFileIds: ['foreign'],
|
||||
files: [foreign],
|
||||
toolResource: 'file_search',
|
||||
userId,
|
||||
}),
|
||||
).toEqual({
|
||||
ownedFiles: [],
|
||||
unlinkOnlyFiles: [{ tool_resource: 'file_search', file_id: 'foreign' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('only unlinks an attached file whose metadata record is gone', () => {
|
||||
expect(
|
||||
partitionAgentResourceFiles({
|
||||
requestedFileIds: ['missing'],
|
||||
attachedFileIds: ['missing'],
|
||||
files: [],
|
||||
toolResource: 'file_search',
|
||||
userId,
|
||||
}),
|
||||
).toEqual({
|
||||
ownedFiles: [],
|
||||
unlinkOnlyFiles: [{ tool_resource: 'file_search', file_id: 'missing' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('does not treat an ownerless record as the caller’s', () => {
|
||||
expect(
|
||||
partitionAgentResourceFiles({
|
||||
requestedFileIds: ['orphan'],
|
||||
attachedFileIds: ['orphan'],
|
||||
files: [input('orphan', null)],
|
||||
toolResource: 'file_search',
|
||||
userId,
|
||||
}),
|
||||
).toEqual({
|
||||
ownedFiles: [],
|
||||
unlinkOnlyFiles: [{ tool_resource: 'file_search', file_id: 'orphan' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('ignores files the tool resource does not hold', () => {
|
||||
expect(
|
||||
partitionAgentResourceFiles({
|
||||
requestedFileIds: ['owned', 'elsewhere'],
|
||||
attachedFileIds: ['owned'],
|
||||
files: [owned, input('elsewhere', userId)],
|
||||
toolResource: 'file_search',
|
||||
userId,
|
||||
}),
|
||||
).toEqual({ ownedFiles: [owned], unlinkOnlyFiles: [] });
|
||||
});
|
||||
|
||||
it('splits a mixed request and counts each file once', () => {
|
||||
expect(
|
||||
partitionAgentResourceFiles({
|
||||
requestedFileIds: ['owned', 'foreign', 'owned'],
|
||||
attachedFileIds: ['owned', 'foreign'],
|
||||
files: [owned, foreign],
|
||||
toolResource: 'ocr',
|
||||
userId,
|
||||
}),
|
||||
).toEqual({
|
||||
ownedFiles: [owned],
|
||||
unlinkOnlyFiles: [{ tool_resource: 'ocr', file_id: 'foreign' }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('delete agent resource files', () => {
|
||||
const userId = 'user-1';
|
||||
|
||||
const makeDeps = (sharedFileIds: string[] = [], calls: string[] = []) => ({
|
||||
getSharedResourceFileIds: jest.fn().mockResolvedValue(sharedFileIds),
|
||||
removeAgentResourceFiles: jest.fn().mockImplementation(() => {
|
||||
calls.push('unlink');
|
||||
return Promise.resolve(undefined);
|
||||
}),
|
||||
deleteFiles: jest.fn().mockImplementation((files: TestFile[]) => {
|
||||
calls.push('destroy');
|
||||
return Promise.resolve({
|
||||
deletedFileIds: files.map((file) => file.file_id),
|
||||
failedFileIds: [],
|
||||
});
|
||||
}),
|
||||
});
|
||||
|
||||
const run = (
|
||||
files: Array<AgentResourceFileInput<TestFile>>,
|
||||
deps: ReturnType<typeof makeDeps>,
|
||||
attachedFileIds?: string[],
|
||||
) =>
|
||||
deleteAgentResourceFiles(
|
||||
{
|
||||
agentId: 'agent_1',
|
||||
agentObjectId: '65f000000000000000000001',
|
||||
toolResource: 'file_search',
|
||||
requestedFileIds: files.map((file) => file.file_id),
|
||||
attachedFileIds: attachedFileIds ?? files.map((file) => file.file_id),
|
||||
files,
|
||||
userId,
|
||||
},
|
||||
deps,
|
||||
);
|
||||
|
||||
it('destroys a file this agent was the last to reference', async () => {
|
||||
const deps = makeDeps();
|
||||
const result = await run([input('owned', userId)], deps);
|
||||
|
||||
expect(deps.deleteFiles).toHaveBeenCalledWith([{ file_id: 'owned', filename: 'owned.txt' }]);
|
||||
expect(deps.removeAgentResourceFiles).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
outcome: { deletedFileIds: ['owned'], failedFileIds: [] },
|
||||
unlinkedFileIds: [],
|
||||
destroyedFileIds: ['owned'],
|
||||
});
|
||||
});
|
||||
|
||||
it('excludes the agent by a globally unique identity, not its logical id', async () => {
|
||||
const deps = makeDeps();
|
||||
await run([input('owned', userId)], deps);
|
||||
|
||||
expect(deps.getSharedResourceFileIds).toHaveBeenCalledWith({
|
||||
file_ids: ['owned'],
|
||||
excludeAgentObjectId: '65f000000000000000000001',
|
||||
excludeToolResource: 'file_search',
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps a file another agent still references, unlinking it here only', async () => {
|
||||
const deps = makeDeps(['shared']);
|
||||
const result = await run([input('shared', userId)], deps);
|
||||
|
||||
expect(deps.deleteFiles).not.toHaveBeenCalled();
|
||||
expect(deps.removeAgentResourceFiles).toHaveBeenCalledWith({
|
||||
agent_id: 'agent_1',
|
||||
files: [{ tool_resource: 'file_search', file_id: 'shared' }],
|
||||
});
|
||||
expect(result).toEqual({
|
||||
outcome: null,
|
||||
unlinkedFileIds: ['shared'],
|
||||
destroyedFileIds: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('destroys the last-reference file and unlinks the shared one in the same request', async () => {
|
||||
const deps = makeDeps(['shared']);
|
||||
const result = await run([input('shared', userId), input('owned', userId)], deps);
|
||||
|
||||
expect(deps.deleteFiles).toHaveBeenCalledWith([{ file_id: 'owned', filename: 'owned.txt' }]);
|
||||
expect(deps.removeAgentResourceFiles).toHaveBeenCalledWith({
|
||||
agent_id: 'agent_1',
|
||||
files: [{ tool_resource: 'file_search', file_id: 'shared' }],
|
||||
});
|
||||
expect(result.unlinkedFileIds).toEqual(['shared']);
|
||||
expect(result.destroyedFileIds).toEqual(['owned']);
|
||||
});
|
||||
|
||||
it('unlinks a file the caller does not own without asking to destroy it', async () => {
|
||||
const deps = makeDeps();
|
||||
const result = await run([input('foreign', 'user-2')], deps);
|
||||
|
||||
expect(deps.getSharedResourceFileIds).not.toHaveBeenCalled();
|
||||
expect(deps.deleteFiles).not.toHaveBeenCalled();
|
||||
expect(deps.removeAgentResourceFiles).toHaveBeenCalledWith({
|
||||
agent_id: 'agent_1',
|
||||
files: [{ tool_resource: 'file_search', file_id: 'foreign' }],
|
||||
});
|
||||
expect(result.outcome).toBeNull();
|
||||
});
|
||||
|
||||
it('touches nothing when the tool resource holds none of the requested files', async () => {
|
||||
const deps = makeDeps();
|
||||
const result = await run([input('owned', userId)], deps, []);
|
||||
|
||||
expect(deps.getSharedResourceFileIds).not.toHaveBeenCalled();
|
||||
expect(deps.removeAgentResourceFiles).not.toHaveBeenCalled();
|
||||
expect(deps.deleteFiles).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({ outcome: null, unlinkedFileIds: [], destroyedFileIds: [] });
|
||||
});
|
||||
|
||||
it('destroys before removing any reference, so a failed destroy leaves the file attached', async () => {
|
||||
const calls: string[] = [];
|
||||
const deps = makeDeps(['shared'], calls);
|
||||
await run([input('shared', userId), input('owned', userId)], deps);
|
||||
|
||||
expect(calls).toEqual(['destroy', 'unlink']);
|
||||
});
|
||||
|
||||
it('reports what the delete pass deleted rather than what it was handed', async () => {
|
||||
const deps = makeDeps();
|
||||
deps.deleteFiles.mockResolvedValue({
|
||||
deletedFileIds: ['owned'],
|
||||
failedFileIds: ['other-owned'],
|
||||
});
|
||||
const result = await run([input('owned', userId), input('other-owned', userId)], deps);
|
||||
|
||||
expect(result.destroyedFileIds).toEqual(['owned']);
|
||||
});
|
||||
|
||||
it('passes a partial delete outcome back to the caller', async () => {
|
||||
const deps = makeDeps();
|
||||
deps.deleteFiles.mockResolvedValue({ deletedFileIds: [], failedFileIds: ['owned'] });
|
||||
const result = await run([input('owned', userId)], deps);
|
||||
|
||||
expect(result.outcome).toEqual({ deletedFileIds: [], failedFileIds: ['owned'] });
|
||||
expect(deps.removeAgentResourceFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -6,6 +6,52 @@ export type FileDeletionOutcome = {
|
|||
failedFileIds?: string[];
|
||||
};
|
||||
|
||||
/** The `{ tool_resource, file_id }` pair an agent resource unlink takes. */
|
||||
export type AgentResourceFileRef = {
|
||||
tool_resource: string;
|
||||
file_id: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* One requested file, as this module needs to see it: an id, an owner already normalized to a
|
||||
* plain string by the caller, and the opaque record the delete pass will be handed. Ownership
|
||||
* arrives as a string so no storage type crosses into this package.
|
||||
*/
|
||||
export type AgentResourceFileInput<TFile> = {
|
||||
file_id: string;
|
||||
owner: string | null;
|
||||
file: TFile;
|
||||
};
|
||||
|
||||
export type AgentResourceDeletion<TFile> = {
|
||||
/** Attached files the caller owns: candidates for storage, vector and metadata deletion. */
|
||||
ownedFiles: Array<AgentResourceFileInput<TFile>>;
|
||||
/** Attached files owned by someone else, or with no metadata left: unlink the association only. */
|
||||
unlinkOnlyFiles: AgentResourceFileRef[];
|
||||
};
|
||||
|
||||
export type AgentResourceDeletionDeps<TFile> = {
|
||||
/** Which of these ids another agent still references; a shared file is never destroyed. */
|
||||
getSharedResourceFileIds: (params: {
|
||||
file_ids: string[];
|
||||
excludeAgentObjectId: string;
|
||||
excludeToolResource: string;
|
||||
}) => Promise<string[]>;
|
||||
removeAgentResourceFiles: (params: {
|
||||
agent_id: string;
|
||||
files: AgentResourceFileRef[];
|
||||
}) => Promise<unknown>;
|
||||
/** The full delete pass, injected: storage, vectors, metadata, and the unlink of what it deleted. */
|
||||
deleteFiles: (files: TFile[]) => Promise<FileDeletionOutcome>;
|
||||
};
|
||||
|
||||
export type AgentResourceDeletionResult = {
|
||||
/** `null` when nothing was destroyed, so the caller answers with the unlink message. */
|
||||
outcome: FileDeletionOutcome | null;
|
||||
unlinkedFileIds: string[];
|
||||
destroyedFileIds: string[];
|
||||
};
|
||||
|
||||
export const PARTIAL_FILE_DELETION_MESSAGE = 'Some files could not be deleted';
|
||||
|
||||
/**
|
||||
|
|
@ -25,3 +71,129 @@ export const buildDeleteFilesResponse = (
|
|||
failedFileIds,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Splits the files a delete request names for one agent tool resource into the ones whose storage
|
||||
* and embeddings may go with the unlink, and the ones only the association can be removed for.
|
||||
*
|
||||
* A file the caller owns is theirs to destroy, so it becomes a candidate for the full delete pass;
|
||||
* a file that belongs to another user, or that no longer has a metadata record, keeps its bytes and
|
||||
* its chunks and loses only its link to the agent. A file the request names but the tool resource
|
||||
* does not hold appears in neither list.
|
||||
*/
|
||||
export const partitionAgentResourceFiles = <TFile>({
|
||||
requestedFileIds,
|
||||
attachedFileIds,
|
||||
files,
|
||||
toolResource,
|
||||
userId,
|
||||
}: {
|
||||
requestedFileIds: string[];
|
||||
attachedFileIds: string[];
|
||||
files: Array<AgentResourceFileInput<TFile>>;
|
||||
toolResource: string;
|
||||
userId: string;
|
||||
}): AgentResourceDeletion<TFile> => {
|
||||
const attached = new Set(attachedFileIds);
|
||||
const inputsById = new Map(files.map((input) => [input.file_id, input]));
|
||||
const seen = new Set<string>();
|
||||
const ownedFiles: Array<AgentResourceFileInput<TFile>> = [];
|
||||
const unlinkOnlyFiles: AgentResourceFileRef[] = [];
|
||||
|
||||
for (const fileId of requestedFileIds) {
|
||||
if (!attached.has(fileId) || seen.has(fileId)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(fileId);
|
||||
const input = inputsById.get(fileId);
|
||||
if (input != null && input.owner === userId) {
|
||||
ownedFiles.push(input);
|
||||
continue;
|
||||
}
|
||||
unlinkOnlyFiles.push({ tool_resource: toolResource, file_id: fileId });
|
||||
}
|
||||
|
||||
return { ownedFiles, unlinkOnlyFiles };
|
||||
};
|
||||
|
||||
/**
|
||||
* Removes files from one agent tool resource, destroying only what this agent was the last holder
|
||||
* of.
|
||||
*
|
||||
* Three outcomes, decided per file. A file the caller does not own is unlinked and left whole. A
|
||||
* file the caller owns that any other `(agent, tool_resource)` pair still references is unlinked here
|
||||
* and left whole too:
|
||||
* duplicating an agent copies `file_ids` rather than the files behind them, so destroying the bytes
|
||||
* would empty the other agent's knowledge without touching its configuration. Only a file the
|
||||
* caller owns and this agent was the last to reference goes through the delete pass, which removes
|
||||
* the storage, the vector chunks and the metadata, and unlinks exactly what it managed to delete.
|
||||
*/
|
||||
export const deleteAgentResourceFiles = async <TFile>(
|
||||
{
|
||||
agentId,
|
||||
agentObjectId,
|
||||
toolResource,
|
||||
requestedFileIds,
|
||||
attachedFileIds,
|
||||
files,
|
||||
userId,
|
||||
}: {
|
||||
agentId: string;
|
||||
/** The agent's globally unique `_id`; `id` alone repeats across tenants. */
|
||||
agentObjectId: string;
|
||||
toolResource: string;
|
||||
requestedFileIds: string[];
|
||||
attachedFileIds: string[];
|
||||
files: Array<AgentResourceFileInput<TFile>>;
|
||||
userId: string;
|
||||
},
|
||||
deps: AgentResourceDeletionDeps<TFile>,
|
||||
): Promise<AgentResourceDeletionResult> => {
|
||||
const { ownedFiles, unlinkOnlyFiles } = partitionAgentResourceFiles({
|
||||
requestedFileIds,
|
||||
attachedFileIds,
|
||||
files,
|
||||
toolResource,
|
||||
userId,
|
||||
});
|
||||
|
||||
const sharedFileIds =
|
||||
ownedFiles.length === 0
|
||||
? new Set<string>()
|
||||
: new Set(
|
||||
await deps.getSharedResourceFileIds({
|
||||
file_ids: ownedFiles.map((input) => input.file_id),
|
||||
excludeAgentObjectId: agentObjectId,
|
||||
excludeToolResource: toolResource,
|
||||
}),
|
||||
);
|
||||
|
||||
const destroyable: Array<AgentResourceFileInput<TFile>> = [];
|
||||
const unlinkOnly = [...unlinkOnlyFiles];
|
||||
for (const input of ownedFiles) {
|
||||
if (sharedFileIds.has(input.file_id)) {
|
||||
unlinkOnly.push({ tool_resource: toolResource, file_id: input.file_id });
|
||||
continue;
|
||||
}
|
||||
destroyable.push(input);
|
||||
}
|
||||
|
||||
/* The destroy runs before any reference is removed, and the delete pass strips the references of
|
||||
the files it actually deleted. A reference removed ahead of a destroy that then fails is what
|
||||
strands a file: it leaves the agent panel while its storage, chunks or metadata remain, and
|
||||
neither this route nor the client's retry queue can name it again. */
|
||||
const outcome =
|
||||
destroyable.length === 0
|
||||
? null
|
||||
: await deps.deleteFiles(destroyable.map((input) => input.file));
|
||||
|
||||
if (unlinkOnly.length > 0) {
|
||||
await deps.removeAgentResourceFiles({ agent_id: agentId, files: unlinkOnly });
|
||||
}
|
||||
|
||||
return {
|
||||
outcome,
|
||||
unlinkedFileIds: unlinkOnly.map((ref) => ref.file_id),
|
||||
destroyedFileIds: outcome?.deletedFileIds ?? [],
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ let revertAgentVersion: AgentMethods['revertAgentVersion'];
|
|||
let addAgentResourceFile: AgentMethods['addAgentResourceFile'];
|
||||
let removeAgentResourceFiles: AgentMethods['removeAgentResourceFiles'];
|
||||
let removeAgentResourceFilesFromAllAgents: AgentMethods['removeAgentResourceFilesFromAllAgents'];
|
||||
let getSharedResourceFileIds: AgentMethods['getSharedResourceFileIds'];
|
||||
let getListAgentsByAccess: AgentMethods['getListAgentsByAccess'];
|
||||
let getAgentManagementListByAccess: AgentMethods['getAgentManagementListByAccess'];
|
||||
let generateActionMetadataHash: AgentMethods['generateActionMetadataHash'];
|
||||
|
|
@ -117,6 +118,7 @@ beforeAll(async () => {
|
|||
addAgentResourceFile = methods.addAgentResourceFile;
|
||||
removeAgentResourceFiles = methods.removeAgentResourceFiles;
|
||||
removeAgentResourceFilesFromAllAgents = methods.removeAgentResourceFilesFromAllAgents;
|
||||
getSharedResourceFileIds = methods.getSharedResourceFileIds;
|
||||
getListAgentsByAccess = methods.getListAgentsByAccess;
|
||||
getAgentManagementListByAccess = methods.getAgentManagementListByAccess;
|
||||
generateActionMetadataHash = methods.generateActionMetadataHash;
|
||||
|
|
@ -4238,6 +4240,146 @@ describe('Agent Methods', () => {
|
|||
).rejects.toThrow('Agent not found for removing resource files');
|
||||
});
|
||||
|
||||
describe('getSharedResourceFileIds', () => {
|
||||
beforeEach(async () => {
|
||||
await Agent.deleteMany({});
|
||||
});
|
||||
|
||||
test('reports a file another agent still references, across tool resources', async () => {
|
||||
const sharedFileId = `file_${uuidv4()}`;
|
||||
const soleFileId = `file_${uuidv4()}`;
|
||||
|
||||
const agent = await createBasicAgent();
|
||||
const duplicate = await createBasicAgent();
|
||||
|
||||
await addAgentResourceFile({
|
||||
agent_id: agent.id,
|
||||
tool_resource: EToolResources.file_search,
|
||||
file_id: sharedFileId,
|
||||
});
|
||||
await addAgentResourceFile({
|
||||
agent_id: agent.id,
|
||||
tool_resource: EToolResources.file_search,
|
||||
file_id: soleFileId,
|
||||
});
|
||||
await addAgentResourceFile({
|
||||
agent_id: duplicate.id,
|
||||
tool_resource: EToolResources.context,
|
||||
file_id: sharedFileId,
|
||||
});
|
||||
|
||||
const shared = await getSharedResourceFileIds({
|
||||
file_ids: [sharedFileId, soleFileId],
|
||||
excludeAgentObjectId: String(agent._id),
|
||||
});
|
||||
|
||||
expect(shared).toEqual([sharedFileId]);
|
||||
});
|
||||
|
||||
test('does not count the excluded agent’s own reference', async () => {
|
||||
const fileId = `file_${uuidv4()}`;
|
||||
const agent = await createBasicAgent();
|
||||
|
||||
await addAgentResourceFile({
|
||||
agent_id: agent.id,
|
||||
tool_resource: EToolResources.file_search,
|
||||
file_id: fileId,
|
||||
});
|
||||
|
||||
expect(
|
||||
await getSharedResourceFileIds({
|
||||
file_ids: [fileId],
|
||||
excludeAgentObjectId: String(agent._id),
|
||||
}),
|
||||
).toEqual([]);
|
||||
expect(await getSharedResourceFileIds({ file_ids: [fileId] })).toEqual([fileId]);
|
||||
});
|
||||
|
||||
test('reports a file the same agent still holds under another tool resource', async () => {
|
||||
const fileId = `file_${uuidv4()}`;
|
||||
const agent = await createBasicAgent();
|
||||
|
||||
await addAgentResourceFile({
|
||||
agent_id: agent.id,
|
||||
tool_resource: EToolResources.file_search,
|
||||
file_id: fileId,
|
||||
});
|
||||
await addAgentResourceFile({
|
||||
agent_id: agent.id,
|
||||
tool_resource: EToolResources.context,
|
||||
file_id: fileId,
|
||||
});
|
||||
|
||||
expect(
|
||||
await getSharedResourceFileIds({
|
||||
file_ids: [fileId],
|
||||
excludeAgentObjectId: String(agent._id),
|
||||
excludeToolResource: EToolResources.file_search,
|
||||
}),
|
||||
).toEqual([fileId]);
|
||||
});
|
||||
|
||||
test('does not count the pair being removed', async () => {
|
||||
const fileId = `file_${uuidv4()}`;
|
||||
const agent = await createBasicAgent();
|
||||
|
||||
await addAgentResourceFile({
|
||||
agent_id: agent.id,
|
||||
tool_resource: EToolResources.file_search,
|
||||
file_id: fileId,
|
||||
});
|
||||
|
||||
expect(
|
||||
await getSharedResourceFileIds({
|
||||
file_ids: [fileId],
|
||||
excludeAgentObjectId: String(agent._id),
|
||||
excludeToolResource: EToolResources.file_search,
|
||||
}),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
test('does not skip another tenant’s agent that shares the logical id', async () => {
|
||||
const sharedId = `agent_${uuidv4()}`;
|
||||
const fileId = `file_${uuidv4()}`;
|
||||
const tenantA = `tenant-${uuidv4()}`;
|
||||
const tenantB = `tenant-${uuidv4()}`;
|
||||
|
||||
/* `id` is unique only with `tenantId`, so two tenants can hold the same logical agent id.
|
||||
Excluding by `id` would call the file unreferenced and destroy tenant B's bytes. */
|
||||
const agentA = await tenantStorage.run({ tenantId: tenantA }, () =>
|
||||
createBasicAgent({ id: sharedId }),
|
||||
);
|
||||
await tenantStorage.run({ tenantId: tenantB }, () => createBasicAgent({ id: sharedId }));
|
||||
|
||||
await tenantStorage.run({ tenantId: tenantA }, () =>
|
||||
addAgentResourceFile({
|
||||
agent_id: sharedId,
|
||||
tool_resource: EToolResources.file_search,
|
||||
file_id: fileId,
|
||||
}),
|
||||
);
|
||||
await tenantStorage.run({ tenantId: tenantB }, () =>
|
||||
addAgentResourceFile({
|
||||
agent_id: sharedId,
|
||||
tool_resource: EToolResources.file_search,
|
||||
file_id: fileId,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(
|
||||
await getSharedResourceFileIds({
|
||||
file_ids: [fileId],
|
||||
excludeAgentObjectId: String(agentA._id),
|
||||
excludeToolResource: EToolResources.file_search,
|
||||
}),
|
||||
).toEqual([fileId]);
|
||||
});
|
||||
|
||||
test('answers without querying when given no file_ids', async () => {
|
||||
expect(await getSharedResourceFileIds({ file_ids: [] })).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('removeAgentResourceFilesFromAllAgents', () => {
|
||||
beforeEach(async () => {
|
||||
await Agent.deleteMany({});
|
||||
|
|
|
|||
|
|
@ -650,6 +650,15 @@ export function createAgentMethods(
|
|||
}: {
|
||||
file_ids: string[];
|
||||
}) => Promise<{ matchedCount: number; modifiedCount: number }>;
|
||||
getSharedResourceFileIds: ({
|
||||
file_ids,
|
||||
excludeAgentObjectId,
|
||||
excludeToolResource,
|
||||
}: {
|
||||
file_ids: string[];
|
||||
excludeAgentObjectId?: string;
|
||||
excludeToolResource?: string;
|
||||
}) => Promise<string[]>;
|
||||
} {
|
||||
const { removeAllPermissions, getActions, getSoleOwnedResourceIds, isExternalSkillId } = deps;
|
||||
|
||||
|
|
@ -1229,6 +1238,68 @@ export function createAgentMethods(
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports which of the given file_ids keep a reference once the caller's own is removed, so a
|
||||
* caller can tell a last reference from a shared one.
|
||||
*
|
||||
* The unit of a reference is the `(agent, tool_resource)` pair rather than the agent:
|
||||
* `addAgentResourceFile` stores `file_ids` per resource, so one agent can hold the same file under
|
||||
* both `file_search` and `context`, and removing it from one leaves the other needing the bytes.
|
||||
* `excludeToolResource` narrows the exclusion to the pair being removed; omitting it excludes the
|
||||
* whole agent.
|
||||
*
|
||||
* The agent is excluded by `_id`, because `id` is unique only together with `tenantId`: matching on
|
||||
* `id` alone would skip another tenant's agent of the same name and call its file unreferenced.
|
||||
*
|
||||
* Duplicating an agent copies `file_ids` rather than the files behind them, so one file record
|
||||
* can back two agents; destroying its bytes on behalf of one agent would empty the other. This
|
||||
* is deliberately not scoped by tenant: a reference is a reference, whoever holds it.
|
||||
*/
|
||||
async function getSharedResourceFileIds({
|
||||
file_ids,
|
||||
excludeAgentObjectId,
|
||||
excludeToolResource,
|
||||
}: {
|
||||
file_ids: string[];
|
||||
excludeAgentObjectId?: string;
|
||||
excludeToolResource?: string;
|
||||
}): Promise<string[]> {
|
||||
if (!file_ids || file_ids.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const Agent = mongoose.models.Agent as Model<IAgent>;
|
||||
const requested = new Set(file_ids);
|
||||
const searchParameter: FilterQuery<IAgent> = {
|
||||
$or: TOOL_RESOURCE_KEYS.map((key) => ({
|
||||
[`tool_resources.${key}.file_ids`]: { $in: file_ids },
|
||||
})),
|
||||
};
|
||||
|
||||
const agents = await Agent.find(searchParameter, { _id: 1, tool_resources: 1 }).lean();
|
||||
const shared = new Set<string>();
|
||||
for (const agent of agents) {
|
||||
const isExcludedAgent =
|
||||
excludeAgentObjectId != null && String(agent._id) === excludeAgentObjectId;
|
||||
for (const key of TOOL_RESOURCE_KEYS) {
|
||||
if (isExcludedAgent && (excludeToolResource == null || key === excludeToolResource)) {
|
||||
continue;
|
||||
}
|
||||
const fileIds = agent.tool_resources?.[key]?.file_ids;
|
||||
if (fileIds == null) {
|
||||
continue;
|
||||
}
|
||||
for (const fileId of fileIds) {
|
||||
if (requested.has(fileId)) {
|
||||
shared.add(fileId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...shared];
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes an agent based on the provided search parameter.
|
||||
*/
|
||||
|
|
@ -1698,6 +1769,7 @@ export function createAgentMethods(
|
|||
generateActionMetadataHash,
|
||||
removeAgentFromUserFavorites,
|
||||
removeAgentResourceFilesFromAllAgents,
|
||||
getSharedResourceFileIds,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue