🗄️ refactor: Honor All-Data Retention for Agent Files (#13424)
Some checks are pending
Docker Dev Branch Images Build / build (Dockerfile, lc-dev, node) (push) Waiting to run
Docker Dev Branch Images Build / build (Dockerfile.multi, lc-dev-api, api-build) (push) Waiting to run
GitNexus Index / index (push) Waiting to run
GitNexus Index / post-index (push) Blocked by required conditions
Docker Dev Images Build / build (Dockerfile, librechat-dev, node) (push) Waiting to run
Docker Dev Images Build / build (Dockerfile.multi, librechat-dev-api, api-build) (push) Waiting to run
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions

* 🛡️ fix: Honor All-Data Retention for Agent Files

* 🧹 fix: Delete Agent Tool Storage on Retention Sweep

* 🧯 fix: Clean Tool Storage After Missing Primary Files

* 🪢 fix: Defer Tool Deletes Until Primary File Resolves

* 🧭 fix: Prefer Session Object Delete for Code Files
This commit is contained in:
Danny Avila 2026-05-30 22:32:22 -04:00 • committed by GitHub
parent de760f6b51
commit e3cc2a9c62
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 475 additions and 31 deletions

View file

@ -58,6 +58,71 @@ async function getCodeOutputDownloadStream(fileIdentifier, identity, req) {
}
}
/**
* Deletes a file from the Code Environment server.
*
* @param {ServerRequest} req - Current authenticated request, used to mint Code API auth.
* @param {MongoFile} file - File metadata containing `metadata.codeEnvRef`.
* @returns {Promise<void>}
*/
async function deleteCodeEnvFile(req, file) {
const ref = file?.metadata?.codeEnvRef;
if (!ref) {
return;
}
let lastError;
const missingOrUnsupportedStatuses = new Set([404, 405]);
try {
const baseURL = getCodeBaseURL();
const query = buildCodeEnvDownloadQuery({
kind: ref.kind,
id: ref.id,
...(ref.kind === 'skill' ? { version: ref.version } : {}),
});
const authHeaders = await getCodeApiAuthHeaders(req);
const baseRequest = {
method: 'delete',
headers: {
'User-Agent': 'LibreChat/1.0',
...authHeaders,
},
httpAgent: codeServerHttpAgent,
httpsAgent: codeServerHttpsAgent,
timeout: 15000,
};
const urls = [
`${baseURL}/sessions/${ref.storage_session_id}/objects/${ref.file_id}${query}`,
`${baseURL}/files/${ref.storage_session_id}/${ref.file_id}${query}`,
];
for (const url of urls) {
try {
await axios({ ...baseRequest, url });
return;
} catch (error) {
lastError = error;
if (!missingOrUnsupportedStatuses.has(error.response?.status)) {
throw error;
}
}
}
} catch (error) {
lastError = error;
}
if (lastError) {
logAxiosError({
error: lastError,
message: `Error deleting code environment file: ${lastError.message}`,
});
if (lastError.response?.status === 404) {
return;
}
throw new Error(lastError.message || 'An error occurred during file deletion.');
}
}
/**
* Uploads a file to the Code Environment server.
*
@ -209,6 +274,7 @@ async function batchUploadCodeEnvFiles({ req, files, kind, id, version, read_onl
}
module.exports = {
deleteCodeEnvFile,
getCodeOutputDownloadStream,
uploadCodeEnvFile,
batchUploadCodeEnvFiles,

View file

@ -61,7 +61,7 @@ const {
codeServerHttpsAgent,
getCodeApiAuthHeaders,
} = require('@librechat/api');
const { getCodeOutputDownloadStream, uploadCodeEnvFile } = require('./crud');
const { deleteCodeEnvFile, getCodeOutputDownloadStream, uploadCodeEnvFile } = require('./crud');
describe('Code CRUD', () => {
beforeEach(() => {
@ -155,6 +155,94 @@ describe('Code CRUD', () => {
});
});
describe('deleteCodeEnvFile', () => {
const req = { user: { id: 'user-123' } };
const file = {
metadata: {
codeEnvRef: {
kind: 'agent',
id: 'agent-abc',
storage_session_id: 'session-1',
file_id: 'file-1',
},
},
};
it('deletes the code environment object with resource identity and auth headers', async () => {
getCodeApiAuthHeaders.mockResolvedValue({ Authorization: 'Bearer codeapi-token' });
mockAxios.mockResolvedValue({ status: 204 });
await deleteCodeEnvFile(req, file);
expect(getCodeApiAuthHeaders).toHaveBeenCalledWith(req);
expect(mockAxios).toHaveBeenCalledWith(
expect.objectContaining({
method: 'delete',
url: 'https://code-api.example.com/sessions/session-1/objects/file-1?kind=agent&id=agent-abc',
headers: expect.objectContaining({
Authorization: 'Bearer codeapi-token',
'User-Agent': 'LibreChat/1.0',
}),
httpAgent: codeServerHttpAgent,
httpsAgent: codeServerHttpsAgent,
timeout: 15000,
}),
);
});
it.each([404, 405])(
'falls back to the legacy code environment delete route after a %s',
async (status) => {
mockAxios
.mockRejectedValueOnce(
Object.assign(new Error('missing route'), { response: { status } }),
)
.mockResolvedValueOnce({ status: 204 });
await deleteCodeEnvFile(req, file);
expect(mockAxios).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
method: 'delete',
url: 'https://code-api.example.com/sessions/session-1/objects/file-1?kind=agent&id=agent-abc',
}),
);
expect(mockAxios).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
method: 'delete',
url: 'https://code-api.example.com/files/session-1/file-1?kind=agent&id=agent-abc',
}),
);
},
);
it('skips files without a code environment ref', async () => {
await deleteCodeEnvFile(req, {});
expect(mockAxios).not.toHaveBeenCalled();
expect(getCodeApiAuthHeaders).not.toHaveBeenCalled();
});
it('treats missing code environment objects as already deleted', async () => {
mockAxios
.mockRejectedValueOnce(Object.assign(new Error('missing'), { response: { status: 404 } }))
.mockRejectedValueOnce(Object.assign(new Error('missing'), { response: { status: 404 } }));
await expect(deleteCodeEnvFile(req, file)).resolves.toBeUndefined();
expect(mockAxios).toHaveBeenCalledTimes(2);
});
it('throws when code environment deletion fails', async () => {
mockAxios.mockRejectedValue(
Object.assign(new Error('unavailable'), { response: { status: 500 } }),
);
await expect(deleteCodeEnvFile(req, file)).rejects.toThrow('unavailable');
});
});
describe('uploadCodeEnvFile', () => {
const baseUploadParams = {
req: { user: { id: 'user-123' } },

View file

@ -10,6 +10,7 @@ const {
imageExtRegex,
EModelEndpoint,
EToolResources,
RetentionMode,
mergeFileConfig,
AgentCapabilities,
checkOpenAIStorage,
@ -71,13 +72,18 @@ const isPersistentAgentResourceUpload = ({ messageAttachment, tool_resource }) =
!messageAttachment && !!tool_resource;
const getAgentFileRetentionExpiry = async ({ req, messageAttachment, tool_resource }) => {
if (isPersistentAgentResourceUpload({ messageAttachment, tool_resource })) {
if (
isPersistentAgentResourceUpload({ messageAttachment, tool_resource }) &&
req?.config?.interfaceConfig?.retentionMode !== RetentionMode.ALL
) {
return {};
}
return await getRetentionExpiry(req);
};
const hasCodeEnvRef = (file) => file?.metadata?.codeEnvRef != null;
const isMissingStorageError = (err) => {
const code = err?.code ?? err?.status ?? err?.statusCode ?? err?.response?.status;
if ([404, '404', 'ENOENT', 'NoSuchKey', 'NotFound', 'ResourceNotFound'].includes(code)) {
@ -155,6 +161,49 @@ function enqueueDeleteOperation({
}
}
const getDeleteMethod = ({ source, deletionMethods }) => {
if (deletionMethods[source]) {
return deletionMethods[source];
}
const { deleteFile } = getStrategyFunctions(source);
if (!deleteFile) {
throw new Error(`Delete function not implemented for ${source}`);
}
deletionMethods[source] = deleteFile;
return deleteFile;
};
const createDeleteFileWithSecondaryStorage = ({ source, deleteFile, deletionMethods }) => {
return async (req, file, openai) => {
const secondaryDeleteMethods = [];
if (file.embedded === true && source !== FileSources.vectordb) {
secondaryDeleteMethods.push(
getDeleteMethod({ source: FileSources.vectordb, deletionMethods }),
);
}
if (hasCodeEnvRef(file) && source !== FileSources.execute_code) {
secondaryDeleteMethods.push(
getDeleteMethod({ source: FileSources.execute_code, deletionMethods }),
);
}
try {
await deleteFile(req, file, openai);
} catch (err) {
if (!isMissingStorageError(err)) {
throw err;
}
logger.warn('Primary file storage was already missing during delete', err);
}
await Promise.all(
secondaryDeleteMethods.map((secondaryDeleteFile) => secondaryDeleteFile(req, file)),
);
};
};
// TODO: refactor as currently only image files can be deleted this way
// as other filetypes will not reside in public path
/**
@ -240,29 +289,11 @@ const processDeleteRequest = async ({ req, files }) => {
promises.push(openai.beta.assistants.files.del(req.body.assistant_id, file.file_id));
}
if (deletionMethods[source]) {
enqueueDeleteOperation({
req,
file,
deleteFile: deletionMethods[source],
promises,
resolvedFileIds,
failedFileIds,
openai,
});
continue;
}
const { deleteFile } = getStrategyFunctions(source);
if (!deleteFile) {
throw new Error(`Delete function not implemented for ${source}`);
}
deletionMethods[source] = deleteFile;
const deleteFile = getDeleteMethod({ source, deletionMethods });
enqueueDeleteOperation({
req,
file,
deleteFile,
deleteFile: createDeleteFileWithSecondaryStorage({ source, deleteFile, deletionMethods }),
promises,
resolvedFileIds,
failedFileIds,

View file

@ -144,7 +144,7 @@ const ODT_MIME = 'application/vnd.oasis.opendocument.text';
const ODP_MIME = 'application/vnd.oasis.opendocument.presentation';
const ODG_MIME = 'application/vnd.oasis.opendocument.graphics';
const makeReq = ({ mimetype = PDF_MIME, ocrConfig = null } = {}) => ({
const makeReq = ({ mimetype = PDF_MIME, ocrConfig = null, interfaceConfig, body } = {}) => ({
user: { id: 'user-123', tenantId: 'tenant-a' },
file: {
path: '/tmp/upload.bin',
@ -152,11 +152,12 @@ const makeReq = ({ mimetype = PDF_MIME, ocrConfig = null } = {}) => ({
filename: 'upload-uuid.bin',
mimetype,
},
body: { model: 'gpt-4o' },
body: { model: 'gpt-4o', ...body },
config: {
fileConfig: {},
fileStrategy: 'local',
ocr: ocrConfig,
...(interfaceConfig ? { interfaceConfig } : {}),
},
});
@ -413,10 +414,15 @@ describe('processAgentFileUpload', () => {
});
describe('retention for agent resource uploads', () => {
test('does not apply retention metadata to persistent agent context files', async () => {
test('skips retention metadata for persistent agent context files outside all-data retention', async () => {
const expiredAt = new Date('2030-01-01T00:00:00.000Z');
getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null });
const req = makeReq({
mimetype: PDF_MIME,
ocrConfig: null,
interfaceConfig: { retentionMode: RetentionMode.TEMPORARY },
body: { conversationId: 'temporary-convo', isTemporary: true },
});
await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() });
@ -430,6 +436,34 @@ describe('processAgentFileUpload', () => {
);
});
test('applies all-data retention metadata to persistent agent context files', async () => {
const expiredAt = new Date('2030-01-01T00:00:00.000Z');
getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
const req = makeReq({
mimetype: PDF_MIME,
ocrConfig: null,
interfaceConfig: { retentionMode: RetentionMode.ALL },
});
await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() });
expect(getRetentionExpiry).toHaveBeenCalledTimes(1);
expect(getRetentionExpiry.mock.calls[0][0]).toBe(req);
expect(db.createFile).toHaveBeenCalledWith(
expect.objectContaining({
expiredAt,
context: FileContext.agents,
}),
true,
);
expect(db.addAgentResourceFile).toHaveBeenCalledWith(
expect.objectContaining({
agent_id: 'agent-abc',
tool_resource: EToolResources.context,
}),
);
});
test('applies retention metadata to context files uploaded as message attachments', async () => {
const expiredAt = new Date('2030-01-01T00:00:00.000Z');
getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
@ -453,7 +487,7 @@ describe('processAgentFileUpload', () => {
expect(db.addAgentResourceFile).not.toHaveBeenCalled();
});
test('does not apply retention metadata to persistent agent file-search files', async () => {
test('skips retention metadata for persistent agent file-search files outside all-data retention', async () => {
const expiredAt = new Date('2030-01-01T00:00:00.000Z');
getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
setupStoredFileUpload();
@ -475,6 +509,40 @@ describe('processAgentFileUpload', () => {
}),
);
});
test('applies all-data retention metadata to persistent agent file-search files', async () => {
const expiredAt = new Date('2030-01-01T00:00:00.000Z');
getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
setupStoredFileUpload();
const req = makeReq({
mimetype: 'text/plain',
ocrConfig: null,
interfaceConfig: { retentionMode: RetentionMode.ALL },
});
await processAgentFileUpload({
req,
res: mockRes,
metadata: { ...makeMetadata(), tool_resource: EToolResources.file_search },
});
expect(uploadVectors).toHaveBeenCalled();
expect(getRetentionExpiry).toHaveBeenCalledTimes(1);
expect(getRetentionExpiry.mock.calls[0][0]).toBe(req);
expect(db.createFile).toHaveBeenCalledWith(
expect.objectContaining({
expiredAt,
context: FileContext.agents,
}),
true,
);
expect(db.addAgentResourceFile).toHaveBeenCalledWith(
expect.objectContaining({
agent_id: 'agent-abc',
tool_resource: EToolResources.file_search,
}),
);
});
});
/* Phase C / option α regression: the upload must persist its sandbox
@ -580,7 +648,7 @@ describe('processAgentFileUpload', () => {
);
});
it('does not apply retention metadata to persistent agent execute_code files', async () => {
it('skips retention metadata for persistent agent execute_code files outside all-data retention', async () => {
const expiredAt = new Date('2030-01-01T00:00:00.000Z');
getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
setupCodeEnvUpload({ storage_session_id: 'sess-4', file_id: 'fid-4' });
@ -606,6 +674,47 @@ describe('processAgentFileUpload', () => {
);
});
it('applies all-data retention metadata to persistent agent execute_code files', async () => {
const expiredAt = new Date('2030-01-01T00:00:00.000Z');
getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
setupCodeEnvUpload({ storage_session_id: 'sess-5', file_id: 'fid-5' });
const req = makeReq({ interfaceConfig: { retentionMode: RetentionMode.ALL } });
await processAgentFileUpload({
req,
res: mockRes,
metadata: {
agent_id: 'agent-abc',
tool_resource: EToolResources.execute_code,
file_id: 'file-uuid',
},
});
expect(getRetentionExpiry).toHaveBeenCalledTimes(1);
expect(getRetentionExpiry.mock.calls[0][0]).toBe(req);
expect(db.createFile).toHaveBeenCalledWith(
expect.objectContaining({
expiredAt,
context: FileContext.agents,
metadata: {
codeEnvRef: {
kind: 'agent',
id: 'agent-abc',
storage_session_id: 'sess-5',
file_id: 'fid-5',
},
},
}),
true,
);
expect(db.addAgentResourceFile).toHaveBeenCalledWith(
expect.objectContaining({
agent_id: 'agent-abc',
tool_resource: EToolResources.execute_code,
}),
);
});
it('does not persist legacy fileIdentifier key (mongoose strict drops it)', async () => {
setupCodeEnvUpload({ storage_session_id: 'sess-3', file_id: 'fid-3' });
const req = makeReq();
@ -942,6 +1051,157 @@ describe('processDeleteRequest', () => {
expect(db.deleteFiles).toHaveBeenCalledWith(['expired-file']);
expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
});
it('deletes vector storage before removing embedded file metadata', async () => {
const primaryDelete = jest.fn().mockResolvedValue(undefined);
const vectorDelete = jest.fn().mockResolvedValue(undefined);
getStrategyFunctions.mockImplementation((source) =>
source === FileSources.vectordb
? { deleteFile: vectorDelete }
: { deleteFile: primaryDelete },
);
db.deleteFiles.mockResolvedValue({ deletedCount: 1 });
const req = {
body: {},
config: {},
user: { id: 'user-123', tenantId: 'tenant-a' },
};
const file = {
file_id: 'embedded-file',
filepath: '/uploads/embedded.txt',
source: FileSources.local,
embedded: true,
};
const result = await processDeleteRequest({ req, files: [file] });
expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
expect(vectorDelete).toHaveBeenCalledWith(req, file);
expect(db.deleteFiles).toHaveBeenCalledWith(['embedded-file']);
expect(result).toEqual({ deletedFileIds: ['embedded-file'], failedFileIds: [] });
});
it('keeps embedded file metadata when vector deletion fails', async () => {
const primaryDelete = jest.fn().mockResolvedValue(undefined);
const vectorDelete = jest.fn().mockRejectedValue(new Error('rag unavailable'));
getStrategyFunctions.mockImplementation((source) =>
source === FileSources.vectordb
? { deleteFile: vectorDelete }
: { deleteFile: primaryDelete },
);
const req = {
body: {},
config: {},
user: { id: 'user-123', tenantId: 'tenant-a' },
};
const file = {
file_id: 'embedded-file',
filepath: '/uploads/embedded.txt',
source: FileSources.local,
embedded: true,
};
const result = await processDeleteRequest({ req, files: [file] });
expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
expect(vectorDelete).toHaveBeenCalledWith(req, file);
expect(db.deleteFiles).not.toHaveBeenCalled();
expect(result).toEqual({ deletedFileIds: [], failedFileIds: ['embedded-file'] });
});
it('does not delete vector storage when primary embedded file deletion fails', async () => {
const primaryDelete = jest.fn().mockRejectedValue(new Error('permission denied'));
const vectorDelete = jest.fn().mockResolvedValue(undefined);
getStrategyFunctions.mockImplementation((source) =>
source === FileSources.vectordb
? { deleteFile: vectorDelete }
: { deleteFile: primaryDelete },
);
const req = {
body: {},
config: {},
user: { id: 'user-123', tenantId: 'tenant-a' },
};
const file = {
file_id: 'embedded-file',
filepath: '/uploads/embedded.txt',
source: FileSources.local,
embedded: true,
};
const result = await processDeleteRequest({ req, files: [file] });
expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
expect(vectorDelete).not.toHaveBeenCalled();
expect(db.deleteFiles).not.toHaveBeenCalled();
expect(result).toEqual({ deletedFileIds: [], failedFileIds: ['embedded-file'] });
});
it('still deletes vector storage when primary embedded file storage is already missing', async () => {
const missingError = Object.assign(new Error('no such file'), { code: 'ENOENT' });
const primaryDelete = jest.fn().mockRejectedValue(missingError);
const vectorDelete = jest.fn().mockResolvedValue(undefined);
getStrategyFunctions.mockImplementation((source) =>
source === FileSources.vectordb
? { deleteFile: vectorDelete }
: { deleteFile: primaryDelete },
);
db.deleteFiles.mockResolvedValue({ deletedCount: 1 });
const req = {
body: {},
config: {},
user: { id: 'user-123', tenantId: 'tenant-a' },
};
const file = {
file_id: 'embedded-file',
filepath: '/uploads/embedded.txt',
source: FileSources.local,
embedded: true,
};
const result = await processDeleteRequest({ req, files: [file] });
expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
expect(vectorDelete).toHaveBeenCalledWith(req, file);
expect(db.deleteFiles).toHaveBeenCalledWith(['embedded-file']);
expect(result).toEqual({ deletedFileIds: ['embedded-file'], failedFileIds: [] });
});
it('deletes code environment storage before removing code resource file metadata', async () => {
const primaryDelete = jest.fn().mockResolvedValue(undefined);
const codeDelete = jest.fn().mockResolvedValue(undefined);
getStrategyFunctions.mockImplementation((source) =>
source === FileSources.execute_code
? { deleteFile: codeDelete }
: { deleteFile: primaryDelete },
);
db.deleteFiles.mockResolvedValue({ deletedCount: 1 });
const req = {
body: {},
config: {},
user: { id: 'user-123', tenantId: 'tenant-a' },
};
const file = {
file_id: 'code-resource-file',
filepath: '/uploads/code-resource.txt',
source: FileSources.local,
metadata: {
codeEnvRef: {
kind: 'agent',
id: 'agent-abc',
storage_session_id: 'sess-1',
file_id: 'fid-1',
},
},
};
const result = await processDeleteRequest({ req, files: [file] });
expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
expect(codeDelete).toHaveBeenCalledWith(req, file);
expect(db.deleteFiles).toHaveBeenCalledWith(['code-resource-file']);
expect(result).toEqual({ deletedFileIds: ['code-resource-file'], failedFileIds: [] });
});
});
describe('sweepExpiredFiles', () => {

View file

@ -72,7 +72,7 @@ const {
processAzureAvatar,
} = require('./Azure');
const { uploadOpenAIFile, deleteOpenAIFile, getOpenAIFileStream } = require('./OpenAI');
const { getCodeOutputDownloadStream, uploadCodeEnvFile } = require('./Code');
const { deleteCodeEnvFile, getCodeOutputDownloadStream, uploadCodeEnvFile } = require('./Code');
const { uploadVectors, deleteVectors } = require('./VectorDB');
/**
@ -221,8 +221,7 @@ const codeOutputStrategy = () => ({
handleImageUpload: null,
/** @type {typeof prepareImagesLocal | null} */
prepareImagePayload: null,
/** @type {typeof deleteLocalFile | null} */
deleteFile: null,
deleteFile: deleteCodeEnvFile,
handleFileUpload: uploadCodeEnvFile,
getDownloadStream: getCodeOutputDownloadStream,
});