🗃️ feat: Retain Agent Files During All-Data Retention (#13477)

* feat: add agent file retention exemption

* refactor: centralize agent file retention policy

api/server/services/Files/process.js

@@ -10,7 +10,6 @@ const {
   imageExtRegex,
   EModelEndpoint,
   EToolResources,
-  RetentionMode,
   mergeFileConfig,
   AgentCapabilities,
   checkOpenAIStorage,
@@ -39,7 +38,7 @@ const { loadAuthValues } = require('~/server/services/Tools/credentials');
 const { getFileStrategy } = require('~/server/utils/getFileStrategy');
 const { checkCapability } = require('~/server/services/Config');
 const { LB_QueueAsyncCall } = require('~/server/utils/queue');
-const { getRetentionExpiry } = require('./retention');
+const { getRetentionExpiry, getAgentFileRetentionExpiry } = require('./retention');
 const { getStrategyFunctions } = require('./strategies');
 const { determineFileType } = require('~/server/utils');
 const { STTService } = require('./Audio/STTService');
@@ -68,20 +67,6 @@ const createSanitizedUploadWrapper = (uploadFunction) => {
   };
 };
 
-const isPersistentAgentResourceUpload = ({ messageAttachment, tool_resource }) =>
-  !messageAttachment && !!tool_resource;
-
-const getAgentFileRetentionExpiry = async ({ req, messageAttachment, tool_resource }) => {
-  if (
-    isPersistentAgentResourceUpload({ messageAttachment, tool_resource }) &&
-    req?.config?.interfaceConfig?.retentionMode !== RetentionMode.ALL
-  ) {
-    return {};
-  }
-
-  return await getRetentionExpiry(req);
-};
-
 const hasCodeEnvRef = (file) => file?.metadata?.codeEnvRef != null;
 
 const isMissingStorageError = (err) => {

api/server/services/Files/process.spec.js

@@ -34,12 +34,27 @@ jest.mock('librechat-data-provider', () => {
 });
 
 jest.mock('@librechat/api', () => {
+  const actualDataProvider = jest.requireActual('librechat-data-provider');
+  const RetentionMode = actualDataProvider.RetentionMode ?? { ALL: 'all', TEMPORARY: 'temporary' };
+  const getRetentionExpiry = jest.fn(() => ({}));
   return {
     sanitizeFilename: jest.fn((n) => n),
     parseText: jest.fn().mockResolvedValue({ text: '', bytes: 0 }),
     processAudioFile: jest.fn(),
     getStorageMetadata: jest.fn(() => ({})),
-    getRetentionExpiry: jest.fn(() => ({})),
+    getRetentionExpiry,
+    getAgentFileRetentionExpiry: jest.fn(({ req, messageAttachment, toolResource }) => {
+      const interfaceConfig = req?.config?.interfaceConfig;
+      if (
+        !messageAttachment &&
+        !!toolResource &&
+        (interfaceConfig?.retentionMode !== RetentionMode.ALL ||
+          interfaceConfig?.retainAgentFiles === true)
+      ) {
+        return {};
+      }
+      return getRetentionExpiry(req);
+    }),
     sweepExpiredFiles: jest.fn().mockResolvedValue({ scanned: 0, deleted: 0, failed: 0 }),
     startExpiredFileSweep: jest.fn().mockReturnValue('sweep-interval'),
   };
@@ -112,6 +127,7 @@ jest.mock('~/server/services/Files/Audio/STTService', () => ({
 
 const {
   getRetentionExpiry,
+  getAgentFileRetentionExpiry,
   sweepExpiredFiles: sweepExpiredFilesWithDeps,
   startExpiredFileSweep: startExpiredFileSweepWithDeps,
 } = require('@librechat/api');
@@ -414,18 +430,45 @@ describe('processAgentFileUpload', () => {
   });
 
   describe('retention for agent resource uploads', () => {
-    test('skips retention metadata for persistent agent context files outside all-data retention', async () => {
+    test('skips retention metadata for persistent agent context files outside all-data retention when retainAgentFiles is disabled', async () => {
       const expiredAt = new Date('2030-01-01T00:00:00.000Z');
-      getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
       const req = makeReq({
         mimetype: PDF_MIME,
         ocrConfig: null,
-        interfaceConfig: { retentionMode: RetentionMode.TEMPORARY },
+        interfaceConfig: { retentionMode: RetentionMode.TEMPORARY, retainAgentFiles: false },
         body: { conversationId: 'temporary-convo', isTemporary: true },
       });
 
       await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() });
 
+      expect(getAgentFileRetentionExpiry).toHaveBeenCalledWith(
+        {
+          req,
+          messageAttachment: false,
+          toolResource: EToolResources.context,
+        },
+        expect.any(Object),
+      );
+      expect(getRetentionExpiry).not.toHaveBeenCalled();
+      expect(db.createFile).toHaveBeenCalledWith(expect.not.objectContaining({ expiredAt }), true);
+      expect(db.addAgentResourceFile).toHaveBeenCalledWith(
+        expect.objectContaining({
+          agent_id: 'agent-abc',
+          tool_resource: EToolResources.context,
+        }),
+      );
+    });
+
+    test('skips retention metadata for persistent agent context files outside all-data retention when retainAgentFiles is enabled', async () => {
+      const expiredAt = new Date('2030-01-01T00:00:00.000Z');
+      const req = makeReq({
+        mimetype: PDF_MIME,
+        ocrConfig: null,
+        interfaceConfig: { retentionMode: RetentionMode.TEMPORARY, retainAgentFiles: true },
+      });
+
+      await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() });
+
       expect(getRetentionExpiry).not.toHaveBeenCalled();
       expect(db.createFile).toHaveBeenCalledWith(expect.not.objectContaining({ expiredAt }), true);
       expect(db.addAgentResourceFile).toHaveBeenCalledWith(
@@ -436,13 +479,13 @@ describe('processAgentFileUpload', () => {
       );
     });
 
-    test('applies all-data retention metadata to persistent agent context files', async () => {
+    test('applies all-data retention metadata to persistent agent context files when retainAgentFiles is disabled', async () => {
       const expiredAt = new Date('2030-01-01T00:00:00.000Z');
       getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
       const req = makeReq({
         mimetype: PDF_MIME,
         ocrConfig: null,
-        interfaceConfig: { retentionMode: RetentionMode.ALL },
+        interfaceConfig: { retentionMode: RetentionMode.ALL, retainAgentFiles: false },
       });
 
       await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() });
@@ -464,6 +507,40 @@ describe('processAgentFileUpload', () => {
       );
     });
 
+    test('skips all-data retention metadata for persistent agent context files when retainAgentFiles is enabled', async () => {
+      const expiredAt = new Date('2030-01-01T00:00:00.000Z');
+      const req = makeReq({
+        mimetype: PDF_MIME,
+        ocrConfig: null,
+        interfaceConfig: { retentionMode: RetentionMode.ALL, retainAgentFiles: true },
+      });
+
+      await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() });
+
+      expect(getAgentFileRetentionExpiry).toHaveBeenCalledWith(
+        {
+          req,
+          messageAttachment: false,
+          toolResource: EToolResources.context,
+        },
+        expect.any(Object),
+      );
+      expect(getRetentionExpiry).not.toHaveBeenCalled();
+      expect(db.createFile).toHaveBeenCalledWith(
+        expect.objectContaining({
+          context: FileContext.agents,
+        }),
+        true,
+      );
+      expect(db.createFile).toHaveBeenCalledWith(expect.not.objectContaining({ expiredAt }), true);
+      expect(db.addAgentResourceFile).toHaveBeenCalledWith(
+        expect.objectContaining({
+          agent_id: 'agent-abc',
+          tool_resource: EToolResources.context,
+        }),
+      );
+    });
+
     test('applies retention metadata to context files uploaded as message attachments', async () => {
       const expiredAt = new Date('2030-01-01T00:00:00.000Z');
       getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
@@ -489,7 +566,6 @@ describe('processAgentFileUpload', () => {
 
     test('skips retention metadata for persistent agent file-search files outside all-data retention', async () => {
       const expiredAt = new Date('2030-01-01T00:00:00.000Z');
-      getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
       setupStoredFileUpload();
       const req = makeReq({ mimetype: 'text/plain', ocrConfig: null });
 
@@ -650,7 +726,6 @@ describe('processAgentFileUpload', () => {
 
     it('skips retention metadata for persistent agent execute_code files outside all-data retention', async () => {
       const expiredAt = new Date('2030-01-01T00:00:00.000Z');
-      getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
       setupCodeEnvUpload({ storage_session_id: 'sess-4', file_id: 'fid-4' });
       const req = makeReq();
 
@@ -822,7 +897,7 @@ describe('processFileURL', () => {
       req: {
         user: { id: 'user-123', tenantId: 'tenant-a' },
         body: {},
-        config: { interfaceConfig: { retentionMode: 'all' } },
+        config: { interfaceConfig: { retentionMode: 'all', retainAgentFiles: true } },
       },
     });
 

api/server/services/Files/retention.js

@@ -1,7 +1,16 @@
-const { getRetentionExpiry: getRetentionExpiryWithDeps } = require('@librechat/api');
+const {
+  getRetentionExpiry: getRetentionExpiryWithDeps,
+  getAgentFileRetentionExpiry: getAgentFileRetentionExpiryWithDeps,
+} = require('@librechat/api');
 const { logger, createTempChatExpirationDate } = require('@librechat/data-schemas');
 const db = require('~/models');
 
+const getRetentionDependencies = () => ({
+  getConvo: db.getConvoRetention ?? db.getConvo,
+  createExpirationDate: createTempChatExpirationDate,
+  logger,
+});
+
 /**
  * Returns `{ expiredAt }` when the request indicates data retention applies, otherwise `{}`.
  * Spread into file data objects before calling createFile.
@@ -9,13 +18,26 @@ const db = require('~/models');
  * @returns {Promise<{ expiredAt?: Date | null }>}
  */
 async function getRetentionExpiry(req) {
-  return getRetentionExpiryWithDeps(req, {
-    getConvo: db.getConvoRetention ?? db.getConvo,
-    createExpirationDate: createTempChatExpirationDate,
-    logger,
-  });
+  return getRetentionExpiryWithDeps(req, getRetentionDependencies());
+}
+
+/**
+ * Returns `{ expiredAt }` for agent file uploads when retention applies, otherwise `{}`.
+ * @param {object} params
+ * @param {ServerRequest} params.req
+ * @param {boolean} [params.messageAttachment]
+ * @param {string} [params.tool_resource]
+ * @param {string} [params.toolResource]
+ * @returns {Promise<{ expiredAt?: Date | null }>}
+ */
+async function getAgentFileRetentionExpiry({ tool_resource, toolResource, ...params }) {
+  return getAgentFileRetentionExpiryWithDeps(
+    { ...params, toolResource: tool_resource ?? toolResource },
+    getRetentionDependencies(),
+  );
 }
 
 module.exports = {
   getRetentionExpiry,
+  getAgentFileRetentionExpiry,
 };