🗃️ feat: Retain Agent Files During All-Data Retention (#13477)

* feat: add agent file retention exemption

* refactor: centralize agent file retention policy

api/server/services/Files/process.js

@@ -10,7 +10,6 @@ const {
   imageExtRegex,
   EModelEndpoint,
   EToolResources,
-  RetentionMode,
   mergeFileConfig,
   AgentCapabilities,
   checkOpenAIStorage,
@@ -39,7 +38,7 @@ const { loadAuthValues } = require('~/server/services/Tools/credentials');
 const { getFileStrategy } = require('~/server/utils/getFileStrategy');
 const { checkCapability } = require('~/server/services/Config');
 const { LB_QueueAsyncCall } = require('~/server/utils/queue');
-const { getRetentionExpiry } = require('./retention');
+const { getRetentionExpiry, getAgentFileRetentionExpiry } = require('./retention');
 const { getStrategyFunctions } = require('./strategies');
 const { determineFileType } = require('~/server/utils');
 const { STTService } = require('./Audio/STTService');
@@ -68,20 +67,6 @@ const createSanitizedUploadWrapper = (uploadFunction) => {
   };
 };
 
-const isPersistentAgentResourceUpload = ({ messageAttachment, tool_resource }) =>
-  !messageAttachment && !!tool_resource;
-
-const getAgentFileRetentionExpiry = async ({ req, messageAttachment, tool_resource }) => {
-  if (
-    isPersistentAgentResourceUpload({ messageAttachment, tool_resource }) &&
-    req?.config?.interfaceConfig?.retentionMode !== RetentionMode.ALL
-  ) {
-    return {};
-  }
-
-  return await getRetentionExpiry(req);
-};
-
 const hasCodeEnvRef = (file) => file?.metadata?.codeEnvRef != null;
 
 const isMissingStorageError = (err) => {

api/server/services/Files/process.spec.js

@@ -34,12 +34,27 @@ jest.mock('librechat-data-provider', () => {
 });
 
 jest.mock('@librechat/api', () => {
+  const actualDataProvider = jest.requireActual('librechat-data-provider');
+  const RetentionMode = actualDataProvider.RetentionMode ?? { ALL: 'all', TEMPORARY: 'temporary' };
+  const getRetentionExpiry = jest.fn(() => ({}));
   return {
     sanitizeFilename: jest.fn((n) => n),
     parseText: jest.fn().mockResolvedValue({ text: '', bytes: 0 }),
     processAudioFile: jest.fn(),
     getStorageMetadata: jest.fn(() => ({})),
-    getRetentionExpiry: jest.fn(() => ({})),
+    getRetentionExpiry,
+    getAgentFileRetentionExpiry: jest.fn(({ req, messageAttachment, toolResource }) => {
+      const interfaceConfig = req?.config?.interfaceConfig;
+      if (
+        !messageAttachment &&
+        !!toolResource &&
+        (interfaceConfig?.retentionMode !== RetentionMode.ALL ||
+          interfaceConfig?.retainAgentFiles === true)
+      ) {
+        return {};
+      }
+      return getRetentionExpiry(req);
+    }),
     sweepExpiredFiles: jest.fn().mockResolvedValue({ scanned: 0, deleted: 0, failed: 0 }),
     startExpiredFileSweep: jest.fn().mockReturnValue('sweep-interval'),
   };
@@ -112,6 +127,7 @@ jest.mock('~/server/services/Files/Audio/STTService', () => ({
 
 const {
   getRetentionExpiry,
+  getAgentFileRetentionExpiry,
   sweepExpiredFiles: sweepExpiredFilesWithDeps,
   startExpiredFileSweep: startExpiredFileSweepWithDeps,
 } = require('@librechat/api');
@@ -414,18 +430,45 @@ describe('processAgentFileUpload', () => {
   });
 
   describe('retention for agent resource uploads', () => {
-    test('skips retention metadata for persistent agent context files outside all-data retention', async () => {
+    test('skips retention metadata for persistent agent context files outside all-data retention when retainAgentFiles is disabled', async () => {
       const expiredAt = new Date('2030-01-01T00:00:00.000Z');
-      getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
       const req = makeReq({
         mimetype: PDF_MIME,
         ocrConfig: null,
-        interfaceConfig: { retentionMode: RetentionMode.TEMPORARY },
+        interfaceConfig: { retentionMode: RetentionMode.TEMPORARY, retainAgentFiles: false },
         body: { conversationId: 'temporary-convo', isTemporary: true },
       });
 
       await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() });
 
+      expect(getAgentFileRetentionExpiry).toHaveBeenCalledWith(
+        {
+          req,
+          messageAttachment: false,
+          toolResource: EToolResources.context,
+        },
+        expect.any(Object),
+      );
+      expect(getRetentionExpiry).not.toHaveBeenCalled();
+      expect(db.createFile).toHaveBeenCalledWith(expect.not.objectContaining({ expiredAt }), true);
+      expect(db.addAgentResourceFile).toHaveBeenCalledWith(
+        expect.objectContaining({
+          agent_id: 'agent-abc',
+          tool_resource: EToolResources.context,
+        }),
+      );
+    });
+
+    test('skips retention metadata for persistent agent context files outside all-data retention when retainAgentFiles is enabled', async () => {
+      const expiredAt = new Date('2030-01-01T00:00:00.000Z');
+      const req = makeReq({
+        mimetype: PDF_MIME,
+        ocrConfig: null,
+        interfaceConfig: { retentionMode: RetentionMode.TEMPORARY, retainAgentFiles: true },
+      });
+
+      await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() });
+
       expect(getRetentionExpiry).not.toHaveBeenCalled();
       expect(db.createFile).toHaveBeenCalledWith(expect.not.objectContaining({ expiredAt }), true);
       expect(db.addAgentResourceFile).toHaveBeenCalledWith(
@@ -436,13 +479,13 @@ describe('processAgentFileUpload', () => {
       );
     });
 
-    test('applies all-data retention metadata to persistent agent context files', async () => {
+    test('applies all-data retention metadata to persistent agent context files when retainAgentFiles is disabled', async () => {
       const expiredAt = new Date('2030-01-01T00:00:00.000Z');
       getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
       const req = makeReq({
         mimetype: PDF_MIME,
         ocrConfig: null,
-        interfaceConfig: { retentionMode: RetentionMode.ALL },
+        interfaceConfig: { retentionMode: RetentionMode.ALL, retainAgentFiles: false },
       });
 
       await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() });
@@ -464,6 +507,40 @@ describe('processAgentFileUpload', () => {
       );
     });
 
+    test('skips all-data retention metadata for persistent agent context files when retainAgentFiles is enabled', async () => {
+      const expiredAt = new Date('2030-01-01T00:00:00.000Z');
+      const req = makeReq({
+        mimetype: PDF_MIME,
+        ocrConfig: null,
+        interfaceConfig: { retentionMode: RetentionMode.ALL, retainAgentFiles: true },
+      });
+
+      await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() });
+
+      expect(getAgentFileRetentionExpiry).toHaveBeenCalledWith(
+        {
+          req,
+          messageAttachment: false,
+          toolResource: EToolResources.context,
+        },
+        expect.any(Object),
+      );
+      expect(getRetentionExpiry).not.toHaveBeenCalled();
+      expect(db.createFile).toHaveBeenCalledWith(
+        expect.objectContaining({
+          context: FileContext.agents,
+        }),
+        true,
+      );
+      expect(db.createFile).toHaveBeenCalledWith(expect.not.objectContaining({ expiredAt }), true);
+      expect(db.addAgentResourceFile).toHaveBeenCalledWith(
+        expect.objectContaining({
+          agent_id: 'agent-abc',
+          tool_resource: EToolResources.context,
+        }),
+      );
+    });
+
     test('applies retention metadata to context files uploaded as message attachments', async () => {
       const expiredAt = new Date('2030-01-01T00:00:00.000Z');
       getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
@@ -489,7 +566,6 @@ describe('processAgentFileUpload', () => {
 
     test('skips retention metadata for persistent agent file-search files outside all-data retention', async () => {
       const expiredAt = new Date('2030-01-01T00:00:00.000Z');
-      getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
       setupStoredFileUpload();
       const req = makeReq({ mimetype: 'text/plain', ocrConfig: null });
 
@@ -650,7 +726,6 @@ describe('processAgentFileUpload', () => {
 
     it('skips retention metadata for persistent agent execute_code files outside all-data retention', async () => {
       const expiredAt = new Date('2030-01-01T00:00:00.000Z');
-      getRetentionExpiry.mockResolvedValueOnce({ expiredAt });
       setupCodeEnvUpload({ storage_session_id: 'sess-4', file_id: 'fid-4' });
       const req = makeReq();
 
@@ -822,7 +897,7 @@ describe('processFileURL', () => {
       req: {
         user: { id: 'user-123', tenantId: 'tenant-a' },
         body: {},
-        config: { interfaceConfig: { retentionMode: 'all' } },
+        config: { interfaceConfig: { retentionMode: 'all', retainAgentFiles: true } },
       },
     });
 

api/server/services/Files/retention.js

@@ -1,7 +1,16 @@
-const { getRetentionExpiry: getRetentionExpiryWithDeps } = require('@librechat/api');
+const {
+  getRetentionExpiry: getRetentionExpiryWithDeps,
+  getAgentFileRetentionExpiry: getAgentFileRetentionExpiryWithDeps,
+} = require('@librechat/api');
 const { logger, createTempChatExpirationDate } = require('@librechat/data-schemas');
 const db = require('~/models');
 
+const getRetentionDependencies = () => ({
+  getConvo: db.getConvoRetention ?? db.getConvo,
+  createExpirationDate: createTempChatExpirationDate,
+  logger,
+});
+
 /**
  * Returns `{ expiredAt }` when the request indicates data retention applies, otherwise `{}`.
  * Spread into file data objects before calling createFile.
@@ -9,13 +18,26 @@ const db = require('~/models');
  * @returns {Promise<{ expiredAt?: Date | null }>}
  */
 async function getRetentionExpiry(req) {
-  return getRetentionExpiryWithDeps(req, {
-    getConvo: db.getConvoRetention ?? db.getConvo,
-    createExpirationDate: createTempChatExpirationDate,
-    logger,
-  });
+  return getRetentionExpiryWithDeps(req, getRetentionDependencies());
+}
+
+/**
+ * Returns `{ expiredAt }` for agent file uploads when retention applies, otherwise `{}`.
+ * @param {object} params
+ * @param {ServerRequest} params.req
+ * @param {boolean} [params.messageAttachment]
+ * @param {string} [params.tool_resource]
+ * @param {string} [params.toolResource]
+ * @returns {Promise<{ expiredAt?: Date | null }>}
+ */
+async function getAgentFileRetentionExpiry({ tool_resource, toolResource, ...params }) {
+  return getAgentFileRetentionExpiryWithDeps(
+    { ...params, toolResource: tool_resource ?? toolResource },
+    getRetentionDependencies(),
+  );
 }
 
 module.exports = {
   getRetentionExpiry,
+  getAgentFileRetentionExpiry,
 };

librechat.example.yaml

@@ -176,6 +176,9 @@ interface:
   # such as "_meiliIndex_1_expiredAt_1" can be dropped from conversations/messages once the new
   # "_meiliIndex_1_isTemporary_1_expiredAt_1" indexes exist.
   # retentionMode: "temporary"
+  # Set retainAgentFiles to true to keep persistent agent resource files from expiring under
+  # retentionMode: "all"; non-agent files still expire.
+  # retainAgentFiles: false
 
 # Example Cloudflare turnstile (optional)
 #turnstile:

packages/api/src/files/retention.spec.ts

@@ -1,6 +1,7 @@
 import { RetentionMode } from 'librechat-data-provider';
 import {
   createMinimalRetentionRequest,
+  getAgentFileRetentionExpiry,
   getConversationExpirationDate,
   getRetentionExpiry,
   getSharedLinkExpiration,
@@ -191,6 +192,129 @@ describe('retention helpers', () => {
     await expect(getRetentionExpiry(undefined, dependencies)).resolves.toEqual({});
   });
 
+  it('skips persistent agent files in temporary retention mode when retainAgentFiles is disabled', async () => {
+    const result = await getAgentFileRetentionExpiry(
+      {
+        req: request({
+          config: {
+            interfaceConfig: {
+              retentionMode: RetentionMode.TEMPORARY,
+              retainAgentFiles: false,
+            },
+          },
+        }),
+        messageAttachment: false,
+        toolResource: 'context',
+      },
+      dependencies,
+    );
+
+    expect(result).toEqual({});
+    expect(dependencies.getConvo).not.toHaveBeenCalled();
+    expect(dependencies.createExpirationDate).not.toHaveBeenCalled();
+  });
+
+  it('skips persistent agent files in temporary retention mode when retainAgentFiles is enabled', async () => {
+    const result = await getAgentFileRetentionExpiry(
+      {
+        req: request({
+          config: {
+            interfaceConfig: {
+              retentionMode: RetentionMode.TEMPORARY,
+              retainAgentFiles: true,
+            },
+          },
+        }),
+        messageAttachment: false,
+        toolResource: 'context',
+      },
+      dependencies,
+    );
+
+    expect(result).toEqual({});
+    expect(dependencies.getConvo).not.toHaveBeenCalled();
+    expect(dependencies.createExpirationDate).not.toHaveBeenCalled();
+  });
+
+  it('applies all-data retention to persistent agent files when retainAgentFiles is disabled', async () => {
+    const result = await getAgentFileRetentionExpiry(
+      {
+        req: request({
+          config: {
+            interfaceConfig: {
+              retentionMode: RetentionMode.ALL,
+              retainAgentFiles: false,
+            },
+          },
+        }),
+        messageAttachment: false,
+        toolResource: 'context',
+      },
+      dependencies,
+    );
+
+    expect(result).toEqual({ expiredAt: expirationDate });
+    expect(dependencies.getConvo).not.toHaveBeenCalled();
+    expect(dependencies.createExpirationDate).toHaveBeenCalledTimes(1);
+  });
+
+  it('keeps current all-data retention behavior when retainAgentFiles is unset', async () => {
+    const result = await getAgentFileRetentionExpiry(
+      {
+        req: request({ config: { interfaceConfig: { retentionMode: RetentionMode.ALL } } }),
+        messageAttachment: false,
+        toolResource: 'context',
+      },
+      dependencies,
+    );
+
+    expect(result).toEqual({ expiredAt: expirationDate });
+    expect(dependencies.createExpirationDate).toHaveBeenCalledTimes(1);
+  });
+
+  it('skips all-data retention for persistent agent files when retainAgentFiles is enabled', async () => {
+    const result = await getAgentFileRetentionExpiry(
+      {
+        req: request({
+          config: {
+            interfaceConfig: {
+              retentionMode: RetentionMode.ALL,
+              retainAgentFiles: true,
+            },
+          },
+        }),
+        messageAttachment: false,
+        toolResource: 'context',
+      },
+      dependencies,
+    );
+
+    expect(result).toEqual({});
+    expect(dependencies.getConvo).not.toHaveBeenCalled();
+    expect(dependencies.createExpirationDate).not.toHaveBeenCalled();
+  });
+
+  it('still applies all-data retention to agent message attachments when retainAgentFiles is enabled', async () => {
+    const result = await getAgentFileRetentionExpiry(
+      {
+        req: request({
+          config: {
+            interfaceConfig: {
+              retentionMode: RetentionMode.ALL,
+              retainAgentFiles: true,
+            },
+          },
+        }),
+        messageAttachment: true,
+        toolResource: 'context',
+      },
+      dependencies,
+    );
+
+    expect(result).toEqual({ expiredAt: expirationDate });
+    expect(dependencies.createExpirationDate).toHaveBeenCalledTimes(1);
+  });
+
   it('parses valid conversation expiration dates and ignores invalid ones', () => {
     expect(getConversationExpirationDate({ expiredAt: expirationDate })).toBe(expirationDate);
     expect(getConversationExpirationDate({ expiredAt: expirationDate.toISOString() })).toEqual(

packages/api/src/files/retention.ts

@@ -34,6 +34,12 @@ export type RetentionExpiry = {
   expiredAt?: Date | null;
 };
 
+export type AgentFileRetentionRequest = {
+  req: RetentionRequest | null | undefined;
+  messageAttachment?: boolean | null;
+  toolResource?: string | null;
+};
+
 export type RetentionLogger = {
   error: (message: string, error?: unknown) => void;
 };
@@ -160,6 +166,35 @@ export async function getRetentionExpiry(
   return promise;
 }
 
+const isPersistentAgentResourceUpload = ({
+  messageAttachment,
+  toolResource,
+}: Omit<AgentFileRetentionRequest, 'req'>): boolean => !messageAttachment && !!toolResource;
+
+const shouldRetainPersistentAgentFile = ({
+  req,
+  messageAttachment,
+  toolResource,
+}: AgentFileRetentionRequest): boolean => {
+  const interfaceConfig = req?.config?.interfaceConfig;
+  return (
+    isPersistentAgentResourceUpload({ messageAttachment, toolResource }) &&
+    (interfaceConfig?.retentionMode !== RetentionMode.ALL ||
+      interfaceConfig?.retainAgentFiles === true)
+  );
+};
+
+export async function getAgentFileRetentionExpiry(
+  params: AgentFileRetentionRequest,
+  dependencies: RetentionDependencies,
+): Promise<RetentionExpiry> {
+  if (shouldRetainPersistentAgentFile(params)) {
+    return {};
+  }
+
+  return await getRetentionExpiry(params.req, dependencies);
+}
+
 /**
  * Resolves the retention deadline for a shared link derived from a conversation.
  *

packages/data-provider/specs/config-schemas.spec.ts

@@ -3,6 +3,7 @@ import {
   agentsEndpointSchema,
   azureEndpointSchema,
   endpointSchema,
+  RetentionMode,
   configSchema,
   interfaceSchema,
   fileStorageSchema,
@@ -690,6 +691,16 @@ describe('interfaceSchema', () => {
     expect(result).not.toHaveProperty('sidePanel');
     expect(result.modelSelect).toBe(false);
   });
+
+  it('accepts retainAgentFiles with all-data retention', () => {
+    const result = interfaceSchema.parse({
+      retentionMode: RetentionMode.ALL,
+      retainAgentFiles: true,
+    });
+
+    expect(result.retentionMode).toBe(RetentionMode.ALL);
+    expect(result.retainAgentFiles).toBe(true);
+  });
 });
 
 describe('summarizationTriggerSchema', () => {

packages/data-provider/src/config.ts

@@ -961,6 +961,7 @@ export const interfaceSchema = z
     temporaryChatRetention: z.number().min(1).max(8760).optional(),
     autoSubmitFromUrl: z.boolean().optional(),
     retentionMode: z.nativeEnum(RetentionMode).default(RetentionMode.TEMPORARY),
+    retainAgentFiles: z.boolean().optional(),
     runCode: z.boolean().optional(),
     webSearch: z.boolean().optional(),
     peoplePicker: z

packages/data-schemas/src/app/interface.spec.ts

@@ -1,4 +1,4 @@
-import { getConfigDefaults } from 'librechat-data-provider';
+import { getConfigDefaults, RetentionMode } from 'librechat-data-provider';
 import type { TCustomConfig } from 'librechat-data-provider';
 import { loadDefaultInterface } from './interface';
 
@@ -65,4 +65,21 @@ describe('loadDefaultInterface', () => {
 
     expect(interfaceConfig).not.toHaveProperty('temporaryChatRetention');
   });
+
+  it('preserves the configured agent file retention exemption', async () => {
+    const config: Partial<TCustomConfig> = {
+      interface: {
+        retentionMode: RetentionMode.ALL,
+        retainAgentFiles: true,
+      },
+    };
+
+    const interfaceConfig = await loadDefaultInterface({
+      config,
+      configDefaults: getConfigDefaults(),
+    });
+
+    expect(interfaceConfig?.retentionMode).toBe(RetentionMode.ALL);
+    expect(interfaceConfig?.retainAgentFiles).toBe(true);
+  });
 });

packages/data-schemas/src/app/interface.ts

@@ -49,6 +49,7 @@ export async function loadDefaultInterface({
     temporaryChat: interfaceConfig?.temporaryChat,
     temporaryChatRetention: interfaceConfig?.temporaryChatRetention,
     retentionMode: interfaceConfig?.retentionMode,
+    retainAgentFiles: interfaceConfig?.retainAgentFiles,
     runCode: interfaceConfig?.runCode,
     webSearch: interfaceConfig?.webSearch,
     fileSearch: interfaceConfig?.fileSearch,