diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 081343d14c..ca8980e8fc 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -10,7 +10,6 @@ const { imageExtRegex, EModelEndpoint, EToolResources, - RetentionMode, mergeFileConfig, AgentCapabilities, checkOpenAIStorage, @@ -39,7 +38,7 @@ const { loadAuthValues } = require('~/server/services/Tools/credentials'); const { getFileStrategy } = require('~/server/utils/getFileStrategy'); const { checkCapability } = require('~/server/services/Config'); const { LB_QueueAsyncCall } = require('~/server/utils/queue'); -const { getRetentionExpiry } = require('./retention'); +const { getRetentionExpiry, getAgentFileRetentionExpiry } = require('./retention'); const { getStrategyFunctions } = require('./strategies'); const { determineFileType } = require('~/server/utils'); const { STTService } = require('./Audio/STTService'); @@ -68,20 +67,6 @@ const createSanitizedUploadWrapper = (uploadFunction) => { }; }; -const isPersistentAgentResourceUpload = ({ messageAttachment, tool_resource }) => - !messageAttachment && !!tool_resource; - -const getAgentFileRetentionExpiry = async ({ req, messageAttachment, tool_resource }) => { - if ( - isPersistentAgentResourceUpload({ messageAttachment, tool_resource }) && - req?.config?.interfaceConfig?.retentionMode !== RetentionMode.ALL - ) { - return {}; - } - - return await getRetentionExpiry(req); -}; - const hasCodeEnvRef = (file) => file?.metadata?.codeEnvRef != null; const isMissingStorageError = (err) => { diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index 0c8e48ec46..1521f873bf 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -34,12 +34,27 @@ jest.mock('librechat-data-provider', () => { }); jest.mock('@librechat/api', () => { + const actualDataProvider = jest.requireActual('librechat-data-provider'); + const RetentionMode = actualDataProvider.RetentionMode ?? { ALL: 'all', TEMPORARY: 'temporary' }; + const getRetentionExpiry = jest.fn(() => ({})); return { sanitizeFilename: jest.fn((n) => n), parseText: jest.fn().mockResolvedValue({ text: '', bytes: 0 }), processAudioFile: jest.fn(), getStorageMetadata: jest.fn(() => ({})), - getRetentionExpiry: jest.fn(() => ({})), + getRetentionExpiry, + getAgentFileRetentionExpiry: jest.fn(({ req, messageAttachment, toolResource }) => { + const interfaceConfig = req?.config?.interfaceConfig; + if ( + !messageAttachment && + !!toolResource && + (interfaceConfig?.retentionMode !== RetentionMode.ALL || + interfaceConfig?.retainAgentFiles === true) + ) { + return {}; + } + return getRetentionExpiry(req); + }), sweepExpiredFiles: jest.fn().mockResolvedValue({ scanned: 0, deleted: 0, failed: 0 }), startExpiredFileSweep: jest.fn().mockReturnValue('sweep-interval'), }; @@ -112,6 +127,7 @@ jest.mock('~/server/services/Files/Audio/STTService', () => ({ const { getRetentionExpiry, + getAgentFileRetentionExpiry, sweepExpiredFiles: sweepExpiredFilesWithDeps, startExpiredFileSweep: startExpiredFileSweepWithDeps, } = require('@librechat/api'); @@ -414,18 +430,45 @@ describe('processAgentFileUpload', () => { }); describe('retention for agent resource uploads', () => { - test('skips retention metadata for persistent agent context files outside all-data retention', async () => { + test('skips retention metadata for persistent agent context files outside all-data retention when retainAgentFiles is disabled', async () => { const expiredAt = new Date('2030-01-01T00:00:00.000Z'); - getRetentionExpiry.mockResolvedValueOnce({ expiredAt }); const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null, - interfaceConfig: { retentionMode: RetentionMode.TEMPORARY }, + interfaceConfig: { retentionMode: RetentionMode.TEMPORARY, retainAgentFiles: false }, body: { conversationId: 'temporary-convo', isTemporary: true }, }); await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + expect(getAgentFileRetentionExpiry).toHaveBeenCalledWith( + { + req, + messageAttachment: false, + toolResource: EToolResources.context, + }, + expect.any(Object), + ); + expect(getRetentionExpiry).not.toHaveBeenCalled(); + expect(db.createFile).toHaveBeenCalledWith(expect.not.objectContaining({ expiredAt }), true); + expect(db.addAgentResourceFile).toHaveBeenCalledWith( + expect.objectContaining({ + agent_id: 'agent-abc', + tool_resource: EToolResources.context, + }), + ); + }); + + test('skips retention metadata for persistent agent context files outside all-data retention when retainAgentFiles is enabled', async () => { + const expiredAt = new Date('2030-01-01T00:00:00.000Z'); + const req = makeReq({ + mimetype: PDF_MIME, + ocrConfig: null, + interfaceConfig: { retentionMode: RetentionMode.TEMPORARY, retainAgentFiles: true }, + }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + expect(getRetentionExpiry).not.toHaveBeenCalled(); expect(db.createFile).toHaveBeenCalledWith(expect.not.objectContaining({ expiredAt }), true); expect(db.addAgentResourceFile).toHaveBeenCalledWith( @@ -436,13 +479,13 @@ describe('processAgentFileUpload', () => { ); }); - test('applies all-data retention metadata to persistent agent context files', async () => { + test('applies all-data retention metadata to persistent agent context files when retainAgentFiles is disabled', async () => { const expiredAt = new Date('2030-01-01T00:00:00.000Z'); getRetentionExpiry.mockResolvedValueOnce({ expiredAt }); const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null, - interfaceConfig: { retentionMode: RetentionMode.ALL }, + interfaceConfig: { retentionMode: RetentionMode.ALL, retainAgentFiles: false }, }); await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); @@ -464,6 +507,40 @@ describe('processAgentFileUpload', () => { ); }); + test('skips all-data retention metadata for persistent agent context files when retainAgentFiles is enabled', async () => { + const expiredAt = new Date('2030-01-01T00:00:00.000Z'); + const req = makeReq({ + mimetype: PDF_MIME, + ocrConfig: null, + interfaceConfig: { retentionMode: RetentionMode.ALL, retainAgentFiles: true }, + }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(getAgentFileRetentionExpiry).toHaveBeenCalledWith( + { + req, + messageAttachment: false, + toolResource: EToolResources.context, + }, + expect.any(Object), + ); + expect(getRetentionExpiry).not.toHaveBeenCalled(); + expect(db.createFile).toHaveBeenCalledWith( + expect.objectContaining({ + context: FileContext.agents, + }), + true, + ); + expect(db.createFile).toHaveBeenCalledWith(expect.not.objectContaining({ expiredAt }), true); + expect(db.addAgentResourceFile).toHaveBeenCalledWith( + expect.objectContaining({ + agent_id: 'agent-abc', + tool_resource: EToolResources.context, + }), + ); + }); + test('applies retention metadata to context files uploaded as message attachments', async () => { const expiredAt = new Date('2030-01-01T00:00:00.000Z'); getRetentionExpiry.mockResolvedValueOnce({ expiredAt }); @@ -489,7 +566,6 @@ describe('processAgentFileUpload', () => { test('skips retention metadata for persistent agent file-search files outside all-data retention', async () => { const expiredAt = new Date('2030-01-01T00:00:00.000Z'); - getRetentionExpiry.mockResolvedValueOnce({ expiredAt }); setupStoredFileUpload(); const req = makeReq({ mimetype: 'text/plain', ocrConfig: null }); @@ -650,7 +726,6 @@ describe('processAgentFileUpload', () => { it('skips retention metadata for persistent agent execute_code files outside all-data retention', async () => { const expiredAt = new Date('2030-01-01T00:00:00.000Z'); - getRetentionExpiry.mockResolvedValueOnce({ expiredAt }); setupCodeEnvUpload({ storage_session_id: 'sess-4', file_id: 'fid-4' }); const req = makeReq(); @@ -822,7 +897,7 @@ describe('processFileURL', () => { req: { user: { id: 'user-123', tenantId: 'tenant-a' }, body: {}, - config: { interfaceConfig: { retentionMode: 'all' } }, + config: { interfaceConfig: { retentionMode: 'all', retainAgentFiles: true } }, }, }); diff --git a/api/server/services/Files/retention.js b/api/server/services/Files/retention.js index 2895d5764a..e7394e2952 100644 --- a/api/server/services/Files/retention.js +++ b/api/server/services/Files/retention.js @@ -1,7 +1,16 @@ -const { getRetentionExpiry: getRetentionExpiryWithDeps } = require('@librechat/api'); +const { + getRetentionExpiry: getRetentionExpiryWithDeps, + getAgentFileRetentionExpiry: getAgentFileRetentionExpiryWithDeps, +} = require('@librechat/api'); const { logger, createTempChatExpirationDate } = require('@librechat/data-schemas'); const db = require('~/models'); +const getRetentionDependencies = () => ({ + getConvo: db.getConvoRetention ?? db.getConvo, + createExpirationDate: createTempChatExpirationDate, + logger, +}); + /** * Returns `{ expiredAt }` when the request indicates data retention applies, otherwise `{}`. * Spread into file data objects before calling createFile. @@ -9,13 +18,26 @@ const db = require('~/models'); * @returns {Promise<{ expiredAt?: Date | null }>} */ async function getRetentionExpiry(req) { - return getRetentionExpiryWithDeps(req, { - getConvo: db.getConvoRetention ?? db.getConvo, - createExpirationDate: createTempChatExpirationDate, - logger, - }); + return getRetentionExpiryWithDeps(req, getRetentionDependencies()); +} + +/** + * Returns `{ expiredAt }` for agent file uploads when retention applies, otherwise `{}`. + * @param {object} params + * @param {ServerRequest} params.req + * @param {boolean} [params.messageAttachment] + * @param {string} [params.tool_resource] + * @param {string} [params.toolResource] + * @returns {Promise<{ expiredAt?: Date | null }>} + */ +async function getAgentFileRetentionExpiry({ tool_resource, toolResource, ...params }) { + return getAgentFileRetentionExpiryWithDeps( + { ...params, toolResource: tool_resource ?? toolResource }, + getRetentionDependencies(), + ); } module.exports = { getRetentionExpiry, + getAgentFileRetentionExpiry, }; diff --git a/librechat.example.yaml b/librechat.example.yaml index 1e5a865d6a..fe11a6ab7b 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -176,6 +176,9 @@ interface: # such as "_meiliIndex_1_expiredAt_1" can be dropped from conversations/messages once the new # "_meiliIndex_1_isTemporary_1_expiredAt_1" indexes exist. # retentionMode: "temporary" + # Set retainAgentFiles to true to keep persistent agent resource files from expiring under + # retentionMode: "all"; non-agent files still expire. + # retainAgentFiles: false # Example Cloudflare turnstile (optional) #turnstile: diff --git a/packages/api/src/files/retention.spec.ts b/packages/api/src/files/retention.spec.ts index 65ede6a5b1..4722710a0c 100644 --- a/packages/api/src/files/retention.spec.ts +++ b/packages/api/src/files/retention.spec.ts @@ -1,6 +1,7 @@ import { RetentionMode } from 'librechat-data-provider'; import { createMinimalRetentionRequest, + getAgentFileRetentionExpiry, getConversationExpirationDate, getRetentionExpiry, getSharedLinkExpiration, @@ -191,6 +192,129 @@ describe('retention helpers', () => { await expect(getRetentionExpiry(undefined, dependencies)).resolves.toEqual({}); }); + it('skips persistent agent files in temporary retention mode when retainAgentFiles is disabled', async () => { + const result = await getAgentFileRetentionExpiry( + { + req: request({ + config: { + interfaceConfig: { + retentionMode: RetentionMode.TEMPORARY, + retainAgentFiles: false, + }, + }, + }), + messageAttachment: false, + toolResource: 'context', + }, + dependencies, + ); + + expect(result).toEqual({}); + expect(dependencies.getConvo).not.toHaveBeenCalled(); + expect(dependencies.createExpirationDate).not.toHaveBeenCalled(); + }); + + it('skips persistent agent files in temporary retention mode when retainAgentFiles is enabled', async () => { + const result = await getAgentFileRetentionExpiry( + { + req: request({ + config: { + interfaceConfig: { + retentionMode: RetentionMode.TEMPORARY, + retainAgentFiles: true, + }, + }, + }), + messageAttachment: false, + toolResource: 'context', + }, + dependencies, + ); + + expect(result).toEqual({}); + expect(dependencies.getConvo).not.toHaveBeenCalled(); + expect(dependencies.createExpirationDate).not.toHaveBeenCalled(); + }); + + it('applies all-data retention to persistent agent files when retainAgentFiles is disabled', async () => { + const result = await getAgentFileRetentionExpiry( + { + req: request({ + config: { + interfaceConfig: { + retentionMode: RetentionMode.ALL, + retainAgentFiles: false, + }, + }, + }), + messageAttachment: false, + toolResource: 'context', + }, + dependencies, + ); + + expect(result).toEqual({ expiredAt: expirationDate }); + expect(dependencies.getConvo).not.toHaveBeenCalled(); + expect(dependencies.createExpirationDate).toHaveBeenCalledTimes(1); + }); + + it('keeps current all-data retention behavior when retainAgentFiles is unset', async () => { + const result = await getAgentFileRetentionExpiry( + { + req: request({ config: { interfaceConfig: { retentionMode: RetentionMode.ALL } } }), + messageAttachment: false, + toolResource: 'context', + }, + dependencies, + ); + + expect(result).toEqual({ expiredAt: expirationDate }); + expect(dependencies.createExpirationDate).toHaveBeenCalledTimes(1); + }); + + it('skips all-data retention for persistent agent files when retainAgentFiles is enabled', async () => { + const result = await getAgentFileRetentionExpiry( + { + req: request({ + config: { + interfaceConfig: { + retentionMode: RetentionMode.ALL, + retainAgentFiles: true, + }, + }, + }), + messageAttachment: false, + toolResource: 'context', + }, + dependencies, + ); + + expect(result).toEqual({}); + expect(dependencies.getConvo).not.toHaveBeenCalled(); + expect(dependencies.createExpirationDate).not.toHaveBeenCalled(); + }); + + it('still applies all-data retention to agent message attachments when retainAgentFiles is enabled', async () => { + const result = await getAgentFileRetentionExpiry( + { + req: request({ + config: { + interfaceConfig: { + retentionMode: RetentionMode.ALL, + retainAgentFiles: true, + }, + }, + }), + messageAttachment: true, + toolResource: 'context', + }, + dependencies, + ); + + expect(result).toEqual({ expiredAt: expirationDate }); + expect(dependencies.createExpirationDate).toHaveBeenCalledTimes(1); + }); + it('parses valid conversation expiration dates and ignores invalid ones', () => { expect(getConversationExpirationDate({ expiredAt: expirationDate })).toBe(expirationDate); expect(getConversationExpirationDate({ expiredAt: expirationDate.toISOString() })).toEqual( diff --git a/packages/api/src/files/retention.ts b/packages/api/src/files/retention.ts index edbebadd90..58ba0df099 100644 --- a/packages/api/src/files/retention.ts +++ b/packages/api/src/files/retention.ts @@ -34,6 +34,12 @@ export type RetentionExpiry = { expiredAt?: Date | null; }; +export type AgentFileRetentionRequest = { + req: RetentionRequest | null | undefined; + messageAttachment?: boolean | null; + toolResource?: string | null; +}; + export type RetentionLogger = { error: (message: string, error?: unknown) => void; }; @@ -160,6 +166,35 @@ export async function getRetentionExpiry( return promise; } +const isPersistentAgentResourceUpload = ({ + messageAttachment, + toolResource, +}: Omit): boolean => !messageAttachment && !!toolResource; + +const shouldRetainPersistentAgentFile = ({ + req, + messageAttachment, + toolResource, +}: AgentFileRetentionRequest): boolean => { + const interfaceConfig = req?.config?.interfaceConfig; + return ( + isPersistentAgentResourceUpload({ messageAttachment, toolResource }) && + (interfaceConfig?.retentionMode !== RetentionMode.ALL || + interfaceConfig?.retainAgentFiles === true) + ); +}; + +export async function getAgentFileRetentionExpiry( + params: AgentFileRetentionRequest, + dependencies: RetentionDependencies, +): Promise { + if (shouldRetainPersistentAgentFile(params)) { + return {}; + } + + return await getRetentionExpiry(params.req, dependencies); +} + /** * Resolves the retention deadline for a shared link derived from a conversation. * diff --git a/packages/data-provider/specs/config-schemas.spec.ts b/packages/data-provider/specs/config-schemas.spec.ts index 4a29484728..33ee6db279 100644 --- a/packages/data-provider/specs/config-schemas.spec.ts +++ b/packages/data-provider/specs/config-schemas.spec.ts @@ -3,6 +3,7 @@ import { agentsEndpointSchema, azureEndpointSchema, endpointSchema, + RetentionMode, configSchema, interfaceSchema, fileStorageSchema, @@ -690,6 +691,16 @@ describe('interfaceSchema', () => { expect(result).not.toHaveProperty('sidePanel'); expect(result.modelSelect).toBe(false); }); + + it('accepts retainAgentFiles with all-data retention', () => { + const result = interfaceSchema.parse({ + retentionMode: RetentionMode.ALL, + retainAgentFiles: true, + }); + + expect(result.retentionMode).toBe(RetentionMode.ALL); + expect(result.retainAgentFiles).toBe(true); + }); }); describe('summarizationTriggerSchema', () => { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 999365e8ff..d471f2031f 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -961,6 +961,7 @@ export const interfaceSchema = z temporaryChatRetention: z.number().min(1).max(8760).optional(), autoSubmitFromUrl: z.boolean().optional(), retentionMode: z.nativeEnum(RetentionMode).default(RetentionMode.TEMPORARY), + retainAgentFiles: z.boolean().optional(), runCode: z.boolean().optional(), webSearch: z.boolean().optional(), peoplePicker: z diff --git a/packages/data-schemas/src/app/interface.spec.ts b/packages/data-schemas/src/app/interface.spec.ts index 9e289d289b..5dbb82f5a0 100644 --- a/packages/data-schemas/src/app/interface.spec.ts +++ b/packages/data-schemas/src/app/interface.spec.ts @@ -1,4 +1,4 @@ -import { getConfigDefaults } from 'librechat-data-provider'; +import { getConfigDefaults, RetentionMode } from 'librechat-data-provider'; import type { TCustomConfig } from 'librechat-data-provider'; import { loadDefaultInterface } from './interface'; @@ -65,4 +65,21 @@ describe('loadDefaultInterface', () => { expect(interfaceConfig).not.toHaveProperty('temporaryChatRetention'); }); + + it('preserves the configured agent file retention exemption', async () => { + const config: Partial = { + interface: { + retentionMode: RetentionMode.ALL, + retainAgentFiles: true, + }, + }; + + const interfaceConfig = await loadDefaultInterface({ + config, + configDefaults: getConfigDefaults(), + }); + + expect(interfaceConfig?.retentionMode).toBe(RetentionMode.ALL); + expect(interfaceConfig?.retainAgentFiles).toBe(true); + }); }); diff --git a/packages/data-schemas/src/app/interface.ts b/packages/data-schemas/src/app/interface.ts index 2dbe387ccf..ece570f6bd 100644 --- a/packages/data-schemas/src/app/interface.ts +++ b/packages/data-schemas/src/app/interface.ts @@ -49,6 +49,7 @@ export async function loadDefaultInterface({ temporaryChat: interfaceConfig?.temporaryChat, temporaryChatRetention: interfaceConfig?.temporaryChatRetention, retentionMode: interfaceConfig?.retentionMode, + retainAgentFiles: interfaceConfig?.retainAgentFiles, runCode: interfaceConfig?.runCode, webSearch: interfaceConfig?.webSearch, fileSearch: interfaceConfig?.fileSearch,