diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 258e349124..4e74116405 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -39,6 +39,7 @@ const { rejectPrivateTextAdmission, bindPrivateTextPersistenceAbort, persistedReasoningOverrideFields, + isTemporaryTurn, } = require('@librechat/api'); const { Constants, @@ -1070,7 +1071,9 @@ class BaseClient { const isAgentResponse = this.clientName === EModelEndpoint.agents || isAgentsEndpoint(this.options.endpoint); const langfuseTraceFields = isAgentResponse - ? await getLangfuseTraceMessageFields(appConfig, responseMessageId) + ? await getLangfuseTraceMessageFields(appConfig, responseMessageId, { + isTemporary: isTemporaryTurn(this.options.req), + }) : undefined; /** @type {TMessage} */ diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js index daac481e4e..f9ea444636 100644 --- a/api/server/controllers/agents/__tests__/openai.spec.js +++ b/api/server/controllers/agents/__tests__/openai.spec.js @@ -177,6 +177,7 @@ jest.mock('@librechat/agents', () => ({ jest.mock('@librechat/api', () => ({ getAgentErrorMetadata: (...args) => jest.requireActual('@librechat/api').getAgentErrorMetadata(...args), + isTemporaryTurn: (...args) => jest.requireActual('@librechat/api').isTemporaryTurn(...args), /* Provisioning moved into this package; the controllers build the callback from it. */ createProvisionFilesCallback: () => async () => {}, createAgentExecutionContext: (context) => context, @@ -910,6 +911,15 @@ describe('OpenAIChatCompletionController', () => { expect(createSubagentUsageSink).toHaveBeenCalledWith(expect.any(Array)); }); + it('runs a temporary conversation as temporary for Langfuse', async () => { + const { createRun } = require('@librechat/api'); + req.resolvedConversation = { isTemporary: true }; + + await OpenAIChatCompletionController(req, res); + + expect(createRun).toHaveBeenCalledWith(expect.objectContaining({ isTemporary: true })); + }); + it('uses collected usage for the non-streaming response', async () => { const { buildNonStreamingResponse } = require('@librechat/api'); diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js index f2754ce74f..2dd726abd5 100644 --- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js +++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js @@ -317,6 +317,7 @@ jest.mock('@librechat/api', () => ({ logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure, resolveFailedTurnContent: jest.requireActual('@librechat/api').resolveFailedTurnContent, getFailedTurnTraceFields: (...args) => mockGetFailedTurnTraceFields(...args), + isTemporaryTurn: jest.requireActual('@librechat/api').isTemporaryTurn, startAgentProjectContextResolution: jest.requireActual('@librechat/api').startAgentProjectContextResolution, assertChatProjectInstructions: jest.requireActual('@librechat/api').assertChatProjectInstructions, @@ -4026,6 +4027,7 @@ describe('ResumableAgentController resume metadata', () => { messageId: 'server-user_', runId: 'server-response-uuid', runCreated: true, + isTemporary: false, }); expect(errorRow).toMatchObject({ error: true, isCreatedByUser: false, ...traceFields }); }); diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js index 15d9fea185..a6923e0145 100644 --- a/api/server/controllers/agents/__tests__/responses.unit.spec.js +++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js @@ -202,6 +202,7 @@ jest.mock('@librechat/api', () => ({ jest.requireActual('@librechat/api').getAgentErrorMetadata(...args), getConversationWriteContext: (...args) => jest.requireActual('@librechat/api').getConversationWriteContext(...args), + isTemporaryTurn: (...args) => jest.requireActual('@librechat/api').isTemporaryTurn(...args), announceReply: jest.fn().mockResolvedValue(undefined), /* Provisioning moved into this package; the controllers build the callback from it. */ createProvisionFilesCallback: () => async () => {}, @@ -1039,7 +1040,9 @@ describe('createResponse controller', () => { await createResponse(req, res); - expect(api.getLangfuseTraceMessageFields).toHaveBeenCalledWith(req.config, 'resp_mock-123'); + expect(api.getLangfuseTraceMessageFields).toHaveBeenCalledWith(req.config, 'resp_mock-123', { + isTemporary: false, + }); expect(saveMessage).toHaveBeenCalledWith( expect.objectContaining({ userId: 'user-123' }), expect.objectContaining({ @@ -1053,6 +1056,24 @@ describe('createResponse controller', () => { ); }); + it.each([false, true])( + 'runs and records a forced-temporary turn as temporary for Langfuse: stream=%s', + async (stream) => { + const api = require('@librechat/api'); + req.config.interfaceConfig = { retentionMode: 'ephemeral', temporaryChatRetention: 1 }; + api.validateResponseRequest.mockReturnValueOnce({ + request: { ...req.body, stream, store: true }, + }); + + await createResponse(req, res); + + expect(api.createRun).toHaveBeenCalledWith(expect.objectContaining({ isTemporary: true })); + expect(api.getLangfuseTraceMessageFields).toHaveBeenCalledWith(req.config, 'resp_mock-123', { + isTemporary: true, + }); + }, + ); + it.each([false, true])( 'stores input and output under the retention write context: stream=%s', async (stream) => { diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index d74f4bba6c..2ec7bf5a11 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -208,6 +208,7 @@ const { markCompactionOutcome, resolvePersistableCodeEnvironmentDecision, getChatProjectContextKey, + isTemporaryTurn, } = require('@librechat/api'); const { Run, @@ -5037,6 +5038,7 @@ class AgentClient extends BaseClient { // Conversation-stable identity for the e2e run hook; a resumed run // carries no messages, so history cannot identify the conversation. conversationId: this.conversationId, + isTemporary: isTemporaryTurn(this.options.req), messages, discoveredToolNames: this.eventActorContinuation === 'warm' ? this.eventActorDiscoveredToolNames : undefined, @@ -5844,6 +5846,7 @@ class AgentClient extends BaseClient { ), agents, conversationId: this.conversationId, + isTemporary: isTemporaryTurn(this.options.req), modelCallbacks: [ modelBoundCallback, attachmentMemoryCallback, diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index c768bdefbc..5d5697a837 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -75,6 +75,7 @@ const { resolveApiConversationProject, resolveAdmittedCodeEnvironmentDecision, createTerminalRunErrorObserver, + isTemporaryTurn, } = require('@librechat/api'); const { buildSummarizationHandlers, @@ -1140,6 +1141,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { user: { ...createSafeUser(req.user), id: userId }, traceContext: { endpoint: EModelEndpoint.agents }, tenantId: principal.tenantId, + isTemporary: isTemporaryTurn(req), /** Bills subagent child-run model calls (reported outside the * streamEvents loop) into the same collectedUsage array. */ subagentUsageSink: createSubagentUsageSink(collectedUsage), diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index f6cc1694ee..807b9debca 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -66,6 +66,7 @@ const { settleExistingRowsBeforeErrorTurn, resolveDisconnectSnapshotMode, markAbortedCompactionContent, + isTemporaryTurn, } = require('@librechat/api'); const { disposeClient } = require('~/server/cleanup'); const { @@ -529,6 +530,7 @@ async function saveErrorTurn( messageId: errorMessageId, runId: liveResponseMessageId, runCreated, + isTemporary: isTemporaryTurn(req), }); const savedErrorMessage = await saveMessage( reqCtx, diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index 21c4076e01..ad183b9406 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -91,6 +91,7 @@ const { createTerminalRunErrorObserver, announceReply, getConversationWriteContext, + isTemporaryTurn, } = require('@librechat/api'); const { createResponsesToolEndCallback, @@ -428,7 +429,9 @@ async function saveResponseOutput( } } - const langfuseTraceFields = await getLangfuseTraceMessageFields(req.config, responseId); + const langfuseTraceFields = await getLangfuseTraceMessageFields(req.config, responseId, { + isTemporary: isTemporaryTurn(req), + }); // Save the assistant message return db.saveMessage( @@ -1357,6 +1360,7 @@ const executeResponse = async (envelope, { req, res }) => { user: { ...createSafeUser(req.user), id: userId }, traceContext: { endpoint: EModelEndpoint.agents }, tenantId: principal.tenantId, + isTemporary: isTemporaryTurn(req), modelCallbacks: [terminalRunError.modelCallback], clientToolNames: clientTools.clientToolNames, /** Bills subagent child-run model calls (reported outside the @@ -1596,6 +1600,7 @@ const executeResponse = async (envelope, { req, res }) => { user: { ...createSafeUser(req.user), id: userId }, traceContext: { endpoint: EModelEndpoint.agents }, tenantId: principal.tenantId, + isTemporary: isTemporaryTurn(req), modelCallbacks: [terminalRunError.modelCallback], clientToolNames: clientTools.clientToolNames, /** Bills subagent child-run model calls (reported outside the diff --git a/api/server/routes/__tests__/messages-feedback.spec.js b/api/server/routes/__tests__/messages-feedback.spec.js index 25064b188d..0f2abc5557 100644 --- a/api/server/routes/__tests__/messages-feedback.spec.js +++ b/api/server/routes/__tests__/messages-feedback.spec.js @@ -143,6 +143,25 @@ describe('PUT /:conversationId/:messageId/feedback', () => { ); }); + it('passes a temporary message to the scorer as temporary', async () => { + updateMessage.mockImplementationOnce((userId, { messageId, feedback }) => + Promise.resolve({ + messageId, + conversationId: 'conversation-1', + endpoint: 'agents', + isTemporary: true, + feedback, + }), + ); + + const response = await request(app) + .put('/api/messages/conversation-1/message-1/feedback') + .send({ feedback: { rating: 'thumbsDown', tag: 'other' } }); + + expect(response.status).toBe(200); + expect(sendFeedbackScore).toHaveBeenCalledWith(expect.objectContaining({ isTemporary: true })); + }); + it('scores the trace of the run a failed turn stands for', async () => { updateMessage.mockImplementationOnce((userId, { messageId, feedback }) => Promise.resolve({ diff --git a/api/server/routes/messages.js b/api/server/routes/messages.js index 89019d3d9f..c667346acf 100644 --- a/api/server/routes/messages.js +++ b/api/server/routes/messages.js @@ -784,6 +784,7 @@ router.put( destinationIds: updatedMessage.langfuseDestinationIds, feedback: updatedMessage.feedback, appConfig: req.config, + isTemporary: updatedMessage.isTemporary, metadata: { messageId: updatedMessage.messageId ?? messageId, parentMessageId: updatedMessage.parentMessageId, diff --git a/librechat.example.yaml b/librechat.example.yaml index f00bd479fa..402e682da8 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -76,6 +76,10 @@ projects: # librechat.endpoint.type, librechat.provider, librechat.model, # librechat.model.label, librechat.spec. # +# Temporary chats, including every chat when interface.retentionMode is +# "ephemeral", are not traced and their feedback is not scored unless +# langfuse.trace.temporaryChats is true. +# # langfuse: # headers: # CF-Access-Client-Id: "${CF_ACCESS_CLIENT_ID}" @@ -84,6 +88,7 @@ projects: # userIdField: email # userMetadataFields: [email, username, role, provider] # conversationMetadataFields: [conversationId, endpoint, model, modelLabel, spec] +# temporaryChats: false # File storage configuration # Single strategy for all file types (legacy format, still supported) diff --git a/packages/api/src/agents/openai/service.spec.ts b/packages/api/src/agents/openai/service.spec.ts index 3eff802d34..e97f3d602d 100644 --- a/packages/api/src/agents/openai/service.spec.ts +++ b/packages/api/src/agents/openai/service.spec.ts @@ -523,6 +523,17 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => { expect(runArgs.appConfig).not.toHaveProperty('interfaceConfig'); }); + it('forwards only the retention mode so forced-temporary runs stay untraced', async () => { + deps.appConfig = { + interfaceConfig: { modelSelect: true, retentionMode: 'ephemeral' }, + } as never; + + await createAgentChatCompletion(createMockReq({ id: 'user-123' }), createMockRes(), deps); + + const runArgs = createRun.mock.calls[0][0] as CreateRunArgs; + expect(runArgs.appConfig).toEqual({ interfaceConfig: { retentionMode: 'ephemeral' } }); + }); + it('forwards the stateful environment allowlist from appConfig to agent initialization', async () => { deps.appConfig = { endpoints: { diff --git a/packages/api/src/agents/openai/service.ts b/packages/api/src/agents/openai/service.ts index 0b4d8a7ad6..72b16a94cb 100644 --- a/packages/api/src/agents/openai/service.ts +++ b/packages/api/src/agents/openai/service.ts @@ -291,7 +291,10 @@ type LoadToolsFn = (params: { /** * Create run function type */ -type CreateRunAppConfig = Pick; +type CreateRunAppConfig = Pick< + AppConfig, + 'endpoints' | 'filters' | 'langfuse' | 'interfaceConfig' | 'messageFilter' +>; type CreateRunFn = (params: { agents: unknown[]; @@ -321,6 +324,8 @@ interface AppConfig { endpoints?: Record; filters?: FiltersConfig; langfuse?: Record; + /** Carries `retentionMode`, which keeps forced-temporary runs out of Langfuse. */ + interfaceConfig?: Record; messageFilter?: MessageFilterConfig; [key: string]: unknown; } @@ -335,6 +340,9 @@ function selectCreateRunAppConfig( ...(appConfig.endpoints !== undefined && { endpoints: appConfig.endpoints }), ...(appConfig.filters !== undefined && { filters: appConfig.filters }), ...(appConfig.langfuse !== undefined && { langfuse: appConfig.langfuse }), + ...(appConfig.interfaceConfig?.retentionMode !== undefined && { + interfaceConfig: { retentionMode: appConfig.interfaceConfig.retentionMode }, + }), ...(appConfig.messageFilter !== undefined && { messageFilter: appConfig.messageFilter }), }; } diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts index 0867a483e1..0232a992ff 100644 --- a/packages/api/src/agents/run.ts +++ b/packages/api/src/agents/run.ts @@ -2138,6 +2138,7 @@ export async function createRun({ user, tenantId, centralTraceExportEnabled, + isTemporary, traceContext, tokenCounter, customHandlers, @@ -2189,6 +2190,8 @@ export async function createRun({ * run. Tenant fanout can still export when tenant routing is available. */ centralTraceExportEnabled?: boolean; + /** A temporary chat's run is not traced unless `langfuse.trace.temporaryChats` opts in. */ + isTemporary?: boolean; /** * Exact tool names the owner approved for the rest of this conversation, read from the * stored conversation (never from the request body). Honored only when @@ -3118,6 +3121,7 @@ export async function createRun({ runId: resolvedRunId, tenantId: tenantId ?? user?.tenantId, centralTraceExportEnabled, + isTemporary, user, traceContext: resolveRunTraceContext({ agents, conversationId, requestBody, traceContext }), }), diff --git a/packages/api/src/conversations/retention.spec.ts b/packages/api/src/conversations/retention.spec.ts index ff10b61742..8eca0277bf 100644 --- a/packages/api/src/conversations/retention.spec.ts +++ b/packages/api/src/conversations/retention.spec.ts @@ -8,6 +8,7 @@ import { resolveImportRetentionFields, resolveImportTagCounts, isTemporaryRecord, + isTemporaryTurn, } from './retention'; describe('applyForcedRetention', () => { @@ -225,6 +226,45 @@ describe('resolveResumableRetention', () => { }); }); +describe('isTemporaryTurn', () => { + it('follows the stored conversation over the request flag', () => { + expect( + isTemporaryTurn({ + body: { isTemporary: false }, + resolvedConversation: { isTemporary: true }, + }), + ).toBe(true); + expect( + isTemporaryTurn({ + body: { isTemporary: true }, + resolvedConversation: { isTemporary: false }, + }), + ).toBe(false); + }); + + it('counts a turn the administrator forces temporary', () => { + expect( + isTemporaryTurn({ + resolvedConversation: { isTemporary: false }, + config: { interfaceConfig: { retentionMode: RetentionMode.EPHEMERAL } }, + }), + ).toBe(true); + }); + + it('counts a legacy conversation that stores only a deadline', () => { + const expiredAt = new Date('2099-01-01T00:00:00.000Z'); + expect(isTemporaryTurn({ resolvedConversation: { expiredAt } })).toBe(true); + expect(isTemporaryTurn({ resolvedConversation: { isTemporary: false, expiredAt } })).toBe( + false, + ); + }); + + it('treats a turn without request state as permanent', () => { + expect(isTemporaryTurn()).toBe(false); + expect(isTemporaryTurn({ body: {} })).toBe(false); + }); +}); + describe('resolveImportRetentionFields', () => { const deps = { createChatExpirationDate: jest.fn(() => new Date('2030-01-01T00:00:00.000Z')), diff --git a/packages/api/src/conversations/retention.ts b/packages/api/src/conversations/retention.ts index 53878e54bf..f0e6f64ce1 100644 --- a/packages/api/src/conversations/retention.ts +++ b/packages/api/src/conversations/retention.ts @@ -78,6 +78,18 @@ export async function persistForcedTemporaryMetadata( await store.updateMetadata(streamId, { isTemporary: true }, createdAt); } +/** + * Whether a turn runs in a temporary chat, counting one the administrator forces temporary + * and a legacy conversation that stores only a deadline. + */ +export function isTemporaryTurn(req?: TurnConversationRequest): boolean { + const { isTemporary, expiredAt, interfaceConfig } = getConversationWriteContext(req); + return ( + isTemporaryRecord({ isTemporary, expiredAt }) || + isForcedTemporaryRetention(interfaceConfig?.retentionMode) + ); +} + /** Captures the effective retention policy for a paused turn without re-reading its records. */ export function resolveResumableRetention( req: TurnConversationRequest, diff --git a/packages/api/src/langfuse/config.spec.ts b/packages/api/src/langfuse/config.spec.ts index e398d32a44..16dda9ec05 100644 --- a/packages/api/src/langfuse/config.spec.ts +++ b/packages/api/src/langfuse/config.spec.ts @@ -172,6 +172,71 @@ describe('buildLangfuseConfig', () => { }); }); + it('does not trace a temporary chat to any destination', async () => { + process.env.LANGFUSE_PUBLIC_KEY = 'pk-env'; + process.env.LANGFUSE_SECRET_KEY = 'sk-env'; + process.env.LANGFUSE_FANOUT_ENABLED = 'true'; + process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318'; + const { encryptV3 } = await import('@librechat/data-schemas'); + const { buildLangfuseConfig } = await import('./config'); + + expect( + buildLangfuseConfig({ + runId: 'run-1', + tenantId: 'tenant-1', + isTemporary: true, + appConfig: { + langfuse: { + enabled: true, + publicKey: 'pk-tenant-1', + secretKey: encryptV3('sk-tenant-1'), + destination: 'us', + }, + } as unknown as AppConfig, + }), + ).toEqual({ + deterministicTraceId: true, + enabled: false, + metadata: { 'librechat.tenant.id': 'tenant-1' }, + tags: ['tenant:tenant-1'], + }); + }); + + it('does not trace a run when retention forces every chat temporary', async () => { + delete process.env.TENANT_ISOLATION_STRICT; + process.env.LANGFUSE_PUBLIC_KEY = 'pk-env'; + process.env.LANGFUSE_SECRET_KEY = 'sk-env'; + const { buildLangfuseConfig } = await import('./config'); + + expect( + buildLangfuseConfig({ + runId: 'run-1', + appConfig: { interfaceConfig: { retentionMode: 'ephemeral' } } as unknown as AppConfig, + }), + ).toMatchObject({ enabled: false }); + }); + + it('traces a temporary chat when the deployment opts in', async () => { + delete process.env.TENANT_ISOLATION_STRICT; + process.env.LANGFUSE_PUBLIC_KEY = 'pk-env'; + process.env.LANGFUSE_SECRET_KEY = 'sk-env'; + process.env.LANGFUSE_BASE_URL = 'https://env.langfuse.example'; + const { buildLangfuseConfig } = await import('./config'); + + expect( + buildLangfuseConfig({ + runId: 'run-1', + isTemporary: true, + appConfig: { langfuse: { trace: { temporaryChats: true } } } as unknown as AppConfig, + }), + ).toEqual({ + deterministicTraceId: true, + publicKey: 'pk-env', + secretKey: 'sk-env', + baseUrl: 'https://env.langfuse.example', + }); + }); + it('does not trace a disabled stored connection in single-tenant mode', async () => { delete process.env.TENANT_ISOLATION_STRICT; const { encryptV3 } = await import('@librechat/data-schemas'); diff --git a/packages/api/src/langfuse/config.ts b/packages/api/src/langfuse/config.ts index 376758e472..10e626ed79 100644 --- a/packages/api/src/langfuse/config.ts +++ b/packages/api/src/langfuse/config.ts @@ -6,6 +6,7 @@ import { isLangfuseCentralMediaUploadDisabled, isLangfuseFanoutEnabled, isLangfuseTenantExportEnabled, + isLangfuseTraceAllowed, isLangfuseTraceSampled, isLangfuseTracingEnabled, usesLangfuseMultiTenantRouting, @@ -250,6 +251,7 @@ export function buildLangfuseConfig({ runId, tenantId, centralTraceExportEnabled = true, + isTemporary, user, traceContext, }: { @@ -262,6 +264,8 @@ export function buildLangfuseConfig({ * to drop the central pipeline while preserving tenant fanout when available. */ centralTraceExportEnabled?: boolean; + /** Disables every destination unless `langfuse.trace.temporaryChats` opts in. */ + isTemporary?: boolean; /** The requesting user, read only for the fields `langfuse.trace` allowlists. */ user?: LangfuseTraceUser; /** Request values `langfuse.trace.conversationMetadataFields` may export. */ @@ -291,6 +295,7 @@ export function buildLangfuseConfig({ if ( !isLangfuseTracingEnabled() || + !isLangfuseTraceAllowed(appConfig, isTemporary) || (runId != null && !isLangfuseTraceSampled(traceIdForMessage(runId))) ) { langfuse.enabled = false; diff --git a/packages/api/src/langfuse/destinations.ts b/packages/api/src/langfuse/destinations.ts index 1345c3abea..d237ec8609 100644 --- a/packages/api/src/langfuse/destinations.ts +++ b/packages/api/src/langfuse/destinations.ts @@ -5,6 +5,7 @@ import { isLangfuseFanoutEnabled, isLangfuseTenantExportEnabled, isLangfuseTracingEnabled, + isLangfuseTraceAllowed, isLangfuseTraceSampled, usesLangfuseMultiTenantRouting, } from './policy'; @@ -366,7 +367,8 @@ export type LangfuseTraceMessageFields = { /** * The sampling record a response stores for its run's trace. `runId` names the * run when it is not the response's own id, as for a failed turn's error row; - * it is then stored too, so feedback and the trace viewer follow that run. + * it is then stored too, so feedback and the trace viewer follow that run. A + * temporary turn's run was not traced unless the deployment opts in. */ export async function getLangfuseTraceMessageFields( appConfig: AppConfig | undefined, @@ -374,14 +376,19 @@ export async function getLangfuseTraceMessageFields( { centralTraceExportEnabled = true, runId = messageId, - }: Pick & { runId?: string } = {}, + isTemporary, + }: Pick & { + runId?: string; + isTemporary?: boolean; + } = {}, ): Promise<{ langfuseSampled: boolean; langfuseDestinationIds?: string[]; langfuseRunId?: string; }> { const traceId = traceIdForMessage(runId); - const langfuseSampled = isLangfuseTraceSampled(traceId); + const langfuseSampled = + isLangfuseTraceAllowed(appConfig, isTemporary) && isLangfuseTraceSampled(traceId); return { langfuseSampled, langfuseDestinationIds: await getLangfuseTraceDestinationIds( @@ -406,13 +413,14 @@ export async function getFailedTurnTraceFields( messageId, runId, runCreated, - }: { messageId: string; runId?: string | null; runCreated: boolean }, + isTemporary, + }: { messageId: string; runId?: string | null; runCreated: boolean; isTemporary?: boolean }, ): Promise { if (!runCreated || typeof runId !== 'string' || runId.length === 0) { return {}; } try { - return await getLangfuseTraceMessageFields(appConfig, messageId, { runId }); + return await getLangfuseTraceMessageFields(appConfig, messageId, { runId, isTemporary }); } catch (error) { logger.warn('[langfuse] Could not record the failed run trace:', error); return {}; diff --git a/packages/api/src/langfuse/feedback.spec.ts b/packages/api/src/langfuse/feedback.spec.ts index 04ea5dcb40..5114535bfc 100644 --- a/packages/api/src/langfuse/feedback.spec.ts +++ b/packages/api/src/langfuse/feedback.spec.ts @@ -259,6 +259,41 @@ describe('Langfuse feedback scores', () => { expect(getFetchMock()).not.toHaveBeenCalled(); }); + it('does not score a temporary chat unless the deployment opts in', async () => { + const { sendFeedbackScore } = await loadFeedback(); + const score = { traceId: 'trace-id', feedback: { rating: 'thumbsUp' as const } }; + + await sendFeedbackScore({ ...score, isTemporary: true }); + expect(getFetchMock()).not.toHaveBeenCalled(); + + await sendFeedbackScore({ + ...score, + isTemporary: true, + appConfig: { langfuse: { trace: { temporaryChats: true } } } as AppConfig, + }); + expect(getFetchMock()).toHaveBeenCalledTimes(1); + }); + + it('does not score a stored permanent row once retention forces chats temporary', async () => { + const { sendFeedbackScore } = await loadFeedback(); + const score = { traceId: 'trace-id', feedback: { rating: 'thumbsUp' as const } }; + const interfaceConfig = { retentionMode: 'ephemeral' }; + + await sendFeedbackScore({ + ...score, + isTemporary: false, + appConfig: { interfaceConfig } as AppConfig, + }); + expect(getFetchMock()).not.toHaveBeenCalled(); + + await sendFeedbackScore({ + ...score, + isTemporary: false, + appConfig: { interfaceConfig, langfuse: { trace: { temporaryChats: true } } } as AppConfig, + }); + expect(getFetchMock()).toHaveBeenCalledTimes(1); + }); + it('preserves a sampled trace when the sample rate decreases', async () => { process.env.LANGFUSE_SAMPLE_RATE = '0.1'; const { sendFeedbackScore } = await loadFeedback(); @@ -440,6 +475,29 @@ describe('Langfuse feedback scores', () => { ).resolves.toMatchObject({ langfuseSampled: true, langfuseRunId: 'run-1' }); }); + it('records a temporary turn as untraced unless the deployment opts in', async () => { + await loadFeedback(); + const { getFailedTurnTraceFields, getLangfuseTraceMessageFields } = await import( + './destinations' + ); + const optIn = { langfuse: { trace: { temporaryChats: true } } } as AppConfig; + + await expect( + getLangfuseTraceMessageFields(undefined, 'message-1', { isTemporary: true }), + ).resolves.toEqual({ langfuseSampled: false, langfuseDestinationIds: [] }); + await expect( + getFailedTurnTraceFields(undefined, { + messageId: 'user-1_', + runId: 'run-1', + runCreated: true, + isTemporary: true, + }), + ).resolves.toMatchObject({ langfuseSampled: false, langfuseRunId: 'run-1' }); + await expect( + getLangfuseTraceMessageFields(optIn, 'message-1', { isTemporary: true }), + ).resolves.toMatchObject({ langfuseSampled: true }); + }); + it('keeps the central destination identity stable when credentials rotate', async () => { const { sendFeedbackScore } = await loadFeedback(); const { getLangfuseTraceDestinationIds } = await import('./destinations'); diff --git a/packages/api/src/langfuse/feedback.ts b/packages/api/src/langfuse/feedback.ts index 77a8d4db6d..31813b5d65 100644 --- a/packages/api/src/langfuse/feedback.ts +++ b/packages/api/src/langfuse/feedback.ts @@ -1,6 +1,7 @@ import { logger } from '@librechat/data-schemas'; import type { AppConfig } from '@librechat/data-schemas'; import { getScoreDestinations, type LangfuseScoreDestination } from './destinations'; +import { isLangfuseTraceAllowed } from './policy'; import { mergeHeaders } from '~/utils/headers'; import { redirectPolicyFor } from './utils'; @@ -27,6 +28,11 @@ export type SendFeedbackScoreParams = { * whose project identity is unknown. */ centralTraceExportEnabled?: boolean; + /** + * Whether the message belongs to a temporary chat. Checked on its own because + * a resumed turn's response stores no trace record to carry the decision. + */ + isTemporary?: boolean | null; }; const ENVIRONMENT = process.env.LANGFUSE_TRACING_ENVIRONMENT; @@ -125,8 +131,9 @@ export async function sendFeedbackScore({ observationId, appConfig, centralTraceExportEnabled, + isTemporary, }: SendFeedbackScoreParams): Promise { - if (!traceId) { + if (!traceId || !isLangfuseTraceAllowed(appConfig, isTemporary)) { return; } diff --git a/packages/api/src/langfuse/policy.ts b/packages/api/src/langfuse/policy.ts index c8ac5368fd..dfa4265efa 100644 --- a/packages/api/src/langfuse/policy.ts +++ b/packages/api/src/langfuse/policy.ts @@ -1,3 +1,5 @@ +import { isForcedTemporaryRetention } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; import { isFalseEnv, isTrueEnv } from './utils'; import { normalizeString } from '~/utils/text'; @@ -58,6 +60,24 @@ function traceIdAccumulation(traceId: string): number { return accumulation; } +/** + * A temporary chat reaches no Langfuse destination unless the deployment opts + * in. `interface.retentionMode: "ephemeral"` makes every chat temporary, so it + * counts even when the caller only knows a row's stored flag. Read strictly, + * since admin config patches persist without validation. + */ +export function isLangfuseTraceAllowed( + appConfig: AppConfig | undefined, + isTemporary?: boolean | null, +): boolean { + if (appConfig?.langfuse?.trace?.temporaryChats === true) { + return true; + } + return ( + isTemporary !== true && !isForcedTemporaryRetention(appConfig?.interfaceConfig?.retentionMode) + ); +} + export function isLangfuseTraceSampled(traceId: string): boolean { if (!isLangfuseTracingEnabled()) { return false; diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 403b94976c..eed8b809a3 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -3918,6 +3918,12 @@ export const langfuseTraceConfigSchema = z.object({ conversationMetadataFields: z .array(z.enum(LANGFUSE_TRACE_CONVERSATION_METADATA_FIELDS)) .optional(), + /** + * Whether temporary chats, including chats forced temporary by + * `interface.retentionMode: "ephemeral"`, are traced and receive feedback + * scores. Defaults to false: a temporary chat reaches no Langfuse destination. + */ + temporaryChats: z.boolean().optional(), }); export type LangfuseTraceConfig = z.infer; diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.ts index aeddac7f10..cd24172a76 100644 --- a/packages/data-schemas/src/methods/message.spec.ts +++ b/packages/data-schemas/src/methods/message.spec.ts @@ -1194,6 +1194,43 @@ describe('Message Operations', () => { expect(result?.langfuseRunId).toBe('run-1'); }); + it('returns the temporary flag with feedback updates', async () => { + await saveMessage({ ...mockCtx, isTemporary: true }, mockMessageData); + + const result = await updateMessage(mockCtx.userId, { + messageId: 'msg123', + feedback: { rating: 'thumbsDown', tag: undefined }, + }); + + expect(result?.isTemporary).toBe(true); + }); + + it('reads a legacy deadline-only row as temporary with feedback updates', async () => { + const expiredAt = new Date(Date.now() + 60 * 60 * 1000); + await Message.collection.insertMany([ + { ...mockMessageData, user: mockCtx.userId, expiredAt }, + { + ...mockMessageData, + messageId: 'msg-permanent', + user: mockCtx.userId, + isTemporary: false, + expiredAt, + }, + ]); + + const legacy = await updateMessage(mockCtx.userId, { + messageId: 'msg123', + feedback: { rating: 'thumbsDown', tag: undefined }, + }); + const permanent = await updateMessage(mockCtx.userId, { + messageId: 'msg-permanent', + feedback: { rating: 'thumbsDown', tag: undefined }, + }); + + expect(legacy?.isTemporary).toBe(true); + expect(permanent?.isTemporary).toBe(false); + }); + it('should throw an error if message is not found', async () => { await expect( updateMessage(mockCtx.userId, { messageId: 'nonexistent', text: 'Test' }), diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts index 37fa26033c..a368f1cf2c 100644 --- a/packages/data-schemas/src/methods/message.ts +++ b/packages/data-schemas/src/methods/message.ts @@ -320,6 +320,18 @@ function buildMessageSaveUpdate( }; } +/** + * A legacy temporary message stores only its deadline; hydration then fills + * `isTemporary` with the schema default `false`, which must not read as permanent. + */ +function isTemporaryMessage(message: IMessage): boolean { + if (message.isTemporary === true) { + return true; + } + const implicit = message.isTemporary == null || message.$isDefault?.('isTemporary') === true; + return implicit && message.expiredAt != null; +} + async function findOneAndMergeMessageProvenance( Message: Model, identity: FilterQuery, @@ -2542,6 +2554,7 @@ export function createMessageMethods( tokenCount: updatedMessage.tokenCount, feedback: updatedMessage.feedback, endpoint: updatedMessage.endpoint, + isTemporary: isTemporaryMessage(updatedMessage), langfuseSampled: updatedMessage.langfuseSampled, langfuseDestinationIds: updatedMessage.langfuseDestinationIds, langfuseRunId: updatedMessage.langfuseRunId,