mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 06:51:59 +00:00
🕶️ fix: Keep Temporary Chats Out of Langfuse (#17202)
* fix: Keep Temporary Chats Out of Langfuse * fix: Gate Forced-Ephemeral and Open Responses Turns From Langfuse * fix: Treat Legacy and Chat Completion Temporary Turns as Untraced --------- Co-authored-by: Joel Hirzel <joel.hirzel@eps.ch>
This commit is contained in:
parent
147aa29807
commit
ab8dc9d3d2
25 changed files with 377 additions and 10 deletions
|
|
@ -39,6 +39,7 @@ const {
|
|||
rejectPrivateTextAdmission,
|
||||
bindPrivateTextPersistenceAbort,
|
||||
persistedReasoningOverrideFields,
|
||||
isTemporaryTurn,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
Constants,
|
||||
|
|
@ -1070,7 +1071,9 @@ class BaseClient {
|
|||
const isAgentResponse =
|
||||
this.clientName === EModelEndpoint.agents || isAgentsEndpoint(this.options.endpoint);
|
||||
const langfuseTraceFields = isAgentResponse
|
||||
? await getLangfuseTraceMessageFields(appConfig, responseMessageId)
|
||||
? await getLangfuseTraceMessageFields(appConfig, responseMessageId, {
|
||||
isTemporary: isTemporaryTurn(this.options.req),
|
||||
})
|
||||
: undefined;
|
||||
|
||||
/** @type {TMessage} */
|
||||
|
|
|
|||
|
|
@ -177,6 +177,7 @@ jest.mock('@librechat/agents', () => ({
|
|||
jest.mock('@librechat/api', () => ({
|
||||
getAgentErrorMetadata: (...args) =>
|
||||
jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),
|
||||
isTemporaryTurn: (...args) => jest.requireActual('@librechat/api').isTemporaryTurn(...args),
|
||||
/* Provisioning moved into this package; the controllers build the callback from it. */
|
||||
createProvisionFilesCallback: () => async () => {},
|
||||
createAgentExecutionContext: (context) => context,
|
||||
|
|
@ -910,6 +911,15 @@ describe('OpenAIChatCompletionController', () => {
|
|||
expect(createSubagentUsageSink).toHaveBeenCalledWith(expect.any(Array));
|
||||
});
|
||||
|
||||
it('runs a temporary conversation as temporary for Langfuse', async () => {
|
||||
const { createRun } = require('@librechat/api');
|
||||
req.resolvedConversation = { isTemporary: true };
|
||||
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
|
||||
expect(createRun).toHaveBeenCalledWith(expect.objectContaining({ isTemporary: true }));
|
||||
});
|
||||
|
||||
it('uses collected usage for the non-streaming response', async () => {
|
||||
const { buildNonStreamingResponse } = require('@librechat/api');
|
||||
|
||||
|
|
|
|||
|
|
@ -317,6 +317,7 @@ jest.mock('@librechat/api', () => ({
|
|||
logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure,
|
||||
resolveFailedTurnContent: jest.requireActual('@librechat/api').resolveFailedTurnContent,
|
||||
getFailedTurnTraceFields: (...args) => mockGetFailedTurnTraceFields(...args),
|
||||
isTemporaryTurn: jest.requireActual('@librechat/api').isTemporaryTurn,
|
||||
startAgentProjectContextResolution:
|
||||
jest.requireActual('@librechat/api').startAgentProjectContextResolution,
|
||||
assertChatProjectInstructions: jest.requireActual('@librechat/api').assertChatProjectInstructions,
|
||||
|
|
@ -4026,6 +4027,7 @@ describe('ResumableAgentController resume metadata', () => {
|
|||
messageId: 'server-user_',
|
||||
runId: 'server-response-uuid',
|
||||
runCreated: true,
|
||||
isTemporary: false,
|
||||
});
|
||||
expect(errorRow).toMatchObject({ error: true, isCreatedByUser: false, ...traceFields });
|
||||
});
|
||||
|
|
|
|||
|
|
@ -202,6 +202,7 @@ jest.mock('@librechat/api', () => ({
|
|||
jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),
|
||||
getConversationWriteContext: (...args) =>
|
||||
jest.requireActual('@librechat/api').getConversationWriteContext(...args),
|
||||
isTemporaryTurn: (...args) => jest.requireActual('@librechat/api').isTemporaryTurn(...args),
|
||||
announceReply: jest.fn().mockResolvedValue(undefined),
|
||||
/* Provisioning moved into this package; the controllers build the callback from it. */
|
||||
createProvisionFilesCallback: () => async () => {},
|
||||
|
|
@ -1039,7 +1040,9 @@ describe('createResponse controller', () => {
|
|||
|
||||
await createResponse(req, res);
|
||||
|
||||
expect(api.getLangfuseTraceMessageFields).toHaveBeenCalledWith(req.config, 'resp_mock-123');
|
||||
expect(api.getLangfuseTraceMessageFields).toHaveBeenCalledWith(req.config, 'resp_mock-123', {
|
||||
isTemporary: false,
|
||||
});
|
||||
expect(saveMessage).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ userId: 'user-123' }),
|
||||
expect.objectContaining({
|
||||
|
|
@ -1053,6 +1056,24 @@ describe('createResponse controller', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
'runs and records a forced-temporary turn as temporary for Langfuse: stream=%s',
|
||||
async (stream) => {
|
||||
const api = require('@librechat/api');
|
||||
req.config.interfaceConfig = { retentionMode: 'ephemeral', temporaryChatRetention: 1 };
|
||||
api.validateResponseRequest.mockReturnValueOnce({
|
||||
request: { ...req.body, stream, store: true },
|
||||
});
|
||||
|
||||
await createResponse(req, res);
|
||||
|
||||
expect(api.createRun).toHaveBeenCalledWith(expect.objectContaining({ isTemporary: true }));
|
||||
expect(api.getLangfuseTraceMessageFields).toHaveBeenCalledWith(req.config, 'resp_mock-123', {
|
||||
isTemporary: true,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([false, true])(
|
||||
'stores input and output under the retention write context: stream=%s',
|
||||
async (stream) => {
|
||||
|
|
|
|||
|
|
@ -208,6 +208,7 @@ const {
|
|||
markCompactionOutcome,
|
||||
resolvePersistableCodeEnvironmentDecision,
|
||||
getChatProjectContextKey,
|
||||
isTemporaryTurn,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
Run,
|
||||
|
|
@ -5037,6 +5038,7 @@ class AgentClient extends BaseClient {
|
|||
// Conversation-stable identity for the e2e run hook; a resumed run
|
||||
// carries no messages, so history cannot identify the conversation.
|
||||
conversationId: this.conversationId,
|
||||
isTemporary: isTemporaryTurn(this.options.req),
|
||||
messages,
|
||||
discoveredToolNames:
|
||||
this.eventActorContinuation === 'warm' ? this.eventActorDiscoveredToolNames : undefined,
|
||||
|
|
@ -5844,6 +5846,7 @@ class AgentClient extends BaseClient {
|
|||
),
|
||||
agents,
|
||||
conversationId: this.conversationId,
|
||||
isTemporary: isTemporaryTurn(this.options.req),
|
||||
modelCallbacks: [
|
||||
modelBoundCallback,
|
||||
attachmentMemoryCallback,
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ const {
|
|||
resolveApiConversationProject,
|
||||
resolveAdmittedCodeEnvironmentDecision,
|
||||
createTerminalRunErrorObserver,
|
||||
isTemporaryTurn,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
buildSummarizationHandlers,
|
||||
|
|
@ -1140,6 +1141,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
user: { ...createSafeUser(req.user), id: userId },
|
||||
traceContext: { endpoint: EModelEndpoint.agents },
|
||||
tenantId: principal.tenantId,
|
||||
isTemporary: isTemporaryTurn(req),
|
||||
/** Bills subagent child-run model calls (reported outside the
|
||||
* streamEvents loop) into the same collectedUsage array. */
|
||||
subagentUsageSink: createSubagentUsageSink(collectedUsage),
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ const {
|
|||
settleExistingRowsBeforeErrorTurn,
|
||||
resolveDisconnectSnapshotMode,
|
||||
markAbortedCompactionContent,
|
||||
isTemporaryTurn,
|
||||
} = require('@librechat/api');
|
||||
const { disposeClient } = require('~/server/cleanup');
|
||||
const {
|
||||
|
|
@ -529,6 +530,7 @@ async function saveErrorTurn(
|
|||
messageId: errorMessageId,
|
||||
runId: liveResponseMessageId,
|
||||
runCreated,
|
||||
isTemporary: isTemporaryTurn(req),
|
||||
});
|
||||
const savedErrorMessage = await saveMessage(
|
||||
reqCtx,
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ const {
|
|||
createTerminalRunErrorObserver,
|
||||
announceReply,
|
||||
getConversationWriteContext,
|
||||
isTemporaryTurn,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
createResponsesToolEndCallback,
|
||||
|
|
@ -428,7 +429,9 @@ async function saveResponseOutput(
|
|||
}
|
||||
}
|
||||
|
||||
const langfuseTraceFields = await getLangfuseTraceMessageFields(req.config, responseId);
|
||||
const langfuseTraceFields = await getLangfuseTraceMessageFields(req.config, responseId, {
|
||||
isTemporary: isTemporaryTurn(req),
|
||||
});
|
||||
|
||||
// Save the assistant message
|
||||
return db.saveMessage(
|
||||
|
|
@ -1357,6 +1360,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
user: { ...createSafeUser(req.user), id: userId },
|
||||
traceContext: { endpoint: EModelEndpoint.agents },
|
||||
tenantId: principal.tenantId,
|
||||
isTemporary: isTemporaryTurn(req),
|
||||
modelCallbacks: [terminalRunError.modelCallback],
|
||||
clientToolNames: clientTools.clientToolNames,
|
||||
/** Bills subagent child-run model calls (reported outside the
|
||||
|
|
@ -1596,6 +1600,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
user: { ...createSafeUser(req.user), id: userId },
|
||||
traceContext: { endpoint: EModelEndpoint.agents },
|
||||
tenantId: principal.tenantId,
|
||||
isTemporary: isTemporaryTurn(req),
|
||||
modelCallbacks: [terminalRunError.modelCallback],
|
||||
clientToolNames: clientTools.clientToolNames,
|
||||
/** Bills subagent child-run model calls (reported outside the
|
||||
|
|
|
|||
|
|
@ -143,6 +143,25 @@ describe('PUT /:conversationId/:messageId/feedback', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('passes a temporary message to the scorer as temporary', async () => {
|
||||
updateMessage.mockImplementationOnce((userId, { messageId, feedback }) =>
|
||||
Promise.resolve({
|
||||
messageId,
|
||||
conversationId: 'conversation-1',
|
||||
endpoint: 'agents',
|
||||
isTemporary: true,
|
||||
feedback,
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await request(app)
|
||||
.put('/api/messages/conversation-1/message-1/feedback')
|
||||
.send({ feedback: { rating: 'thumbsDown', tag: 'other' } });
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(sendFeedbackScore).toHaveBeenCalledWith(expect.objectContaining({ isTemporary: true }));
|
||||
});
|
||||
|
||||
it('scores the trace of the run a failed turn stands for', async () => {
|
||||
updateMessage.mockImplementationOnce((userId, { messageId, feedback }) =>
|
||||
Promise.resolve({
|
||||
|
|
|
|||
|
|
@ -784,6 +784,7 @@ router.put(
|
|||
destinationIds: updatedMessage.langfuseDestinationIds,
|
||||
feedback: updatedMessage.feedback,
|
||||
appConfig: req.config,
|
||||
isTemporary: updatedMessage.isTemporary,
|
||||
metadata: {
|
||||
messageId: updatedMessage.messageId ?? messageId,
|
||||
parentMessageId: updatedMessage.parentMessageId,
|
||||
|
|
|
|||
|
|
@ -76,6 +76,10 @@ projects:
|
|||
# librechat.endpoint.type, librechat.provider, librechat.model,
|
||||
# librechat.model.label, librechat.spec.
|
||||
#
|
||||
# Temporary chats, including every chat when interface.retentionMode is
|
||||
# "ephemeral", are not traced and their feedback is not scored unless
|
||||
# langfuse.trace.temporaryChats is true.
|
||||
#
|
||||
# langfuse:
|
||||
# headers:
|
||||
# CF-Access-Client-Id: "${CF_ACCESS_CLIENT_ID}"
|
||||
|
|
@ -84,6 +88,7 @@ projects:
|
|||
# userIdField: email
|
||||
# userMetadataFields: [email, username, role, provider]
|
||||
# conversationMetadataFields: [conversationId, endpoint, model, modelLabel, spec]
|
||||
# temporaryChats: false
|
||||
|
||||
# File storage configuration
|
||||
# Single strategy for all file types (legacy format, still supported)
|
||||
|
|
|
|||
|
|
@ -523,6 +523,17 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => {
|
|||
expect(runArgs.appConfig).not.toHaveProperty('interfaceConfig');
|
||||
});
|
||||
|
||||
it('forwards only the retention mode so forced-temporary runs stay untraced', async () => {
|
||||
deps.appConfig = {
|
||||
interfaceConfig: { modelSelect: true, retentionMode: 'ephemeral' },
|
||||
} as never;
|
||||
|
||||
await createAgentChatCompletion(createMockReq({ id: 'user-123' }), createMockRes(), deps);
|
||||
|
||||
const runArgs = createRun.mock.calls[0][0] as CreateRunArgs;
|
||||
expect(runArgs.appConfig).toEqual({ interfaceConfig: { retentionMode: 'ephemeral' } });
|
||||
});
|
||||
|
||||
it('forwards the stateful environment allowlist from appConfig to agent initialization', async () => {
|
||||
deps.appConfig = {
|
||||
endpoints: {
|
||||
|
|
|
|||
|
|
@ -291,7 +291,10 @@ type LoadToolsFn = (params: {
|
|||
/**
|
||||
* Create run function type
|
||||
*/
|
||||
type CreateRunAppConfig = Pick<AppConfig, 'endpoints' | 'filters' | 'langfuse' | 'messageFilter'>;
|
||||
type CreateRunAppConfig = Pick<
|
||||
AppConfig,
|
||||
'endpoints' | 'filters' | 'langfuse' | 'interfaceConfig' | 'messageFilter'
|
||||
>;
|
||||
|
||||
type CreateRunFn = (params: {
|
||||
agents: unknown[];
|
||||
|
|
@ -321,6 +324,8 @@ interface AppConfig {
|
|||
endpoints?: Record<string, unknown>;
|
||||
filters?: FiltersConfig;
|
||||
langfuse?: Record<string, unknown>;
|
||||
/** Carries `retentionMode`, which keeps forced-temporary runs out of Langfuse. */
|
||||
interfaceConfig?: Record<string, unknown>;
|
||||
messageFilter?: MessageFilterConfig;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
|
@ -335,6 +340,9 @@ function selectCreateRunAppConfig(
|
|||
...(appConfig.endpoints !== undefined && { endpoints: appConfig.endpoints }),
|
||||
...(appConfig.filters !== undefined && { filters: appConfig.filters }),
|
||||
...(appConfig.langfuse !== undefined && { langfuse: appConfig.langfuse }),
|
||||
...(appConfig.interfaceConfig?.retentionMode !== undefined && {
|
||||
interfaceConfig: { retentionMode: appConfig.interfaceConfig.retentionMode },
|
||||
}),
|
||||
...(appConfig.messageFilter !== undefined && { messageFilter: appConfig.messageFilter }),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2138,6 +2138,7 @@ export async function createRun({
|
|||
user,
|
||||
tenantId,
|
||||
centralTraceExportEnabled,
|
||||
isTemporary,
|
||||
traceContext,
|
||||
tokenCounter,
|
||||
customHandlers,
|
||||
|
|
@ -2189,6 +2190,8 @@ export async function createRun({
|
|||
* run. Tenant fanout can still export when tenant routing is available.
|
||||
*/
|
||||
centralTraceExportEnabled?: boolean;
|
||||
/** A temporary chat's run is not traced unless `langfuse.trace.temporaryChats` opts in. */
|
||||
isTemporary?: boolean;
|
||||
/**
|
||||
* Exact tool names the owner approved for the rest of this conversation, read from the
|
||||
* stored conversation (never from the request body). Honored only when
|
||||
|
|
@ -3118,6 +3121,7 @@ export async function createRun({
|
|||
runId: resolvedRunId,
|
||||
tenantId: tenantId ?? user?.tenantId,
|
||||
centralTraceExportEnabled,
|
||||
isTemporary,
|
||||
user,
|
||||
traceContext: resolveRunTraceContext({ agents, conversationId, requestBody, traceContext }),
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import {
|
|||
resolveImportRetentionFields,
|
||||
resolveImportTagCounts,
|
||||
isTemporaryRecord,
|
||||
isTemporaryTurn,
|
||||
} from './retention';
|
||||
|
||||
describe('applyForcedRetention', () => {
|
||||
|
|
@ -225,6 +226,45 @@ describe('resolveResumableRetention', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('isTemporaryTurn', () => {
|
||||
it('follows the stored conversation over the request flag', () => {
|
||||
expect(
|
||||
isTemporaryTurn({
|
||||
body: { isTemporary: false },
|
||||
resolvedConversation: { isTemporary: true },
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isTemporaryTurn({
|
||||
body: { isTemporary: true },
|
||||
resolvedConversation: { isTemporary: false },
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('counts a turn the administrator forces temporary', () => {
|
||||
expect(
|
||||
isTemporaryTurn({
|
||||
resolvedConversation: { isTemporary: false },
|
||||
config: { interfaceConfig: { retentionMode: RetentionMode.EPHEMERAL } },
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('counts a legacy conversation that stores only a deadline', () => {
|
||||
const expiredAt = new Date('2099-01-01T00:00:00.000Z');
|
||||
expect(isTemporaryTurn({ resolvedConversation: { expiredAt } })).toBe(true);
|
||||
expect(isTemporaryTurn({ resolvedConversation: { isTemporary: false, expiredAt } })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('treats a turn without request state as permanent', () => {
|
||||
expect(isTemporaryTurn()).toBe(false);
|
||||
expect(isTemporaryTurn({ body: {} })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveImportRetentionFields', () => {
|
||||
const deps = {
|
||||
createChatExpirationDate: jest.fn(() => new Date('2030-01-01T00:00:00.000Z')),
|
||||
|
|
|
|||
|
|
@ -78,6 +78,18 @@ export async function persistForcedTemporaryMetadata(
|
|||
await store.updateMetadata(streamId, { isTemporary: true }, createdAt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a turn runs in a temporary chat, counting one the administrator forces temporary
|
||||
* and a legacy conversation that stores only a deadline.
|
||||
*/
|
||||
export function isTemporaryTurn(req?: TurnConversationRequest): boolean {
|
||||
const { isTemporary, expiredAt, interfaceConfig } = getConversationWriteContext(req);
|
||||
return (
|
||||
isTemporaryRecord({ isTemporary, expiredAt }) ||
|
||||
isForcedTemporaryRetention(interfaceConfig?.retentionMode)
|
||||
);
|
||||
}
|
||||
|
||||
/** Captures the effective retention policy for a paused turn without re-reading its records. */
|
||||
export function resolveResumableRetention(
|
||||
req: TurnConversationRequest,
|
||||
|
|
|
|||
|
|
@ -172,6 +172,71 @@ describe('buildLangfuseConfig', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('does not trace a temporary chat to any destination', async () => {
|
||||
process.env.LANGFUSE_PUBLIC_KEY = 'pk-env';
|
||||
process.env.LANGFUSE_SECRET_KEY = 'sk-env';
|
||||
process.env.LANGFUSE_FANOUT_ENABLED = 'true';
|
||||
process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';
|
||||
const { encryptV3 } = await import('@librechat/data-schemas');
|
||||
const { buildLangfuseConfig } = await import('./config');
|
||||
|
||||
expect(
|
||||
buildLangfuseConfig({
|
||||
runId: 'run-1',
|
||||
tenantId: 'tenant-1',
|
||||
isTemporary: true,
|
||||
appConfig: {
|
||||
langfuse: {
|
||||
enabled: true,
|
||||
publicKey: 'pk-tenant-1',
|
||||
secretKey: encryptV3('sk-tenant-1'),
|
||||
destination: 'us',
|
||||
},
|
||||
} as unknown as AppConfig,
|
||||
}),
|
||||
).toEqual({
|
||||
deterministicTraceId: true,
|
||||
enabled: false,
|
||||
metadata: { 'librechat.tenant.id': 'tenant-1' },
|
||||
tags: ['tenant:tenant-1'],
|
||||
});
|
||||
});
|
||||
|
||||
it('does not trace a run when retention forces every chat temporary', async () => {
|
||||
delete process.env.TENANT_ISOLATION_STRICT;
|
||||
process.env.LANGFUSE_PUBLIC_KEY = 'pk-env';
|
||||
process.env.LANGFUSE_SECRET_KEY = 'sk-env';
|
||||
const { buildLangfuseConfig } = await import('./config');
|
||||
|
||||
expect(
|
||||
buildLangfuseConfig({
|
||||
runId: 'run-1',
|
||||
appConfig: { interfaceConfig: { retentionMode: 'ephemeral' } } as unknown as AppConfig,
|
||||
}),
|
||||
).toMatchObject({ enabled: false });
|
||||
});
|
||||
|
||||
it('traces a temporary chat when the deployment opts in', async () => {
|
||||
delete process.env.TENANT_ISOLATION_STRICT;
|
||||
process.env.LANGFUSE_PUBLIC_KEY = 'pk-env';
|
||||
process.env.LANGFUSE_SECRET_KEY = 'sk-env';
|
||||
process.env.LANGFUSE_BASE_URL = 'https://env.langfuse.example';
|
||||
const { buildLangfuseConfig } = await import('./config');
|
||||
|
||||
expect(
|
||||
buildLangfuseConfig({
|
||||
runId: 'run-1',
|
||||
isTemporary: true,
|
||||
appConfig: { langfuse: { trace: { temporaryChats: true } } } as unknown as AppConfig,
|
||||
}),
|
||||
).toEqual({
|
||||
deterministicTraceId: true,
|
||||
publicKey: 'pk-env',
|
||||
secretKey: 'sk-env',
|
||||
baseUrl: 'https://env.langfuse.example',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not trace a disabled stored connection in single-tenant mode', async () => {
|
||||
delete process.env.TENANT_ISOLATION_STRICT;
|
||||
const { encryptV3 } = await import('@librechat/data-schemas');
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
isLangfuseCentralMediaUploadDisabled,
|
||||
isLangfuseFanoutEnabled,
|
||||
isLangfuseTenantExportEnabled,
|
||||
isLangfuseTraceAllowed,
|
||||
isLangfuseTraceSampled,
|
||||
isLangfuseTracingEnabled,
|
||||
usesLangfuseMultiTenantRouting,
|
||||
|
|
@ -250,6 +251,7 @@ export function buildLangfuseConfig({
|
|||
runId,
|
||||
tenantId,
|
||||
centralTraceExportEnabled = true,
|
||||
isTemporary,
|
||||
user,
|
||||
traceContext,
|
||||
}: {
|
||||
|
|
@ -262,6 +264,8 @@ export function buildLangfuseConfig({
|
|||
* to drop the central pipeline while preserving tenant fanout when available.
|
||||
*/
|
||||
centralTraceExportEnabled?: boolean;
|
||||
/** Disables every destination unless `langfuse.trace.temporaryChats` opts in. */
|
||||
isTemporary?: boolean;
|
||||
/** The requesting user, read only for the fields `langfuse.trace` allowlists. */
|
||||
user?: LangfuseTraceUser;
|
||||
/** Request values `langfuse.trace.conversationMetadataFields` may export. */
|
||||
|
|
@ -291,6 +295,7 @@ export function buildLangfuseConfig({
|
|||
|
||||
if (
|
||||
!isLangfuseTracingEnabled() ||
|
||||
!isLangfuseTraceAllowed(appConfig, isTemporary) ||
|
||||
(runId != null && !isLangfuseTraceSampled(traceIdForMessage(runId)))
|
||||
) {
|
||||
langfuse.enabled = false;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
isLangfuseFanoutEnabled,
|
||||
isLangfuseTenantExportEnabled,
|
||||
isLangfuseTracingEnabled,
|
||||
isLangfuseTraceAllowed,
|
||||
isLangfuseTraceSampled,
|
||||
usesLangfuseMultiTenantRouting,
|
||||
} from './policy';
|
||||
|
|
@ -366,7 +367,8 @@ export type LangfuseTraceMessageFields = {
|
|||
/**
|
||||
* The sampling record a response stores for its run's trace. `runId` names the
|
||||
* run when it is not the response's own id, as for a failed turn's error row;
|
||||
* it is then stored too, so feedback and the trace viewer follow that run.
|
||||
* it is then stored too, so feedback and the trace viewer follow that run. A
|
||||
* temporary turn's run was not traced unless the deployment opts in.
|
||||
*/
|
||||
export async function getLangfuseTraceMessageFields(
|
||||
appConfig: AppConfig | undefined,
|
||||
|
|
@ -374,14 +376,19 @@ export async function getLangfuseTraceMessageFields(
|
|||
{
|
||||
centralTraceExportEnabled = true,
|
||||
runId = messageId,
|
||||
}: Pick<LangfuseScoreDestinationOptions, 'centralTraceExportEnabled'> & { runId?: string } = {},
|
||||
isTemporary,
|
||||
}: Pick<LangfuseScoreDestinationOptions, 'centralTraceExportEnabled'> & {
|
||||
runId?: string;
|
||||
isTemporary?: boolean;
|
||||
} = {},
|
||||
): Promise<{
|
||||
langfuseSampled: boolean;
|
||||
langfuseDestinationIds?: string[];
|
||||
langfuseRunId?: string;
|
||||
}> {
|
||||
const traceId = traceIdForMessage(runId);
|
||||
const langfuseSampled = isLangfuseTraceSampled(traceId);
|
||||
const langfuseSampled =
|
||||
isLangfuseTraceAllowed(appConfig, isTemporary) && isLangfuseTraceSampled(traceId);
|
||||
return {
|
||||
langfuseSampled,
|
||||
langfuseDestinationIds: await getLangfuseTraceDestinationIds(
|
||||
|
|
@ -406,13 +413,14 @@ export async function getFailedTurnTraceFields(
|
|||
messageId,
|
||||
runId,
|
||||
runCreated,
|
||||
}: { messageId: string; runId?: string | null; runCreated: boolean },
|
||||
isTemporary,
|
||||
}: { messageId: string; runId?: string | null; runCreated: boolean; isTemporary?: boolean },
|
||||
): Promise<LangfuseTraceMessageFields> {
|
||||
if (!runCreated || typeof runId !== 'string' || runId.length === 0) {
|
||||
return {};
|
||||
}
|
||||
try {
|
||||
return await getLangfuseTraceMessageFields(appConfig, messageId, { runId });
|
||||
return await getLangfuseTraceMessageFields(appConfig, messageId, { runId, isTemporary });
|
||||
} catch (error) {
|
||||
logger.warn('[langfuse] Could not record the failed run trace:', error);
|
||||
return {};
|
||||
|
|
|
|||
|
|
@ -259,6 +259,41 @@ describe('Langfuse feedback scores', () => {
|
|||
expect(getFetchMock()).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not score a temporary chat unless the deployment opts in', async () => {
|
||||
const { sendFeedbackScore } = await loadFeedback();
|
||||
const score = { traceId: 'trace-id', feedback: { rating: 'thumbsUp' as const } };
|
||||
|
||||
await sendFeedbackScore({ ...score, isTemporary: true });
|
||||
expect(getFetchMock()).not.toHaveBeenCalled();
|
||||
|
||||
await sendFeedbackScore({
|
||||
...score,
|
||||
isTemporary: true,
|
||||
appConfig: { langfuse: { trace: { temporaryChats: true } } } as AppConfig,
|
||||
});
|
||||
expect(getFetchMock()).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not score a stored permanent row once retention forces chats temporary', async () => {
|
||||
const { sendFeedbackScore } = await loadFeedback();
|
||||
const score = { traceId: 'trace-id', feedback: { rating: 'thumbsUp' as const } };
|
||||
const interfaceConfig = { retentionMode: 'ephemeral' };
|
||||
|
||||
await sendFeedbackScore({
|
||||
...score,
|
||||
isTemporary: false,
|
||||
appConfig: { interfaceConfig } as AppConfig,
|
||||
});
|
||||
expect(getFetchMock()).not.toHaveBeenCalled();
|
||||
|
||||
await sendFeedbackScore({
|
||||
...score,
|
||||
isTemporary: false,
|
||||
appConfig: { interfaceConfig, langfuse: { trace: { temporaryChats: true } } } as AppConfig,
|
||||
});
|
||||
expect(getFetchMock()).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('preserves a sampled trace when the sample rate decreases', async () => {
|
||||
process.env.LANGFUSE_SAMPLE_RATE = '0.1';
|
||||
const { sendFeedbackScore } = await loadFeedback();
|
||||
|
|
@ -440,6 +475,29 @@ describe('Langfuse feedback scores', () => {
|
|||
).resolves.toMatchObject({ langfuseSampled: true, langfuseRunId: 'run-1' });
|
||||
});
|
||||
|
||||
it('records a temporary turn as untraced unless the deployment opts in', async () => {
|
||||
await loadFeedback();
|
||||
const { getFailedTurnTraceFields, getLangfuseTraceMessageFields } = await import(
|
||||
'./destinations'
|
||||
);
|
||||
const optIn = { langfuse: { trace: { temporaryChats: true } } } as AppConfig;
|
||||
|
||||
await expect(
|
||||
getLangfuseTraceMessageFields(undefined, 'message-1', { isTemporary: true }),
|
||||
).resolves.toEqual({ langfuseSampled: false, langfuseDestinationIds: [] });
|
||||
await expect(
|
||||
getFailedTurnTraceFields(undefined, {
|
||||
messageId: 'user-1_',
|
||||
runId: 'run-1',
|
||||
runCreated: true,
|
||||
isTemporary: true,
|
||||
}),
|
||||
).resolves.toMatchObject({ langfuseSampled: false, langfuseRunId: 'run-1' });
|
||||
await expect(
|
||||
getLangfuseTraceMessageFields(optIn, 'message-1', { isTemporary: true }),
|
||||
).resolves.toMatchObject({ langfuseSampled: true });
|
||||
});
|
||||
|
||||
it('keeps the central destination identity stable when credentials rotate', async () => {
|
||||
const { sendFeedbackScore } = await loadFeedback();
|
||||
const { getLangfuseTraceDestinationIds } = await import('./destinations');
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import { getScoreDestinations, type LangfuseScoreDestination } from './destinations';
|
||||
import { isLangfuseTraceAllowed } from './policy';
|
||||
import { mergeHeaders } from '~/utils/headers';
|
||||
import { redirectPolicyFor } from './utils';
|
||||
|
||||
|
|
@ -27,6 +28,11 @@ export type SendFeedbackScoreParams = {
|
|||
* whose project identity is unknown.
|
||||
*/
|
||||
centralTraceExportEnabled?: boolean;
|
||||
/**
|
||||
* Whether the message belongs to a temporary chat. Checked on its own because
|
||||
* a resumed turn's response stores no trace record to carry the decision.
|
||||
*/
|
||||
isTemporary?: boolean | null;
|
||||
};
|
||||
|
||||
const ENVIRONMENT = process.env.LANGFUSE_TRACING_ENVIRONMENT;
|
||||
|
|
@ -125,8 +131,9 @@ export async function sendFeedbackScore({
|
|||
observationId,
|
||||
appConfig,
|
||||
centralTraceExportEnabled,
|
||||
isTemporary,
|
||||
}: SendFeedbackScoreParams): Promise<void> {
|
||||
if (!traceId) {
|
||||
if (!traceId || !isLangfuseTraceAllowed(appConfig, isTemporary)) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import { isForcedTemporaryRetention } from 'librechat-data-provider';
|
||||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import { isFalseEnv, isTrueEnv } from './utils';
|
||||
import { normalizeString } from '~/utils/text';
|
||||
|
||||
|
|
@ -58,6 +60,24 @@ function traceIdAccumulation(traceId: string): number {
|
|||
return accumulation;
|
||||
}
|
||||
|
||||
/**
|
||||
* A temporary chat reaches no Langfuse destination unless the deployment opts
|
||||
* in. `interface.retentionMode: "ephemeral"` makes every chat temporary, so it
|
||||
* counts even when the caller only knows a row's stored flag. Read strictly,
|
||||
* since admin config patches persist without validation.
|
||||
*/
|
||||
export function isLangfuseTraceAllowed(
|
||||
appConfig: AppConfig | undefined,
|
||||
isTemporary?: boolean | null,
|
||||
): boolean {
|
||||
if (appConfig?.langfuse?.trace?.temporaryChats === true) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
isTemporary !== true && !isForcedTemporaryRetention(appConfig?.interfaceConfig?.retentionMode)
|
||||
);
|
||||
}
|
||||
|
||||
export function isLangfuseTraceSampled(traceId: string): boolean {
|
||||
if (!isLangfuseTracingEnabled()) {
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -3918,6 +3918,12 @@ export const langfuseTraceConfigSchema = z.object({
|
|||
conversationMetadataFields: z
|
||||
.array(z.enum(LANGFUSE_TRACE_CONVERSATION_METADATA_FIELDS))
|
||||
.optional(),
|
||||
/**
|
||||
* Whether temporary chats, including chats forced temporary by
|
||||
* `interface.retentionMode: "ephemeral"`, are traced and receive feedback
|
||||
* scores. Defaults to false: a temporary chat reaches no Langfuse destination.
|
||||
*/
|
||||
temporaryChats: z.boolean().optional(),
|
||||
});
|
||||
|
||||
export type LangfuseTraceConfig = z.infer<typeof langfuseTraceConfigSchema>;
|
||||
|
|
|
|||
|
|
@ -1194,6 +1194,43 @@ describe('Message Operations', () => {
|
|||
expect(result?.langfuseRunId).toBe('run-1');
|
||||
});
|
||||
|
||||
it('returns the temporary flag with feedback updates', async () => {
|
||||
await saveMessage({ ...mockCtx, isTemporary: true }, mockMessageData);
|
||||
|
||||
const result = await updateMessage(mockCtx.userId, {
|
||||
messageId: 'msg123',
|
||||
feedback: { rating: 'thumbsDown', tag: undefined },
|
||||
});
|
||||
|
||||
expect(result?.isTemporary).toBe(true);
|
||||
});
|
||||
|
||||
it('reads a legacy deadline-only row as temporary with feedback updates', async () => {
|
||||
const expiredAt = new Date(Date.now() + 60 * 60 * 1000);
|
||||
await Message.collection.insertMany([
|
||||
{ ...mockMessageData, user: mockCtx.userId, expiredAt },
|
||||
{
|
||||
...mockMessageData,
|
||||
messageId: 'msg-permanent',
|
||||
user: mockCtx.userId,
|
||||
isTemporary: false,
|
||||
expiredAt,
|
||||
},
|
||||
]);
|
||||
|
||||
const legacy = await updateMessage(mockCtx.userId, {
|
||||
messageId: 'msg123',
|
||||
feedback: { rating: 'thumbsDown', tag: undefined },
|
||||
});
|
||||
const permanent = await updateMessage(mockCtx.userId, {
|
||||
messageId: 'msg-permanent',
|
||||
feedback: { rating: 'thumbsDown', tag: undefined },
|
||||
});
|
||||
|
||||
expect(legacy?.isTemporary).toBe(true);
|
||||
expect(permanent?.isTemporary).toBe(false);
|
||||
});
|
||||
|
||||
it('should throw an error if message is not found', async () => {
|
||||
await expect(
|
||||
updateMessage(mockCtx.userId, { messageId: 'nonexistent', text: 'Test' }),
|
||||
|
|
|
|||
|
|
@ -320,6 +320,18 @@ function buildMessageSaveUpdate(
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A legacy temporary message stores only its deadline; hydration then fills
|
||||
* `isTemporary` with the schema default `false`, which must not read as permanent.
|
||||
*/
|
||||
function isTemporaryMessage(message: IMessage): boolean {
|
||||
if (message.isTemporary === true) {
|
||||
return true;
|
||||
}
|
||||
const implicit = message.isTemporary == null || message.$isDefault?.('isTemporary') === true;
|
||||
return implicit && message.expiredAt != null;
|
||||
}
|
||||
|
||||
async function findOneAndMergeMessageProvenance(
|
||||
Message: Model<IMessage>,
|
||||
identity: FilterQuery<IMessage>,
|
||||
|
|
@ -2542,6 +2554,7 @@ export function createMessageMethods(
|
|||
tokenCount: updatedMessage.tokenCount,
|
||||
feedback: updatedMessage.feedback,
|
||||
endpoint: updatedMessage.endpoint,
|
||||
isTemporary: isTemporaryMessage(updatedMessage),
|
||||
langfuseSampled: updatedMessage.langfuseSampled,
|
||||
langfuseDestinationIds: updatedMessage.langfuseDestinationIds,
|
||||
langfuseRunId: updatedMessage.langfuseRunId,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue