diff --git a/api/server/controllers/agents/__tests__/callbacks.spec.js b/api/server/controllers/agents/__tests__/callbacks.spec.js index 5004823b4c..b92e1f53ca 100644 --- a/api/server/controllers/agents/__tests__/callbacks.spec.js +++ b/api/server/controllers/agents/__tests__/callbacks.spec.js @@ -6,6 +6,8 @@ jest.mock('nanoid', () => ({ })); jest.mock('@librechat/api', () => ({ + captureResponseAccountingUsage: (...args) => + jest.requireActual('@librechat/api').captureResponseAccountingUsage(...args), writeGuarded: (res, chunk) => { if (res.writableEnded === true || res.destroyed === true) { return false; diff --git a/api/server/controllers/agents/__tests__/modelEndHandler.spec.js b/api/server/controllers/agents/__tests__/modelEndHandler.spec.js index 5f794347f2..fb8b124fd8 100644 --- a/api/server/controllers/agents/__tests__/modelEndHandler.spec.js +++ b/api/server/controllers/agents/__tests__/modelEndHandler.spec.js @@ -2,7 +2,8 @@ jest.mock('@librechat/data-schemas', () => ({ logger: { error: jest.fn(), debug: jest.fn(), info: jest.fn() }, })); jest.mock('@librechat/api', () => { - const { getModelRefusalInfo, getModelEndUsage } = jest.requireActual('@librechat/api'); + const { getModelRefusalInfo, getModelEndUsage, captureResponseAccountingUsage } = + jest.requireActual('@librechat/api'); return { sendEvent: jest.fn(), emitEvent: jest.fn(), @@ -10,6 +11,7 @@ jest.mock('@librechat/api', () => { markSummarizationUsage: (usage) => usage, getModelRefusalInfo, getModelEndUsage, + captureResponseAccountingUsage, }; }); jest.mock('~/server/services/Files/Citations', () => ({ @@ -36,8 +38,30 @@ describe('ModelEndHandler — Vertex thoughtSignature capture (issue #13006 foll it('leaves usage usable when graph context is unavailable', () => { const usage = { input_tokens: 10, output_tokens: 5 }; - expect(contextualizeModelUsage(usage, undefined, undefined)).toEqual(usage); - expect(contextualizeModelUsage(usage, undefined, null)).toEqual(usage); + expect(contextualizeModelUsage(usage, undefined, undefined)).toEqual({ + ...usage, + accountingIdentity: { modelSource: 'unknown' }, + }); + expect(contextualizeModelUsage(usage, undefined, null)).toEqual({ + ...usage, + accountingIdentity: { modelSource: 'unknown' }, + }); + }); + + it('captures configured provenance before billing fills the model and keeps it on repeated contextualization', () => { + const usage = { input_tokens: 10, output_tokens: 5 }; + const context = { provider: 'bedrock', clientOptions: { model: 'configured-model' } }; + const first = contextualizeModelUsage(usage, undefined, context); + expect(first.model).toBe('configured-model'); + expect(first.accountingIdentity).toEqual({ + model: 'configured-model', + provider: 'bedrock', + modelSource: 'configured', + }); + expect(contextualizeModelUsage(first, undefined, context).accountingIdentity).toEqual( + first.accountingIdentity, + ); + expect(usage).not.toHaveProperty('accountingIdentity'); }); it('prefers the actually invoked fallback provider and model', () => { @@ -60,6 +84,11 @@ describe('ModelEndHandler — Vertex thoughtSignature capture (issue #13006 foll provider: 'anthropic', model: 'claude-fallback', agentId: 'agent-1', + accountingIdentity: { + model: 'claude-fallback', + provider: 'anthropic', + modelSource: 'runtime_reported', + }, }); }); diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js index d68161b18a..0dbd7e5b40 100644 --- a/api/server/controllers/agents/__tests__/responses.unit.spec.js +++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js @@ -150,13 +150,19 @@ const mockResponsesUsage = { const mockBuildResponsesUsage = jest.fn().mockReturnValue(mockResponsesUsage); const mockEnrollAgentExecution = jest.fn(); let mockExecution; +let mockRealExecutionManager; function resetMockExecution() { const controller = new AbortController(); + const writes = []; mockExecution = { signal: controller.signal, + terminalOutcome: 'complete', abort: jest.fn((reason) => controller.abort(reason)), - track: jest.fn((promise) => promise), + track: jest.fn((promise) => { + writes.push(promise); + return promise; + }), beginProviderExecution: jest.fn(async () => { if (controller.signal.aborted) { throw Object.assign(new Error('request disconnected'), { @@ -165,7 +171,9 @@ function resetMockExecution() { }); } }), - settle: jest.fn().mockResolvedValue(undefined), + settle: jest.fn(async () => { + await Promise.all(writes); + }), }; mockEnrollAgentExecution.mockResolvedValue(mockExecution); } @@ -179,6 +187,7 @@ jest.mock('uuid', () => ({ })); jest.mock('@librechat/data-schemas', () => ({ + ...jest.requireActual('@librechat/data-schemas'), logger: { debug: jest.fn(), info: jest.fn(), @@ -198,6 +207,30 @@ jest.mock('@librechat/agents', () => ({ })); jest.mock('@librechat/api', () => ({ + captureResponseAccountingUsage: (...args) => + jest.requireActual('@librechat/api').captureResponseAccountingUsage(...args), + resolveResponseContinuation: (...args) => + jest.requireActual('@librechat/api').resolveResponseContinuation(...args), + buildResponseInputChain: (...args) => + jest.requireActual('@librechat/api').buildResponseInputChain(...args), + loadForegroundResponseHistory: (...args) => + jest.requireActual('@librechat/api').loadForegroundResponseHistory(...args), + projectStoredResponseMessage: (...args) => + jest.requireActual('@librechat/api').projectStoredResponseMessage(...args), + requireResponseStorageWrite: (...args) => + jest.requireActual('@librechat/api').requireResponseStorageWrite(...args), + saveForegroundResponseInputs: (...args) => + jest.requireActual('@librechat/api').saveForegroundResponseInputs(...args), + stageForegroundResponse: (...args) => + jest.requireActual('@librechat/api').stageForegroundResponse(...args), + createForegroundResponseTerminal: (...args) => + jest.requireActual('@librechat/api').createForegroundResponseTerminal(...args), + createStoredForegroundResponseReader: (...args) => + jest.requireActual('@librechat/api').createStoredForegroundResponseReader(...args), + createResponseAccountingSnapshot: (...args) => + jest.requireActual('@librechat/api').createResponseAccountingSnapshot(...args), + withResponsePersistenceErrors: (...args) => + jest.requireActual('@librechat/api').withResponsePersistenceErrors(...args), createFileProvisionRecovery: () => jest.requireActual('@librechat/api').createFileProvisionRecovery(), getAgentErrorMetadata: (...args) => @@ -342,7 +375,7 @@ jest.mock('@librechat/api', () => ({ resolveRecursionLimit: jest.fn().mockReturnValue(50), // Responses API writeDone: jest.fn(), - buildResponse: jest.fn().mockReturnValue({ id: 'resp_123', output: [] }), + buildResponse: jest.fn().mockReturnValue({ id: 'resp_mock-123', output: [] }), generateResponseId: jest.fn().mockReturnValue('resp_mock-123'), isValidationFailure: jest.fn().mockReturnValue(false), inspectContent: mockInspectContent, @@ -395,7 +428,7 @@ jest.mock('@librechat/api', () => ({ request: { model: 'agent-123', input: 'Hello', stream: false }, }), buildAggregatedResponse: jest.fn().mockReturnValue({ - id: 'resp_123', + id: 'resp_mock-123', status: 'completed', output: [], usage: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }, @@ -414,6 +447,17 @@ jest.mock('@librechat/api', () => ({ on_chat_model_end: { handle: jest.fn() }, }, finalizeStream: jest.fn(), + failStream: jest.fn(), + prepareStreamResponse: jest.fn((usage, accounting) => ({ + id: 'resp_mock-123', + usage, + accounting, + object: 'response', + output: [], + model: 'agent-123', + status: 'completed', + store: false, + })), }), createAggregatorEventHandlers: jest.fn().mockReturnValue({ on_message_delta: { handle: jest.fn() }, @@ -422,16 +466,22 @@ jest.mock('@librechat/api', () => ({ on_run_step_delta: { handle: jest.fn() }, on_chat_model_end: { handle: jest.fn() }, }), - executeAgentRun: async ({ - envelope, - runId, - conversationId, - connection, - isPrincipalActive, - execute, - handleExecutionError, - beforeSettle, - }) => { + executeAgentRun: async (params) => { + if (mockRealExecutionManager) + return jest + .requireActual('@librechat/api') + .executeAgentRun(params, { manager: mockRealExecutionManager }); + const { + envelope, + runId, + conversationId, + connection, + isPrincipalActive, + execute, + handleExecutionError, + beforeSettle, + afterSettle, + } = params; let execution; let executionError; let closed = connection?.isClosed() ?? false; @@ -461,6 +511,7 @@ jest.mock('@librechat/api', () => ({ if (execution) { await beforeSettle?.(execution, executionError); await execution.settle(executionError); + await afterSettle?.(execution, executionError); } } }, @@ -537,6 +588,8 @@ jest.mock('~/cache', () => ({ logViolation: jest.fn(), })); +jest.mock('~/server/services/Files/process', () => ({ saveBase64Image: jest.fn() })); + jest.mock('~/server/services/Files/strategies', () => ({ getStrategyFunctions: jest.fn().mockReturnValue({}), })); @@ -558,7 +611,8 @@ jest.mock('~/models', () => ({ getProjectFiles: jest.fn(), getFiles: jest.fn(), getMessages: jest.fn().mockResolvedValue([]), - saveMessage: jest.fn().mockResolvedValue({}), + getMessage: jest.fn().mockResolvedValue(null), + saveMessage: jest.fn().mockImplementation(async (_context, message) => ({ ...message })), updateFilesUsage: jest.fn(), getUserKeyValues: jest.fn(), getUserCodeFiles: jest.fn(), @@ -572,7 +626,7 @@ jest.mock('~/models', () => ({ getCacheMultiplier: mockGetCacheMultiplier, getConvoFiles: jest.fn().mockResolvedValue([]), getFormattedMemories: jest.fn().mockResolvedValue({ withKeys: '', withoutKeys: '' }), - saveConvo: jest.fn().mockResolvedValue({}), + saveConvo: jest.fn().mockImplementation(async (_context, conversation) => ({ ...conversation })), getConvo: jest.fn().mockResolvedValue(null), readAdmittedConvoCodeEnvironmentDecision: jest.fn().mockResolvedValue(null), isSubagentOwnerAdmissible: jest.fn().mockResolvedValue(true), @@ -652,6 +706,7 @@ describe('createResponse controller', () => { codeWorkspaces: selections, }); await createResponse(req, res); + expect(api.sendResponsesErrorResponse.mock.calls).toEqual([]); const fencedRead = require('~/models').readAdmittedConvoCodeEnvironmentDecision; if (movesEnabled) expect(fencedRead).toHaveBeenCalledTimes(1); else expect(fencedRead).not.toHaveBeenCalled(); @@ -803,16 +858,21 @@ describe('createResponse controller', () => { }); it('does not initialize a provider after disconnecting during enrollment', async () => { + let enteredEnrollment; + const enrollmentReady = new Promise((resolve) => { + enteredEnrollment = resolve; + }); let finishEnrollment; mockEnrollAgentExecution.mockImplementationOnce( () => new Promise((resolve) => { finishEnrollment = resolve; + enteredEnrollment(); }), ); const request = createResponse(req, res); - await Promise.resolve(); + await enrollmentReady; res.once.mock.calls[0][1](); finishEnrollment(mockExecution); await request; @@ -1093,8 +1153,10 @@ describe('createResponse controller', () => { { role: 'user', content: 'Hello', messageId: 'input-123' }, ]); const savedResponse = { messageId: 'resp_mock-123', isTemporary: true }; - db.saveMessage.mockResolvedValueOnce({ messageId: 'input-123' }); - db.saveMessage.mockResolvedValueOnce(savedResponse); + db.saveMessage.mockImplementationOnce(async (_context, message) => ({ ...message })); + db.saveMessage.mockImplementationOnce(async (_context, message) => + Object.assign(savedResponse, { conversationId: message.conversationId }), + ); await createResponse(req, res); @@ -2630,6 +2692,7 @@ describe('createResponse controller', () => { expect.anything(), expect.anything(), mockResponsesUsage, + expect.objectContaining({ status: 'unavailable', calls: expect.any(Array) }), ); }); @@ -2643,7 +2706,339 @@ describe('createResponse controller', () => { const finalizeStream = api.createResponsesEventHandlers.mock.results.at(-1).value.finalizeStream; - expect(finalizeStream).toHaveBeenCalledWith(mockResponsesUsage); + expect(finalizeStream).toHaveBeenCalledWith( + mockResponsesUsage, + expect.objectContaining({ calls: expect.any(Array) }), + expect.objectContaining({ store: false }), + ); + }); + }); + + describe('foreground production composition', () => { + const names = [ + 'createResponseContext', + 'createResponseTracker', + 'createResponseAggregator', + 'createResponsesEventHandlers', + 'createAggregatorEventHandlers', + 'buildAggregatedResponse', + 'buildResponsesUsage', + 'convertInputToMessages', + 'setupStreamingResponse', + 'emitResponseCreated', + 'emitResponseInProgress', + 'createOwnedToolEndHandler', + 'sendResponsesErrorResponse', + ]; + let originals; + let originalRun; + let originalContextualize; + let originalToolEnd; + beforeEach(() => { + const api = require('@librechat/api'); + const actual = jest.requireActual('@librechat/api'); + originals = new Map(names.map((name) => [name, api[name].getMockImplementation()])); + for (const name of names) api[name].mockImplementation(actual[name]); + const sdk = require('@librechat/agents'); + originalToolEnd = sdk.ToolEndHandler.getMockImplementation(); + sdk.ToolEndHandler.mockImplementation( + (...args) => new (jest.requireActual('@librechat/agents').ToolEndHandler)(...args), + ); + originalRun = api.createRun.getMockImplementation(); + const callbacks = require('~/server/controllers/agents/callbacks'); + originalContextualize = callbacks.contextualizeModelUsage.getMockImplementation(); + callbacks.contextualizeModelUsage.mockImplementation( + jest.requireActual('~/server/controllers/agents/callbacks').contextualizeModelUsage, + ); + mockRealExecutionManager = new actual.GenerationJobManagerClass(); + mockRealExecutionManager.configure({ + jobStore: new actual.InMemoryJobStore(), + eventTransport: new actual.InMemoryEventTransport(), + }); + mockRealExecutionManager.initialize(); + jest.spyOn(mockRealExecutionManager, 'createJob'); + api.createRun.mockImplementation(async ({ customHandlers }) => ({ + processStream: async () => { + customHandlers.on_run_step.handle('on_run_step', { + id: 'step-tool', + type: 'tool_calls', + index: 0, + stepDetails: { + type: 'tool_calls', + tool_calls: [{ id: 'call', name: 'lookup', args: '{}' }], + }, + }); + customHandlers.on_run_step_completed.handle('on_run_step_completed', { + result: { + id: 'step-tool', + index: 0, + tool_call: { + id: 'call', + name: 'lookup', + args: '{}', + output: 'retained-tool-result', + type: 'tool_call', + }, + }, + }); + await customHandlers.on_tool_end.handle( + 'on_tool_end', + { + output: new (require('@librechat/agents/langchain/messages').ToolMessage)({ + content: 'retained-tool-result', + tool_call_id: 'call', + }), + }, + {}, + { toolCallStepIds: new Map([['call', 'step-tool']]) }, + ); + customHandlers.on_message_delta.handle('on_message_delta', { + delta: { content: [{ type: 'text', text: 'Answer' }] }, + }); + customHandlers.on_chat_model_end.handle( + 'on_chat_model_end', + { output: { usage_metadata: { input_tokens: 2, output_tokens: 1 } } }, + {}, + { + getAgentContext: () => ({ + clientOptions: { model: 'configured-provider-model' }, + provider: 'openAI', + }), + }, + ); + }, + })); + }); + afterEach(async () => { + await mockRealExecutionManager?.destroy(); + mockRealExecutionManager = undefined; + const api = require('@librechat/api'); + for (const [name, implementation] of originals) api[name].mockImplementation(implementation); + api.createRun.mockImplementation(originalRun); + require('@librechat/agents').ToolEndHandler.mockImplementation(originalToolEnd); + require('~/server/controllers/agents/callbacks').contextualizeModelUsage.mockImplementation( + originalContextualize, + ); + }); + const completed = (res, stream) => { + if (!stream) return res.json.mock.calls.at(-1)?.[0]; + return res.write.mock.calls + .map(([chunk]) => chunk) + .filter((chunk) => chunk.startsWith('data: {')) + .map((chunk) => JSON.parse(chunk.slice(6).trim())) + .find((event) => event.type === 'response.completed')?.response; + }; + it.each([false, true])( + 'publishes tool results, accounting and successful storage after real host drain: stream=%s', + async (stream) => { + const api = require('@librechat/api'); + const db = require('~/models'); + api.validateResponseRequest.mockReturnValueOnce({ + request: { model: 'agent-123', input: 'Hello', stream, store: true }, + }); + const writers = [db.saveMessage, db.saveConvo, api.announceReply]; + const implementations = writers.map((writer) => writer.getMockImplementation()); + writers.forEach((writer, index) => { + writer.mockImplementation(async (...args) => { + const job = await mockRealExecutionManager.getJob('resp_mock-123'); + expect(job.metadata.providerDrained).toBe(false); + expect( + await mockRealExecutionManager.getCleanupBlockingJobIdsForUser(req.user.id), + ).toContain('resp_mock-123'); + return implementations[index](...args); + }); + }); + try { + await createResponse(req, res); + } finally { + writers.forEach((writer, index) => writer.mockImplementation(implementations[index])); + } + expect(api.sendResponsesErrorResponse.mock.calls).toEqual([]); + const response = completed(res, stream); + expect(response).toMatchObject({ + id: 'resp_mock-123', + store: true, + accounting: { + calls: [ + expect.objectContaining({ + model: 'configured-provider-model', + modelSource: 'configured', + inputTokens: 2, + outputTokens: 1, + }), + ], + }, + }); + expect(response.output).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'function_call_output', + call_id: 'call', + output: 'retained-tool-result', + }), + ]), + ); + const message = db.saveMessage.mock.calls.find(([, value]) => !value.isCreatedByUser)?.[1]; + expect(message).toMatchObject({ + parentMessageId: expect.any(String), + content: expect.arrayContaining([ + expect.objectContaining({ + tool_call: expect.objectContaining({ output: 'retained-tool-result' }), + }), + ]), + metadata: { responseReceipt: { accounting: response.accounting, usage: response.usage } }, + }); + expect( + require('~/server/controllers/agents/callbacks')[ + stream ? 'createResponsesToolEndCallback' : 'createToolEndCallback' + ], + ).toHaveBeenCalled(); + const callbacks = require('~/server/controllers/agents/callbacks'); + const artifactCallback = + callbacks[ + stream ? 'createResponsesToolEndCallback' : 'createToolEndCallback' + ].mock.results.at(-1).value; + expect(artifactCallback).toHaveBeenCalledWith( + expect.objectContaining({ output: expect.objectContaining({ tool_call_id: 'call' }) }), + expect.objectContaining({ stepId: 'step-tool' }), + ); + expect( + response.output.filter( + (item) => item.type === 'function_call_output' && item.call_id === 'call', + ), + ).toHaveLength(1); + expect(mockRecordCollectedUsage).toHaveBeenCalledTimes(1); + }, + ); + it.each([false, true])( + 'retains provider output with store false on resolved conversation save failure: stream=%s', + async (stream) => { + const api = require('@librechat/api'); + const db = require('~/models'); + api.validateResponseRequest.mockReturnValueOnce({ + request: { model: 'agent-123', input: 'Hello', stream, store: true }, + }); + db.saveConvo.mockResolvedValueOnce({ message: 'private database failure' }); + await createResponse(req, res); + expect(completed(res, stream)).toMatchObject({ + store: false, + output: expect.arrayContaining([ + expect.objectContaining({ + type: 'function_call_output', + output: 'retained-tool-result', + }), + ]), + }); + expect(db.saveMessage).not.toHaveBeenCalled(); + }, + ); + it.each([false, true])( + 'closes canonical cancellation without emitting success or persisting output: stream=%s', + async (stream) => { + const api = require('@librechat/api'); + const db = require('~/models'); + const provider = api.createRun.getMockImplementation(); + api.validateResponseRequest.mockReturnValueOnce({ + request: { model: 'agent-123', input: 'Hello', stream, store: true }, + }); + api.createRun.mockImplementationOnce(async (options) => { + const run = await provider(options); + return { + ...run, + processStream: async (...args) => { + await run.processStream(...args); + const job = await mockRealExecutionManager.getJob('resp_mock-123'); + await mockRealExecutionManager.abortJob('resp_mock-123', { + expectedCreatedAt: job.createdAt, + awaitProviderDrain: false, + }); + }, + }; + }); + await createResponse(req, res); + expect(db.saveMessage).not.toHaveBeenCalled(); + if (stream) { + expect(completed(res, true)).toBeUndefined(); + expect(res.write.mock.calls.map(([value]) => value).join('')).toContain( + 'response_execution_cancelled', + ); + expect(res.end).toHaveBeenCalled(); + } else { + expect(res.status).toHaveBeenCalledWith(409); + expect(res.json).toHaveBeenCalledWith({ + error: expect.objectContaining({ code: 'response_execution_cancelled' }), + }); + } + }, + ); + it.each([false, true])( + 'closes outcome readback failure with a safe error after provider drain: stream=%s', + async (stream) => { + const api = require('@librechat/api'); + const db = require('~/models'); + api.validateResponseRequest.mockReturnValueOnce({ + request: { model: 'agent-123', input: 'Hello', stream, store: true }, + }); + const complete = mockRealExecutionManager.completeJob.bind(mockRealExecutionManager); + jest + .spyOn(mockRealExecutionManager, 'completeJob') + .mockImplementationOnce(async (...args) => { + const result = await complete(...args); + jest + .spyOn(mockRealExecutionManager, 'getJob') + .mockRejectedValueOnce(new Error('private-outage-secret')); + return result; + }); + await expect(createResponse(req, res)).rejects.toThrow('private-outage-secret'); + expect(db.saveMessage).toHaveBeenCalledTimes(2); + const body = stream + ? res.write.mock.calls.map(([value]) => value).join('') + : JSON.stringify(res.json.mock.calls); + expect(body).toContain('response_execution_unavailable'); + expect(body).not.toContain('private-outage-secret'); + expect(completed(res, stream)?.store).not.toBe(true); + expect(await mockRealExecutionManager.getCleanupBlockingJobIdsForUser(req.user.id)).toEqual( + [], + ); + }, + ); + + it('resolves a returned response ID before real host conversation admission and links its next input', async () => { + const api = require('@librechat/api'); + const db = require('~/models'); + const priorConversation = '6834b9eb-9939-40d2-847a-ac9ad95fd88d'; + api.validateResponseRequest.mockReturnValueOnce({ + request: { + model: 'agent-123', + input: 'Next', + stream: false, + store: true, + previous_response_id: 'resp_prior', + }, + }); + db.getMessage.mockResolvedValueOnce({ + messageId: 'resp_prior', + conversationId: priorConversation, + isCreatedByUser: false, + }); + db.getConvo.mockResolvedValueOnce({ conversationId: priorConversation, user: req.user.id }); + db.getMessages.mockResolvedValueOnce([ + { + messageId: 'resp_prior', + conversationId: priorConversation, + parentMessageId: null, + text: 'Earlier answer', + isCreatedByUser: false, + }, + ]); + await createResponse(req, res); + expect(api.sendResponsesErrorResponse.mock.calls).toEqual([]); + expect(db.getConvo).toHaveBeenCalledWith(req.user.id, priorConversation); + expect( + db.saveMessage.mock.calls.find(([, message]) => message.isCreatedByUser)?.[1], + ).toMatchObject({ conversationId: priorConversation, parentMessageId: 'resp_prior' }); + expect(mockRealExecutionManager.createJob.mock.calls[0][2]).toBe(priorConversation); + expect(await mockRealExecutionManager.getJob('resp_mock-123')).toBeUndefined(); }); }); diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js index 07e4c405eb..20b5c73f68 100644 --- a/api/server/controllers/agents/callbacks.js +++ b/api/server/controllers/agents/callbacks.js @@ -21,6 +21,7 @@ const { sendEvent, getModelEndUsage, computeUsageCostUSD, + captureResponseAccountingUsage, GenerationJobManager, waitForGenerationSettled, writeAttachmentEvent, @@ -1581,7 +1582,7 @@ function markSummarizationUsage(usage, metadata) { * not consistently include it, and cache normalization depends on it. */ function contextualizeModelUsage(usage, metadata, agentContext = {}) { - const taggedUsage = { ...usage }; + const taggedUsage = captureResponseAccountingUsage(usage, metadata, agentContext); const context = agentContext ?? {}; const invokedProvider = metadata?.__invoked_provider; const invokedModel = metadata?.__invoked_model; diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index 33cc2ee369..6c058a7cf8 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -1,5 +1,4 @@ const { nanoid } = require('nanoid'); -const { v4: uuidv4 } = require('uuid'); const { logger } = require('@librechat/data-schemas'); const { Callback, formatAgentMessages } = require('@librechat/agents'); const { @@ -60,8 +59,15 @@ const { getRemoteAgentPermissions, resolveAgentScopedSkillIds, // Responses API + resolveResponseContinuation, + buildResponseInputChain, + loadForegroundResponseHistory, + createForegroundResponseTerminal, + stageForegroundResponse, + createStoredForegroundResponseReader, + createResponseAccountingSnapshot, + withResponsePersistenceErrors, writeDone, - buildResponse, generateResponseId, isValidationFailure, emitResponseCreated, @@ -88,10 +94,8 @@ const { resolveToolRoleGrants, resolveApiConversationProject, resolveAdmittedCodeEnvironmentDecision, - resolvePersistableCodeEnvironmentDecision, createTerminalRunErrorObserver, announceReply, - getConversationWriteContext, isTemporaryTurn, getModelEndUsage, hasEntityEmbeddings, @@ -314,158 +318,10 @@ function extractResponseRequestContent(request, messageFragments) { * @param {string} userId - The user ID * @returns {Promise} Messages from the conversation */ -async function loadPreviousMessages(conversationId, userId) { - try { - const messages = await db.getMessages({ conversationId, user: userId }); - if (!messages || messages.length === 0) { - return []; - } - - // Convert stored messages to internal format - return messages.map((msg) => { - let text; - if (typeof msg.text === 'string') { - text = msg.text; - } else if (msg.text != null) { - text = String(msg.text); - } - const internalMsg = { - role: msg.isCreatedByUser ? 'user' : 'assistant', - content: Array.isArray(msg.content) ? msg.content : (text ?? ''), - messageId: msg.messageId, - isCreatedByUser: msg.isCreatedByUser === true, - ...(text !== undefined && { text }), - ...(typeof msg.isUserSubmitted === 'boolean' && { - isUserSubmitted: msg.isUserSubmitted, - }), - ...(Array.isArray(msg.userSubmittedPaths) && { - userSubmittedPaths: msg.userSubmittedPaths, - }), - ...(Array.isArray(msg.userSubmittedMessageFieldPaths) && { - userSubmittedMessageFieldPaths: msg.userSubmittedMessageFieldPaths, - }), - }; - - return internalMsg; - }); - } catch (error) { - logger.error('[Responses API] Error loading previous messages:', getSafeErrorMetadata(error)); - return []; - } -} - -/** - * Save input messages to database - * @param {import('express').Request} req - * @param {string} conversationId - * @param {Array} inputMessages - Internal format messages - * @param {string} agentId - * @returns {Promise} - */ -async function saveInputMessages(req, conversationId, inputMessages, agentId) { - for (const msg of inputMessages) { - if (msg.role === 'user') { - await db.saveMessage( - getConversationWriteContext(req), - { - messageId: msg.messageId || nanoid(), - conversationId, - parentMessageId: null, - isCreatedByUser: true, - text: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content), - sender: 'User', - endpoint: EModelEndpoint.agents, - model: agentId, - }, - { context: 'Responses API - save user input' }, - ); - } - } -} - -/** - * Save response output to database - * @param {import('express').Request} req - * @param {string} conversationId - * @param {string} responseId - * @param {import('@librechat/api').Response} response - * @param {string} agentId - * @param {number | undefined} visibleOutputTokens - * @returns {Promise} - */ -async function saveResponseOutput( - req, - conversationId, - responseId, - response, - agentId, - visibleOutputTokens, -) { - // Extract text content from output items - let responseText = ''; - for (const item of response.output) { - if (item.type === 'message' && item.content) { - for (const part of item.content) { - if (part.type === 'output_text' && part.text) { - responseText += part.text; - } - } - } - } - - const langfuseTraceFields = await getLangfuseTraceMessageFields(req.config, responseId, { - isTemporary: isTemporaryTurn(req), - }); - - // Save the assistant message - return db.saveMessage( - getConversationWriteContext(req), - { - messageId: responseId, - conversationId, - parentMessageId: null, - isCreatedByUser: false, - ...langfuseTraceFields, - text: responseText, - sender: 'Agent', - endpoint: EModelEndpoint.agents, - model: agentId, - finish_reason: response.status === 'completed' ? 'stop' : response.status, - tokenCount: visibleOutputTokens ?? response.usage?.output_tokens, - }, - { context: 'Responses API - save assistant response' }, - ); -} - -/** - * Save or update conversation - * @param {import('express').Request} req - * @param {string} conversationId - * @param {string} agentId - * @param {object} agent - * @param {import('@librechat/api').ConversationCodeEnvironmentDecision} codeEnvironmentDecision - * @returns {Promise} - */ -async function saveConversation(req, conversationId, agentId, agent, codeEnvironmentDecision) { - const title = resolveConversationTitle(req, agent?.name || 'Open Responses Conversation'); - await db.saveConvo( - getConversationWriteContext(req), - { - conversationId, - endpoint: EModelEndpoint.agents, - agent_id: agentId, - ...resolvePersistableCodeEnvironmentDecision({ - conversationId, - decision: codeEnvironmentDecision, - conversation: req.resolvedConversation, - }), - ...(title != null && { title }), - model: agent?.model, - }, - { - context: 'Responses API - save conversation', - initialAgentId: agent?.id === agentId ? agentId : null, - }, +async function loadPreviousMessages(conversationId, userId, previousResponseId) { + return loadForegroundResponseHistory( + { conversationId, userId, previousResponseId }, + { getMessages: db.getMessages }, ); } @@ -474,29 +330,6 @@ async function saveConversation(req, conversationId, agentId, agent, codeEnviron * @param {Array} messages - Stored messages * @returns {Array} Output items */ -function convertMessagesToOutputItems(messages) { - const output = []; - - for (const msg of messages) { - if (!msg.isCreatedByUser) { - output.push({ - type: 'message', - id: msg.messageId, - role: 'assistant', - status: 'completed', - content: [ - { - type: 'output_text', - text: msg.text || '', - annotations: [], - }, - ], - }); - } - } - - return output; -} /** * Runs a validated Responses envelope in the current process. @@ -620,11 +453,21 @@ const executeResponse = async (envelope, { req, res }) => { `[Responses API] Request ${responseId} started for agent ${agentId}, stream: ${isStreaming}`, ); - const conversationId = request.previous_response_id ?? uuidv4(); + const { conversationId, parentMessageId: previousMessageId } = await resolveResponseContinuation( + request.previous_response_id, + principal.userId, + { getMessage: db.getMessage }, + ); + const terminal = createForegroundResponseTerminal({ + context, + requestedStore: request.store, + logger, + }); /** @type {Promise[]} */ const artifactPromises = []; let artifactWritesCovered = false; return executeAgentRun({ + captureTerminalOutcome: true, envelope, runId: responseId, conversationId, @@ -644,7 +487,8 @@ const executeResponse = async (envelope, { req, res }) => { /** Conversation delete-all uses the shared owner-admission fence. Remote * execution must observe it after durable enrollment and before provider work. */ isPrincipalActive: db.isSubagentOwnerAdmissible, - beforeSettle: (execution) => { + afterSettle: terminal.afterSettle, + beforeSettle: (execution, error) => { if (!artifactWritesCovered && artifactPromises.length > 0) { execution.track( waitForAgentExecutionWrites(artifactPromises).catch((artifactError) => { @@ -655,10 +499,9 @@ const executeResponse = async (envelope, { req, res }) => { }), ); } + terminal.beforeSettle(execution, error); }, - onSettlementError: (error) => { - logger.error('[Responses API] Failed to settle execution:', getSafeErrorMetadata(error)); - }, + onSettlementError: terminal.onSettlementError, handleExecutionError: (error, signal) => { terminalRunError.log(error, signal); return handleExecutionError({ error, res, appConfig }); @@ -673,7 +516,7 @@ const executeResponse = async (envelope, { req, res }) => { { userId: principal.userId, tenantId: principal.tenantId, - conversationId: request.previous_response_id, + conversationId, rejectSubagentThread: true, }, { @@ -738,7 +581,6 @@ const executeResponse = async (envelope, { req, res }) => { chatProjectContext: req.chatProjectContext, }); - const parentMessageId = null; const mcpRequestBody = createMCPRuntimeRequestBody({ messageId: responseId, conversationId, @@ -751,8 +593,14 @@ const executeResponse = async (envelope, { req, res }) => { const backgroundCompletionResultMaxChars = agentsEConfig?.backgroundTasks?.completionResultMaxChars; const previousMessages = request.previous_response_id - ? await loadPreviousMessages(request.previous_response_id, principal.userId) + ? await loadPreviousMessages(conversationId, principal.userId, previousMessageId) : []; + const inputChain = buildResponseInputChain( + inputMessages, + previousMessageId, + previousMessages, + ); + const parentMessageId = inputChain.parentMessageId; if (request.previous_response_id) { assertModelBoundContent({ onTraversalFailure: reportLocatorTraversalFailure, @@ -1192,6 +1040,18 @@ const executeResponse = async (envelope, { req, res }) => { const tracker = actuallyStreaming ? createResponseTracker() : null; const aggregator = actuallyStreaming ? null : createResponseAggregator(); + const collectedUsage = []; + const snapshotAccounting = createResponseAccountingSnapshot({ + collectedUsage, + emittedUsage: [], + billingEnabled: getTransactionsConfig(appConfig)?.enabled === true, + billing: { + pricing: { getMultiplier: db.getMultiplier, getCacheMultiplier: db.getCacheMultiplier }, + getValueKey: db.getValueKey, + resolveEndpointTokenConfig, + }, + }); + // Set up response for streaming if (actuallyStreaming) { setupStreamingResponse(res); @@ -1214,11 +1074,12 @@ const executeResponse = async (envelope, { req, res }) => { const { handlers: responsesHandlers, finalizeStream, + prepareStreamResponse, + failStream, emitClientToolDeferral, } = createResponsesEventHandlers(handlerConfig); // Collect usage for balance tracking - const collectedUsage = []; // Artifact promises for processing tool outputs // Use Responses API-specific callback that emits librechat:attachment events @@ -1297,7 +1158,7 @@ const executeResponse = async (envelope, { req, res }) => { }, }, on_tool_end: createOwnedToolEndHandler(toolEndCallback, logger), - on_run_step_completed: { handle: () => {} }, + on_run_step_completed: responsesHandlers.on_run_step_completed, on_chain_stream: { handle: () => {} }, on_chain_end: { handle: () => {} }, on_agent_update: { handle: () => {} }, @@ -1405,51 +1266,37 @@ const executeResponse = async (envelope, { req, res }) => { const usage = buildResponsesUsage(collectedUsage); - // Finalize the stream - finalizeStream(usage); - res.end(); - - const duration = Date.now() - requestStartTime; - logger.debug( - `[Responses API] Request ${responseId} completed in ${duration}ms (streaming)`, + const accounting = snapshotAccounting(); + const finalResponse = prepareStreamResponse(usage, accounting); + stageForegroundResponse( + { + terminal, + request: req, + response: finalResponse, + conversationId, + agentId, + agent, + codeEnvironmentDecision, + messages: inputChain.messages, + parentMessageId, + visibleOutputTokens: tracker.usage.outputTokens, + transport: { + type: 'stream', + response: res, + finalize: finalizeStream, + fail: failStream, + }, + }, + { + saveConvo: db.saveConvo, + saveMessage: db.saveMessage, + announceReply: announceReply.bind(null, db), + resolveTitle: resolveConversationTitle, + traceFields: getLangfuseTraceMessageFields, + }, ); - // Save to database if store: true - if (request.store === true) { - try { - // Save conversation - await saveConversation(req, conversationId, agentId, agent, codeEnvironmentDecision); - - // Save input messages - await saveInputMessages(req, conversationId, inputMessages, agentId); - - // Build response for saving (use tracker with buildResponse for streaming) - const finalResponse = buildResponse(context, tracker, 'completed'); - const savedResponse = await saveResponseOutput( - req, - conversationId, - responseId, - finalResponse, - agentId, - tracker.usage.outputTokens, - ); - await announceReply(db, { - userId: req?.user?.id, - conversationId, - reply: savedResponse, - context: 'Responses API - announce stored reply', - }); - - logger.debug( - `[Responses API] Stored response ${responseId} in conversation ${conversationId}`, - ); - } catch (saveError) { - logger.error('[Responses API] Error saving response:', getSafeErrorMetadata(saveError)); - // Don't fail the request if saving fails - } - } - - // The HTTP response is complete, while destructive cleanup still waits for artifacts. + // Register artifact writes before the host drains and publishes the terminal response. if (artifactPromises.length > 0) { execution.track( waitForAgentExecutionWrites(artifactPromises).catch((artifactError) => { @@ -1465,7 +1312,6 @@ const executeResponse = async (envelope, { req, res }) => { const aggregatorHandlers = createAggregatorEventHandlers(aggregator); // Collect usage for balance tracking - const collectedUsage = []; const toolEndCallback = createToolEndCallback({ req, @@ -1540,7 +1386,7 @@ const executeResponse = async (envelope, { req, res }) => { }, }, on_tool_end: createOwnedToolEndHandler(toolEndCallback, logger), - on_run_step_completed: { handle: () => {} }, + on_run_step_completed: aggregatorHandlers.on_run_step_completed, on_chain_stream: { handle: () => {} }, on_chain_end: { handle: () => {} }, on_agent_update: { handle: () => {} }, @@ -1658,39 +1504,30 @@ const executeResponse = async (envelope, { req, res }) => { context, aggregator, buildResponsesUsage(collectedUsage), + snapshotAccounting(), + ); + stageForegroundResponse( + { + terminal, + request: req, + response: response, + conversationId, + agentId, + agent, + codeEnvironmentDecision, + messages: inputChain.messages, + parentMessageId, + visibleOutputTokens: aggregator.usage.outputTokens, + transport: { type: 'json', response: res }, + }, + { + saveConvo: db.saveConvo, + saveMessage: db.saveMessage, + announceReply: announceReply.bind(null, db), + resolveTitle: resolveConversationTitle, + traceFields: getLangfuseTraceMessageFields, + }, ); - - if (request.store === true) { - try { - await saveConversation(req, conversationId, agentId, agent, codeEnvironmentDecision); - - await saveInputMessages(req, conversationId, inputMessages, agentId); - - const savedResponse = await saveResponseOutput( - req, - conversationId, - responseId, - response, - agentId, - aggregator.usage.outputTokens, - ); - await announceReply(db, { - userId: req?.user?.id, - conversationId, - reply: savedResponse, - context: 'Responses API - announce stored reply', - }); - - logger.debug( - `[Responses API] Stored response ${responseId} in conversation ${conversationId}`, - ); - } catch (saveError) { - logger.error('[Responses API] Error saving response:', getSafeErrorMetadata(saveError)); - // Don't fail the request if saving fails - } - } - - res.json(response); const duration = Date.now() - requestStartTime; logger.debug( @@ -1803,104 +1640,15 @@ const listModels = async (req, res) => { * @param {import('express').Request} req * @param {import('express').Response} res */ -const getResponse = async (req, res) => { - try { - const responseId = req.params.id; - const userId = req.user?.id; - - if (!responseId) { - return sendResponsesErrorResponse(res, 400, 'Response ID is required'); - } - - // The responseId could be either the response ID or the conversation ID - // Try to find a conversation with this ID - const conversation = await db.getConvo(userId, responseId); - - if (!conversation) { - return sendResponsesErrorResponse( - res, - 404, - `Response not found: ${responseId}`, - 'not_found', - 'response_not_found', - ); - } - - // Load messages for this conversation - const messages = await db.getMessages({ conversationId: responseId, user: userId }); - - if (!messages || messages.length === 0) { - return sendResponsesErrorResponse( - res, - 404, - `No messages found for response: ${responseId}`, - 'not_found', - 'response_not_found', - ); - } - - // Convert messages to Open Responses output format - const output = convertMessagesToOutputItems(messages); - - // Find the last assistant message for usage info - const lastAssistantMessage = messages.filter((m) => !m.isCreatedByUser).pop(); - - // Build the response object - const response = { - id: responseId, - object: 'response', - created_at: Math.floor(new Date(conversation.createdAt || Date.now()).getTime() / 1000), - completed_at: Math.floor(new Date(conversation.updatedAt || Date.now()).getTime() / 1000), - status: 'completed', - incomplete_details: null, - model: conversation.agentId || conversation.model || 'unknown', - previous_response_id: null, - instructions: null, - output, - error: null, - tools: [], - tool_choice: 'auto', - truncation: 'disabled', - parallel_tool_calls: true, - text: { format: { type: 'text' } }, - temperature: 1, - top_p: 1, - presence_penalty: 0, - frequency_penalty: 0, - top_logprobs: null, - reasoning: null, - user: userId, - usage: lastAssistantMessage?.tokenCount - ? { - input_tokens: 0, - output_tokens: lastAssistantMessage.tokenCount, - total_tokens: lastAssistantMessage.tokenCount, - } - : null, - max_output_tokens: null, - max_tool_calls: null, - store: true, - background: false, - service_tier: 'default', - metadata: {}, - safety_identifier: null, - prompt_cache_key: null, - }; - - res.json(response); - } catch (error) { - logger.error('[Responses API] Error getting response:', getSafeErrorMetadata(error)); - sendResponsesErrorResponse( - res, - 500, - error instanceof Error ? error.message : 'Failed to get response', - 'server_error', - ); - } -}; +const getResponse = createStoredForegroundResponseReader({ + getMessage: db.getMessage, + getMessages: db.getMessages, + getConvo: db.getConvo, + logger, +}); module.exports = { - createResponse, + createResponse: withResponsePersistenceErrors(createResponse), getResponse, listModels, }; diff --git a/packages/api/src/agents/remote/host.ts b/packages/api/src/agents/remote/host.ts index 2ce0935da8..387f12dd63 100644 --- a/packages/api/src/agents/remote/host.ts +++ b/packages/api/src/agents/remote/host.ts @@ -18,6 +18,7 @@ export interface ExecuteAgentRunParams { conversationId: string; clientRequestId?: string; retainTerminalPersistence?: true; + captureTerminalOutcome?: true; /** Server-validated journal identity only; never copied from an API request body. */ resume?: ResumedAgentExecutionIdentity; connection?: AgentExecutionConnection; @@ -52,6 +53,7 @@ export async function executeAgentRun( conversationId, clientRequestId, retainTerminalPersistence, + captureTerminalOutcome, resume, connection, isPrincipalActive, @@ -87,6 +89,7 @@ export async function executeAgentRun( protocol: envelope.protocol, clientRequestId, retainTerminalPersistence, + captureTerminalOutcome, isPrincipalActive, }; execution = diff --git a/packages/api/src/agents/remote/lifecycle.spec.ts b/packages/api/src/agents/remote/lifecycle.spec.ts index 6fdaacf5bf..c10e644ef7 100644 --- a/packages/api/src/agents/remote/lifecycle.spec.ts +++ b/packages/api/src/agents/remote/lifecycle.spec.ts @@ -71,7 +71,9 @@ describe('Agent execution enrollment', () => { it('preserves the existing custom failure message while using guarded terminal settlement', async () => { const enrolled = await enrollAgentExecution(enrollmentParams('assistant-failure'), { manager }); const job = (await manager.getJob(enrolled.runId))!; - const execution = new AgentExecutionEnrollment(manager, job, 'Assistant run failed'); + const execution = new AgentExecutionEnrollment(manager, job, 'Assistant run failed', { + captureTerminalOutcome: true, + }); await execution.beginProviderExecution(); await execution.settle(new Error('private provider diagnostic')); expect(await manager.getJob(execution.runId)).toMatchObject({ @@ -79,6 +81,52 @@ describe('Agent execution enrollment', () => { error: 'Assistant run failed', metadata: { providerDrained: true }, }); + expect(execution.terminalOutcome).toBe('error'); + }); + + it.each(['complete', 'signal-only', 'aborted'] as const)( + 'captures %s before default cleanup aborts the runtime signal', + async (outcome) => { + manager.configure({ + jobStore: new InMemoryJobStore(), + eventTransport: new InMemoryEventTransport(), + }); + const execution = await enrollAgentExecution( + { ...enrollmentParams('render-outcome'), captureTerminalOutcome: true }, + { manager }, + ); + expect(execution.terminalOutcome).toBeUndefined(); + await execution.beginProviderExecution(); + if (outcome === 'signal-only') execution.abort(); + if (outcome === 'aborted') + await manager.abortJob(execution.runId, { + expectedCreatedAt: execution.createdAt, + awaitProviderDrain: false, + }); + await execution.settle(); + expect(execution.terminalOutcome).toBe(outcome === 'aborted' ? 'aborted' : 'complete'); + expect(await manager.getJob('render-outcome')).toBeUndefined(); + }, + ); + + it('drains and releases the provider tracker even when rendering outcome readback fails', async () => { + const execution = await enrollAgentExecution( + { ...enrollmentParams('render-readback'), captureTerminalOutcome: true }, + { manager }, + ); + await execution.beginProviderExecution(); + const complete = manager.completeJob.bind(manager); + jest.spyOn(manager, 'completeJob').mockImplementationOnce(async (...args) => { + const result = await complete(...args); + jest + .spyOn(manager, 'getJob') + .mockRejectedValueOnce(new Error('outcome readback unavailable')); + return result; + }); + await expect(execution.settle()).rejects.toThrow('outcome readback unavailable'); + expect(execution.terminalOutcome).toBeUndefined(); + expect((await manager.getJob('render-readback'))?.metadata.providerDrained).toBe(true); + expect(await manager.getCleanupBlockingJobIdsForUser('user-1')).toEqual([]); }); it('retains a drained paused Stop when its required journal write fails with default cleanup', async () => { diff --git a/packages/api/src/agents/remote/lifecycle.ts b/packages/api/src/agents/remote/lifecycle.ts index 0fa5a22fa5..3c778224c8 100644 --- a/packages/api/src/agents/remote/lifecycle.ts +++ b/packages/api/src/agents/remote/lifecycle.ts @@ -28,6 +28,8 @@ export interface AgentExecutionEnrollmentParams { isPrincipalActive: (userId: string) => Promise; clientRequestId?: string; retainTerminalPersistence?: true; + /** Request a rendering outcome captured before transient generation cleanup. */ + captureTerminalOutcome?: true; } export interface AgentExecutionEnrollmentDeps { @@ -58,12 +60,19 @@ export class AgentExecutionEnrollment { private readonly attemptController?: AbortController; private readonly ownerId?: string; private readonly retainTerminalPersistence?: true; + private readonly captureTerminalOutcome?: true; private readonly trailingWrites: Promise[] = []; private providerStarted = false; private providerStartRejected = false; private settlement?: Promise; private pendingPause?: string; private paused = false; + private settledTerminalOutcome?: 'complete' | 'error' | 'aborted'; + + /** Rendering outcome only; this is not durable cleanup or journal authority. */ + get terminalOutcome(): 'complete' | 'error' | 'aborted' | undefined { + return this.settledTerminalOutcome; + } get isPaused(): boolean { return this.paused; @@ -82,7 +91,11 @@ export class AgentExecutionEnrollment { manager: GenerationJobManagerClass, job: GenerationJob, private readonly failureMessage: string = REMOTE_EXECUTION_ERROR, - options: { ownerId?: string; retainTerminalPersistence?: true } = {}, + options: { + ownerId?: string; + retainTerminalPersistence?: true; + captureTerminalOutcome?: true; + } = {}, ) { const providerExecutionId = job.metadata.providerExecutionId; if (!providerExecutionId) { @@ -91,6 +104,7 @@ export class AgentExecutionEnrollment { this.manager = manager; this.ownerId = options.ownerId; this.retainTerminalPersistence = options.retainTerminalPersistence; + this.captureTerminalOutcome = options.captureTerminalOutcome; this.runId = job.streamId; this.createdAt = job.createdAt; this.providerExecutionId = providerExecutionId; @@ -354,6 +368,26 @@ export class AgentExecutionEnrollment { } } + let terminalOutcome: 'complete' | 'error' | 'aborted' | undefined; + let outcomeError: unknown; + if (this.captureTerminalOutcome && this.providerStarted && terminalError == null) { + try { + const terminal = await this.manager.getJob(this.runId); + if ( + terminal?.createdAt !== this.createdAt || + terminal.metadata.providerExecutionId !== this.providerExecutionId || + (this.ownerId != null && terminal.metadata.providerExecutionOwnerId !== this.ownerId) || + (terminal.status !== 'complete' && + terminal.status !== 'error' && + terminal.status !== 'aborted') + ) + throw new Error('Agent terminal outcome could not be confirmed'); + terminalOutcome = terminal.status; + } catch (error) { + // Disposition readback cannot strand an already-finished provider's drain tracker. + outcomeError = error; + } + } let drainError: unknown; if (this.providerStarted && terminalError != null) { /** Provider work and trailing writes have settled, but terminalization gave up, so no @@ -391,6 +425,8 @@ export class AgentExecutionEnrollment { ) throw new Error('Agent failed execution outcome could not be confirmed'); } + if (outcomeError != null) throw outcomeError; + this.settledTerminalOutcome = terminalOutcome; return persistenceError == null ? {} : { persistenceError }; } @@ -531,6 +567,7 @@ export async function enrollAgentExecution( return new AgentExecutionEnrollment(deps.manager, job, undefined, { retainTerminalPersistence: params.retainTerminalPersistence, + captureTerminalOutcome: params.captureTerminalOutcome, }); } export interface ResumedAgentExecutionIdentity { @@ -570,5 +607,6 @@ export async function enrollResumedAgentExecution( return new AgentExecutionEnrollment(deps.manager, job, undefined, { ownerId: randomUUID(), retainTerminalPersistence: params.retainTerminalPersistence, + captureTerminalOutcome: params.captureTerminalOutcome, }); } diff --git a/packages/api/src/agents/responses/__tests__/service.test.ts b/packages/api/src/agents/responses/__tests__/service.test.ts index 1407288eb9..f0dcd0c149 100644 --- a/packages/api/src/agents/responses/__tests__/service.test.ts +++ b/packages/api/src/agents/responses/__tests__/service.test.ts @@ -16,6 +16,7 @@ import { import { clientToolDeferralContent, createClientToolExecuteHandler } from '../clientTools'; import { extractMessageContent } from '~/protection/adapters/messages'; import { buildResponse, createResponseTracker } from '../handlers'; +import { createResponseAccountingCollector } from '../accounting'; import { buildRunToolSet } from '~/agents/tools'; describe('response usage aggregation', () => { @@ -105,6 +106,31 @@ describe('response usage aggregation', () => { const completed = writes.find((chunk) => chunk.startsWith('data: {')); expect(JSON.parse(completed?.slice(6) ?? '{}').response.usage).toEqual(usage); }); + + it('uses the same separate accounting metadata for terminal JSON and SSE only', () => { + const writes: string[] = []; + const res = { write: (chunk: string) => writes.push(chunk) } as unknown as ServerResponse; + const tracker = createResponseTracker(); + const collector = createResponseAccountingCollector(); + collector.record({ input_tokens: 5, output_tokens: 2, model: 'actual-model' }); + const accounting = collector.snapshot(); + expect( + buildResponse(context, tracker, 'in_progress', undefined, accounting), + ).not.toHaveProperty('accounting'); + const response = buildAggregatedResponse( + context, + createResponseAggregator(), + undefined, + accounting, + ); + createResponsesEventHandlers({ res, context, tracker }).finalizeStream(undefined, accounting); + const completed = writes.find((chunk) => chunk.startsWith('data: {')); + expect(JSON.parse(completed?.slice(6) ?? '{}').response.accounting).toEqual( + response.accounting, + ); + expect(response.accounting?.calls[0].model).toBe('actual-model'); + expect(response.model).toBe('agent_test'); + }); }); /** diff --git a/packages/api/src/agents/responses/accounting.spec.ts b/packages/api/src/agents/responses/accounting.spec.ts new file mode 100644 index 0000000000..f17b4b2773 --- /dev/null +++ b/packages/api/src/agents/responses/accounting.spec.ts @@ -0,0 +1,452 @@ +import type { UsageMetadata } from '~/stream/interfaces/IJobStore'; +import type { AccountingQuote } from './accounting'; +import type { PricingFns } from '../transactions'; +import { + buildCollectedResponseAccounting, + combineResponseAccounting, + captureResponseAccountingUsage, + buildPersistedResponseAccounting, + createApplicationBillingQuote, + createResponseAccountingCollector, +} from './accounting'; +import { computeUsageCostUSD } from '../usage'; + +const usage: UsageMetadata = { + input_tokens: 100, + output_tokens: 20, + total_tokens: 120, + provider: 'openAI', + model: 'gpt-4o', +}; + +describe('Responses accounting', () => { + it('does not add metadata when no collected usage exists', () => { + expect(buildPersistedResponseAccounting({ emittedUsage: [] })).toEqual({}); + expect(buildPersistedResponseAccounting({ collectedUsage: [], emittedUsage: [] })).toEqual({}); + }); + + it.each(['activity-label', 'activity-phase', 'reasoning-label'] as const)( + 'retains auxiliary-only %s usage without inventing complete primary coverage', + (usageType) => { + const result = buildPersistedResponseAccounting({ + collectedUsage: [], + emittedUsage: [ + { + input_tokens: 2, + output_tokens: 1, + provider: 'openAI', + model: 'label-model', + usage_type: usageType, + runId: 'label', + seq: -1, + }, + ], + }).accounting; + expect(result).toMatchObject({ + status: 'unavailable', + calls: [ + expect.objectContaining({ + model: 'label-model', + usageType, + inputTokens: 2, + outputTokens: 1, + }), + ], + }); + expect(result?.coverage).not.toBe('complete'); + expect(result).not.toHaveProperty('amount'); + }, + ); + + it('only quotes persisted accounting when application billing is explicitly enabled', () => { + const getValueKey = jest.fn(() => 'gpt-4o'); + const billing = { + getValueKey, + pricing: { + getMultiplier: () => 1, + getCacheMultiplier: () => 1, + }, + }; + const disabled = buildPersistedResponseAccounting({ + collectedUsage: [usage], + emittedUsage: [], + billing, + }); + expect(disabled.accounting).toMatchObject({ status: 'unavailable' }); + expect(disabled.accounting).not.toHaveProperty('amount'); + expect(getValueKey).not.toHaveBeenCalled(); + + const enabled = buildPersistedResponseAccounting({ + collectedUsage: [usage], + emittedUsage: [], + billing, + billingEnabled: true, + }); + expect(getValueKey).toHaveBeenCalledWith('gpt-4o'); + expect(enabled.accounting).toMatchObject({ status: 'available', basis: 'application_billing' }); + }); + + it('does not claim zero dollars when no model usage was reported', () => { + expect(createResponseAccountingCollector().snapshot()).toEqual({ + status: 'unavailable', + currency: 'USD', + coverage: 'none', + reason: 'usage_unavailable', + calls: [], + }); + }); + + it('retains actual models and marks configured fallbacks honestly', () => { + const collector = createResponseAccountingCollector(); + collector.record( + usage, + { ls_model_name: 'actual-model' }, + { + clientOptions: { model: 'configured-model' }, + }, + ); + collector.record({ input_tokens: 1, output_tokens: 2 }, undefined, { + provider: 'anthropic', + clientOptions: { model: 'configured-model' }, + }); + expect(collector.snapshot().calls).toEqual([ + expect.objectContaining({ model: 'actual-model', modelSource: 'runtime_reported' }), + expect.objectContaining({ model: 'configured-model', modelSource: 'configured' }), + ]); + expect(collector.snapshot()).not.toHaveProperty('amount'); + }); + + it('adds decimal quotes exactly and reports their billing basis', () => { + const collector = createResponseAccountingCollector({ + quote: (entry) => ({ + amount: entry.output_tokens === 20 ? '0.1' : '0.2', + basis: 'application_billing', + }), + }); + collector.record(usage); + collector.record({ ...usage, output_tokens: 21 }); + expect(collector.snapshot()).toMatchObject({ + status: 'available', + coverage: 'complete', + amount: '0.3', + basis: 'application_billing', + }); + }); + + it('quotes handoffs using their billed model while retaining invocation provenance', () => { + const quote = jest.fn((): AccountingQuote => ({ amount: '0', basis: 'application_billing' })); + const collector = createResponseAccountingCollector({ quote }); + collector.record(usage, { __invoked_model: 'handoff-model' }, { agentId: 'handoff-agent' }); + expect(quote).toHaveBeenCalledWith( + expect.objectContaining({ + model: usage.model, + agentId: 'handoff-agent', + }), + ); + expect(collector.snapshot()).toMatchObject({ + status: 'available', + amount: '0', + calls: [expect.objectContaining({ model: 'handoff-model', modelSource: 'runtime_reported' })], + }); + }); + + it('dedupes stable model response IDs but not distinct identical calls', () => { + const collector = createResponseAccountingCollector({ + quote: () => ({ amount: '0.01', basis: 'provider_reported' }), + }); + collector.record(usage, undefined, undefined, 'model-call-1'); + collector.record(usage, undefined, undefined, 'model-call-1'); + collector.record(usage, undefined, undefined, 'model-call-2'); + expect(collector.snapshot().calls).toHaveLength(2); + expect(collector.snapshot().amount).toBe('0.02'); + }); + + it('fails closed on conflicting repeated model responses', () => { + const collector = createResponseAccountingCollector({ + quote: () => ({ amount: '1', basis: 'provider_reported' }), + }); + collector.record(usage, undefined, undefined, 'same'); + collector.record({ ...usage, output_tokens: 40 }, undefined, undefined, 'same'); + expect(collector.snapshot()).toMatchObject({ + status: 'unavailable', + reason: 'conflicting_usage', + }); + expect(collector.snapshot()).not.toHaveProperty('amount'); + }); + + it('does not publish partial or mixed-basis quotes as a run total', () => { + const quote = jest + .fn() + .mockReturnValueOnce({ amount: '0.2', basis: 'provider_reported' }) + .mockReturnValueOnce(undefined) + .mockReturnValueOnce({ amount: '0.3', basis: 'application_billing' }); + const collector = createResponseAccountingCollector({ quote }); + collector.record(usage); + collector.record({ ...usage, usage_type: 'subagent' }); + expect(collector.snapshot()).toMatchObject({ coverage: 'partial', reason: 'partial_coverage' }); + expect(collector.snapshot()).not.toHaveProperty('amount'); + collector.record(usage); + expect(collector.snapshot().reason).toBe('mixed_bases'); + expect(collector.snapshot()).not.toHaveProperty('amount'); + }); + + it('keeps subagent and summarization cache accounting provider-aware', () => { + const collector = createResponseAccountingCollector(); + collector.record({ ...usage, usage_type: 'subagent', input_token_details: { cache_read: 30 } }); + collector.record({ + ...usage, + provider: 'bedrock', + usage_type: 'summarization', + total_tokens: 155, + input_token_details: { cache_read: 30, cache_creation: 5 }, + }); + expect(collector.snapshot().calls).toEqual([ + expect.objectContaining({ usageType: 'subagent', inputTokens: 100, cacheReadTokens: 30 }), + expect.objectContaining({ + usageType: 'summarization', + inputTokens: 135, + outputTokens: 20, + cacheReadTokens: 30, + cacheWriteTokens: 5, + }), + ]); + }); + + it('treats missing/invalid usage and failed pricing as unavailable, not free', () => { + const quote = jest.fn(() => { + throw new Error('pricing unavailable'); + }); + const collector = createResponseAccountingCollector({ quote }); + collector.record(undefined); + collector.record({ ...usage, input_tokens: -1 }); + collector.record(usage); + expect(quote).toHaveBeenCalledTimes(1); + expect(collector.snapshot().calls.map((call) => call.reason)).toEqual([ + 'usage_unavailable', + 'invalid_usage', + 'pricing_unavailable', + ]); + expect(collector.snapshot()).not.toHaveProperty('amount'); + }); + + it.each(['-1', 'NaN', 'Infinity', '1e-4', '0.1234567890123456789'])( + 'rejects invalid USD quote %s', + (amount) => { + const collector = createResponseAccountingCollector({ + quote: () => ({ amount, basis: 'provider_reported' }), + }); + collector.record(usage); + expect(collector.snapshot().status).toBe('unavailable'); + }, + ); + + it('returns defensive snapshots', () => { + const collector = createResponseAccountingCollector(); + collector.record(usage); + collector.snapshot().calls[0].model = 'tampered'; + expect(collector.snapshot().calls[0].model).toBe('gpt-4o'); + }); + + it('persists real parent/child provenance and includes auxiliary calls without guessing their endpoint price', () => { + const result = buildCollectedResponseAccounting( + [usage, { ...usage, model: 'child-model', usage_type: 'subagent' }], + [ + { + input_tokens: 2, + output_tokens: 1, + provider: 'openAI', + model: 'label-model', + usage_type: 'activity-label', + runId: 'label', + seq: -1, + }, + ], + { quote: () => ({ amount: '0.1', basis: 'application_billing' }) }, + ); + expect(result.coverage).toBe('partial'); + expect(result).not.toHaveProperty('amount'); + expect(result.calls.map((call) => [call.model, call.usageType, call.modelSource])).toEqual([ + ['gpt-4o', 'message', 'unknown'], + ['child-model', 'subagent', 'unknown'], + ['label-model', 'activity-label', 'unknown'], + ]); + }); +}); + +describe('application-billing quotes', () => { + const pricing: PricingFns = { + getMultiplier: ({ tokenType }) => (tokenType === 'prompt' ? 2 : 4), + getCacheMultiplier: ({ cacheType }) => { + if (cacheType === 'write1h') return 5; + return cacheType === 'write' ? 3 : 1; + }, + }; + + it('uses the existing billing computation and identifies its basis', () => { + const quote = createApplicationBillingQuote({ pricing, getValueKey: () => 'gpt-4o' }); + expect(quote(usage)).toEqual({ amount: '0.000280000000', basis: 'application_billing' }); + expect( + quote({ ...usage, input_token_details: { cache_read: 20, cache_creation: 10 } }), + ).toEqual({ amount: '0.000270000000', basis: 'application_billing' }); + }); + + it('does not mistake the unknown-model default multiplier for a real price', () => { + const quote = createApplicationBillingQuote({ pricing, getValueKey: () => undefined }); + expect(quote({ ...usage, model: 'unknown-model' })).toBeUndefined(); + expect(quote({ ...usage, model: undefined })).toBeUndefined(); + }); + + it('preserves the distinct one-hour cache-write rate from model-end usage', () => { + const quote = createApplicationBillingQuote({ pricing, getValueKey: () => 'gpt-4o' }); + expect( + quote({ + ...usage, + input_token_details: { cache_read: 20, cache_creation: 10, cache_creation_1h: 5 }, + }), + ).toEqual({ amount: '0.000280000000', basis: 'application_billing' }); + }); + + it('prices the configured billing alias while preserving the provider identity', () => { + const endpoint = { 'billed-alias': { prompt: 2, completion: 4, context: 1000 } }; + const collector = createResponseAccountingCollector({ + quote: createApplicationBillingQuote({ + pricing, + getValueKey: () => undefined, + resolveEndpointTokenConfig: (entry) => + entry.provider === 'custom-endpoint' ? endpoint : undefined, + }), + }); + collector.record({ + ...usage, + model: 'billed-alias', + provider: 'custom-endpoint', + accountingIdentity: { + model: 'provider-model', + provider: 'openAI', + modelSource: 'provider_reported', + }, + }); + expect(collector.snapshot()).toMatchObject({ + status: 'available', + basis: 'application_billing', + calls: [ + expect.objectContaining({ model: 'provider-model', provider: 'openAI', reason: 'priced' }), + ], + }); + expect(Number(collector.snapshot().amount)).toBe( + computeUsageCostUSD( + { ...usage, model: 'billed-alias', provider: 'custom-endpoint' }, + pricing, + endpoint, + ), + ); + }); + + it('supports explicit endpoint prices for nonstandard models', () => { + const quote = createApplicationBillingQuote({ + pricing, + getValueKey: () => undefined, + resolveEndpointTokenConfig: () => ({ custom: { prompt: 2, completion: 4, context: 1000 } }), + }); + expect(quote({ ...usage, model: 'custom' })?.basis).toBe('application_billing'); + }); +}); + +describe('accounting lineage and invocation receipts', () => { + const priced = () => { + const collector = createResponseAccountingCollector({ + quote: () => ({ amount: '0.25', basis: 'application_billing' }), + }); + collector.record(usage); + return collector.snapshot(); + }; + it('never upgrades missing historical coverage over arbitrarily many priced continuations', () => { + let result = combineResponseAccounting(undefined, priced()); + for (let segment = 0; segment < 5; segment++) { + result = combineResponseAccounting(result, priced()); + expect(result).toMatchObject({ + status: 'unavailable', + coverage: 'partial', + reason: 'partial_coverage', + }); + expect(result).not.toHaveProperty('amount'); + } + expect(result.calls).toHaveLength(6); + const resumed = createResponseAccountingCollector({ + seed: result, + quote: () => ({ amount: '0.1', basis: 'application_billing' }), + }); + resumed.record(usage); + expect(resumed.snapshot()).toMatchObject({ + status: 'unavailable', + coverage: 'partial', + reason: 'partial_coverage', + }); + }); + it('retains incomplete coverage even when a later segment has conflicting usage', () => { + const conflict = createResponseAccountingCollector({ + quote: () => ({ amount: '0.25', basis: 'application_billing' }), + }); + conflict.record(usage, undefined, undefined, 'call'); + conflict.record({ ...usage, input_tokens: 999 }, undefined, undefined, 'call'); + const combined = combineResponseAccounting( + combineResponseAccounting(undefined, priced()), + conflict.snapshot(), + ); + expect(combineResponseAccounting(combined, priced())).toMatchObject({ + status: 'unavailable', + coverage: 'partial', + reason: 'conflicting_usage', + }); + }); + it('preserves complete priced folds and original historical quotes', () => { + expect( + combineResponseAccounting(combineResponseAccounting(priced(), priced()), priced()), + ).toMatchObject({ status: 'available', coverage: 'complete', amount: '0.75' }); + const old = priced(); + const resumed = createResponseAccountingCollector({ + seed: old, + quote: () => ({ amount: '1', basis: 'application_billing' }), + }); + resumed.record(usage); + expect(resumed.snapshot()).toMatchObject({ status: 'available', amount: '1.25' }); + expect(old.amount).toBe('0.25'); + }); + it.each([ + [ + { __invoked_model: 'runtime' }, + { clientOptions: { model: 'configured' } }, + 'runtime', + 'runtime_reported', + ], + [{}, { clientOptions: { model: 'configured' } }, 'provider', 'provider_reported'], + ] as const)( + 'preserves captured invocation identity after billing flattens it: %s', + (metadata, context, model, modelSource) => { + const raw = { ...usage, model: 'provider' }; + const captured = captureResponseAccountingUsage(raw, metadata, context); + captured.model = 'billing-fallback'; + const result = buildCollectedResponseAccounting([captured], []); + expect(result.calls[0]).toMatchObject({ model, modelSource }); + expect(raw.model).toBe('provider'); + }, + ); + it('captures configured-only provenance instead of upgrading it to runtime-reported', () => { + const captured = captureResponseAccountingUsage( + { input_tokens: 1, output_tokens: 1 }, + undefined, + { clientOptions: { model: 'configured' } }, + ); + captured.model = 'configured'; + const repeated = captureResponseAccountingUsage(captured, { + __invoked_model: 'flattened-model', + }); + expect(buildCollectedResponseAccounting([repeated], []).calls[0]).toMatchObject({ + model: 'configured', + modelSource: 'configured', + }); + expect( + buildCollectedResponseAccounting([{ ...usage, model: 'legacy-flat' }], []).calls[0], + ).toMatchObject({ model: 'legacy-flat', modelSource: 'unknown' }); + }); +}); diff --git a/packages/api/src/agents/responses/accounting.ts b/packages/api/src/agents/responses/accounting.ts new file mode 100644 index 0000000000..5a0ff61416 --- /dev/null +++ b/packages/api/src/agents/responses/accounting.ts @@ -0,0 +1,408 @@ +import { GraphNodeKeys } from '@librechat/agents'; +import { tokenValues } from '@librechat/data-schemas'; +import type { TTokenUsageEvent } from 'librechat-data-provider'; +import type { + AccountingBasis, + AccountingModelSource, + ResponseAccounting, + ResponseAccountingCall, +} from './types'; +import type { UsageMetadata } from '~/stream/interfaces/IJobStore'; +import type { EndpointTokenConfig } from '~/types/tokens'; +import type { PricingFns } from '../transactions'; +import { aggregateCollectedUsage, computeUsageCostUSD } from '../usage'; + +export interface AccountingQuote { + /** Unsigned, non-exponential USD decimal. */ + amount: string; + basis: AccountingBasis; +} + +export interface AccountingCollectorOptions { + quote?: (usage: UsageMetadata) => AccountingQuote | undefined; + /** Server-persisted call provenance; retain original quotes rather than repricing past work. */ + seed?: ResponseAccounting; +} + +/** Private receipt captured before billing applies configured provider/model fallbacks. */ +export interface AccountingInvocationIdentity { + model?: string; + provider?: string; + modelSource: AccountingModelSource; +} + +export interface CollectedResponseAccountingUsage extends UsageMetadata { + accountingIdentity?: AccountingInvocationIdentity; +} + +export type AccountingUsage = Omit & { + usage_type?: ResponseAccountingCall['usageType']; +}; + +interface InvocationMetadata { + ls_model_name?: string; + __invoked_model?: string; + __invoked_provider?: string; + langgraph_node?: string; +} + +interface InvocationContext { + provider?: string; + agentId?: string; + clientOptions?: { model?: string }; +} + +function invocationIdentity( + usage: Pick | undefined, + metadata?: InvocationMetadata, + context?: InvocationContext | null, +): AccountingInvocationIdentity { + const model = + metadata?.ls_model_name || + metadata?.__invoked_model || + usage?.model || + context?.clientOptions?.model; + const provider = metadata?.__invoked_provider || usage?.provider || context?.provider; + let modelSource: AccountingModelSource = 'unknown'; + if (metadata?.ls_model_name || metadata?.__invoked_model) modelSource = 'runtime_reported'; + else if (usage?.model) modelSource = 'provider_reported'; + else if (context?.clientOptions?.model) modelSource = 'configured'; + return { ...(model ? { model } : {}), ...(provider ? { provider } : {}), modelSource }; +} + +export function captureResponseAccountingUsage( + usage: CollectedResponseAccountingUsage | undefined, + metadata?: InvocationMetadata, + context?: InvocationContext | null, +): CollectedResponseAccountingUsage { + return { + ...usage, + accountingIdentity: { + ...(usage?.accountingIdentity ?? invocationIdentity(usage, metadata, context)), + }, + }; +} + +/** Missing historical accounting cannot be repaired by appending later priced calls. */ +function missingCoverage(value: ResponseAccounting | undefined): boolean { + return ( + value == null || + value.coverage === 'partial' || + value.reason === 'partial_coverage' || + (value.calls.length === 0 && value.status === 'unavailable') + ); +} + +const decimal = /^(?:0|[1-9]\d*)(?:\.\d{1,18})?$/; + +/** Fixed-point addition avoids accumulating binary floating-point USD drift. */ +function sumAmounts(amounts: string[]): string { + const scale = Math.max(0, ...amounts.map((amount) => amount.split('.')[1]?.length ?? 0)); + const total = amounts.reduce((sum, amount) => { + const [whole, fraction = ''] = amount.split('.'); + return sum + BigInt(whole + fraction.padEnd(scale, '0')); + }, BigInt(0)); + if (scale === 0) { + return total.toString(); + } + const digits = total.toString().padStart(scale + 1, '0'); + return `${digits.slice(0, -scale)}.${digits.slice(-scale)}`.replace(/\.?0+$/, ''); +} + +function validUsage(usage: UsageMetadata): boolean { + const counts = [ + usage.input_tokens, + usage.output_tokens, + usage.total_tokens, + ...Object.values(usage.input_token_details ?? {}), + ...Object.values(usage.output_token_details ?? {}), + usage.cache_creation_input_tokens, + usage.cache_read_input_tokens, + usage.cache_write_tokens, + ]; + return counts.every((value) => value == null || (Number.isSafeInteger(value) && value >= 0)); +} + +/** + * Capture actual invocation provenance before the legacy billing contextualizer + * replaces provider metadata with configured fallbacks. No requested Agent ID + * or configured model is presented as a provider-reported model. + */ +export function createResponseAccountingCollector(options: AccountingCollectorOptions = {}) { + const calls: ResponseAccountingCall[] = options.seed?.calls.map((call) => ({ ...call })) ?? []; + const callIds = new Map(); + let conflictingUsage = options.seed?.reason === 'conflicting_usage'; + const inheritedMissingCoverage = options.seed != null && missingCoverage(options.seed); + + return { + record( + usage: AccountingUsage | undefined, + metadata?: InvocationMetadata, + context?: InvocationContext | null, + callId?: string, + ): void { + const { model, provider, modelSource } = + usage?.accountingIdentity ?? invocationIdentity(usage, metadata, context); + const normalized: UsageMetadata = { + ...usage, + agentId: context?.agentId || usage?.agentId, + usage_type: usage?.usage_type === 'subagent' ? 'subagent' : undefined, + }; + const totals = aggregateCollectedUsage(usage == null ? [] : [normalized]).total; + const hasUsage = usage?.input_tokens != null && usage?.output_tokens != null; + const usable = hasUsage && validUsage(normalized); + let quote: AccountingQuote | undefined; + const auxiliary = + usage?.usage_type === 'activity-label' || + usage?.usage_type === 'activity-phase' || + usage?.usage_type === 'reasoning-label'; + if (usable && !auxiliary) { + try { + quote = options.quote?.(normalized); + } catch { + // Optional accounting must not fail a successful generation. + } + } + if (quote && (!decimal.test(quote.amount) || quote.amount.length > 64)) { + quote = undefined; + } + let reason: ResponseAccountingCall['reason'] = 'pricing_unavailable'; + if (!hasUsage) { + reason = 'usage_unavailable'; + } else if (!usable) { + reason = 'invalid_usage'; + } else if (quote) { + reason = 'priced'; + } + const call: ResponseAccountingCall = { + ...(model ? { model } : {}), + ...(provider ? { provider } : {}), + modelSource, + usageType: metadata?.langgraph_node?.startsWith(GraphNodeKeys.SUMMARIZE) + ? 'summarization' + : (usage?.usage_type ?? 'message'), + inputTokens: usable ? totals.inputTokens : 0, + outputTokens: usable ? totals.outputTokens : 0, + cacheReadTokens: usable ? totals.cacheReadTokens : 0, + cacheWriteTokens: usable + ? (usage?.input_token_details?.cache_creation ?? + usage?.input_token_details?.cache_write_tokens ?? + usage?.cache_creation_input_tokens ?? + usage?.cache_write_tokens ?? + 0) + : 0, + ...(quote ? quote : {}), + reason, + }; + // Only a model response's stable ID is a dedupe key. An Agent/run ID + // would discard distinct sequential calls and under-report the bill. + if (callId) { + const serialized = JSON.stringify(call); + const previous = callIds.get(callId); + if (previous != null) { + conflictingUsage ||= previous !== serialized; + return; + } + callIds.set(callId, serialized); + } + calls.push(call); + }, + snapshot(): ResponseAccounting { + const priced = calls.filter((call) => call.amount != null); + const bases = new Set(priced.map((call) => call.basis)); + const complete = + calls.length > 0 && priced.length === calls.length && !inheritedMissingCoverage; + const available = complete && bases.size === 1 && !conflictingUsage; + let coverage: ResponseAccounting['coverage'] = 'none'; + if (complete) { + coverage = 'complete'; + } else if (priced.length > 0) { + coverage = 'partial'; + } + let reason: ResponseAccounting['reason'] = 'pricing_unavailable'; + if (conflictingUsage) { + reason = 'conflicting_usage'; + } else if (available) { + reason = 'priced'; + } else if (bases.size > 1) { + reason = 'mixed_bases'; + } else if (calls.length === 0 || calls.every((call) => call.reason === 'usage_unavailable')) { + reason = 'usage_unavailable'; + } else if (priced.length > 0) { + reason = 'partial_coverage'; + } + return { + status: available ? 'available' : 'unavailable', + currency: 'USD', + ...(available + ? { + amount: sumAmounts(priced.map((call) => call.amount ?? '0')), + basis: priced[0].basis, + } + : {}), + coverage, + reason, + calls: calls.map((call) => ({ ...call })), + }; + }, + }; +} + +/** Combine observed segment provenance without fabricating calls for missing historical accounting. */ +export function combineResponseAccounting( + previous: ResponseAccounting | undefined, + current: ResponseAccounting | undefined, +): ResponseAccounting { + const calls = [...(previous?.calls ?? []), ...(current?.calls ?? [])]; + const conflicting = + previous?.reason === 'conflicting_usage' || current?.reason === 'conflicting_usage'; + const snapshot = createResponseAccountingCollector({ + seed: { + status: 'unavailable', + currency: 'USD', + coverage: 'none', + reason: conflicting ? 'conflicting_usage' : 'usage_unavailable', + calls, + }, + }).snapshot(); + const missing = [previous, current].some(missingCoverage); + if (!missing) return snapshot; + const priced = calls.some((call) => call.amount != null); + let reason: ResponseAccounting['reason'] = priced ? 'partial_coverage' : 'usage_unavailable'; + if (conflicting) reason = 'conflicting_usage'; + return { + status: 'unavailable', + currency: 'USD', + coverage: priced ? 'partial' : 'none', + reason, + calls: snapshot.calls, + }; +} + +/** Persist complete call provenance before the ephemeral generation is removed. */ +export function buildCollectedResponseAccounting( + collectedUsage: ReadonlyArray, + emittedUsage: ReadonlyArray, + options: AccountingCollectorOptions = {}, +): ResponseAccounting { + const collector = createResponseAccountingCollector(options); + for (const entry of collectedUsage) { + collector.record({ + ...entry, + accountingIdentity: entry?.accountingIdentity ?? { + ...(entry?.model ? { model: entry.model } : {}), + ...(entry?.provider ? { provider: entry.provider } : {}), + modelSource: 'unknown', + }, + }); + } + for (const entry of emittedUsage) { + if ( + entry.usage_type !== 'activity-label' && + entry.usage_type !== 'activity-phase' && + entry.usage_type !== 'reasoning-label' + ) + continue; + // Label calls can use a distinct endpoint config that the collected parent + // batch does not retain. Preserve their usage, but never price them with + // the primary Agent's config or report an incomplete total as complete. + collector.record( + { + ...entry, + accountingIdentity: { + model: entry.model, + provider: entry.provider, + modelSource: 'unknown', + }, + }, + undefined, + undefined, + entry.runId && entry.seq != null ? `${entry.runId}:${entry.seq}` : undefined, + ); + } + return collector.snapshot(); +} + +/** Plain metadata patch shared by completion and abort saves after transient jobs are removed. */ +export function buildPersistedResponseAccounting(options: { + collectedUsage?: ReadonlyArray; + emittedUsage: ReadonlyArray; + billingEnabled?: boolean; + billing?: Parameters[0]; +}): { accounting?: ResponseAccounting } { + const hasPrimary = Boolean(options.collectedUsage?.length); + const hasAuxiliary = options.emittedUsage.some( + (entry) => + entry.usage_type === 'activity-label' || + entry.usage_type === 'activity-phase' || + entry.usage_type === 'reasoning-label', + ); + if (!hasPrimary && !hasAuxiliary) return {}; + const quote = + options.billingEnabled === true && typeof options.billing?.getValueKey === 'function' + ? createApplicationBillingQuote(options.billing) + : undefined; + return { + accounting: buildCollectedResponseAccounting( + options.collectedUsage ?? [], + options.emittedUsage, + { + ...(!hasPrimary + ? { + seed: { + status: 'unavailable' as const, + currency: 'USD' as const, + coverage: 'none' as const, + reason: 'usage_unavailable' as const, + calls: [], + }, + } + : {}), + quote, + }, + ), + }; +} + +/** + * Reuse application billing, but reject its unknown-model default-rate fallback. + * This is an application charge, not a claim about a provider invoice. + */ +export function createApplicationBillingQuote(options: { + pricing: PricingFns; + getValueKey: (model: string) => string | undefined; + resolveEndpointTokenConfig?: (usage: UsageMetadata) => EndpointTokenConfig | undefined; +}): (usage: UsageMetadata) => AccountingQuote | undefined { + return (usage) => { + if (!usage.model) { + return undefined; + } + const endpointTokenConfig = options.resolveEndpointTokenConfig?.(usage); + const configured = endpointTokenConfig?.[usage.model]; + const valueKey = options.getValueKey(usage.model); + const rates = configured ?? (valueKey ? tokenValues[valueKey] : undefined); + if ( + !rates || + !Number.isFinite(rates.prompt) || + rates.prompt < 0 || + !Number.isFinite(rates.completion) || + rates.completion < 0 + ) { + return undefined; + } + const amount = computeUsageCostUSD(usage, options.pricing, endpointTokenConfig); + if (!Number.isFinite(amount) || amount < 0 || amount >= 1e15) { + return undefined; + } + return { amount: amount.toFixed(12), basis: 'application_billing' }; + }; +} + +/** Read the final collected batch once; billing still consumes only new invocation usage. */ +export function createResponseAccountingSnapshot( + options: Parameters[0], +): () => ResponseAccounting { + return (): ResponseAccounting => + buildPersistedResponseAccounting(options).accounting ?? + createResponseAccountingCollector().snapshot(); +} diff --git a/packages/api/src/agents/responses/clientTools.ts b/packages/api/src/agents/responses/clientTools.ts index 6bfc4e5fa7..448ab60c77 100644 --- a/packages/api/src/agents/responses/clientTools.ts +++ b/packages/api/src/agents/responses/clientTools.ts @@ -27,9 +27,9 @@ * require. * * Replay is the only continuation this endpoint supports for a tool exchange. - * `previous_response_id` does not carry one: the server persists a turn as - * text (`saveResponseOutput` keeps `output_text`, `saveInputMessages` keeps - * `role: 'user'`), so neither item survives, and the id is rejected outright + * `previous_response_id` does not carry an unanswered client exchange: stored + * assistant turns retain completed server tool pairs, not unexecuted client calls, + * and `saveInputMessages` keeps only `role: 'user'`. The id is rejected outright * with 404 unless the first request also asked for `store: true`. Sending * `previous_response_id` with a bare `function_call_output` therefore produces * a tool result whose call is missing. diff --git a/packages/api/src/agents/responses/completion.ts b/packages/api/src/agents/responses/completion.ts new file mode 100644 index 0000000000..25177f066f --- /dev/null +++ b/packages/api/src/agents/responses/completion.ts @@ -0,0 +1,198 @@ +import { EModelEndpoint } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import type { Response as HttpResponse } from 'express'; +import type { TurnConversationRequest, ConversationWriteContext } from '~/conversations/save'; +import type { PersistedReply, ReplyAnnouncement } from '~/conversations/announce'; +import type { getLangfuseTraceMessageFields } from '~/langfuse/destinations'; +import type { ConversationCodeEnvironmentDecision } from '~/code/decision'; +import type { Response, Usage, ResponseAccounting } from './types'; +import type { ForegroundResponseTerminal } from './foreground'; +import type { InternalMessage } from './service'; +import { + saveForegroundResponseInputs, + projectStoredResponseMessage, + requireResponseStorageWrite, +} from './foreground'; +import { resolvePersistableCodeEnvironmentDecision } from '~/code/decision'; +import { getConversationWriteContext } from '~/conversations/save'; +import { isTemporaryTurn } from '~/conversations/retention'; +import { sendResponsesErrorResponse } from './service'; + +interface CompletionRequest + extends Omit { + config?: AppConfig; + resolvedConversation?: + | (ReturnType & { + conversationId?: string; + isTemporary?: boolean; + expiredAt?: Date; + }) + | null; +} +type InputMessage = Parameters< + Parameters>[1]['saveMessage'] +>[1]; +type OutputMessage = ReturnType & { + messageId: string; + conversationId: string; + isCreatedByUser: false; + sender: string; + endpoint: string; + model: string; + finish_reason: string; + tokenCount?: number; +}; +interface StoredReply extends PersistedReply { + conversationId: string; + messageId: string; +} +interface CompletionDependencies { + saveMessage: ( + context: ConversationWriteContext, + message: InputMessage | OutputMessage, + options: { context: string }, + ) => Promise; + saveConvo: ( + context: ConversationWriteContext, + fields: ReturnType & { + conversationId: string; + endpoint: string; + agent_id: string; + title?: string; + model?: string; + }, + options: { context: string; initialAgentId: string | null }, + ) => Promise; + announceReply: (announcement: ReplyAnnouncement) => Promise; + resolveTitle: (request: CompletionRequest, candidate: string) => string | null; + traceFields: typeof getLangfuseTraceMessageFields; +} + +/** Own storage ordering and transport selection in the same typed completion boundary. */ +export function stageForegroundResponse( + { + terminal, + request, + response, + conversationId, + agentId, + agent, + codeEnvironmentDecision, + messages, + parentMessageId, + visibleOutputTokens, + transport, + }: { + terminal: ForegroundResponseTerminal; + request: CompletionRequest; + response: Response; + conversationId: string; + agentId: string; + agent: { id?: string; name?: string; model?: string }; + codeEnvironmentDecision?: ConversationCodeEnvironmentDecision; + messages: (InternalMessage & { messageId: string; parentMessageId: string })[]; + parentMessageId: string; + visibleOutputTokens?: number; + transport: + | { type: 'json'; response: HttpResponse } + | { + type: 'stream'; + response: HttpResponse; + finalize: (usage?: Usage, accounting?: ResponseAccounting, response?: Response) => void; + fail: Parameters['stage']>[0]['fail']; + }; + }, + deps: CompletionDependencies, +): void { + terminal.stage({ + response, + save: async (result) => { + const context = getConversationWriteContext(request); + const title = deps.resolveTitle(request, agent.name || 'Open Responses Conversation'); + await requireResponseStorageWrite( + deps.saveConvo( + context, + { + conversationId, + endpoint: EModelEndpoint.agents, + agent_id: agentId, + ...resolvePersistableCodeEnvironmentDecision({ + conversationId, + decision: codeEnvironmentDecision, + conversation: + request.resolvedConversation?.conversationId != null + ? { + conversationId: request.resolvedConversation.conversationId, + codeEnvironmentMode: request.resolvedConversation.codeEnvironmentMode, + codeWorkspaces: request.resolvedConversation.codeWorkspaces, + } + : undefined, + }), + ...(title != null && { title }), + model: agent.model, + }, + { + context: 'Responses API - save conversation', + initialAgentId: agent.id === agentId ? agentId : null, + }, + ), + { conversationId }, + ); + await saveForegroundResponseInputs( + { context, conversationId, messages, agentId }, + { saveMessage: deps.saveMessage }, + ); + const traceFields = await deps.traceFields(request.config, result.id, { + isTemporary: isTemporaryTurn(request), + }); + return requireResponseStorageWrite( + deps.saveMessage( + context, + { + messageId: result.id, + conversationId, + ...projectStoredResponseMessage(result, parentMessageId), + isCreatedByUser: false, + ...traceFields, + sender: 'Agent', + endpoint: EModelEndpoint.agents, + model: agentId, + finish_reason: result.status === 'completed' ? 'stop' : result.status, + tokenCount: visibleOutputTokens ?? result.usage?.output_tokens, + }, + { context: 'Responses API - save assistant response' }, + ), + { conversationId, messageId: result.id }, + ).then((saved) => { + if (!saved) throw new Error('Response write returned no reply'); + return saved; + }); + }, + afterStore: async (reply) => { + await deps.announceReply({ + userId: request.user?.id, + conversationId, + reply, + context: 'Responses API - announce stored reply', + }); + }, + fail: (failure) => { + if (transport.type === 'stream') return transport.fail(failure); + sendResponsesErrorResponse( + transport.response, + failure.status, + failure.message, + 'server_error', + failure.code, + ); + }, + publish: (result) => { + if (transport.type === 'json') { + transport.response.json(result); + return; + } + transport.finalize(result.usage ?? undefined, result.accounting, result); + transport.response.end(); + }, + }); +} diff --git a/packages/api/src/agents/responses/foreground.spec.ts b/packages/api/src/agents/responses/foreground.spec.ts new file mode 100644 index 0000000000..d64b345b0a --- /dev/null +++ b/packages/api/src/agents/responses/foreground.spec.ts @@ -0,0 +1,550 @@ +import express from 'express'; +import request from 'supertest'; +import mongoose from 'mongoose'; +import { formatAgentMessages } from '@librechat/agents'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { ToolMessage } from '@librechat/agents/langchain/messages'; +import { createModels, createMethods, tenantStorage, runAsSystem } from '@librechat/data-schemas'; +import type { Response } from './types'; +import { + createForegroundResponseTerminal, + projectStoredResponseMessage, + saveForegroundResponseInputs, + requireResponseStorageWrite, + loadForegroundResponseHistory, +} from './foreground'; +import { + createResponseContext, + createResponseAggregator, + buildAggregatedResponse, +} from './service'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { buildResponseInputChain, resolveResponseContinuation } from './persistence'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import { createStoredForegroundResponseReader } from './stored'; +import { stopUserGenerations } from '~/user/generations'; +import { createAgentRunEnvelope } from '../envelope'; +import { executeAgentRun } from '../remote/host'; + +const owner = { userId: '507f1f77bcf86cd799439011', tenantId: 'foreground' }; +const conversationId = '43528be3-42bb-47b8-b0e8-7cc02ef32f45'; +const responseId = 'resp_foreground'; +const scoped = (work: () => Promise) => tenantStorage.run(owner, work); +let mongo: MongoMemoryReplSet; +let db: ReturnType; +const logger = { error: jest.fn() }; + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + db = createMethods(mongoose); +}); +beforeEach(async () => { + jest.clearAllMocks(); + await runAsSystem(async () => { + await Promise.all([ + mongoose.models.Message.deleteMany({}), + mongoose.models.Conversation.deleteMany({}), + mongoose.models.User.deleteMany({}), + ]); + }); + await scoped(() => + mongoose.models.User.create({ + _id: owner.userId, + tenantId: owner.tenantId, + email: 'foreground@example.com', + provider: 'local', + }), + ); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); + +function result(): Response { + const context = createResponseContext( + { model: 'agent', input: 'input', store: true }, + responseId, + ); + const aggregator = createResponseAggregator(); + aggregator.toolCalls.set('call', { id: 'call', name: 'lookup', arguments: '{"key":"a"}' }); + aggregator.toolOutputs.set('call', 'tool-result-only-in-canonical-content'); + return buildAggregatedResponse(context, aggregator, undefined, { + status: 'unavailable', + currency: 'USD', + coverage: 'none', + reason: 'pricing_unavailable', + calls: [ + { + model: 'provider-model', + modelSource: 'provider_reported', + usageType: 'message', + inputTokens: 2, + outputTokens: 1, + cacheReadTokens: 0, + cacheWriteTokens: 0, + reason: 'pricing_unavailable', + }, + ], + }); +} +async function save(response: Response) { + const context = { userId: owner.userId }; + await requireResponseStorageWrite( + db.saveConvo(context, { conversationId, endpoint: 'agents', agent_id: 'agent' }), + { conversationId }, + ); + const chain = buildResponseInputChain( + [{ role: 'user' as const, content: 'input' }], + '00000000-0000-0000-0000-000000000000', + ); + await saveForegroundResponseInputs( + { context, conversationId, messages: chain.messages, agentId: 'agent' }, + db, + ); + return requireResponseStorageWrite( + db.saveMessage(context, { + messageId: response.id, + conversationId, + ...projectStoredResponseMessage(response, chain.parentMessageId), + isCreatedByUser: false, + sender: 'Agent', + endpoint: 'agents', + model: response.model, + }), + { conversationId, messageId: response.id }, + ); +} +function appFor( + scope = owner, + overrides: Partial[0]> = {}, +) { + const app = express(); + app.use((req, _res, next) => { + Object.assign(req, { user: { id: scope.userId } }); + tenantStorage.run(scope, next); + }); + app.get('/:id', createStoredForegroundResponseReader({ ...db, logger, ...overrides })); + return app; +} + +it.each([conversationId, responseId])('preserves legacy tokenCount usage for %s', (id) => + scoped(async () => { + await db.saveConvo({ userId: owner.userId }, { conversationId, endpoint: 'agents' }); + await db.saveMessage( + { userId: owner.userId }, + { + messageId: responseId, + conversationId, + text: 'Legacy reply', + isCreatedByUser: false, + tokenCount: 17, + sender: 'Agent', + endpoint: 'agents', + model: 'agent', + }, + ); + const read = await request(appFor()) + .get('/' + id) + .expect(200); + expect(read.body.usage).toMatchObject({ input_tokens: 0, output_tokens: 17, total_tokens: 17 }); + expect(read.body.accounting).toMatchObject({ status: 'unavailable', calls: [] }); + }), +); + +it('stores canonical tool history and accounting, resolves the returned response ID and retrieves that exact turn', () => + scoped(async () => { + const response = result(); + const saved = await save(response); + expect(saved?.metadata).toMatchObject({ + responseReceipt: { accounting: response.accounting, usage: response.usage }, + }); + expect(JSON.stringify(saved?.metadata)).not.toContain('tool-result-only-in-canonical-content'); + const reference = await resolveResponseContinuation(responseId, owner.userId, db); + expect(reference).toEqual({ conversationId, parentMessageId: responseId }); + const history = await loadForegroundResponseHistory( + { conversationId, userId: owner.userId, previousResponseId: responseId }, + db, + ); + expect(history).toHaveLength(2); + expect(history[1].content).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + tool_call: expect.objectContaining({ output: 'tool-result-only-in-canonical-content' }), + }), + ]), + ); + const read = await request(appFor()) + .get('/' + responseId) + .expect(200); + expect(read.body).toMatchObject({ + id: responseId, + store: true, + accounting: response.accounting, + usage: response.usage, + }); + expect(read.body.output).toEqual(response.output); + })); + +it('retains submitted system/assistant tool replay provenance in the next selected history', () => + scoped(async () => { + const context = { userId: owner.userId }; + await requireResponseStorageWrite( + db.saveConvo(context, { conversationId, endpoint: 'agents' }), + { conversationId }, + ); + const chain = buildResponseInputChain( + [ + { role: 'system' as const, content: 'Caller context' }, + { + role: 'assistant' as const, + content: [ + { + type: 'tool_call', + tool_call: { + id: 'replay-call', + name: 'lookup', + args: '{}', + output: 'caller-tool-result', + }, + }, + ], + }, + { role: 'user' as const, content: 'Continue' }, + ], + '00000000-0000-0000-0000-000000000000', + ); + await saveForegroundResponseInputs( + { context, conversationId, messages: chain.messages, agentId: 'agent' }, + db, + ); + const response = result(); + await requireResponseStorageWrite( + db.saveMessage(context, { + messageId: responseId, + conversationId, + ...projectStoredResponseMessage(response, chain.parentMessageId), + isCreatedByUser: false, + sender: 'Agent', + }), + { conversationId, messageId: responseId }, + ); + const history = await loadForegroundResponseHistory( + { conversationId, userId: owner.userId, previousResponseId: responseId }, + db, + ); + expect(history.map((message) => message.role)).toEqual([ + 'system', + 'assistant', + 'user', + 'assistant', + ]); + expect(history.slice(0, 3).every((message) => message.isUserSubmitted === true)).toBe(true); + const sdk = formatAgentMessages( + history.map((message) => ({ ...message })), + undefined, + new Set(['lookup']), + ); + expect( + sdk.messages + .filter((message) => message instanceof ToolMessage) + .map((message) => message.content), + ).toContain('caller-tool-result'); + const read = await request(appFor()) + .get('/' + chain.messages[1].messageId) + .expect(404); + expect(JSON.stringify(read.body)).not.toContain('caller-tool-result'); + })); + +it('returns 404 for a foreign tenant or erased conversation without disclosing stored output', () => + scoped(async () => { + await save(result()); + const foreign = await request(appFor({ ...owner, tenantId: 'other' })) + .get('/' + responseId) + .expect(404); + expect(JSON.stringify(foreign.body)).not.toContain('tool-result'); + await mongoose.models.Conversation.deleteOne({ conversationId, user: owner.userId }); + await request(appFor()) + .get('/' + responseId) + .expect(404); + })); + +it('maps storage outages to a safe 503 and never returns an empty successful history', async () => { + const failure = new Error('Authorization: Bearer private-secret'); + await expect( + loadForegroundResponseHistory( + { conversationId, userId: owner.userId }, + { + getMessages: async () => { + throw failure; + }, + }, + ), + ).rejects.toMatchObject({ code: 'response_storage_unavailable' }); + const read = await request( + appFor(owner, { + getMessage: async () => { + throw failure; + }, + }), + ) + .get('/' + responseId) + .expect(503); + expect(JSON.stringify(read.body)).not.toContain('private-secret'); + expect(JSON.stringify(logger.error.mock.calls)).not.toContain('private-secret'); +}); + +it.each([null, undefined, { message: 'database unavailable' }, { conversationId: 'another' }])( + 'rejects a resolved unsuccessful write %p', + async (value) => { + await expect( + requireResponseStorageWrite(Promise.resolve(value), { conversationId }), + ).rejects.toMatchObject({ code: 'response_storage_unavailable' }); + }, +); + +async function settleTerminal( + terminal: Pick< + ReturnType, + 'beforeSettle' | 'afterSettle' | 'onSettlementError' + >, + outcome: 'complete' | 'error' | 'aborted' = 'complete', +): Promise { + const manager = new GenerationJobManagerClass(); + manager.configure({ + jobStore: new InMemoryJobStore(), + eventTransport: new InMemoryEventTransport(), + }); + manager.initialize(); + try { + await executeAgentRun( + { + envelope: createAgentRunEnvelope({ + protocol: 'responses', + requestId: 'terminal-unit', + receivedAt: Date.now(), + principal: { id: owner.userId }, + payload: { model: 'agent', input: '' }, + }), + runId: responseId, + conversationId, + captureTerminalOutcome: true, + isPrincipalActive: async () => true, + execute: async (execution) => { + if (outcome === 'error') throw new Error('provider failed'); + if (outcome === 'aborted') + await manager.abortJob(responseId, { + expectedCreatedAt: execution.createdAt, + awaitProviderDrain: false, + }); + }, + handleExecutionError: () => undefined, + beforeSettle: terminal.beforeSettle, + afterSettle: terminal.afterSettle, + onSettlementError: terminal.onSettlementError, + }, + { manager }, + ); + } finally { + await manager.destroy(); + } +} + +it.each([true, false])( + 'publishes storage truth after settlement and keeps notification failure separate: requested=%s', + async (requestedStore) => { + const response = result(); + const context = createResponseContext({ model: 'agent', input: '' }, response.id); + const terminal = createForegroundResponseTerminal({ context, requestedStore, logger }); + const publish = jest.fn(); + const saveResult = jest.fn(async () => ({ conversationId, messageId: response.id })); + terminal.stage({ + fail: jest.fn(), + response, + save: saveResult, + afterStore: async () => { + throw new Error('private-secret'); + }, + publish, + }); + expect(publish).not.toHaveBeenCalled(); + await settleTerminal(terminal); + expect(saveResult).toHaveBeenCalledTimes(requestedStore ? 1 : 0); + expect(publish).toHaveBeenCalledWith(expect.objectContaining({ store: requestedStore })); + expect(JSON.stringify(logger.error.mock.calls)).not.toContain('private-secret'); + }, +); + +it('retains provider output but reports store false on a failed optional save', async () => { + const response = result(); + const context = createResponseContext({ model: 'agent', input: '' }, response.id); + const terminal = createForegroundResponseTerminal({ context, requestedStore: true, logger }); + const publish = jest.fn(); + terminal.stage({ + fail: jest.fn(), + response, + save: async () => { + throw new Error('storage outage'); + }, + publish, + }); + await settleTerminal(terminal); + expect(publish).toHaveBeenCalledWith({ ...response, store: false }); +}); + +it.each(['error', 'cancelled'])( + 'does not save or publish a staged success after %s', + async (outcome) => { + const response = result(); + const terminal = createForegroundResponseTerminal({ + context: createResponseContext({ model: 'agent', input: '' }, response.id), + requestedStore: true, + logger, + }); + const saveResult = jest.fn(); + const publish = jest.fn(); + terminal.stage({ fail: jest.fn(), response, save: saveResult, publish }); + await settleTerminal(terminal, outcome === 'cancelled' ? 'aborted' : 'error'); + expect(saveResult).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + }, +); + +it('cannot restage a completed foreground result and repeat persistence', async () => { + const response = result(); + const terminal = createForegroundResponseTerminal({ + context: createResponseContext({ model: 'agent', input: '' }, response.id), + requestedStore: true, + logger, + }); + const saveResult = jest.fn(async () => ({ conversationId, messageId: response.id })); + const publish = jest.fn(); + const frame = { response, save: saveResult, publish, fail: jest.fn() }; + terminal.stage(frame); + await settleTerminal(terminal); + expect(() => terminal.stage(frame)).toThrow('already staged'); + expect(saveResult).toHaveBeenCalledTimes(1); + expect(publish).toHaveBeenCalledTimes(1); +}); + +it('fails closed if a caller skips the tracked-write phase', async () => { + const response = result(); + const terminal = createForegroundResponseTerminal({ + context: createResponseContext({ model: 'agent', input: '' }, response.id), + requestedStore: true, + logger, + }); + const saveResult = jest.fn(); + const publish = jest.fn(); + const fail = jest.fn(); + terminal.stage({ response, save: saveResult, publish, fail }); + await terminal.afterSettle({ terminalOutcome: 'complete' }, undefined); + expect(saveResult).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + expect(fail).toHaveBeenCalledWith( + expect.objectContaining({ code: 'response_execution_unavailable' }), + ); +}); + +it('keeps foreground storage inside the account-erasure cleanup barrier', () => + scoped(async () => { + const manager = new GenerationJobManagerClass(); + manager.configure({ + jobStore: new InMemoryJobStore(), + eventTransport: new InMemoryEventTransport(), + }); + manager.initialize(); + let enteredSave!: () => void; + let releaseSave!: () => void; + const saving = new Promise((resolve) => { + enteredSave = resolve; + }); + const writeGate = new Promise((resolve) => { + releaseSave = resolve; + }); + const response = result(); + const terminal = createForegroundResponseTerminal({ + context: createResponseContext({ model: 'agent', input: '' }, response.id), + requestedStore: true, + logger, + }); + const publish = jest.fn(); + const run = executeAgentRun( + { + envelope: createAgentRunEnvelope({ + protocol: 'responses', + requestId: 'erase-foreground', + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: { model: 'agent', input: 'input' }, + }), + runId: response.id, + conversationId, + captureTerminalOutcome: true, + isPrincipalActive: db.isSubagentOwnerAdmissible, + execute: async () => { + terminal.stage({ + response, + save: async (value) => { + enteredSave(); + await writeGate; + return save(value); + }, + publish, + fail: jest.fn(), + }); + }, + beforeSettle: terminal.beforeSettle, + afterSettle: terminal.afterSettle, + onSettlementError: terminal.onSettlementError, + }, + { manager }, + ); + void run.catch(() => undefined); + let cleanup: Promise | undefined; + try { + await saving; + expect(await manager.getCleanupBlockingJobIdsForUser(owner.userId, owner.tenantId)).toContain( + response.id, + ); + let enteredAbort!: () => void; + const stopping = new Promise((resolve) => { + enteredAbort = resolve; + }); + const abort = manager.abortJob.bind(manager); + jest.spyOn(manager, 'abortJob').mockImplementation(async (...args) => { + enteredAbort(); + return abort(...args); + }); + let erased = false; + cleanup = (async () => { + await db.beginAgentTriggerUserDeletion(owner.userId, new Date()); + await stopUserGenerations(owner.userId, owner.tenantId, manager); + await db.deleteConvos(owner.userId, {}, { allowEmpty: true }); + await db.deleteMessages({ user: owner.userId }); + await db.deleteUserById(owner.userId); + erased = true; + })(); + void cleanup.catch(() => undefined); + await stopping; + expect(erased).toBe(false); + releaseSave(); + await Promise.all([run, cleanup]); + expect(publish).not.toHaveBeenCalled(); + await runAsSystem(async () => { + expect(await mongoose.models.Message.countDocuments({ user: owner.userId })).toBe(0); + expect(await mongoose.models.Conversation.countDocuments({ user: owner.userId })).toBe(0); + expect(await mongoose.models.User.countDocuments({ _id: owner.userId })).toBe(0); + }); + } finally { + releaseSave(); + await Promise.allSettled([run, cleanup ?? Promise.resolve()]); + await manager.destroy(); + } + })); diff --git a/packages/api/src/agents/responses/foreground.ts b/packages/api/src/agents/responses/foreground.ts new file mode 100644 index 0000000000..26f3be9a6a --- /dev/null +++ b/packages/api/src/agents/responses/foreground.ts @@ -0,0 +1,349 @@ +import { ContentTypes, EModelEndpoint } from 'librechat-data-provider'; +import type { RequestHandler } from 'express'; +import type { AgentExecutionEnrollment } from '../remote/lifecycle'; +import type { StoredResponseMessage } from './persistence'; +import type { Response, ResponseContext } from './types'; +import type { InternalMessage } from './service'; +import { + ResponsePersistenceError, + selectResponseHistory, + projectResponseMessage, +} from './persistence'; +import { sendResponsesErrorResponse } from './service'; +import { getSafeErrorMetadata } from '~/utils/errors'; + +const inputSenders: Record = { + user: 'User', + assistant: 'Agent', + system: 'System', +}; + +interface Diagnostics { + error: (message: string, metadata: ReturnType) => void; +} + +/** Translate only the storage contract's safe error at the HTTP boundary. */ +export function withResponsePersistenceErrors(handler: RequestHandler): RequestHandler { + return async (req, res, next) => { + try { + await handler(req, res, next); + } catch (error) { + if (!(error instanceof ResponsePersistenceError)) throw error; + sendResponsesErrorResponse(res, error.status, error.message, 'server_error', error.code); + } + }; +} + +export interface ResponseHistoryMessage extends StoredResponseMessage { + content?: unknown; + metadata?: Record; + isUserSubmitted?: boolean; + userSubmittedPaths?: string[]; + userSubmittedMessageFieldPaths?: { path: string; field: string }[]; +} + +export type ForegroundHistoryMessage = InternalMessage & + Pick< + ResponseHistoryMessage, + | 'messageId' + | 'isCreatedByUser' + | 'text' + | 'isUserSubmitted' + | 'userSubmittedPaths' + | 'userSubmittedMessageFieldPaths' + >; + +function internalHistoryContent(message: ResponseHistoryMessage): InternalMessage['content'] { + if (!Array.isArray(message.content)) return message.text ?? ''; + const content = message.content; + if ( + !content.every( + (part): part is { type: string; text?: string; image_url?: unknown } => + part != null && + typeof part === 'object' && + 'type' in part && + typeof part.type === 'string' && + (!('text' in part) || typeof part.text === 'string'), + ) + ) + throw new ResponsePersistenceError(); + return content; +} + +function historyRole(message: ResponseHistoryMessage): InternalMessage['role'] { + const role = message.isUserSubmitted ? message.metadata?.responseInputRole : undefined; + if (role === 'system' || role === 'assistant') return role; + return message.isCreatedByUser ? 'user' : 'assistant'; +} + +/** A lookup outage is not an empty conversation; model history must fail closed. */ +export async function loadForegroundResponseHistory( + input: { conversationId: string; userId: string; previousResponseId?: string }, + deps: { + getMessages: (query: { + conversationId: string; + user: string; + }) => Promise; + }, +): Promise { + try { + const messages = selectResponseHistory( + (await deps.getMessages({ conversationId: input.conversationId, user: input.userId })) ?? [], + input.previousResponseId, + ); + if (input.previousResponseId?.startsWith('resp_') && messages.length === 0) + throw new ResponsePersistenceError(); + return messages.map((message) => ({ + role: historyRole(message), + content: internalHistoryContent(message), + messageId: message.messageId, + isCreatedByUser: message.isCreatedByUser === true, + ...(message.text !== undefined && { text: message.text }), + ...(typeof message.isUserSubmitted === 'boolean' && { + isUserSubmitted: message.isUserSubmitted, + }), + ...(message.userSubmittedPaths && { userSubmittedPaths: message.userSubmittedPaths }), + ...(message.userSubmittedMessageFieldPaths && { + userSubmittedMessageFieldPaths: message.userSubmittedMessageFieldPaths, + }), + })); + } catch { + throw new ResponsePersistenceError(); + } +} + +export interface StoredResponseOutputLayout { + id: string; + type: 'message' | 'function_call' | 'function_call_output'; + callId?: string; + textParts?: number; +} + +/** Keep accounting beside canonical message content, without a second copy of private output. */ +export function projectStoredResponseMessage( + response: Response, + parentMessageId: string, +): ReturnType & { + parentMessageId: string; + metadata: { + responseReceipt: Pick & { + previousResponseId: Response['previous_response_id']; + createdAt: number; + completedAt: number | null; + outputLayout: StoredResponseOutputLayout[]; + }; + }; +} { + return { + ...projectResponseMessage(response), + parentMessageId, + metadata: { + responseReceipt: { + outputLayout: response.output.flatMap((item): StoredResponseOutputLayout[] => { + if (item.type === 'reasoning') return []; + return [ + { + id: item.id, + type: item.type, + ...(item.type === 'message' + ? { textParts: item.content.filter((part) => part.type === 'output_text').length } + : { callId: item.call_id }), + }, + ]; + }), + ...(response.accounting && { accounting: response.accounting }), + usage: response.usage, + previousResponseId: response.previous_response_id, + createdAt: response.created_at, + completedAt: response.completed_at, + }, + }, + }; +} + +export async function saveForegroundResponseInputs( + input: { + context: WriteContext; + conversationId: string; + messages: (InternalMessage & { messageId: string; parentMessageId: string })[]; + agentId: string; + }, + deps: { + saveMessage: ( + context: WriteContext, + message: { + messageId: string; + conversationId: string; + parentMessageId: string; + isCreatedByUser: boolean; + isUserSubmitted: true; + metadata: { responseInputRole: InternalMessage['role'] }; + text: string; + content: Exclude; + sender: string; + endpoint: string; + model: string; + }, + options: { context: string }, + ) => Promise; + }, +): Promise { + for (const message of input.messages) { + await requireResponseStorageWrite( + deps.saveMessage( + input.context, + { + messageId: message.messageId, + conversationId: input.conversationId, + parentMessageId: message.parentMessageId, + isCreatedByUser: message.role === 'user', + isUserSubmitted: true, + metadata: { responseInputRole: message.role }, + text: + typeof message.content === 'string' ? message.content : JSON.stringify(message.content), + content: + typeof message.content === 'string' + ? [{ type: ContentTypes.TEXT, text: message.content }] + : message.content, + sender: inputSenders[message.role], + endpoint: EModelEndpoint.agents, + model: input.agentId, + }, + { context: 'Responses API - save response input' }, + ), + { messageId: message.messageId, conversationId: input.conversationId }, + ); + } +} + +export interface ForegroundResponseFailure { + status: 409 | 503; + code: 'response_execution_cancelled' | 'response_execution_unavailable'; + message: string; +} + +interface ForegroundTerminalFrame { + response: Response; + save: (response: Response) => Promise; + afterStore?: (saved: Saved) => Promise; + publish: (response: Response) => void; + fail: (failure: ForegroundResponseFailure) => void; +} + +export interface ForegroundResponseTerminal { + onSettlementError: (error: unknown) => void; + beforeSettle: ( + execution: Pick, + error: unknown, + ) => void; + stage: (frame: ForegroundTerminalFrame) => void; + afterSettle: ( + execution: Pick, + error: unknown, + ) => Promise; +} + +/** Legacy storage can resolve absence/error records; those are not successful persistence. */ +export async function requireResponseStorageWrite( + operation: Promise, + identity: { conversationId: string; messageId?: string }, +): Promise { + const value = await operation; + if ( + value == null || + typeof value !== 'object' || + !('conversationId' in value) || + value.conversationId !== identity.conversationId || + (identity.messageId != null && + (!('messageId' in value) || value.messageId !== identity.messageId)) + ) + throw new ResponsePersistenceError(); + return value; +} + +/** Persist while the provider owns cleanup; render only after the host confirms drain. */ +export function createForegroundResponseTerminal({ + context, + requestedStore, + logger, +}: { + context: ResponseContext; + requestedStore?: boolean; + logger: Diagnostics; +}): ForegroundResponseTerminal { + let staged: ForegroundTerminalFrame | undefined; + let phase: 'open' | 'staged' | 'prepared' | 'done' = 'open'; + const fail = (cancelled = false): void => { + const frame = staged; + staged = undefined; + phase = 'done'; + frame?.fail( + cancelled + ? { + status: 409, + code: 'response_execution_cancelled', + message: 'Response execution was cancelled.', + } + : { + status: 503, + code: 'response_execution_unavailable', + message: 'Response execution is unavailable. Please try again.', + }, + ); + }; + const persist = async (frame: ForegroundTerminalFrame): Promise => { + let saved: { value: Saved } | undefined; + try { + saved = { value: await frame.save(frame.response) }; + context.stored = true; + } catch (saveError) { + logger.error('[Responses API] Error saving response', getSafeErrorMetadata(saveError)); + } + if (!saved || !frame.afterStore) return; + try { + await frame.afterStore(saved.value); + } catch (notificationError) { + logger.error( + '[Responses API] Error announcing stored response', + getSafeErrorMetadata(notificationError), + ); + } + }; + return { + onSettlementError(error: unknown): void { + logger.error('[Responses API] Failed to settle execution', getSafeErrorMetadata(error)); + fail(); + }, + stage(frame: ForegroundTerminalFrame): void { + if (phase !== 'open') throw new Error('Response terminal result was already staged'); + staged = frame; + phase = 'staged'; + }, + beforeSettle( + execution: Pick, + error: unknown, + ): void { + if (!staged) return; + if (phase !== 'staged') throw new Error('Response persistence was already prepared'); + phase = 'prepared'; + context.stored = false; + if (requestedStore !== true || error != null || execution.signal.aborted) return; + // Account/conversation cleanup must still see this generation until all data writes finish. + execution.track(persist(staged)); + }, + async afterSettle( + execution: Pick, + error: unknown, + ): Promise { + if (!staged) return; + if (phase !== 'prepared' || error != null || execution.terminalOutcome !== 'complete') { + fail(execution.terminalOutcome === 'aborted'); + return; + } + const frame = staged; + staged = undefined; + phase = 'done'; + frame.publish({ ...frame.response, store: context.stored === true }); + }, + }; +} diff --git a/packages/api/src/agents/responses/handlers.ts b/packages/api/src/agents/responses/handlers.ts index fc0977ad48..5f486b0e0f 100644 --- a/packages/api/src/agents/responses/handlers.ts +++ b/packages/api/src/agents/responses/handlers.ts @@ -7,6 +7,7 @@ import type { Response as ServerResponse } from 'express'; import type { Response, + ResponseAccounting, ResponseContext, ResponseEvent, OutputItem, @@ -127,6 +128,7 @@ export function buildResponse( tracker: ResponseTracker, status: ResponseStatus = 'in_progress', usageOverride?: Usage, + accounting?: ResponseAccounting, ): Response { const isCompleted = status === 'completed'; @@ -134,6 +136,7 @@ export function buildResponse( // Required fields id: context.responseId, object: 'response', + ...(isCompleted && accounting ? { accounting } : {}), created_at: context.createdAt, completed_at: isCompleted ? Math.floor(Date.now() / 1000) : null, status, @@ -166,7 +169,7 @@ export function buildResponse( : null, max_output_tokens: null, max_tool_calls: null, - store: false, + store: context.stored === true, background: false, service_tier: 'default', metadata: {}, @@ -318,10 +321,16 @@ export function emitResponseInProgress(config: StreamHandlerConfig): void { /** * Emit response.completed event */ -export function emitResponseCompleted(config: StreamHandlerConfig, usage?: Usage): void { +export function emitResponseCompleted( + config: StreamHandlerConfig, + usage?: Usage, + accounting?: ResponseAccounting, + preparedResponse?: Response, +): void { const { res, context, tracker } = config; tracker.status = 'completed'; - const response = buildResponse(context, tracker, 'completed', usage); + const response = + preparedResponse ?? buildResponse(context, tracker, 'completed', usage, accounting); writeEvent(res, { type: 'response.completed', sequence_number: tracker.nextSequence(), diff --git a/packages/api/src/agents/responses/index.ts b/packages/api/src/agents/responses/index.ts index 88bbc7a2f6..e6ad4ebb91 100644 --- a/packages/api/src/agents/responses/index.ts +++ b/packages/api/src/agents/responses/index.ts @@ -5,6 +5,36 @@ * @see https://openresponses.org/specification */ +export { + ResponsePersistenceError, + selectResponseHistory, + selectStoredResponse, + buildResponseInputChain, + projectResponseMessage, + resolveResponseContinuation, +} from './persistence'; + +export { + createResponseAccountingCollector, + createResponseAccountingSnapshot, + captureResponseAccountingUsage, + createApplicationBillingQuote, + buildCollectedResponseAccounting, + buildPersistedResponseAccounting, +} from './accounting'; +export type { + AccountingQuote, + AccountingCollectorOptions, + AccountingInvocationIdentity, + CollectedResponseAccountingUsage, +} from './accounting'; +export type { + AccountingBasis, + AccountingModelSource, + ResponseAccountingCall, + ResponseAccounting, +} from './types'; + // Types export type { // Enums @@ -181,3 +211,8 @@ export { // Client-side tool execution export { createClientToolHandoff, type ClientToolHandoff } from './clientTools'; + +export * from './foreground'; +export * from './stored'; + +export * from './completion'; diff --git a/packages/api/src/agents/responses/persistence.spec.ts b/packages/api/src/agents/responses/persistence.spec.ts new file mode 100644 index 0000000000..d108263bdf --- /dev/null +++ b/packages/api/src/agents/responses/persistence.spec.ts @@ -0,0 +1,269 @@ +import { formatAgentMessages } from '@librechat/agents'; +import { Constants, ContentTypes } from 'librechat-data-provider'; +import { ToolMessage } from '@librechat/agents/langchain/messages'; +import type { StoredResponseMessage } from './persistence'; +import type { Response } from './types'; +import { + selectStoredResponse, + selectResponseHistory, + buildResponseInputChain, + projectResponseMessage, + resolveResponseContinuation, +} from './persistence'; + +describe('Responses assistant history projection', () => { + it('marks paused output unfinished and explicitly clears that flag on terminal publication', () => { + expect(projectResponseMessage({ output: [], status: 'in_progress' }).unfinished).toBe(true); + expect(projectResponseMessage({ output: [], status: 'completed' }).unfinished).toBe(false); + expect(projectResponseMessage({ output: [] })).not.toHaveProperty('unfinished'); + }); + const response: Pick = { + output: [ + { + type: 'function_call', + id: 'fc_server', + call_id: 'call-server', + name: 'lookup', + arguments: '{"key":"a"}', + status: 'completed', + }, + { + type: 'function_call_output', + id: 'fco_server', + call_id: 'call-server', + output: '{"result":"server-only-result"}', + status: 'completed', + }, + { + type: 'function_call', + id: 'fc_client', + call_id: 'call-client', + name: 'client_action', + arguments: '{}', + status: 'completed', + }, + { + type: 'message', + id: 'message', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'Done.', annotations: [], logprobs: [] }], + }, + ], + }; + it('retains exact server results, not unanswered client calls, in the normal browser message shape', () => { + const projected = projectResponseMessage(response); + expect(projected).toEqual({ + text: 'Done.', + content: [ + { + type: ContentTypes.TOOL_CALL, + tool_call: { + id: 'call-server', + name: 'lookup', + args: '{"key":"a"}', + output: '{"result":"server-only-result"}', + }, + }, + { type: ContentTypes.TEXT, text: 'Done.' }, + ], + }); + expect(JSON.stringify(projected)).not.toContain('client_action'); + }); + it('round-trips stored content into an actual SDK ToolMessage on continuation', () => { + const projected = projectResponseMessage(response); + const { messages } = formatAgentMessages( + [ + { role: 'user', content: 'Look up a key' }, + { role: 'assistant', content: projected.content }, + { role: 'user', content: 'What exactly did the lookup return?' }, + ], + undefined, + new Set(['lookup']), + ); + const toolMessages = messages.filter((message) => message instanceof ToolMessage); + expect(toolMessages).toHaveLength(1); + expect(toolMessages[0].content).toBe('{"result":"server-only-result"}'); + }); + it('preserves empty server results instead of making the call look unanswered', () => { + const projected = projectResponseMessage({ + output: [ + response.output[0], + { + type: 'function_call_output', + id: 'fco_empty', + call_id: 'call-server', + output: '', + status: 'completed', + }, + ], + }); + expect(projected.content).toEqual([ + expect.objectContaining({ tool_call: expect.objectContaining({ output: '' }) }), + ]); + }); +}); + +describe('Responses persistence references', () => { + const messages: StoredResponseMessage[] = [ + { messageId: 'u1', parentMessageId: String(Constants.NO_PARENT), isCreatedByUser: true }, + { messageId: 'resp_first', parentMessageId: 'u1', text: 'first answer' }, + { messageId: 'u2', parentMessageId: 'resp_first', isCreatedByUser: true }, + { messageId: 'resp_second', parentMessageId: 'u2', text: 'second answer' }, + { messageId: 'branch-user', parentMessageId: 'resp_first', isCreatedByUser: true }, + { messageId: 'resp_branch', parentMessageId: 'branch-user', text: 'branch answer' }, + ]; + + it('gives a new conversation a UI-compatible root parent', async () => { + const getMessage = jest.fn(); + const result = await resolveResponseContinuation(undefined, 'owner', { + getMessage, + }); + expect(result.conversationId).toMatch(/^[a-f0-9-]{36}$/); + expect(result.parentMessageId).toBe(Constants.NO_PARENT); + expect(getMessage).not.toHaveBeenCalled(); + }); + + it('resolves a returned response ID using an owner-scoped message lookup', async () => { + const getMessage = jest.fn().mockResolvedValue({ + messageId: 'resp_first', + conversationId: 'conversation', + isCreatedByUser: false, + }); + await expect( + resolveResponseContinuation('resp_first', 'owner', { getMessage }), + ).resolves.toEqual({ + conversationId: 'conversation', + parentMessageId: 'resp_first', + }); + expect(getMessage).toHaveBeenCalledWith({ user: 'owner', messageId: 'resp_first' }); + }); + + it('does not resolve a foreign response into its conversation', async () => { + const getMessage = jest.fn().mockResolvedValue(null); + await expect( + resolveResponseContinuation('resp_foreign', 'owner', { getMessage }), + ).resolves.toEqual({ + conversationId: 'resp_foreign', + parentMessageId: Constants.NO_PARENT, + }); + }); + + it('does not accept a user message as a response reference', async () => { + const getMessage = jest.fn().mockResolvedValue({ + messageId: 'resp_user', + conversationId: 'conversation', + isCreatedByUser: true, + }); + expect(await resolveResponseContinuation('resp_user', 'owner', { getMessage })).toEqual({ + conversationId: 'resp_user', + parentMessageId: Constants.NO_PARENT, + }); + }); + + it('propagates database failures rather than starting a new conversation', async () => { + const getMessage = jest.fn().mockRejectedValue(new Error('database unavailable')); + await expect( + resolveResponseContinuation('resp_first', 'owner', { getMessage }), + ).rejects.toThrow('Response storage is unavailable'); + }); + + it('links legacy conversation-ID continuation to its last assistant reply', async () => { + const getMessage = jest.fn().mockResolvedValue(null); + const continuation = await resolveResponseContinuation('conversation', 'owner', { getMessage }); + expect(continuation.conversationId).toBe('conversation'); + expect( + buildResponseInputChain([], continuation.parentMessageId, messages).parentMessageId, + ).toBe('resp_branch'); + }); + + it('preserves old null-parent history when a new reply continues an old response', () => { + const legacy = messages.slice(0, 2).map((message) => ({ ...message, parentMessageId: null })); + expect(selectResponseHistory([...legacy, ...messages.slice(2, 4)], 'resp_second')).toEqual([ + ...legacy, + ...messages.slice(2, 4), + ]); + }); + + it('replays only the selected response ancestry, without subsequent or sibling turns', () => { + expect( + selectResponseHistory(messages, 'resp_second').map((message) => message.messageId), + ).toEqual(['u1', 'resp_first', 'u2', 'resp_second']); + expect( + selectResponseHistory(messages, 'resp_first').map((message) => message.messageId), + ).toEqual(['u1', 'resp_first']); + expect( + selectResponseHistory(messages, 'resp_branch').map((message) => message.messageId), + ).toEqual(['u1', 'resp_first', 'branch-user', 'resp_branch']); + }); + + it('does not load history for an unknown response or user message', () => { + expect(selectResponseHistory(messages, 'resp_missing')).toEqual([]); + expect( + selectResponseHistory([{ messageId: 'resp_user', isCreatedByUser: true }], 'resp_user'), + ).toEqual([]); + }); + + it('does not treat caller-submitted assistant input as a generated response', () => { + const imported = { messageId: 'resp_imported', isCreatedByUser: false, isUserSubmitted: true }; + expect(selectResponseHistory([imported], imported.messageId)).toEqual([]); + expect(selectStoredResponse([imported], imported.messageId)).toEqual([]); + expect(selectStoredResponse([messages[1], imported], 'legacy-conversation')).toEqual([ + messages[1], + ]); + }); + + it('detects cyclic or incomplete message ancestry', () => { + expect(() => + selectResponseHistory( + [{ messageId: 'resp_cycle', parentMessageId: 'resp_cycle' }], + 'resp_cycle', + ), + ).toThrow('cycle'); + expect(() => + selectResponseHistory( + [{ messageId: 'resp_broken', parentMessageId: 'missing' }], + 'resp_broken', + ), + ).toThrow('incomplete'); + }); + + it('retains legacy conversation-ID history and bounds legacy null-parent response history', () => { + expect(selectResponseHistory(messages, 'legacy-conversation')).toBe(messages); + const legacy = messages.map((message) => ({ ...message, parentMessageId: null })); + expect(selectResponseHistory(legacy, 'resp_first')).toEqual(legacy.slice(0, 2)); + }); + + it('builds user and assistant parent links without mutating inputs', () => { + const input = [ + { role: 'user', messageId: 'u1' }, + { role: 'user', messageId: 'u2' }, + ]; + expect(buildResponseInputChain(input, 'resp_previous')).toEqual({ + messages: [ + { role: 'user', messageId: 'u1', parentMessageId: 'resp_previous' }, + { role: 'user', messageId: 'u2', parentMessageId: 'u1' }, + ], + parentMessageId: 'u2', + }); + expect(input[0]).not.toHaveProperty('parentMessageId'); + }); + + it('assigns missing IDs and preserves submitted roles in the parent chain', () => { + const chain = buildResponseInputChain( + [{ role: 'system' }, { role: 'user' }], + String(Constants.NO_PARENT), + ); + expect(chain.messages).toHaveLength(2); + expect(chain.messages.map((message) => message.role)).toEqual(['system', 'user']); + expect(chain.messages[0].parentMessageId).toBe(Constants.NO_PARENT); + expect(chain.messages[1].parentMessageId).toBe(chain.messages[0].messageId); + expect(chain.parentMessageId).toBe(chain.messages[1].messageId); + }); + + it('retrieves exactly the requested response rather than the entire conversation', () => { + expect(selectStoredResponse(messages, 'resp_first')).toEqual([messages[1]]); + expect(selectStoredResponse(messages, 'resp_missing')).toEqual([]); + expect(selectStoredResponse(messages, 'legacy-conversation')).toEqual([messages[5]]); + }); +}); diff --git a/packages/api/src/agents/responses/persistence.ts b/packages/api/src/agents/responses/persistence.ts new file mode 100644 index 0000000000..975b1ca659 --- /dev/null +++ b/packages/api/src/agents/responses/persistence.ts @@ -0,0 +1,173 @@ +import { v4 as uuidv4 } from 'uuid'; +import { Constants, ContentTypes } from 'librechat-data-provider'; +import type { Agents } from 'librechat-data-provider'; +import type { Response } from './types'; + +const rootParent = String(Constants.NO_PARENT); + +/** Retain completed server tool pairs for browser rendering and later model turns. */ +export function projectResponseMessage( + response: Pick & Partial>, +): { + text: string; + content: Agents.MessageContentComplex[]; + unfinished?: boolean; +} { + const results = new Map(); + for (const item of response.output) + if (item.type === 'function_call_output') results.set(item.call_id, item.output); + const content: Agents.MessageContentComplex[] = []; + let text = ''; + for (const item of response.output) { + if (item.type === 'function_call' && results.has(item.call_id)) { + content.push({ + type: ContentTypes.TOOL_CALL, + tool_call: { + id: item.call_id, + name: item.name, + args: item.arguments, + output: results.get(item.call_id)!, + }, + }); + } + if (item.type !== 'message' || item.role !== 'assistant') continue; + for (const part of item.content) { + if (part.type !== 'output_text') continue; + text += part.text; + content.push({ type: ContentTypes.TEXT, text: part.text }); + } + } + return { + text, + content, + ...(response.status == null + ? {} + : { unfinished: response.status === 'in_progress' || response.status === 'queued' }), + }; +} + +export type StoredResponseMessage = { + messageId: string; + conversationId?: string; + parentMessageId?: string | null; + isCreatedByUser?: boolean; + isUserSubmitted?: boolean; + text?: string; + tokenCount?: number; + createdAt?: Date | string; +}; + +type ResponseReferenceDeps = { + getMessage: (query: { user: string; messageId: string }) => Promise; +}; + +export class ResponsePersistenceError extends Error { + readonly status = 503; + readonly code = 'response_storage_unavailable'; + + constructor() { + super('Response storage is unavailable. Please try again.'); + this.name = 'ResponsePersistenceError'; + } +} + +/** Response IDs identify assistant messages, not the conversation containing them. */ +export async function resolveResponseContinuation( + previousResponseId: string | undefined, + userId: string, + deps: ResponseReferenceDeps, +): Promise<{ conversationId: string; parentMessageId: string }> { + try { + return await resolveStoredContinuation(previousResponseId, userId, deps); + } catch { + throw new ResponsePersistenceError(); + } +} + +async function resolveStoredContinuation( + previousResponseId: string | undefined, + userId: string, + deps: ResponseReferenceDeps, +): Promise<{ conversationId: string; parentMessageId: string }> { + if (!previousResponseId) { + return { conversationId: uuidv4(), parentMessageId: rootParent }; + } + const message = await deps.getMessage({ user: userId, messageId: previousResponseId }); + if (message && !message.isCreatedByUser && !message.isUserSubmitted && message.conversationId) { + return { conversationId: message.conversationId, parentMessageId: message.messageId }; + } + // Keep legacy conversation-ID callers working. The existing owner/project + // admission check rejects absent or foreign conversations before execution. + return { conversationId: previousResponseId, parentMessageId: rootParent }; +} + +/** Follow the selected reply's ancestry so continuation cannot replay a later sibling turn. */ +export function selectResponseHistory( + messages: T[], + previousResponseId?: string, +): T[] { + if (!previousResponseId || !previousResponseId.startsWith('resp_')) return messages; + const byId = new Map(messages.map((message) => [message.messageId, message])); + const anchor = byId.get(previousResponseId); + if (!anchor || anchor.isCreatedByUser || anchor.isUserSubmitted) return []; + // Old Responses writes had null parents; preserve their original linear + // history, but never include a message written after the selected response. + if (anchor.parentMessageId == null) { + return messages.slice(0, messages.indexOf(anchor) + 1); + } + const chain: T[] = []; + const visited = new Set(); + let current: T | undefined = anchor; + while (current) { + if (visited.has(current.messageId)) throw new Error('Stored response history contains a cycle'); + visited.add(current.messageId); + chain.push(current); + const parent = current.parentMessageId; + if (parent == null) { + return [...messages.slice(0, messages.indexOf(current)), ...chain.reverse()]; + } + if (!parent || parent === Constants.NO_PARENT) break; + current = byId.get(parent); + if (!current) throw new Error('Stored response history is incomplete'); + } + return chain.reverse(); +} + +export function buildResponseInputChain( + messages: T[], + previousMessageId: string, + history: { messageId?: string }[] = [], +): { messages: (T & { messageId: string; parentMessageId: string })[]; parentMessageId: string } { + const inputs: (T & { messageId: string; parentMessageId: string })[] = []; + let parentMessageId = + previousMessageId === rootParent + ? history[history.length - 1]?.messageId || previousMessageId + : previousMessageId; + for (const message of messages) { + const input = { + ...message, + messageId: message.messageId || uuidv4(), + parentMessageId, + }; + inputs.push(input); + parentMessageId = input.messageId; + } + return { messages: inputs, parentMessageId }; +} + +/** GET returns the selected turn, not every assistant reply in the conversation. */ +export function selectStoredResponse( + messages: T[], + responseId: string, + responseReference: boolean = responseId.startsWith('resp_'), +): T[] { + if (responseReference) { + return messages.filter( + (message) => + message.messageId === responseId && !message.isCreatedByUser && !message.isUserSubmitted, + ); + } + return messages + .filter((message) => !message.isCreatedByUser && !message.isUserSubmitted) + .slice(-1); +} diff --git a/packages/api/src/agents/responses/service.ts b/packages/api/src/agents/responses/service.ts index 269bf24138..1e6adfdf54 100644 --- a/packages/api/src/agents/responses/service.ts +++ b/packages/api/src/agents/responses/service.ts @@ -9,6 +9,7 @@ import { isCodeEnvironmentMode, isCodeWorkspaceSelections, } from 'librechat-data-provider'; +import type { ProcessedToolCall } from '@librechat/agents'; import type { Response as ServerResponse } from 'express'; import type { FunctionCallOutputItemParam, @@ -20,13 +21,16 @@ import type { ModelContent, InputItem, Response, + ResponseAccounting, Usage, } from './types'; import type { UsageMetadata } from '~/stream/interfaces/IJobStore'; import type { TextContentFragment } from '~/protection/types'; import { writeDone, + buildResponse, emitResponseCompleted, + emitError, emitMessageItemAdded, emitMessageItemDone, emitTextContentPartAdded, @@ -67,6 +71,30 @@ interface ModelUsageMetadata { cache_read_input_tokens?: number; } +/** The SDK owns result filtering/serialization; never serialize raw tool artifacts here. */ +function completedRunStepTool( + data: unknown, +): (Pick & { args?: string }) | undefined { + if (data == null || typeof data !== 'object' || !('result' in data)) return undefined; + const result = data.result; + if (result == null || typeof result !== 'object' || !('tool_call' in result)) return undefined; + const call = result.tool_call; + if ( + call == null || + typeof call !== 'object' || + !('id' in call) || + !('output' in call) || + typeof call.id !== 'string' || + typeof call.output !== 'string' + ) + return undefined; + return { + id: call.id, + output: call.output, + ...('args' in call && typeof call.args === 'string' ? { args: call.args } : {}), + }; +} + function accumulateResponseUsage( target: ResponseUsageAccumulator, usage: ModelUsageMetadata, @@ -124,6 +152,12 @@ export function validateResponseRequest(body: unknown): RequestValidationResult if (request.stream !== undefined && typeof request.stream !== 'boolean') { return { valid: false, error: 'stream must be a boolean' }; } + if (request.background !== undefined && typeof request.background !== 'boolean') { + return { valid: false, error: 'background must be a boolean' }; + } + if (request.background === true && request.store === false) { + return { valid: false, error: 'background responses require store: true' }; + } if (request.temperature !== undefined) { const temp = request.temperature as number; @@ -710,7 +744,9 @@ function createToolCallChunkResolver(): ToolCallChunkResolver { export function createResponsesEventHandlers(config: StreamHandlerConfig): { handlers: Record void }>; state: StreamState; - finalizeStream: (usage?: Usage) => void; + finalizeStream: (usage?: Usage, accounting?: ResponseAccounting, response?: Response) => void; + prepareStreamResponse: (usage?: Usage, accounting?: ResponseAccounting) => Response; + failStream: (failure: { message: string; code: string }) => void; emitClientToolDeferral: (callId: string, output: string) => void; } { const state: StreamState = { @@ -724,6 +760,21 @@ export function createResponsesEventHandlers(config: StreamHandlerConfig): { }; const chunkResolver = createToolCallChunkResolver(); + const completeToolCall = (callId: string, output: string, args?: string): void => { + if (!state.activeToolCalls.has(callId) || state.completedToolCalls.has(callId)) return; + const streamed = config.tracker.accumulatedArguments.get(callId) ?? ''; + if (args != null && args.startsWith(streamed) && args.length > streamed.length) + emitFunctionCallArgumentsDelta(config, callId, args.slice(streamed.length)); + else if (args != null && args !== streamed) { + config.tracker.accumulatedArguments.set(callId, args); + const item = config.tracker.functionCalls.get(callId); + if (item) item.arguments = args; + } + state.completedToolCalls.add(callId); + emitFunctionCallArgumentsDone(config, callId); + emitFunctionCallItemDone(config, callId); + emitFunctionCallOutputItem(config, callId, output); + }; let modelEndedClientCalls: Set | undefined; let pendingClientToolDeferrals: Map | undefined; @@ -812,12 +863,8 @@ export function createResponsesEventHandlers(config: StreamHandlerConfig): { * waits for the terminating event before running the tool, as the streaming * lifecycle tells it to, would wait forever. * - * Deliberately limited to those calls. A server tool left open is the - * separate, pre-existing symptom of `on_tool_end` never reaching this module - * (the controller replaces the handler rather than composing with it); fixing - * that belongs at the wiring, not here, and closing such calls from - * finalization would change the event stream of every request that declares - * no client tool. + * Server-owned calls close only on SDK-reported completion. Finalization must + * not fabricate a result for a tool that never finished. * * Idempotent in both directions: a call already closed by `on_tool_end` is * skipped, and marking it closed keeps `on_tool_end` from emitting a second @@ -966,7 +1013,7 @@ export function createResponsesEventHandlers(config: StreamHandlerConfig): { // Resolved before the empty-args check so an id-bearing opening chunk is recorded. const callId = chunkResolver.resolve(deltaData?.id ?? '', tc); const args = typeof tc.args === 'string' ? tc.args : ''; - if (!args || !callId) { + if (!args || !callId || state.completedToolCalls.has(callId)) { continue; } @@ -985,16 +1032,14 @@ export function createResponsesEventHandlers(config: StreamHandlerConfig): { const callId = toolData?.tool_call_id; const output = toolData?.output ?? ''; - if (callId && state.activeToolCalls.has(callId) && !state.completedToolCalls.has(callId)) { - state.completedToolCalls.add(callId); + if (callId) completeToolCall(callId, output); + }, + }, - // Complete the function call item - emitFunctionCallArgumentsDone(config, callId); - emitFunctionCallItemDone(config, callId); - - // Emit the function call output (internal tool result) - emitFunctionCallOutputItem(config, callId, output); - } + on_run_step_completed: { + handle: (_event: string, data: unknown): void => { + const call = completedRunStepTool(data); + if (call) completeToolCall(call.id, call.output, call.args); }, }, @@ -1015,7 +1060,7 @@ export function createResponsesEventHandlers(config: StreamHandlerConfig): { } for (const { id, args } of completedToolCallArguments(data)) { - if (!state.activeToolCalls.has(id)) { + if (!state.activeToolCalls.has(id) || state.completedToolCalls.has(id)) { continue; } const streamed = config.tracker.accumulatedArguments.get(id) ?? ''; @@ -1041,7 +1086,7 @@ export function createResponsesEventHandlers(config: StreamHandlerConfig): { /** * Finalize the stream - close open items and emit completed */ - const finalizeStream = (usage?: Usage): void => { + const prepareStreamResponse = (usage?: Usage, accounting?: ResponseAccounting): Response => { closeOpenStreams(); for (const [callId, output] of pendingClientToolDeferrals ?? []) { deliverClientToolDeferral(callId, output); @@ -1049,11 +1094,31 @@ export function createResponsesEventHandlers(config: StreamHandlerConfig): { // A later step can announce a sibling before its arguments or deferral // are settled. Only terminate handoffs after the run has completed. closeOpenClientToolCalls(); - emitResponseCompleted(config, usage); + return buildResponse(config.context, config.tracker, 'completed', usage, accounting); + }; + const finalizeStream = ( + usage?: Usage, + accounting?: ResponseAccounting, + response?: Response, + ): void => { + const prepared = response ?? prepareStreamResponse(usage, accounting); + emitResponseCompleted(config, usage, accounting, prepared); writeDone(config.res); }; - return { handlers, state, finalizeStream, emitClientToolDeferral }; + const failStream = (failure: { message: string; code: string }): void => { + emitError(config, { type: 'server_error', message: failure.message, code: failure.code }); + writeDone(config.res); + config.res.end(); + }; + return { + handlers, + state, + finalizeStream, + prepareStreamResponse, + failStream, + emitClientToolDeferral, + }; } /* ============================================================================= @@ -1122,6 +1187,7 @@ export function buildAggregatedResponse( context: ResponseContext, aggregator: ResponseAggregator, usageOverride?: Usage, + accounting?: ResponseAccounting, ): Response { const output: Response['output'] = []; @@ -1149,12 +1215,12 @@ export function buildAggregatedResponse( }); const toolOutput = aggregator.toolOutputs.get(callId); - if (toolOutput) { + if (aggregator.toolOutputs.has(callId)) { output.push({ type: 'function_call_output', id: `fco_${Date.now().toString(36)}${Math.random().toString(36).substring(2, 6)}`, call_id: callId, - output: toolOutput, + output: toolOutput ?? '', status: 'completed', }); } @@ -1176,6 +1242,7 @@ export function buildAggregatedResponse( // Required fields per Open Responses spec id: context.responseId, object: 'response', + ...(accounting ? { accounting } : {}), created_at: context.createdAt, completed_at: Math.floor(Date.now() / 1000), status: 'completed', @@ -1207,7 +1274,7 @@ export function buildAggregatedResponse( }, max_output_tokens: null, max_tool_calls: null, - store: false, + store: context.stored === true, background: false, service_tier: 'default', metadata: {}, @@ -1333,7 +1400,7 @@ export function createAggregatorEventHandlers(aggregator: ResponseAggregator): R // Resolved before the empty-args check so an id-bearing opening chunk is recorded. const callId = chunkResolver.resolve(deltaData?.id ?? '', tc); const args = typeof tc.args === 'string' ? tc.args : ''; - if (!args || !callId) { + if (!args || !callId || aggregator.toolOutputs.has(callId)) { continue; } @@ -1352,12 +1419,22 @@ export function createAggregatorEventHandlers(aggregator: ResponseAggregator): R const callId = toolData?.tool_call_id; const output = toolData?.output ?? ''; - if (callId) { + if (callId && !aggregator.toolOutputs.has(callId)) { aggregator.toolOutputs.set(callId, output); } }, }, + on_run_step_completed: { + handle: (_event: string, data: unknown): void => { + const call = completedRunStepTool(data); + if (call && aggregator.toolCalls.has(call.id) && !aggregator.toolOutputs.has(call.id)) { + if (call.args != null) aggregator.toolCalls.get(call.id)!.arguments = call.args; + aggregator.toolOutputs.set(call.id, call.output); + } + }, + }, + on_chat_model_end: { handle: (_event: string, data: unknown): void => { const endData = data as { diff --git a/packages/api/src/agents/responses/stored.ts b/packages/api/src/agents/responses/stored.ts new file mode 100644 index 0000000000..42dc72cf6c --- /dev/null +++ b/packages/api/src/agents/responses/stored.ts @@ -0,0 +1,287 @@ +import { z } from 'zod'; +import { ContentTypes } from 'librechat-data-provider'; +import type { RequestHandler } from 'express'; +import type { Response, ResponseAccounting, OutputItem } from './types'; +import type { StoredResponseOutputLayout } from './foreground'; +import type { StoredResponseMessage } from './persistence'; +import { + createResponseContext, + buildAggregatedResponse, + createResponseAggregator, + sendResponsesErrorResponse, +} from './service'; +import { createResponseAccountingCollector } from './accounting'; +import { getSafeErrorMetadata } from '~/utils/errors'; +import { selectStoredResponse } from './persistence'; + +const count = z.number().int().nonnegative().safe(); +const totals = z.object({ input_tokens: count, output_tokens: count, total_tokens: count }); +const accounting: z.ZodType = z.object({ + status: z.enum(['available', 'unavailable']), + currency: z.literal('USD'), + amount: z.string().optional(), + basis: z.enum(['provider_reported', 'application_billing']).optional(), + coverage: z.enum(['complete', 'partial', 'none']), + reason: z.enum([ + 'priced', + 'pricing_unavailable', + 'usage_unavailable', + 'partial_coverage', + 'mixed_bases', + 'conflicting_usage', + ]), + calls: z.array( + z.object({ + provider: z.string().optional(), + model: z.string().optional(), + modelSource: z.enum(['provider_reported', 'runtime_reported', 'configured', 'unknown']), + usageType: z.enum([ + 'message', + 'summarization', + 'subagent', + 'sequential', + 'activity-label', + 'activity-phase', + 'reasoning-label', + ]), + inputTokens: count, + outputTokens: count, + cacheReadTokens: count, + cacheWriteTokens: count, + amount: z.string().optional(), + basis: z.enum(['provider_reported', 'application_billing']).optional(), + reason: z.enum(['priced', 'pricing_unavailable', 'usage_unavailable', 'invalid_usage']), + }), + ), +}); +const receiptSchema = z.object({ + accounting: accounting.optional(), + usage: totals + .extend({ + input_tokens_details: z.object({ cached_tokens: count }), + output_tokens_details: z.object({ reasoning_tokens: count }), + primary: totals.optional(), + subagent: totals.optional(), + }) + .nullable(), + outputLayout: z + .array( + z.object({ + id: z.string(), + type: z.enum(['message', 'function_call', 'function_call_output']), + callId: z.string().optional(), + textParts: count.optional(), + }), + ) + .optional(), + previousResponseId: z.string().nullable(), + createdAt: z.number(), + completedAt: z.number().nullable(), +}); +const textPart = z.object({ type: z.literal(ContentTypes.TEXT), text: z.string() }); +const toolPart = z.object({ + type: z.literal(ContentTypes.TOOL_CALL), + tool_call: z.object({ + id: z.string().min(1), + name: z.string(), + args: z.unknown().optional(), + output: z.unknown().optional(), + }), +}); + +interface SavedMessage extends StoredResponseMessage { + content?: unknown[]; + metadata?: Record; + model?: string; + tokenCount?: number; +} +interface SavedConversation { + createdAt?: Date | string; + updatedAt?: Date | string; + agent_id?: string; + model?: string; +} +export interface StoredForegroundResponseDependencies { + getMessage: (query: { user: string; messageId: string }) => Promise; + getConvo: (userId: string, conversationId: string) => Promise; + getMessages: (query: { user: string; conversationId: string }) => Promise; + logger: { error: (message: string, metadata: ReturnType) => void }; +} + +/** Project canonical message content rather than retaining an unredacted duplicate response body. */ +function output(message: SavedMessage): OutputItem[] { + const items: OutputItem[] = []; + for (const [index, part] of (message.content ?? []).entries()) { + const text = textPart.safeParse(part); + if (text.success) { + items.push({ + type: 'message', + id: `msg_${message.messageId}_${index}`, + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: text.data.text, annotations: [], logprobs: [] }], + }); + continue; + } + const parsed = toolPart.safeParse(part); + if (!parsed.success || parsed.data.tool_call.output == null) continue; + const call = parsed.data.tool_call; + items.push({ + type: 'function_call', + id: `fc_${message.messageId}_${index}`, + call_id: call.id, + name: call.name, + arguments: typeof call.args === 'string' ? call.args : JSON.stringify(call.args ?? {}), + status: 'completed', + }); + items.push({ + type: 'function_call_output', + id: `fco_${message.messageId}_${index}`, + call_id: call.id, + output: typeof call.output === 'string' ? call.output : JSON.stringify(call.output), + status: 'completed', + }); + } + if (!items.length && message.text) + items.push({ + type: 'message', + id: `msg_${message.messageId}`, + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: message.text, annotations: [], logprobs: [] }], + }); + return items; +} + +/** Identity/layout receipts contain no output text; content still comes from the privacy projection. */ +function restoreOutputIdentity( + items: OutputItem[], + layout?: StoredResponseOutputLayout[], +): OutputItem[] { + if (!layout) return items; + const tools = new Map( + items.flatMap((item) => + item.type === 'function_call' || item.type === 'function_call_output' + ? [[`${item.type}:${item.call_id}`, item] as const] + : [], + ), + ); + const texts = items.filter((item) => item.type === 'message'); + const consumed = new Set(); + const output: OutputItem[] = []; + let textIndex = 0; + for (const entry of layout) { + if (entry.type === 'message') { + const count = entry.textParts ?? 1; + const messages = texts.slice(textIndex, textIndex + count); + textIndex += count; + for (const message of messages) consumed.add(message.id); + if (messages.length > 0 || count === 0) + output.push({ + type: 'message', + id: entry.id, + role: 'assistant', + status: 'completed', + content: messages.flatMap((message) => message.content), + }); + continue; + } + const item = tools.get(`${entry.type}:${entry.callId}`); + if (item) { + output.push({ ...item, id: entry.id }); + consumed.add(item.id); + tools.delete(`${entry.type}:${entry.callId}`); + } + } + for (const item of items) if (!consumed.has(item.id)) output.push(item); + return output; +} + +/** Both response-ID and legacy conversation-ID reads remain owner/tenant scoped. */ +export function createStoredForegroundResponseReader( + deps: StoredForegroundResponseDependencies, +): RequestHandler { + return async (req, res) => { + const responseId = req.params.id; + const userId = + req.user && 'id' in req.user && typeof req.user.id === 'string' ? req.user.id : undefined; + const missing = () => + sendResponsesErrorResponse( + res, + 404, + 'Response not found.', + 'not_found', + 'response_not_found', + ); + if (!responseId || !userId) return missing(); + try { + const selected = responseId.startsWith('resp_') + ? await deps.getMessage({ user: userId, messageId: responseId }) + : undefined; + if ( + selected && + (selected.isCreatedByUser || selected.isUserSubmitted || !selected.conversationId) + ) + return missing(); + const conversationId = selected?.conversationId ?? responseId; + const [conversation, messages] = await Promise.all([ + deps.getConvo(userId, conversationId), + selected ? Promise.resolve([selected]) : deps.getMessages({ user: userId, conversationId }), + ]); + const message = selectStoredResponse(messages ?? [], responseId, selected != null)[0]; + if (!conversation || !message) return missing(); + const receipt = receiptSchema.safeParse(message.metadata?.responseReceipt); + const context = createResponseContext( + { + model: message.model ?? conversation.agent_id ?? conversation.model ?? 'unknown', + input: '', + }, + message.messageId, + ); + context.stored = true; + const response: Response = buildAggregatedResponse(context, createResponseAggregator()); + response.output = restoreOutputIdentity( + output(message), + receipt.success ? receipt.data.outputLayout : undefined, + ); + const legacyTokens = count.safeParse(message.tokenCount); + response.usage = null; + if (receipt.success) response.usage = receipt.data.usage; + else if (legacyTokens.success) { + response.usage = { + input_tokens: 0, + output_tokens: legacyTokens.data, + total_tokens: legacyTokens.data, + input_tokens_details: { cached_tokens: 0 }, + output_tokens_details: { reasoning_tokens: 0 }, + }; + } + response.accounting = + receipt.success && receipt.data.accounting + ? receipt.data.accounting + : createResponseAccountingCollector().snapshot(); + response.previous_response_id = receipt.success ? receipt.data.previousResponseId : null; + response.created_at = receipt.success + ? receipt.data.createdAt + : Math.floor( + new Date(message.createdAt ?? conversation.createdAt ?? Date.now()).getTime() / 1000, + ); + response.completed_at = receipt.success + ? receipt.data.completedAt + : Math.floor(new Date(conversation.updatedAt ?? Date.now()).getTime() / 1000); + res.json(response); + } catch (error) { + deps.logger.error( + '[Responses API] Error getting stored response', + getSafeErrorMetadata(error), + ); + sendResponsesErrorResponse( + res, + 503, + 'Response storage is unavailable. Please try again.', + 'server_error', + 'response_storage_unavailable', + ); + } + }; +}