diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index a7ad089d20..13abe8443b 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -356,6 +356,32 @@ class BaseClient { update.summary = tokenCountMap.summaryMessage.content; update.summaryTokenCount = tokenCountMap.summaryMessage.tokenCount; + + const summaryContentBlock = { + type: ContentTypes.SUMMARY, + content: [{ type: ContentTypes.TEXT, text: tokenCountMap.summaryMessage.content }], + tokenCount: tokenCountMap.summaryMessage.tokenCount, + createdAt: new Date().toISOString(), + }; + let existingContent = []; + if (Array.isArray(message.content)) { + existingContent = [...message.content]; + } else if (message.content) { + existingContent = [{ type: ContentTypes.TEXT, text: message.content }]; + } + let existingSummaryIndex = -1; + for (let index = existingContent.length - 1; index >= 0; index--) { + if (existingContent[index]?.type === ContentTypes.SUMMARY) { + existingSummaryIndex = index; + break; + } + } + if (existingSummaryIndex >= 0) { + existingContent[existingSummaryIndex] = summaryContentBlock; + } else { + existingContent.push(summaryContentBlock); + } + update.content = existingContent; } if (message.tokenCount && !update.summaryTokenCount) { @@ -934,10 +960,24 @@ class BaseClient { return _messages; } - // Find the latest message with a 'summary' property for (let i = _messages.length - 1; i >= 0; i--) { - if (_messages[i]?.summary) { - this.previous_summary = _messages[i]; + const msg = _messages[i]; + if (!msg) { + continue; + } + + const summaryBlock = BaseClient.findSummaryContentBlock(msg); + if (summaryBlock) { + this.previous_summary = { + ...msg, + summary: BaseClient.getSummaryText(summaryBlock), + summaryTokenCount: summaryBlock.tokenCount, + }; + break; + } + + if (msg.summary) { + this.previous_summary = msg; break; } } @@ -1041,6 +1081,32 @@ class BaseClient { await db.updateMessage(this.options?.req?.user?.id, message); } + /** Extracts text from a summary block (handles both legacy `text` field and new `content` array format). */ + static getSummaryText(summaryBlock) { + if (Array.isArray(summaryBlock.content)) { + return summaryBlock.content.map((b) => b.text ?? '').join(''); + } + if (typeof summaryBlock.content === 'string') { + return summaryBlock.content; + } + return summaryBlock.text ?? ''; + } + + /** Finds the last summary content block in a message's content array (last-summary-wins). */ + static findSummaryContentBlock(message) { + if (!Array.isArray(message?.content)) { + return null; + } + let lastSummary = null; + for (const part of message.content) { + // Accepts both new format (content: []) and legacy DB format (text: string) + if (part?.type === ContentTypes.SUMMARY && (part.content || part.text)) { + lastSummary = part; + } + } + return lastSummary; + } + /** * Iterate through messages, building an array based on the parentMessageId. * @@ -1095,20 +1161,35 @@ class BaseClient { break; } - if (summary && message.summary) { - message.role = 'system'; - message.text = message.summary; + let resolved = message; + let hasSummary = false; + if (summary) { + const summaryBlock = BaseClient.findSummaryContentBlock(message); + if (summaryBlock) { + const summaryText = BaseClient.getSummaryText(summaryBlock); + resolved = { + ...message, + role: 'system', + content: [{ type: ContentTypes.TEXT, text: summaryText }], + tokenCount: summaryBlock.tokenCount, + }; + hasSummary = true; + } else if (message.summary) { + resolved = { + ...message, + role: 'system', + content: [{ type: ContentTypes.TEXT, text: message.summary }], + tokenCount: message.summaryTokenCount ?? message.tokenCount, + }; + hasSummary = true; + } } - if (summary && message.summaryTokenCount) { - message.tokenCount = message.summaryTokenCount; - } - - const shouldMap = mapMethod != null && (mapCondition != null ? mapCondition(message) : true); - const processedMessage = shouldMap ? mapMethod(message) : message; + const shouldMap = mapMethod != null && (mapCondition != null ? mapCondition(resolved) : true); + const processedMessage = shouldMap ? mapMethod(resolved) : resolved; orderedMessages.push(processedMessage); - if (summary && message.summary) { + if (hasSummary) { break; } diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index f13c9979ac..3d57e1fbe1 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -355,7 +355,8 @@ describe('BaseClient', () => { id: '3', parentMessageId: '2', role: 'system', - text: 'Summary for Message 3', + text: 'Message 3', + content: [{ type: 'text', text: 'Summary for Message 3' }], summary: 'Summary for Message 3', }, { id: '4', parentMessageId: '3', text: 'Message 4' }, @@ -380,7 +381,8 @@ describe('BaseClient', () => { id: '4', parentMessageId: '3', role: 'system', - text: 'Summary for Message 4', + text: 'Message 4', + content: [{ type: 'text', text: 'Summary for Message 4' }], summary: 'Summary for Message 4', }, { id: '5', parentMessageId: '4', text: 'Message 5' }, @@ -405,12 +407,171 @@ describe('BaseClient', () => { id: '4', parentMessageId: '3', role: 'system', - text: 'Summary for Message 4', + text: 'Message 4', + content: [{ type: 'text', text: 'Summary for Message 4' }], summary: 'Summary for Message 4', }, { id: '5', parentMessageId: '4', text: 'Message 5' }, ]); }); + + it('should detect summary content block and use it over legacy fields (summary mode)', () => { + const messagesWithContentBlock = [ + { id: '3', parentMessageId: '2', text: 'Message 3' }, + { + id: '2', + parentMessageId: '1', + text: 'Message 2', + content: [ + { type: 'text', text: 'Original text' }, + { type: 'summary', text: 'Content block summary', tokenCount: 42 }, + ], + }, + { id: '1', parentMessageId: null, text: 'Message 1' }, + ]; + const result = TestClient.constructor.getMessagesForConversation({ + messages: messagesWithContentBlock, + parentMessageId: '3', + summary: true, + }); + expect(result).toHaveLength(2); + expect(result[0].role).toBe('system'); + expect(result[0].content).toEqual([{ type: 'text', text: 'Content block summary' }]); + expect(result[0].tokenCount).toBe(42); + }); + + it('should prefer content block summary over legacy summary field', () => { + const messagesWithBoth = [ + { id: '2', parentMessageId: '1', text: 'Message 2' }, + { + id: '1', + parentMessageId: null, + text: 'Message 1', + summary: 'Legacy summary', + summaryTokenCount: 10, + content: [{ type: 'summary', text: 'Content block summary', tokenCount: 20 }], + }, + ]; + const result = TestClient.constructor.getMessagesForConversation({ + messages: messagesWithBoth, + parentMessageId: '2', + summary: true, + }); + expect(result).toHaveLength(2); + expect(result[0].content).toEqual([{ type: 'text', text: 'Content block summary' }]); + expect(result[0].tokenCount).toBe(20); + }); + + it('should fallback to legacy summary when no content block exists', () => { + const messagesWithLegacy = [ + { id: '2', parentMessageId: '1', text: 'Message 2' }, + { + id: '1', + parentMessageId: null, + text: 'Message 1', + summary: 'Legacy summary only', + summaryTokenCount: 15, + }, + ]; + const result = TestClient.constructor.getMessagesForConversation({ + messages: messagesWithLegacy, + parentMessageId: '2', + summary: true, + }); + expect(result).toHaveLength(2); + expect(result[0].content).toEqual([{ type: 'text', text: 'Legacy summary only' }]); + expect(result[0].tokenCount).toBe(15); + }); + }); + + describe('findSummaryContentBlock', () => { + it('should find a summary block in the content array', () => { + const message = { + content: [ + { type: 'text', text: 'some text' }, + { type: 'summary', text: 'Summary of conversation', tokenCount: 50 }, + ], + }; + const result = TestClient.constructor.findSummaryContentBlock(message); + expect(result).toBeTruthy(); + expect(result.text).toBe('Summary of conversation'); + expect(result.tokenCount).toBe(50); + }); + + it('should return null when no summary block exists', () => { + const message = { + content: [ + { type: 'text', text: 'some text' }, + { type: 'tool_call', tool_call: {} }, + ], + }; + expect(TestClient.constructor.findSummaryContentBlock(message)).toBeNull(); + }); + + it('should return null for string content', () => { + const message = { content: 'just a string' }; + expect(TestClient.constructor.findSummaryContentBlock(message)).toBeNull(); + }); + + it('should return null for missing content', () => { + expect(TestClient.constructor.findSummaryContentBlock({})).toBeNull(); + expect(TestClient.constructor.findSummaryContentBlock(null)).toBeNull(); + }); + + it('should skip summary blocks with no text', () => { + const message = { + content: [{ type: 'summary', tokenCount: 10 }], + }; + expect(TestClient.constructor.findSummaryContentBlock(message)).toBeNull(); + }); + }); + + describe('handleTokenCountMap', () => { + it('should update the last summary content block when multiple summary blocks exist', async () => { + const updateMessageInDatabaseSpy = jest + .spyOn(TestClient, 'updateMessageInDatabase') + .mockResolvedValue(undefined); + + const targetMessage = { + messageId: 'assistant-message-1', + content: [ + { type: 'summary', text: 'old-summary-1', tokenCount: 10 }, + { type: 'text', text: 'middle' }, + { type: 'summary', text: 'old-summary-2', tokenCount: 20 }, + ], + }; + const trailingUserMessage = { + messageId: 'user-message-1', + content: 'latest user message', + }; + + TestClient.currentMessages = [targetMessage, trailingUserMessage]; + + await TestClient.handleTokenCountMap({ + summaryMessage: { + messageId: targetMessage.messageId, + content: 'new-summary', + tokenCount: 77, + }, + [targetMessage.messageId]: 123, + }); + + expect(updateMessageInDatabaseSpy).toHaveBeenCalledTimes(1); + const updateArg = updateMessageInDatabaseSpy.mock.calls[0][0]; + expect(updateArg.content[0]).toEqual({ + type: 'summary', + text: 'old-summary-1', + tokenCount: 10, + }); + expect(updateArg.content[2]).toEqual( + expect.objectContaining({ + type: 'summary', + content: [{ type: 'text', text: 'new-summary' }], + tokenCount: 77, + }), + ); + expect(updateArg.content[2].createdAt).toBeDefined(); + }); }); describe('sendMessage', () => { diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js index 0bb935795d..fbda09fe12 100644 --- a/api/server/controllers/agents/callbacks.js +++ b/api/server/controllers/agents/callbacks.js @@ -1,7 +1,13 @@ const { nanoid } = require('nanoid'); const { logger } = require('@librechat/data-schemas'); -const { Constants, EnvVar, GraphEvents, ToolEndHandler } = require('@librechat/agents'); const { Tools, StepTypes, FileContext, ErrorTypes } = require('librechat-data-provider'); +const { + EnvVar, + Constants, + GraphEvents, + GraphNodeKeys, + ToolEndHandler, +} = require('@librechat/agents'); const { sendEvent, GenerationJobManager, @@ -71,6 +77,8 @@ class ModelEndHandler { usage.model = modelName; } + markSummarizationUsage(usage, metadata); + this.collectedUsage.push(usage); } catch (error) { logger.error('Error handling model end event:', error); @@ -133,6 +141,7 @@ function getDefaultHandlers({ collectedUsage, streamId = null, toolExecuteOptions = null, + summarizationOptions = null, }) { if (!res || !aggregateContent) { throw new Error( @@ -245,6 +254,46 @@ function getDefaultHandlers({ handlers[GraphEvents.ON_TOOL_EXECUTE] = createToolExecuteHandler(toolExecuteOptions); } + if (summarizationOptions?.enabled !== false) { + handlers[GraphEvents.ON_SUMMARIZE_START] = { + handle: async (_event, data) => { + await emitEvent(res, streamId, { + event: GraphEvents.ON_SUMMARIZE_START, + data, + }); + }, + }; + handlers[GraphEvents.ON_SUMMARIZE_DELTA] = { + handle: async (_event, data) => { + aggregateContent({ event: GraphEvents.ON_SUMMARIZE_DELTA, data }); + await emitEvent(res, streamId, { + event: GraphEvents.ON_SUMMARIZE_DELTA, + data, + }); + }, + }; + handlers[GraphEvents.ON_SUMMARIZE_COMPLETE] = { + handle: async (_event, data) => { + aggregateContent({ event: GraphEvents.ON_SUMMARIZE_COMPLETE, data }); + await emitEvent(res, streamId, { + event: GraphEvents.ON_SUMMARIZE_COMPLETE, + data, + }); + }, + }; + } + + handlers[GraphEvents.ON_AGENT_LOG] = { + handle: (_event, data) => { + const logFn = typeof logger[data.level] === 'function' ? logger[data.level] : logger.info; + logFn(`[agentus:${data.scope}] ${data.message}`, { + ...data.data, + runId: data.runId, + agentId: data.agentId, + }); + }, + }; + return handlers; } @@ -668,8 +717,16 @@ function createResponsesToolEndCallback({ req, res, tracker, artifactPromises }) }; } +function markSummarizationUsage(usage, metadata) { + const node = metadata?.langgraph_node; + if (typeof node === 'string' && node.startsWith(GraphNodeKeys.SUMMARIZE)) { + usage.usage_type = 'summarization'; + } +} + module.exports = { getDefaultHandlers, createToolEndCallback, createResponsesToolEndCallback, + markSummarizationUsage, }; diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 7d3aab7a3a..5bb2680056 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -25,6 +25,7 @@ const { loadAgent: loadAgentFn, createMultiAgentMapper, filterMalformedContentParts, + hydrateMissingIndexTokenCounts, } = require('@librechat/api'); const { Callback, @@ -61,9 +62,6 @@ class AgentClient extends BaseClient { * @type {string} */ this.clientName = EModelEndpoint.agents; - /** @type {'discard' | 'summarize'} */ - this.contextStrategy = 'discard'; - /** @deprecated @type {true} - Is a Chat Completion Request */ this.isChatCompletion = true; @@ -215,7 +213,6 @@ class AgentClient extends BaseClient { })) : []), ]; - if (this.options.attachments) { const attachments = await this.options.attachments; const latestMessage = orderedMessages[orderedMessages.length - 1]; @@ -242,6 +239,9 @@ class AgentClient extends BaseClient { ); } + /** @type {Record} */ + const canonicalTokenCountMap = {}; + let promptTokenTotal = 0; const formattedMessages = orderedMessages.map((message, i) => { const formattedMessage = formatMessage({ message, @@ -261,8 +261,7 @@ class AgentClient extends BaseClient { } } - const needsTokenCount = - (this.contextStrategy && !orderedMessages[i].tokenCount) || message.fileContext; + const needsTokenCount = !orderedMessages[i].tokenCount || message.fileContext; /* If tokens were never counted, or, is a Vision request and the message has files, count again */ if (needsTokenCount || (this.isVisionModel && (message.image_urls || message.files))) { @@ -288,9 +287,18 @@ class AgentClient extends BaseClient { } } + const tokenCount = Number(orderedMessages[i].tokenCount); + const normalizedTokenCount = Number.isFinite(tokenCount) && tokenCount > 0 ? tokenCount : 0; + canonicalTokenCountMap[i] = normalizedTokenCount; + promptTokenTotal += normalizedTokenCount; + return formattedMessage; }); + payload = formattedMessages; + messages = orderedMessages; + promptTokens = promptTokenTotal; + /** * Build shared run context - applies to ALL agents in the run. * This includes: file context (latest message), augmented prompt (RAG), memory context. @@ -323,16 +331,8 @@ class AgentClient extends BaseClient { /** @type {Record | undefined} */ let tokenCountMap; - if (this.contextStrategy) { - ({ payload, promptTokens, tokenCountMap, messages } = await this.handleContextStrategy({ - orderedMessages, - formattedMessages, - })); - } - - for (let i = 0; i < messages.length; i++) { - this.indexTokenCountMap[i] = messages[i].tokenCount; - } + /** Preserve canonical pre-format token counts for all history entering graph formatting */ + this.indexTokenCountMap = canonicalTokenCountMap; const result = { tokenCountMap, @@ -743,11 +743,17 @@ class AgentClient extends BaseClient { }; const toolSet = buildToolSet(this.options.agent); - let { messages: initialMessages, indexTokenCountMap } = formatAgentMessages( - payload, - this.indexTokenCountMap, - toolSet, - ); + const tokenCounter = createTokenCounter(this.getEncoding()); + let { + messages: initialMessages, + indexTokenCountMap, + summary: initialSummary, + } = formatAgentMessages(payload, this.indexTokenCountMap, toolSet); + indexTokenCountMap = hydrateMissingIndexTokenCounts({ + messages: initialMessages, + indexTokenCountMap, + tokenCounter, + }); /** * @param {BaseMessage[]} messages @@ -805,12 +811,14 @@ class AgentClient extends BaseClient { agents, messages, indexTokenCountMap, + initialSummary, runId: this.responseMessageId, signal: abortController.signal, customHandlers: this.options.eventHandlers, requestBody: config.configurable.requestBody, user: createSafeUser(this.options.req?.user), - tokenCounter: createTokenCounter(this.getEncoding()), + summarizationConfig: appConfig?.summarization, + tokenCounter, }); if (!run) { @@ -1056,6 +1064,7 @@ class AgentClient extends BaseClient { titlePrompt: endpointConfig?.titlePrompt, titlePromptTemplate: endpointConfig?.titlePromptTemplate, chainOptions: { + runName: 'TitleRun', signal: abortController.signal, callbacks: [ { diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index 4e3d10e8e6..41a806f66d 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -1818,7 +1818,7 @@ describe('AgentClient - titleConvo', () => { /** Traversal stops at msg-2 (has summary), so we get msg-4 -> msg-3 -> msg-2 */ expect(result).toHaveLength(3); - expect(result[0].text).toBe('Summary of conversation'); + expect(result[0].content).toEqual([{ type: 'text', text: 'Summary of conversation' }]); expect(result[0].role).toBe('system'); expect(result[0].mapped).toBe(true); expect(result[1].mapped).toBe(true); diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index 4a38f6f097..c01e585244 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -1,7 +1,7 @@ const { nanoid } = require('nanoid'); const { logger } = require('@librechat/data-schemas'); -const { Callback, ToolEndHandler, formatAgentMessages } = require('@librechat/agents'); const { EModelEndpoint, ResourceType, PermissionBits } = require('librechat-data-provider'); +const { Callback, ToolEndHandler, formatAgentMessages } = require('@librechat/agents'); const { writeSSE, createRun, @@ -22,7 +22,10 @@ const { isChatCompletionValidationFailure, } = require('@librechat/api'); const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService'); -const { createToolEndCallback } = require('~/server/controllers/agents/callbacks'); +const { + createToolEndCallback, + markSummarizationUsage, +} = require('~/server/controllers/agents/callbacks'); const { findAccessibleResources } = require('~/server/services/PermissionService'); const db = require('~/models'); @@ -266,14 +269,16 @@ const OpenAIChatCompletionController = async (req, res) => { toolEndCallback, }; + const summarizationConfig = appConfig?.summarization; + const openaiMessages = convertMessages(request.messages); const toolSet = buildToolSet(primaryConfig); - const { messages: formattedMessages, indexTokenCountMap } = formatAgentMessages( - openaiMessages, - {}, - toolSet, - ); + const { + messages: formattedMessages, + indexTokenCountMap, + summary: initialSummary, + } = formatAgentMessages(openaiMessages, {}, toolSet); /** * Create a simple handler that processes data @@ -416,15 +421,18 @@ const OpenAIChatCompletionController = async (req, res) => { }), // Usage tracking - on_chat_model_end: createHandler((data) => { - const usage = data?.output?.usage_metadata; - if (usage) { - collectedUsage.push(usage); - const target = isStreaming ? tracker : aggregator; - target.usage.promptTokens += usage.input_tokens ?? 0; - target.usage.completionTokens += usage.output_tokens ?? 0; - } - }), + on_chat_model_end: { + handle: (_event, data, metadata) => { + const usage = data?.output?.usage_metadata; + if (usage) { + markSummarizationUsage(usage, metadata); + collectedUsage.push(usage); + const target = isStreaming ? tracker : aggregator; + target.usage.promptTokens += usage.input_tokens ?? 0; + target.usage.completionTokens += usage.output_tokens ?? 0; + } + }, + }, on_run_step_completed: createHandler(), // Use proper ToolEndHandler for processing artifacts (images, file citations, code output) on_tool_end: new ToolEndHandler(toolEndCallback, logger), @@ -434,6 +442,31 @@ const OpenAIChatCompletionController = async (req, res) => { on_custom_event: createHandler(), // Event-driven tool execution handler on_tool_execute: createToolExecuteHandler(toolExecuteOptions), + ...(summarizationConfig?.enabled !== false + ? { + on_summarize_start: { + handle: async (_event, data) => { + if (isStreaming && !res.writableEnded) { + res.write(`event: on_summarize_start\ndata: ${JSON.stringify(data)}\n\n`); + } + }, + }, + on_summarize_delta: { + handle: async (_event, data) => { + if (isStreaming && !res.writableEnded) { + res.write(`event: on_summarize_delta\ndata: ${JSON.stringify(data)}\n\n`); + } + }, + }, + on_summarize_complete: { + handle: async (_event, data) => { + if (isStreaming && !res.writableEnded) { + res.write(`event: on_summarize_complete\ndata: ${JSON.stringify(data)}\n\n`); + } + }, + }, + } + : {}), }; // Create and run the agent @@ -446,7 +479,9 @@ const OpenAIChatCompletionController = async (req, res) => { agents: [primaryConfig], messages: formattedMessages, indexTokenCountMap, + initialSummary, runId: responseId, + summarizationConfig, signal: abortController.signal, customHandlers: handlers, requestBody: { diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index 6f629d2a81..351450dd5f 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -33,6 +33,7 @@ const { const { createResponsesToolEndCallback, createToolEndCallback, + markSummarizationUsage, } = require('~/server/controllers/agents/callbacks'); const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService'); const { findAccessibleResources } = require('~/server/services/PermissionService'); @@ -49,6 +50,17 @@ function setAppConfig(config) { appConfig = config; } +const agentLogHandler = { + handle: (_event, data) => { + const logFn = typeof logger[data.level] === 'function' ? logger[data.level] : logger.info; + logFn(`[agentus:${data.scope}] ${data.message}`, { + ...data.data, + runId: data.runId, + agentId: data.agentId, + }); + }, +}; + /** * Creates a tool loader function for the agent. * @param {AbortSignal} signal - The abort signal @@ -277,6 +289,7 @@ const createResponse = async (req, res) => { const request = validation.request; const agentId = request.model; const isStreaming = request.stream === true; + const summarizationConfig = req.config?.summarization; // Look up the agent const agent = await db.getAgent({ id: agentId }); @@ -374,11 +387,11 @@ const createResponse = async (req, res) => { const allMessages = [...previousMessages, ...inputMessages]; const toolSet = buildToolSet(primaryConfig); - const { messages: formattedMessages, indexTokenCountMap } = formatAgentMessages( - allMessages, - {}, - toolSet, - ); + const { + messages: formattedMessages, + indexTokenCountMap, + summary: initialSummary, + } = formatAgentMessages(allMessages, {}, toolSet); // Create tracker for streaming or aggregator for non-streaming const tracker = actuallyStreaming ? createResponseTracker() : null; @@ -441,10 +454,11 @@ const createResponse = async (req, res) => { on_run_step: responsesHandlers.on_run_step, on_run_step_delta: responsesHandlers.on_run_step_delta, on_chat_model_end: { - handle: (event, data) => { + handle: (event, data, metadata) => { responsesHandlers.on_chat_model_end.handle(event, data); const usage = data?.output?.usage_metadata; if (usage) { + markSummarizationUsage(usage, metadata); collectedUsage.push(usage); } }, @@ -456,6 +470,32 @@ const createResponse = async (req, res) => { on_agent_update: { handle: () => {} }, on_custom_event: { handle: () => {} }, on_tool_execute: createToolExecuteHandler(toolExecuteOptions), + on_agent_log: agentLogHandler, + ...(summarizationConfig?.enabled !== false + ? { + on_summarize_start: { + handle: async (_event, data) => { + if (actuallyStreaming && !res.writableEnded) { + res.write(`event: on_summarize_start\ndata: ${JSON.stringify(data)}\n\n`); + } + }, + }, + on_summarize_delta: { + handle: async (_event, data) => { + if (actuallyStreaming && !res.writableEnded) { + res.write(`event: on_summarize_delta\ndata: ${JSON.stringify(data)}\n\n`); + } + }, + }, + on_summarize_complete: { + handle: async (_event, data) => { + if (actuallyStreaming && !res.writableEnded) { + res.write(`event: on_summarize_complete\ndata: ${JSON.stringify(data)}\n\n`); + } + }, + }, + } + : {}), }; // Create and run the agent @@ -466,7 +506,9 @@ const createResponse = async (req, res) => { agents: [primaryConfig], messages: formattedMessages, indexTokenCountMap, + initialSummary, runId: responseId, + summarizationConfig, signal: abortController.signal, customHandlers: handlers, requestBody: { @@ -597,10 +639,11 @@ const createResponse = async (req, res) => { on_run_step: aggregatorHandlers.on_run_step, on_run_step_delta: aggregatorHandlers.on_run_step_delta, on_chat_model_end: { - handle: (event, data) => { + handle: (event, data, metadata) => { aggregatorHandlers.on_chat_model_end.handle(event, data); const usage = data?.output?.usage_metadata; if (usage) { + markSummarizationUsage(usage, metadata); collectedUsage.push(usage); } }, @@ -612,6 +655,14 @@ const createResponse = async (req, res) => { on_agent_update: { handle: () => {} }, on_custom_event: { handle: () => {} }, on_tool_execute: createToolExecuteHandler(toolExecuteOptions), + on_agent_log: agentLogHandler, + ...(summarizationConfig?.enabled !== false + ? { + on_summarize_start: { handle: () => {} }, + on_summarize_delta: { handle: () => {} }, + on_summarize_complete: { handle: () => {} }, + } + : {}), }; const userId = req.user?.id ?? 'api-user'; @@ -621,7 +672,9 @@ const createResponse = async (req, res) => { agents: [primaryConfig], messages: formattedMessages, indexTokenCountMap, + initialSummary, runId: responseId, + summarizationConfig, signal: abortController.signal, customHandlers: handlers, requestBody: { diff --git a/api/server/routes/files/files.agents.test.js b/api/server/routes/files/files.agents.test.js index 5a01df022d..cb0e4ff3d2 100644 --- a/api/server/routes/files/files.agents.test.js +++ b/api/server/routes/files/files.agents.test.js @@ -2,15 +2,14 @@ const express = require('express'); const request = require('supertest'); const mongoose = require('mongoose'); const { v4: uuidv4 } = require('uuid'); -const { createMethods } = require('@librechat/data-schemas'); const { MongoMemoryServer } = require('mongodb-memory-server'); +const { createMethods, SystemCapabilities } = require('@librechat/data-schemas'); const { SystemRoles, AccessRoleIds, ResourceType, PrincipalType, } = require('librechat-data-provider'); -const { SystemCapabilities } = require('@librechat/data-schemas'); const { createAgent, createFile } = require('~/models'); // Only mock the external dependencies that we don't want to test diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 8e2cab0ab9..e1adfd8225 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -78,6 +78,14 @@ function createToolLoader(signal, streamId = null, definitionsOnly = false) { }; } +/** + * Initializes the AgentClient for a given request/response cycle. + * @param {Object} params + * @param {Express.Request} params.req + * @param {Express.Response} params.res + * @param {AbortSignal} params.signal + * @param {Object} params.endpointOption + */ const initializeClient = async ({ req, res, signal, endpointOption }) => { if (!endpointOption) { throw new Error('Endpoint option not provided'); @@ -131,9 +139,13 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { toolEndCallback, }; + const summarizationOptions = + appConfig?.summarization?.enabled === false ? { enabled: false } : { enabled: true }; + const eventHandlers = getDefaultHandlers({ res, toolExecuteOptions, + summarizationOptions, aggregateContent, toolEndCallback, collectedUsage, diff --git a/api/server/services/Endpoints/index.js b/api/server/services/Endpoints/index.js deleted file mode 100644 index 3cabfe1c58..0000000000 --- a/api/server/services/Endpoints/index.js +++ /dev/null @@ -1,77 +0,0 @@ -const { Providers } = require('@librechat/agents'); -const { EModelEndpoint } = require('librechat-data-provider'); -const { getCustomEndpointConfig } = require('@librechat/api'); -const initAnthropic = require('~/server/services/Endpoints/anthropic/initialize'); -const getBedrockOptions = require('~/server/services/Endpoints/bedrock/options'); -const initOpenAI = require('~/server/services/Endpoints/openAI/initialize'); -const initCustom = require('~/server/services/Endpoints/custom/initialize'); -const initGoogle = require('~/server/services/Endpoints/google/initialize'); - -/** Check if the provider is a known custom provider - * @param {string | undefined} [provider] - The provider string - * @returns {boolean} - True if the provider is a known custom provider, false otherwise - */ -function isKnownCustomProvider(provider) { - return [Providers.XAI, Providers.DEEPSEEK, Providers.OPENROUTER, Providers.MOONSHOT].includes( - provider?.toLowerCase() || '', - ); -} - -const providerConfigMap = { - [Providers.XAI]: initCustom, - [Providers.DEEPSEEK]: initCustom, - [Providers.MOONSHOT]: initCustom, - [Providers.OPENROUTER]: initCustom, - [EModelEndpoint.openAI]: initOpenAI, - [EModelEndpoint.google]: initGoogle, - [EModelEndpoint.azureOpenAI]: initOpenAI, - [EModelEndpoint.anthropic]: initAnthropic, - [EModelEndpoint.bedrock]: getBedrockOptions, -}; - -/** - * Get the provider configuration and override endpoint based on the provider string - * @param {Object} params - * @param {string} params.provider - The provider string - * @param {AppConfig} params.appConfig - The application configuration - * @returns {{ - * getOptions: (typeof providerConfigMap)[keyof typeof providerConfigMap], - * overrideProvider: string, - * customEndpointConfig?: TEndpoint - * }} - */ -function getProviderConfig({ provider, appConfig }) { - let getOptions = providerConfigMap[provider]; - let overrideProvider = provider; - /** @type {TEndpoint | undefined} */ - let customEndpointConfig; - - if (!getOptions && providerConfigMap[provider.toLowerCase()] != null) { - overrideProvider = provider.toLowerCase(); - getOptions = providerConfigMap[overrideProvider]; - } else if (!getOptions) { - customEndpointConfig = getCustomEndpointConfig({ endpoint: provider, appConfig }); - if (!customEndpointConfig) { - throw new Error(`Provider ${provider} not supported`); - } - getOptions = initCustom; - overrideProvider = Providers.OPENAI; - } - - if (isKnownCustomProvider(overrideProvider) && !customEndpointConfig) { - customEndpointConfig = getCustomEndpointConfig({ endpoint: provider, appConfig }); - if (!customEndpointConfig) { - throw new Error(`Provider ${provider} not supported`); - } - } - - return { - getOptions, - overrideProvider, - customEndpointConfig, - }; -} - -module.exports = { - getProviderConfig, -}; diff --git a/client/src/a11y/LiveAnnouncer.tsx b/client/src/a11y/LiveAnnouncer.tsx index 0eac8089bc..ac83ff2962 100644 --- a/client/src/a11y/LiveAnnouncer.tsx +++ b/client/src/a11y/LiveAnnouncer.tsx @@ -21,6 +21,9 @@ const LiveAnnouncer: React.FC = ({ children }) => { start: localize('com_a11y_start'), end: localize('com_a11y_end'), composing: localize('com_a11y_ai_composing'), + summarize_started: localize('com_a11y_summarize_started'), + summarize_completed: localize('com_a11y_summarize_completed'), + summarize_failed: localize('com_a11y_summarize_failed'), }), [localize], ); diff --git a/client/src/components/Chat/Messages/Content/Part.tsx b/client/src/components/Chat/Messages/Content/Part.tsx index 7bce7ac11d..d0c7d2af37 100644 --- a/client/src/components/Chat/Messages/Content/Part.tsx +++ b/client/src/components/Chat/Messages/Content/Part.tsx @@ -8,7 +8,15 @@ import { } from 'librechat-data-provider'; import { memo } from 'react'; import type { TMessageContentParts, TAttachment } from 'librechat-data-provider'; -import { OpenAIImageGen, EmptyText, Reasoning, ExecuteCode, AgentUpdate, Text } from './Parts'; +import { + OpenAIImageGen, + ExecuteCode, + AgentUpdate, + EmptyText, + Reasoning, + Summary, + Text, +} from './Parts'; import { ErrorMessage } from './MessageContent'; import RetrievalCall from './RetrievalCall'; import { getCachedPreview } from '~/utils'; @@ -100,6 +108,16 @@ const Part = memo(function Part({ return null; } return ; + } else if (part.type === ContentTypes.SUMMARY) { + return ( + + ); } else if (part.type === ContentTypes.TOOL_CALL) { const toolCall = part[ContentTypes.TOOL_CALL]; diff --git a/client/src/components/Chat/Messages/Content/Parts/Summary.tsx b/client/src/components/Chat/Messages/Content/Parts/Summary.tsx new file mode 100644 index 0000000000..1d2b25f55f --- /dev/null +++ b/client/src/components/Chat/Messages/Content/Parts/Summary.tsx @@ -0,0 +1,318 @@ +import { memo, useMemo, useState, useCallback, useRef, useId, useEffect } from 'react'; +import { useAtomValue } from 'jotai'; +import { Clipboard, CheckMark, TooltipAnchor } from '@librechat/client'; +import { ScrollText, ChevronDown, ChevronUp } from 'lucide-react'; +import type { MouseEvent, FocusEvent } from 'react'; +import { fontSizeAtom } from '~/store/fontSize'; +import { useMessageContext } from '~/Providers'; +import { useLocalize } from '~/hooks'; +import { cn } from '~/utils'; + +function useCopyToClipboard(content?: string) { + const [isCopied, setIsCopied] = useState(false); + const timerRef = useRef>(); + useEffect(() => () => clearTimeout(timerRef.current), []); + const handleCopy = useCallback( + (e: MouseEvent) => { + e.stopPropagation(); + if (content) { + navigator.clipboard.writeText(content); + clearTimeout(timerRef.current); + setIsCopied(true); + timerRef.current = setTimeout(() => setIsCopied(false), 2000); + } + }, + [content], + ); + return { isCopied, handleCopy }; +} + +type ContentBlock = { type?: string; text?: string }; + +type SummaryProps = { + content?: ContentBlock[]; + model?: string; + provider?: string; + tokenCount?: number; + summarizing?: boolean; +}; + +const SummaryContent = memo(({ children, meta }: { children: React.ReactNode; meta?: string }) => { + const fontSize = useAtomValue(fontSizeAtom); + + return ( +
+ {meta && {meta}} +

{children}

+
+ ); +}); + +const SummaryButton = memo( + ({ + isExpanded, + onClick, + label, + content, + contentId, + showCopyButton = true, + }: { + isExpanded: boolean; + onClick: (e: MouseEvent) => void; + label: string; + content?: string; + contentId: string; + showCopyButton?: boolean; + }) => { + const localize = useLocalize(); + const fontSize = useAtomValue(fontSizeAtom); + const { isCopied, handleCopy } = useCopyToClipboard(content); + + return ( +
+ + {content && showCopyButton && ( + + )} +
+ ); + }, +); + +const FloatingSummaryBar = memo( + ({ + isVisible, + isExpanded, + onClick, + content, + contentId, + }: { + isVisible: boolean; + isExpanded: boolean; + onClick: (e: MouseEvent) => void; + content?: string; + contentId: string; + }) => { + const localize = useLocalize(); + const { isCopied, handleCopy } = useCopyToClipboard(content); + + const collapseTooltip = isExpanded + ? localize('com_ui_collapse_summary') + : localize('com_ui_expand_summary'); + + const copyTooltip = isCopied + ? localize('com_ui_copied_to_clipboard') + : localize('com_ui_copy_summary'); + + return ( +
+ + {isExpanded ? ( +
+ ); + }, +); + +const Summary = memo(({ content, model, provider, tokenCount, summarizing }: SummaryProps) => { + const contentId = useId(); + const localize = useLocalize(); + const [isExpanded, setIsExpanded] = useState(false); + const [isBarVisible, setIsBarVisible] = useState(false); + const containerRef = useRef(null); + const { isSubmitting, isLatestMessage } = useMessageContext(); + + const text = useMemo(() => (content ?? []).map((block) => block.text ?? '').join(''), [content]); + + const handleClick = useCallback((e: MouseEvent) => { + e.preventDefault(); + setIsExpanded((prev) => !prev); + }, []); + + const handleFocus = useCallback(() => setIsBarVisible(true), []); + const handleBlur = useCallback((e: FocusEvent) => { + if (!containerRef.current?.contains(e.relatedTarget as Node)) { + setIsBarVisible(false); + } + }, []); + const handleMouseEnter = useCallback(() => setIsBarVisible(true), []); + const handleMouseLeave = useCallback(() => { + if (!containerRef.current?.contains(document.activeElement)) { + setIsBarVisible(false); + } + }, []); + + const effectiveIsSubmitting = isLatestMessage ? isSubmitting : false; + const isActivelyStreaming = !!summarizing && !!effectiveIsSubmitting; + + const meta = useMemo(() => { + const parts: string[] = []; + if (provider || model) { + parts.push([provider, model].filter(Boolean).join('/')); + } + if (tokenCount != null && tokenCount > 0) { + parts.push(`${tokenCount} ${localize('com_ui_tokens')}`); + } + return parts.length > 0 ? parts.join(' \u00b7 ') : undefined; + }, [model, provider, tokenCount, localize]); + + const label = useMemo( + () => + isActivelyStreaming + ? localize('com_ui_summarizing') + : localize('com_ui_conversation_summarized'), + [isActivelyStreaming, localize], + ); + + if (!summarizing && !text) { + return null; + } + + return ( +
+
+
+ +
+
+
+ {text} + +
+
+
+
+ ); +}); + +SummaryContent.displayName = 'SummaryContent'; +SummaryButton.displayName = 'SummaryButton'; +FloatingSummaryBar.displayName = 'FloatingSummaryBar'; +Summary.displayName = 'Summary'; + +export default Summary; diff --git a/client/src/components/Chat/Messages/Content/Parts/index.ts b/client/src/components/Chat/Messages/Content/Parts/index.ts index 8788201e65..b0a418c819 100644 --- a/client/src/components/Chat/Messages/Content/Parts/index.ts +++ b/client/src/components/Chat/Messages/Content/Parts/index.ts @@ -6,5 +6,6 @@ export { default as Reasoning } from './Reasoning'; export { default as EmptyText } from './EmptyText'; export { default as LogContent } from './LogContent'; export { default as ExecuteCode } from './ExecuteCode'; +export { default as Summary } from './Summary'; export { default as AgentUpdate } from './AgentUpdate'; export { default as EditTextPart } from './EditTextPart'; diff --git a/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts b/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts index cbe13f3910..1a6a6be7d2 100644 --- a/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts +++ b/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts @@ -1,5 +1,5 @@ import { renderHook, act } from '@testing-library/react'; -import { StepTypes, ContentTypes, ToolCallTypes } from 'librechat-data-provider'; +import { StepTypes, StepEvents, ContentTypes, ToolCallTypes } from 'librechat-data-provider'; import type { TMessageContentParts, EventSubmission, @@ -155,7 +155,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -174,7 +174,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(consoleSpy).toHaveBeenCalledWith('No message id found in run step event'); @@ -194,7 +194,7 @@ describe('useStepHandler', () => { }); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -210,7 +210,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -235,7 +235,7 @@ describe('useStepHandler', () => { act(() => { result.current.stepHandler( - { event: 'on_message_delta', data: createMessageDelta(stepId, 'Hello') }, + { event: StepEvents.ON_MESSAGE_DELTA, data: createMessageDelta(stepId, 'Hello') }, submission, ); }); @@ -245,7 +245,7 @@ describe('useStepHandler', () => { const runStep = createRunStep({ id: stepId }); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -266,7 +266,7 @@ describe('useStepHandler', () => { const submission = createSubmission({ userMessage: userMsg }); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -289,7 +289,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); const lastCall = mockSetMessages.mock.calls[mockSetMessages.mock.calls.length - 1][0]; @@ -315,7 +315,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); mockSetMessages.mockClear(); @@ -330,7 +330,10 @@ describe('useStepHandler', () => { }; act(() => { - result.current.stepHandler({ event: 'on_agent_update', data: agentUpdate }, submission); + result.current.stepHandler( + { event: StepEvents.ON_AGENT_UPDATE, data: agentUpdate }, + submission, + ); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -352,7 +355,10 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_agent_update', data: agentUpdate }, submission); + result.current.stepHandler( + { event: StepEvents.ON_AGENT_UPDATE, data: agentUpdate }, + submission, + ); }); expect(consoleSpy).toHaveBeenCalledWith('No message id found in agent update event'); @@ -371,7 +377,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); mockSetMessages.mockClear(); @@ -379,7 +385,10 @@ describe('useStepHandler', () => { const messageDelta = createMessageDelta('step-1', 'Hello'); act(() => { - result.current.stepHandler({ event: 'on_message_delta', data: messageDelta }, submission); + result.current.stepHandler( + { event: StepEvents.ON_MESSAGE_DELTA, data: messageDelta }, + submission, + ); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -397,7 +406,10 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_message_delta', data: messageDelta }, submission); + result.current.stepHandler( + { event: StepEvents.ON_MESSAGE_DELTA, data: messageDelta }, + submission, + ); }); expect(mockSetMessages).not.toHaveBeenCalled(); @@ -413,19 +425,19 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); act(() => { result.current.stepHandler( - { event: 'on_message_delta', data: createMessageDelta('step-1', 'Hello ') }, + { event: StepEvents.ON_MESSAGE_DELTA, data: createMessageDelta('step-1', 'Hello ') }, submission, ); }); act(() => { result.current.stepHandler( - { event: 'on_message_delta', data: createMessageDelta('step-1', 'World') }, + { event: StepEvents.ON_MESSAGE_DELTA, data: createMessageDelta('step-1', 'World') }, submission, ); }); @@ -447,7 +459,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); mockSetMessages.mockClear(); @@ -458,7 +470,10 @@ describe('useStepHandler', () => { }; act(() => { - result.current.stepHandler({ event: 'on_message_delta', data: messageDelta }, submission); + result.current.stepHandler( + { event: StepEvents.ON_MESSAGE_DELTA, data: messageDelta }, + submission, + ); }); expect(mockSetMessages).not.toHaveBeenCalled(); @@ -476,7 +491,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); mockSetMessages.mockClear(); @@ -485,7 +500,7 @@ describe('useStepHandler', () => { act(() => { result.current.stepHandler( - { event: 'on_reasoning_delta', data: reasoningDelta }, + { event: StepEvents.ON_REASONING_DELTA, data: reasoningDelta }, submission, ); }); @@ -506,7 +521,7 @@ describe('useStepHandler', () => { act(() => { result.current.stepHandler( - { event: 'on_reasoning_delta', data: reasoningDelta }, + { event: StepEvents.ON_REASONING_DELTA, data: reasoningDelta }, submission, ); }); @@ -524,19 +539,19 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); act(() => { result.current.stepHandler( - { event: 'on_reasoning_delta', data: createReasoningDelta('step-1', 'First ') }, + { event: StepEvents.ON_REASONING_DELTA, data: createReasoningDelta('step-1', 'First ') }, submission, ); }); act(() => { result.current.stepHandler( - { event: 'on_reasoning_delta', data: createReasoningDelta('step-1', 'thought') }, + { event: StepEvents.ON_REASONING_DELTA, data: createReasoningDelta('step-1', 'thought') }, submission, ); }); @@ -560,7 +575,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); mockSetMessages.mockClear(); @@ -574,7 +589,10 @@ describe('useStepHandler', () => { }; act(() => { - result.current.stepHandler({ event: 'on_run_step_delta', data: runStepDelta }, submission); + result.current.stepHandler( + { event: StepEvents.ON_RUN_STEP_DELTA, data: runStepDelta }, + submission, + ); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -593,7 +611,10 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step_delta', data: runStepDelta }, submission); + result.current.stepHandler( + { event: StepEvents.ON_RUN_STEP_DELTA, data: runStepDelta }, + submission, + ); }); expect(mockSetMessages).not.toHaveBeenCalled(); @@ -609,7 +630,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); mockSetMessages.mockClear(); @@ -625,7 +646,10 @@ describe('useStepHandler', () => { }; act(() => { - result.current.stepHandler({ event: 'on_run_step_delta', data: runStepDelta }, submission); + result.current.stepHandler( + { event: StepEvents.ON_RUN_STEP_DELTA, data: runStepDelta }, + submission, + ); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -649,7 +673,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); mockSetMessages.mockClear(); @@ -671,8 +695,8 @@ describe('useStepHandler', () => { act(() => { result.current.stepHandler( { - event: 'on_run_step_completed', - data: completedEvent as unknown as Agents.ToolEndEvent, + event: StepEvents.ON_RUN_STEP_COMPLETED, + data: completedEvent as { result: Agents.ToolEndEvent }, }, submission, ); @@ -710,8 +734,8 @@ describe('useStepHandler', () => { act(() => { result.current.stepHandler( { - event: 'on_run_step_completed', - data: completedEvent as unknown as Agents.ToolEndEvent, + event: StepEvents.ON_RUN_STEP_COMPLETED, + data: completedEvent as { result: Agents.ToolEndEvent }, }, submission, ); @@ -735,7 +759,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); act(() => { @@ -746,7 +770,7 @@ describe('useStepHandler', () => { act(() => { result.current.stepHandler( - { event: 'on_message_delta', data: createMessageDelta('step-1', 'Test') }, + { event: StepEvents.ON_MESSAGE_DELTA, data: createMessageDelta('step-1', 'Test') }, submission, ); }); @@ -772,12 +796,12 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); act(() => { result.current.stepHandler( - { event: 'on_message_delta', data: createMessageDelta('step-1', ' more') }, + { event: StepEvents.ON_MESSAGE_DELTA, data: createMessageDelta('step-1', ' more') }, submission, ); }); @@ -824,7 +848,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockAnnouncePolite).toHaveBeenCalledWith({ message: 'composing', isStatus: true }); @@ -842,7 +866,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockAnnouncePolite).not.toHaveBeenCalled(); @@ -872,7 +896,7 @@ describe('useStepHandler', () => { }); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -891,15 +915,15 @@ describe('useStepHandler', () => { act(() => { result.current.stepHandler( - { event: 'on_message_delta', data: createMessageDelta(stepId, 'First ') }, + { event: StepEvents.ON_MESSAGE_DELTA, data: createMessageDelta(stepId, 'First ') }, submission, ); result.current.stepHandler( - { event: 'on_message_delta', data: createMessageDelta(stepId, 'Second ') }, + { event: StepEvents.ON_MESSAGE_DELTA, data: createMessageDelta(stepId, 'Second ') }, submission, ); result.current.stepHandler( - { event: 'on_message_delta', data: createMessageDelta(stepId, 'Third') }, + { event: StepEvents.ON_MESSAGE_DELTA, data: createMessageDelta(stepId, 'Third') }, submission, ); }); @@ -909,7 +933,7 @@ describe('useStepHandler', () => { const runStep = createRunStep({ id: stepId }); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -931,11 +955,14 @@ describe('useStepHandler', () => { act(() => { result.current.stepHandler( - { event: 'on_reasoning_delta', data: createReasoningDelta(stepId, 'Thinking...') }, + { + event: StepEvents.ON_REASONING_DELTA, + data: createReasoningDelta(stepId, 'Thinking...'), + }, submission, ); result.current.stepHandler( - { event: 'on_message_delta', data: createMessageDelta(stepId, 'Response') }, + { event: StepEvents.ON_MESSAGE_DELTA, data: createMessageDelta(stepId, 'Response') }, submission, ); }); @@ -945,7 +972,7 @@ describe('useStepHandler', () => { const runStep = createRunStep({ id: stepId }); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -971,7 +998,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); const textDelta: Agents.MessageDeltaEvent = { @@ -980,7 +1007,10 @@ describe('useStepHandler', () => { }; act(() => { - result.current.stepHandler({ event: 'on_message_delta', data: textDelta }, submission); + result.current.stepHandler( + { event: StepEvents.ON_MESSAGE_DELTA, data: textDelta }, + submission, + ); }); expect(consoleSpy).toHaveBeenCalledWith( @@ -1004,7 +1034,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -1019,7 +1049,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -1035,7 +1065,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); const messageDelta: Agents.MessageDeltaEvent = { @@ -1049,7 +1079,10 @@ describe('useStepHandler', () => { }; act(() => { - result.current.stepHandler({ event: 'on_message_delta', data: messageDelta }, submission); + result.current.stepHandler( + { event: StepEvents.ON_MESSAGE_DELTA, data: messageDelta }, + submission, + ); }); expect(mockSetMessages).toHaveBeenCalled(); @@ -1065,7 +1098,7 @@ describe('useStepHandler', () => { const submission = createSubmission(); act(() => { - result.current.stepHandler({ event: 'on_run_step', data: runStep }, submission); + result.current.stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, submission); }); mockSetMessages.mockClear(); @@ -1076,7 +1109,10 @@ describe('useStepHandler', () => { }; act(() => { - result.current.stepHandler({ event: 'on_message_delta', data: messageDelta }, submission); + result.current.stepHandler( + { event: StepEvents.ON_MESSAGE_DELTA, data: messageDelta }, + submission, + ); }); expect(mockSetMessages).not.toHaveBeenCalled(); diff --git a/client/src/hooks/SSE/useResumableSSE.ts b/client/src/hooks/SSE/useResumableSSE.ts index 831bf042ad..4604139e3c 100644 --- a/client/src/hooks/SSE/useResumableSSE.ts +++ b/client/src/hooks/SSE/useResumableSSE.ts @@ -8,6 +8,7 @@ import { Constants, QueryKeys, ErrorTypes, + StepEvents, apiBaseUrl, createPayload, ViolationTypes, @@ -234,7 +235,7 @@ export default function useResumableSSE( // Replay run steps if (data.resumeState?.runSteps) { for (const runStep of data.resumeState.runSteps) { - stepHandler({ event: 'on_run_step', data: runStep }, { + stepHandler({ event: StepEvents.ON_RUN_STEP, data: runStep }, { ...currentSubmission, userMessage, } as EventSubmission); diff --git a/client/src/hooks/SSE/useStepHandler.ts b/client/src/hooks/SSE/useStepHandler.ts index c3b48cb107..45c001ddb6 100644 --- a/client/src/hooks/SSE/useStepHandler.ts +++ b/client/src/hooks/SSE/useStepHandler.ts @@ -2,6 +2,7 @@ import { useCallback, useRef } from 'react'; import { Constants, StepTypes, + StepEvents, ContentTypes, ToolCallTypes, getNonEmptyValue, @@ -13,6 +14,7 @@ import type { ContentMetadata, EventSubmission, TMessageContentParts, + SummaryContentPart, } from 'librechat-data-provider'; import type { SetterOrUpdater } from 'recoil'; import type { AnnounceOptions } from '~/common'; @@ -27,20 +29,16 @@ type TUseStepHandler = { lastAnnouncementTimeRef: React.MutableRefObject; }; -type TStepEvent = { - event: string; - data: - | Agents.MessageDeltaEvent - | Agents.ReasoningDeltaEvent - | Agents.RunStepDeltaEvent - | Agents.AgentUpdate - | Agents.RunStep - | Agents.ToolEndEvent - | { - runId?: string; - message: string; - }; -}; +type TStepEvent = + | { event: StepEvents.ON_RUN_STEP; data: Agents.RunStep } + | { event: StepEvents.ON_AGENT_UPDATE; data: Agents.AgentUpdate } + | { event: StepEvents.ON_MESSAGE_DELTA; data: Agents.MessageDeltaEvent } + | { event: StepEvents.ON_REASONING_DELTA; data: Agents.ReasoningDeltaEvent } + | { event: StepEvents.ON_RUN_STEP_DELTA; data: Agents.RunStepDeltaEvent } + | { event: StepEvents.ON_RUN_STEP_COMPLETED; data: { result: Agents.ToolEndEvent } } + | { event: StepEvents.ON_SUMMARIZE_START; data: Agents.SummarizeStartEvent } + | { event: StepEvents.ON_SUMMARIZE_DELTA; data: Agents.SummarizeDeltaEvent } + | { event: StepEvents.ON_SUMMARIZE_COMPLETE; data: Agents.SummarizeCompleteEvent }; type MessageDeltaUpdate = { type: ContentTypes.TEXT; text: string; tool_call_ids?: string[] }; @@ -52,6 +50,7 @@ type AllContentTypes = | ContentTypes.TOOL_CALL | ContentTypes.IMAGE_FILE | ContentTypes.IMAGE_URL + | ContentTypes.SUMMARY | ContentTypes.ERROR; export default function useStepHandler({ @@ -138,7 +137,7 @@ export default function useStepHandler({ text: (currentContent.text || '') + contentPart.text, }; - if (contentPart.tool_call_ids != null) { + if ('tool_call_ids' in contentPart && contentPart.tool_call_ids != null) { update.tool_call_ids = contentPart.tool_call_ids; } updatedContent[index] = update; @@ -173,6 +172,13 @@ export default function useStepHandler({ updatedContent[index] = { ...currentContent, }; + } else if (contentType === ContentTypes.SUMMARY) { + const currentSummary = updatedContent[index] as SummaryContentPart | undefined; + const incoming = contentPart as SummaryContentPart; + updatedContent[index] = { + ...incoming, + content: [...(currentSummary?.content ?? []), ...(incoming.content ?? [])], + }; } else if (contentType === ContentTypes.TOOL_CALL && 'tool_call' in contentPart) { const existingContent = updatedContent[index] as Agents.ToolCallContent | undefined; const existingToolCall = existingContent?.tool_call; @@ -243,7 +249,7 @@ export default function useStepHandler({ }; const stepHandler = useCallback( - ({ event, data }: TStepEvent, submission: EventSubmission) => { + (stepEvent: TStepEvent, submission: EventSubmission) => { const messages = getMessages() || []; const { userMessage } = submission; let parentMessageId = userMessage.messageId; @@ -260,8 +266,8 @@ export default function useStepHandler({ initialContent = submission?.initialResponse?.content ?? initialContent; } - if (event === 'on_run_step') { - const runStep = data as Agents.RunStep; + if (stepEvent.event === StepEvents.ON_RUN_STEP) { + const runStep = stepEvent.data; let responseMessageId = runStep.runId ?? ''; if (responseMessageId === Constants.USE_PRELIM_RESPONSE_MESSAGE_ID) { responseMessageId = submission?.initialResponse?.messageId ?? ''; @@ -355,15 +361,38 @@ export default function useStepHandler({ setMessages(updatedMessages); } + if (runStep.summary != null) { + const summaryPart: SummaryContentPart = { + type: ContentTypes.SUMMARY, + content: [], + summarizing: true, + model: runStep.summary.model, + provider: runStep.summary.provider, + }; + + let updatedResponse = { ...(messageMap.current.get(responseMessageId) ?? response) }; + updatedResponse = updateContent( + updatedResponse, + contentIndex, + summaryPart, + false, + getStepMetadata(runStep), + ); + + messageMap.current.set(responseMessageId, updatedResponse); + const currentMessages = getMessages() || []; + setMessages([...currentMessages.slice(0, -1), updatedResponse]); + } + const bufferedDeltas = pendingDeltaBuffer.current.get(runStep.id); if (bufferedDeltas && bufferedDeltas.length > 0) { pendingDeltaBuffer.current.delete(runStep.id); for (const bufferedDelta of bufferedDeltas) { - stepHandler({ event: bufferedDelta.event, data: bufferedDelta.data }, submission); + stepHandler(bufferedDelta, submission); } } - } else if (event === 'on_agent_update') { - const { agent_update } = data as Agents.AgentUpdate; + } else if (stepEvent.event === StepEvents.ON_AGENT_UPDATE) { + const { agent_update } = stepEvent.data; let responseMessageId = agent_update.runId || ''; if (responseMessageId === Constants.USE_PRELIM_RESPONSE_MESSAGE_ID) { responseMessageId = submission?.initialResponse?.messageId ?? ''; @@ -385,7 +414,7 @@ export default function useStepHandler({ const updatedResponse = updateContent( response, currentIndex, - data, + stepEvent.data, false, agentUpdateMeta, ); @@ -393,8 +422,8 @@ export default function useStepHandler({ const currentMessages = getMessages() || []; setMessages([...currentMessages.slice(0, -1), updatedResponse]); } - } else if (event === 'on_message_delta') { - const messageDelta = data as Agents.MessageDeltaEvent; + } else if (stepEvent.event === StepEvents.ON_MESSAGE_DELTA) { + const messageDelta = stepEvent.data; const runStep = stepMap.current.get(messageDelta.id); let responseMessageId = runStep?.runId ?? ''; if (responseMessageId === Constants.USE_PRELIM_RESPONSE_MESSAGE_ID) { @@ -404,7 +433,7 @@ export default function useStepHandler({ if (!runStep || !responseMessageId) { const buffer = pendingDeltaBuffer.current.get(messageDelta.id) ?? []; - buffer.push({ event: 'on_message_delta', data: messageDelta }); + buffer.push({ event: StepEvents.ON_MESSAGE_DELTA, data: messageDelta }); pendingDeltaBuffer.current.set(messageDelta.id, buffer); return; } @@ -436,8 +465,8 @@ export default function useStepHandler({ const currentMessages = getMessages() || []; setMessages([...currentMessages.slice(0, -1), updatedResponse]); } - } else if (event === 'on_reasoning_delta') { - const reasoningDelta = data as Agents.ReasoningDeltaEvent; + } else if (stepEvent.event === StepEvents.ON_REASONING_DELTA) { + const reasoningDelta = stepEvent.data; const runStep = stepMap.current.get(reasoningDelta.id); let responseMessageId = runStep?.runId ?? ''; if (responseMessageId === Constants.USE_PRELIM_RESPONSE_MESSAGE_ID) { @@ -447,7 +476,7 @@ export default function useStepHandler({ if (!runStep || !responseMessageId) { const buffer = pendingDeltaBuffer.current.get(reasoningDelta.id) ?? []; - buffer.push({ event: 'on_reasoning_delta', data: reasoningDelta }); + buffer.push({ event: StepEvents.ON_REASONING_DELTA, data: reasoningDelta }); pendingDeltaBuffer.current.set(reasoningDelta.id, buffer); return; } @@ -479,8 +508,8 @@ export default function useStepHandler({ const currentMessages = getMessages() || []; setMessages([...currentMessages.slice(0, -1), updatedResponse]); } - } else if (event === 'on_run_step_delta') { - const runStepDelta = data as Agents.RunStepDeltaEvent; + } else if (stepEvent.event === StepEvents.ON_RUN_STEP_DELTA) { + const runStepDelta = stepEvent.data; const runStep = stepMap.current.get(runStepDelta.id); let responseMessageId = runStep?.runId ?? ''; if (responseMessageId === Constants.USE_PRELIM_RESPONSE_MESSAGE_ID) { @@ -490,7 +519,7 @@ export default function useStepHandler({ if (!runStep || !responseMessageId) { const buffer = pendingDeltaBuffer.current.get(runStepDelta.id) ?? []; - buffer.push({ event: 'on_run_step_delta', data: runStepDelta }); + buffer.push({ event: StepEvents.ON_RUN_STEP_DELTA, data: runStepDelta }); pendingDeltaBuffer.current.set(runStepDelta.id, buffer); return; } @@ -538,8 +567,8 @@ export default function useStepHandler({ setMessages(updatedMessages); } - } else if (event === 'on_run_step_completed') { - const { result } = data as unknown as { result: Agents.ToolEndEvent }; + } else if (stepEvent.event === StepEvents.ON_RUN_STEP_COMPLETED) { + const { result } = stepEvent.data; const { id: stepId } = result; @@ -581,6 +610,95 @@ export default function useStepHandler({ setMessages(updatedMessages); } + } else if (stepEvent.event === StepEvents.ON_SUMMARIZE_START) { + announcePolite({ message: 'summarize_started', isStatus: true }); + } else if (stepEvent.event === StepEvents.ON_SUMMARIZE_DELTA) { + const deltaData = stepEvent.data; + const runStep = stepMap.current.get(deltaData.id); + let responseMessageId = runStep?.runId ?? ''; + if (responseMessageId === Constants.USE_PRELIM_RESPONSE_MESSAGE_ID) { + responseMessageId = submission?.initialResponse?.messageId ?? ''; + parentMessageId = submission?.initialResponse?.parentMessageId ?? ''; + } + + if (!runStep || !responseMessageId) { + const buffer = pendingDeltaBuffer.current.get(deltaData.id) ?? []; + buffer.push({ event: StepEvents.ON_SUMMARIZE_DELTA, data: deltaData }); + pendingDeltaBuffer.current.set(deltaData.id, buffer); + return; + } + + const response = messageMap.current.get(responseMessageId); + if (response) { + const contentPart: SummaryContentPart = { + ...deltaData.delta.summary, + summarizing: true, + }; + + const contentIndex = runStep.index + initialContent.length; + const updatedResponse = updateContent( + response, + contentIndex, + contentPart, + false, + getStepMetadata(runStep), + ); + messageMap.current.set(responseMessageId, updatedResponse); + const currentMessages = getMessages() || []; + setMessages([...currentMessages.slice(0, -1), updatedResponse]); + } + } else if (stepEvent.event === StepEvents.ON_SUMMARIZE_COMPLETE) { + const completeData = stepEvent.data; + const completeRunStep = stepMap.current.get(completeData.id); + let completeMessageId = completeRunStep?.runId ?? ''; + if (completeMessageId === Constants.USE_PRELIM_RESPONSE_MESSAGE_ID) { + completeMessageId = submission?.initialResponse?.messageId ?? ''; + } + + const targetMessage = messageMap.current.get(completeMessageId); + if (!targetMessage || !Array.isArray(targetMessage.content)) { + return; + } + + const currentMessages = getMessages() || []; + const targetIndex = currentMessages.findIndex((m) => m.messageId === completeMessageId); + + if (completeData.error) { + announcePolite({ message: 'summarize_failed', isStatus: true }); + const filtered = targetMessage.content.filter( + (part) => + part?.type !== ContentTypes.SUMMARY || !(part as SummaryContentPart).summarizing, + ); + if (filtered.length !== targetMessage.content.length) { + const cleaned = { ...targetMessage, content: filtered }; + messageMap.current.set(completeMessageId, cleaned); + if (targetIndex >= 0) { + const updated = [...currentMessages]; + updated[targetIndex] = cleaned; + setMessages(updated); + } + } + } else { + announcePolite({ message: 'summarize_completed', isStatus: true }); + let didFinalize = false; + const updatedContent = targetMessage.content.map((part) => { + if (part?.type === ContentTypes.SUMMARY && (part as SummaryContentPart).summarizing) { + if (!completeData.summary) { + return part; + } + didFinalize = true; + return { ...completeData.summary, summarizing: false } as SummaryContentPart; + } + return part; + }); + if (didFinalize && targetIndex >= 0) { + const finalized = { ...targetMessage, content: updatedContent }; + messageMap.current.set(completeMessageId, finalized); + const updated = [...currentMessages]; + updated[targetIndex] = finalized; + setMessages(updated); + } + } } return () => { diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index 35d8300489..aabca5a142 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -4,6 +4,9 @@ "com_a11y_ai_composing": "The AI is still composing.", "com_a11y_end": "The AI has finished their reply.", "com_a11y_selected": "selected", + "com_a11y_summarize_completed": "Context summarized.", + "com_a11y_summarize_failed": "Summarization failed, continuing with available context.", + "com_a11y_summarize_started": "Summarizing context.", "com_a11y_start": "The AI has started their reply.", "com_agents_agent_card_label": "{{name}} agent. {{description}}", "com_agents_all": "All Agents", @@ -824,6 +827,7 @@ "com_ui_code": "Code", "com_ui_collapse": "Collapse", "com_ui_collapse_chat": "Collapse Chat", + "com_ui_collapse_summary": "Collapse Summary", "com_ui_collapse_thoughts": "Collapse Thoughts", "com_ui_command_placeholder": "Optional: Enter a command for the prompt or name will be used", "com_ui_command_usage_placeholder": "Select a Prompt by command or name", @@ -846,6 +850,7 @@ "com_ui_conversation": "conversation", "com_ui_conversation_label": "{{title}} conversation", "com_ui_conversation_not_found": "Conversation not found", + "com_ui_conversation_summarized": "Conversation summarized", "com_ui_conversations": "conversations", "com_ui_convo_archived": "Conversation archived", "com_ui_convo_delete_error": "Failed to delete conversation", @@ -856,6 +861,7 @@ "com_ui_copy_code": "Copy code", "com_ui_copy_link": "Copy link", "com_ui_copy_stack_trace": "Copy stack trace", + "com_ui_copy_summary": "Copy summary to clipboard", "com_ui_copy_thoughts_to_clipboard": "Copy thoughts to clipboard", "com_ui_copy_to_clipboard": "Copy to clipboard", "com_ui_copy_url_to_clipboard": "Copy URL to clipboard", @@ -977,6 +983,7 @@ "com_ui_examples": "Examples", "com_ui_expand": "Expand", "com_ui_expand_chat": "Expand Chat", + "com_ui_expand_summary": "Expand Summary", "com_ui_expand_thoughts": "Expand Thoughts", "com_ui_export_convo_modal": "Export Conversation Modal", "com_ui_feedback_more": "More...", @@ -1408,6 +1415,7 @@ "com_ui_storage": "Storage", "com_ui_storage_filter_sort": "Filter and Sort by Storage", "com_ui_submit": "Submit", + "com_ui_summarizing": "Summarizing...", "com_ui_support_contact": "Support Contact", "com_ui_support_contact_email": "Email", "com_ui_support_contact_email_invalid": "Please enter a valid email address", diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.ts index 01310a09c4..f9982a6e46 100644 --- a/packages/api/src/agents/__tests__/initialize.test.ts +++ b/packages/api/src/agents/__tests__/initialize.test.ts @@ -190,7 +190,7 @@ describe('initializeAgent — maxContextTokens', () => { db, ); - const expected = Math.round((modelDefault - maxOutputTokens) * 0.9); + const expected = Math.round((modelDefault - maxOutputTokens) * 0.95); expect(result.maxContextTokens).toBe(expected); }); @@ -222,7 +222,7 @@ describe('initializeAgent — maxContextTokens', () => { // optionalChainWithEmptyCheck(0, 200000, 18000) returns 0 (not null/undefined), // then Number(0) || 18000 = 18000 (the fallback default). expect(result.maxContextTokens).not.toBe(0); - const expected = Math.round((18000 - maxOutputTokens) * 0.9); + const expected = Math.round((18000 - maxOutputTokens) * 0.95); expect(result.maxContextTokens).toBe(expected); }); @@ -278,7 +278,59 @@ describe('initializeAgent — maxContextTokens', () => { db, ); - // Should NOT be overridden to Math.round((128000 - 4096) * 0.9) = 111,514 + // Should NOT be overridden to Math.round((128000 - 4096) * 0.95) = 117,709 expect(result.maxContextTokens).toBe(userValue); }); + + it('sets baseContextTokens to agentMaxContextNum minus maxOutputTokensNum', async () => { + const modelDefault = 200000; + const maxOutputTokens = 4096; + const { agent, req, res, loadTools, db } = createMocks({ + maxContextTokens: undefined, + modelDefault, + maxOutputTokens, + }); + + const result = await initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: true, + }, + db, + ); + + expect(result.baseContextTokens).toBe(modelDefault - maxOutputTokens); + }); + + it('clamps maxContextTokens to at least 1024 for tiny models', async () => { + const modelDefault = 1100; + const maxOutputTokens = 1050; + const { agent, req, res, loadTools, db } = createMocks({ + maxContextTokens: undefined, + modelDefault, + maxOutputTokens, + }); + + const result = await initializeAgent( + { + req, + res, + agent, + loadTools, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: true, + }, + db, + ); + + // baseContextTokens = 1100 - 1050 = 50, formula would give ~47.5 rounded + // but Math.max(1024, ...) clamps it + expect(result.maxContextTokens).toBe(1024); + }); }); diff --git a/packages/api/src/agents/__tests__/run-summarization.test.ts b/packages/api/src/agents/__tests__/run-summarization.test.ts new file mode 100644 index 0000000000..62f919787c --- /dev/null +++ b/packages/api/src/agents/__tests__/run-summarization.test.ts @@ -0,0 +1,296 @@ +import type { SummarizationConfig } from 'librechat-data-provider'; +import { createRun } from '~/agents/run'; + +// Mock winston logger +jest.mock('winston', () => ({ + createLogger: jest.fn(() => ({ + debug: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + info: jest.fn(), + })), + format: { combine: jest.fn(), colorize: jest.fn(), simple: jest.fn() }, + transports: { Console: jest.fn() }, +})); + +// Mock env utilities so header resolution doesn't fail +jest.mock('~/utils/env', () => ({ + resolveHeaders: jest.fn((opts: { headers: unknown }) => opts?.headers ?? {}), + createSafeUser: jest.fn(() => ({})), +})); + +// Mock Run.create to capture the graphConfig it receives +jest.mock('@librechat/agents', () => { + const actual = jest.requireActual('@librechat/agents'); + return { + ...actual, + Run: { + create: jest.fn().mockResolvedValue({ + processStream: jest.fn().mockResolvedValue(undefined), + }), + }, + }; +}); + +import { Run } from '@librechat/agents'; + +/** Minimal RunAgent factory */ +function makeAgent( + overrides?: Record, +): Record & { id: string; provider: string; model: string } { + return { + id: 'agent_1', + provider: 'openAI', + endpoint: 'openAI', + model: 'gpt-4o', + tools: [], + model_parameters: { model: 'gpt-4o' }, + maxContextTokens: 100_000, + toolContextMap: {}, + ...overrides, + }; +} + +/** Helper: call createRun and return the captured agentInputs array */ +async function callAndCapture( + opts: { + agents?: ReturnType[]; + summarizationConfig?: SummarizationConfig; + initialSummary?: { text: string; tokenCount: number }; + } = {}, +) { + const agents = opts.agents ?? [makeAgent()]; + const signal = new AbortController().signal; + + await createRun({ + agents: agents as never, + signal, + summarizationConfig: opts.summarizationConfig, + initialSummary: opts.initialSummary, + streaming: true, + streamUsage: true, + }); + + const createMock = Run.create as jest.Mock; + expect(createMock).toHaveBeenCalledTimes(1); + const callArgs = createMock.mock.calls[0][0]; + return callArgs.graphConfig.agents as Array>; +} + +beforeEach(() => { + jest.clearAllMocks(); +}); + +// --------------------------------------------------------------------------- +// Suite 1: reserveRatio +// --------------------------------------------------------------------------- +describe('reserveRatio', () => { + it('applies ratio from config using baseContextTokens, capped at maxContextTokens', async () => { + const agents = await callAndCapture({ + agents: [makeAgent({ baseContextTokens: 200_000, maxContextTokens: 200_000 })], + summarizationConfig: { reserveRatio: 0.03, provider: 'anthropic', model: 'claude' }, + }); + // Math.round(200000 * 0.97) = 194000, min(200000, 194000) = 194000 + expect(agents[0].maxContextTokens).toBe(194_000); + }); + + it('never exceeds user-configured maxContextTokens even when ratio computes higher', async () => { + const agents = await callAndCapture({ + agents: [makeAgent({ baseContextTokens: 200_000, maxContextTokens: 50_000 })], + summarizationConfig: { reserveRatio: 0.03, provider: 'anthropic', model: 'claude' }, + }); + // Math.round(200000 * 0.97) = 194000, but min(50000, 194000) = 50000 + expect(agents[0].maxContextTokens).toBe(50_000); + }); + + it('falls back to maxContextTokens when ratio is not set', async () => { + const agents = await callAndCapture({ + agents: [makeAgent({ maxContextTokens: 100_000, baseContextTokens: 200_000 })], + summarizationConfig: { provider: 'anthropic', model: 'claude' }, + }); + expect(agents[0].maxContextTokens).toBe(100_000); + }); + + it('falls back to maxContextTokens when ratio is 0', async () => { + const agents = await callAndCapture({ + agents: [makeAgent({ maxContextTokens: 100_000, baseContextTokens: 200_000 })], + summarizationConfig: { reserveRatio: 0, provider: 'anthropic', model: 'claude' }, + }); + expect(agents[0].maxContextTokens).toBe(100_000); + }); + + it('falls back to maxContextTokens when ratio is 1', async () => { + const agents = await callAndCapture({ + agents: [makeAgent({ maxContextTokens: 100_000, baseContextTokens: 200_000 })], + summarizationConfig: { reserveRatio: 1, provider: 'anthropic', model: 'claude' }, + }); + expect(agents[0].maxContextTokens).toBe(100_000); + }); + + it('falls back to maxContextTokens when baseContextTokens is undefined', async () => { + const agents = await callAndCapture({ + agents: [makeAgent({ maxContextTokens: 100_000 })], + summarizationConfig: { reserveRatio: 0.05, provider: 'anthropic', model: 'claude' }, + }); + expect(agents[0].maxContextTokens).toBe(100_000); + }); + + it('clamps to 1024 minimum but still capped at maxContextTokens', async () => { + const agents = await callAndCapture({ + agents: [makeAgent({ baseContextTokens: 500, maxContextTokens: 2000 })], + summarizationConfig: { reserveRatio: 0.99, provider: 'anthropic', model: 'claude' }, + }); + // Math.round(500 * 0.01) = 5 → clamped to 1024, min(2000, 1024) = 1024 + expect(agents[0].maxContextTokens).toBe(1024); + }); +}); + +// --------------------------------------------------------------------------- +// Suite 2: maxSummaryTokens passthrough +// --------------------------------------------------------------------------- +describe('maxSummaryTokens passthrough', () => { + it('forwards global maxSummaryTokens value', async () => { + const agents = await callAndCapture({ + summarizationConfig: { + provider: 'anthropic', + model: 'claude', + maxSummaryTokens: 4096, + }, + }); + const config = agents[0].summarizationConfig as Record; + expect(config.maxSummaryTokens).toBe(4096); + }); +}); + +// --------------------------------------------------------------------------- +// Suite 3: summarizationEnabled resolution +// --------------------------------------------------------------------------- +describe('summarizationEnabled resolution', () => { + it('true with provider + model + enabled', async () => { + const agents = await callAndCapture({ + summarizationConfig: { + enabled: true, + provider: 'anthropic', + model: 'claude-3-haiku', + }, + }); + expect(agents[0].summarizationEnabled).toBe(true); + }); + + it('false when provider is empty string', async () => { + const agents = await callAndCapture({ + summarizationConfig: { + enabled: true, + provider: '', + model: 'claude-3-haiku', + }, + }); + expect(agents[0].summarizationEnabled).toBe(false); + }); + + it('false when enabled is explicitly false', async () => { + const agents = await callAndCapture({ + summarizationConfig: { + enabled: false, + provider: 'anthropic', + model: 'claude-3-haiku', + }, + }); + expect(agents[0].summarizationEnabled).toBe(false); + }); + + it('true with self-summarize default when summarizationConfig is undefined', async () => { + const agents = await callAndCapture({ + summarizationConfig: undefined, + }); + expect(agents[0].summarizationEnabled).toBe(true); + const config = agents[0].summarizationConfig as Record; + expect(config.provider).toBe('openAI'); + expect(config.model).toBe('gpt-4o'); + }); +}); + +// --------------------------------------------------------------------------- +// Suite 4: summarizationConfig field passthrough +// --------------------------------------------------------------------------- +describe('summarizationConfig field passthrough', () => { + it('all fields pass through to agentInputs', async () => { + const agents = await callAndCapture({ + summarizationConfig: { + enabled: true, + trigger: { type: 'token_count', value: 8000 }, + provider: 'anthropic', + model: 'claude-3-haiku', + parameters: { temperature: 0.2 }, + prompt: 'Summarize this conversation', + updatePrompt: 'Update the existing summary with new messages', + reserveRatio: 0.1, + maxSummaryTokens: 4096, + }, + }); + const config = agents[0].summarizationConfig as Record; + expect(config).toBeDefined(); + expect(config.enabled).toBe(true); + expect(config.trigger).toEqual({ type: 'token_count', value: 8000 }); + expect(config.provider).toBe('anthropic'); + expect(config.model).toBe('claude-3-haiku'); + expect(config.parameters).toEqual({ temperature: 0.2 }); + expect(config.prompt).toBe('Summarize this conversation'); + expect(config.updatePrompt).toBe('Update the existing summary with new messages'); + expect(config.reserveRatio).toBe(0.1); + expect(config.maxSummaryTokens).toBe(4096); + }); + + it('uses self-summarize default when no config provided', async () => { + const agents = await callAndCapture({ + summarizationConfig: undefined, + }); + const config = agents[0].summarizationConfig as Record; + expect(config).toBeDefined(); + expect(config.enabled).toBe(true); + expect(config.provider).toBe('openAI'); + expect(config.model).toBe('gpt-4o'); + }); +}); + +// --------------------------------------------------------------------------- +// Suite 5: Multi-agent + per-agent overrides +// --------------------------------------------------------------------------- +describe('multi-agent + per-agent overrides', () => { + it('different agents get different effectiveMaxContextTokens', async () => { + const agents = await callAndCapture({ + agents: [ + makeAgent({ id: 'agent_1', baseContextTokens: 200_000, maxContextTokens: 100_000 }), + makeAgent({ id: 'agent_2', baseContextTokens: 100_000, maxContextTokens: 50_000 }), + ], + summarizationConfig: { + reserveRatio: 0.1, + provider: 'anthropic', + model: 'claude', + }, + }); + // agent_1: Math.round(200000 * 0.9) = 180000, but capped at user's maxContextTokens (100000) + expect(agents[0].maxContextTokens).toBe(100_000); + // agent_2: Math.round(100000 * 0.9) = 90000, but capped at user's maxContextTokens (50000) + expect(agents[1].maxContextTokens).toBe(50_000); + }); +}); + +// --------------------------------------------------------------------------- +// Suite 6: initialSummary passthrough +// --------------------------------------------------------------------------- +describe('initialSummary passthrough', () => { + it('forwarded to agent inputs', async () => { + const summary = { text: 'Previous conversation summary', tokenCount: 500 }; + const agents = await callAndCapture({ + initialSummary: summary, + summarizationConfig: { provider: 'anthropic', model: 'claude' }, + }); + expect(agents[0].initialSummary).toEqual(summary); + }); + + it('undefined when not provided', async () => { + const agents = await callAndCapture({}); + expect(agents[0].initialSummary).toBeUndefined(); + }); +}); diff --git a/packages/api/src/agents/__tests__/summarization.e2e.test.ts b/packages/api/src/agents/__tests__/summarization.e2e.test.ts new file mode 100644 index 0000000000..aaf5600e3b --- /dev/null +++ b/packages/api/src/agents/__tests__/summarization.e2e.test.ts @@ -0,0 +1,594 @@ +/** + * E2E Backend Integration Tests for Summarization + * + * Exercises the FULL LibreChat -> agents pipeline: + * LibreChat's createRun (@librechat/api) + * -> agents package Run.create (@librechat/agents) + * -> graph execution -> summarization node -> events + * + * Uses real AI providers, real formatAgentMessages, real token accounting. + * Tracks summaries both mid-run and between runs. + * + * Run from packages/api: + * npx jest summarization.e2e --no-coverage --testTimeout=180000 + * + * Requires real API keys in the environment (ANTHROPIC_API_KEY, OPENAI_API_KEY). + */ +import { + Providers, + Calculator, + GraphEvents, + ToolEndHandler, + ModelEndHandler, + createTokenCounter, + formatAgentMessages, + ChatModelStreamHandler, + createContentAggregator, +} from '@librechat/agents'; +import type { + SummarizeCompleteEvent, + MessageContentComplex, + SummaryContentBlock, + SummarizeStartEvent, + TokenCounter, + EventHandler, +} from '@librechat/agents'; +import { hydrateMissingIndexTokenCounts } from '~/utils'; +import { createRun } from '~/agents'; + +// @librechat/api opens a Redis connection on import; force exit after all tests. +afterAll(() => { + setTimeout(() => process.exit(0), 1000); +}); + +// --------------------------------------------------------------------------- +// Shared test infrastructure +// --------------------------------------------------------------------------- + +interface Spies { + onMessageDelta: jest.Mock; + onRunStep: jest.Mock; + onSummarizeStart: jest.Mock; + onSummarizeDelta: jest.Mock; + onSummarizeComplete: jest.Mock; +} + +type PayloadMessage = { + role: string; + content: string | Array>; +}; + +function getSummaryText(summary: SummaryContentBlock): string { + if (Array.isArray(summary.content)) { + return summary.content + .map((b: MessageContentComplex) => ('text' in b ? (b as { text: string }).text : '')) + .join(''); + } + return ''; +} + +function createSpies(): Spies { + return { + onMessageDelta: jest.fn(), + onRunStep: jest.fn(), + onSummarizeStart: jest.fn(), + onSummarizeDelta: jest.fn(), + onSummarizeComplete: jest.fn(), + }; +} + +function buildHandlers( + collectedUsage: ConstructorParameters[0], + aggregateContent: (params: { event: string; data: unknown }) => void, + spies: Spies, +): Record { + return { + [GraphEvents.TOOL_END]: new ToolEndHandler(), + [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage), + [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(), + [GraphEvents.ON_RUN_STEP]: { + handle: (event: string, data: unknown) => { + spies.onRunStep(event, data); + aggregateContent({ event, data }); + }, + }, + [GraphEvents.ON_RUN_STEP_COMPLETED]: { + handle: (event: string, data: unknown) => { + aggregateContent({ event, data }); + }, + }, + [GraphEvents.ON_RUN_STEP_DELTA]: { + handle: (event: string, data: unknown) => { + aggregateContent({ event, data }); + }, + }, + [GraphEvents.ON_MESSAGE_DELTA]: { + handle: (event: string, data: unknown, metadata?: Record) => { + spies.onMessageDelta(event, data, metadata); + aggregateContent({ event, data }); + }, + }, + [GraphEvents.TOOL_START]: { + handle: () => {}, + }, + [GraphEvents.ON_SUMMARIZE_START]: { + handle: (_event: string, data: unknown) => { + spies.onSummarizeStart(data); + }, + }, + [GraphEvents.ON_SUMMARIZE_DELTA]: { + handle: (_event: string, data: unknown) => { + spies.onSummarizeDelta(data); + aggregateContent({ event: GraphEvents.ON_SUMMARIZE_DELTA, data }); + }, + }, + [GraphEvents.ON_SUMMARIZE_COMPLETE]: { + handle: (_event: string, data: unknown) => { + spies.onSummarizeComplete(data); + }, + }, + }; +} + +function getDefaultModel(provider: string): string { + switch (provider) { + case Providers.ANTHROPIC: + return 'claude-haiku-4-5-20251001'; + case Providers.OPENAI: + return 'gpt-4.1-mini'; + default: + return 'gpt-4.1-mini'; + } +} + +// --------------------------------------------------------------------------- +// Turn runner — mirrors AgentClient.chatCompletion() message flow +// --------------------------------------------------------------------------- + +interface RunFullTurnParams { + payload: PayloadMessage[]; + agentProvider: string; + summarizationProvider: string; + summarizationModel?: string; + maxContextTokens: number; + instructions: string; + spies: Spies; + tokenCounter: TokenCounter; + model?: string; +} + +async function runFullTurn({ + payload, + agentProvider, + summarizationProvider, + summarizationModel, + maxContextTokens, + instructions, + spies, + tokenCounter, + model, +}: RunFullTurnParams) { + const collectedUsage: ConstructorParameters[0] = []; + const { contentParts, aggregateContent } = createContentAggregator(); + + const formatted = formatAgentMessages(payload as never, {}); + const { messages: initialMessages, summary: initialSummary } = formatted; + let { indexTokenCountMap } = formatted; + + indexTokenCountMap = hydrateMissingIndexTokenCounts({ + messages: initialMessages, + indexTokenCountMap: indexTokenCountMap as Record, + tokenCounter, + }); + + const abortController = new AbortController(); + const agent = { + id: `test-agent-${agentProvider}`, + name: 'Test Agent', + provider: agentProvider, + instructions, + tools: [new Calculator()], + maxContextTokens, + model_parameters: { + model: model || getDefaultModel(agentProvider), + streaming: true, + streamUsage: true, + }, + }; + + const summarizationConfig = { + enabled: true, + provider: summarizationProvider, + model: summarizationModel || getDefaultModel(summarizationProvider), + prompt: + 'You are a summarization assistant. Summarize the following conversation messages concisely, preserving key facts, decisions, and context needed to continue the conversation. Do not include preamble -- output only the summary.', + }; + + const run = await createRun({ + agents: [agent] as never, + messages: initialMessages, + indexTokenCountMap, + initialSummary, + runId: `e2e-${Date.now()}`, + signal: abortController.signal, + customHandlers: buildHandlers(collectedUsage, aggregateContent, spies) as never, + summarizationConfig, + tokenCounter, + }); + + const streamConfig = { + configurable: { thread_id: `e2e-${Date.now()}` }, + recursionLimit: 100, + streamMode: 'values' as const, + version: 'v2' as const, + }; + let result: unknown; + let processError: Error | undefined; + try { + result = await run.processStream({ messages: initialMessages }, streamConfig); + } catch (err) { + processError = err as Error; + } + const runMessages = run.getRunMessages() || []; + + return { + result, + processError, + runMessages, + collectedUsage, + contentParts, + indexTokenCountMap, + }; +} + +function getLastContent(runMessages: Array<{ content: string | unknown }>): string { + const last = runMessages[runMessages.length - 1]; + if (!last) { + return ''; + } + return typeof last.content === 'string' ? last.content : JSON.stringify(last.content); +} + +// --------------------------------------------------------------------------- +// Anthropic Tests +// --------------------------------------------------------------------------- + +const hasAnthropic = + process.env.ANTHROPIC_API_KEY != null && process.env.ANTHROPIC_API_KEY !== 'test'; +(hasAnthropic ? describe : describe.skip)('Anthropic Summarization E2E (LibreChat)', () => { + jest.setTimeout(180_000); + + const instructions = + 'You are an expert math tutor. You MUST use the calculator tool for ALL computations. Keep answers to 1-2 sentences.'; + + test('multi-turn triggers summarization, summary persists across runs', async () => { + const spies = createSpies(); + const tokenCounter = await createTokenCounter(); + const conversationPayload: PayloadMessage[] = []; + + const addTurn = async (userMsg: string, maxTokens: number) => { + conversationPayload.push({ role: 'user', content: userMsg }); + const result = await runFullTurn({ + payload: conversationPayload, + agentProvider: Providers.ANTHROPIC, + summarizationProvider: Providers.ANTHROPIC, + summarizationModel: 'claude-haiku-4-5-20251001', + maxContextTokens: maxTokens, + instructions, + spies, + tokenCounter, + }); + conversationPayload.push({ role: 'assistant', content: getLastContent(result.runMessages) }); + return result; + }; + + await addTurn('What is 12345 * 6789? Use the calculator.', 2000); + await addTurn( + 'Now divide that result by 137. Then multiply by 42. Calculator for each step.', + 2000, + ); + await addTurn( + 'Compute step by step: 1) 9876543 - 1234567 2) sqrt of result 3) Add 100. Calculator for each.', + 1500, + ); + await addTurn('What is 2^20? Calculator. Then list everything we calculated so far.', 800); + + if (spies.onSummarizeStart.mock.calls.length === 0) { + await addTurn('Calculate 355 / 113. Calculator.', 600); + } + if (spies.onSummarizeStart.mock.calls.length === 0) { + await addTurn('What is 999 * 999? Calculator.', 400); + } + + const startCalls = spies.onSummarizeStart.mock.calls.length; + const completeCalls = spies.onSummarizeComplete.mock.calls.length; + + expect(startCalls).toBeGreaterThanOrEqual(1); + expect(completeCalls).toBeGreaterThanOrEqual(1); + + const startPayload = spies.onSummarizeStart.mock.calls[0][0] as SummarizeStartEvent; + expect(startPayload.agentId).toBeDefined(); + expect(startPayload.provider).toBeDefined(); + expect(startPayload.messagesToRefineCount).toBeGreaterThan(0); + expect(startPayload.summaryVersion).toBeGreaterThanOrEqual(1); + + const completePayload = spies.onSummarizeComplete.mock.calls[0][0] as SummarizeCompleteEvent; + expect(completePayload.summary).toBeDefined(); + expect(getSummaryText(completePayload.summary!).length).toBeGreaterThan(10); + expect(completePayload.summary!.tokenCount).toBeGreaterThan(0); + expect(completePayload.summary!.tokenCount!).toBeLessThan(2000); + expect(completePayload.summary!.provider).toBeDefined(); + expect(completePayload.summary!.createdAt).toBeDefined(); + expect(completePayload.summary!.summaryVersion).toBeGreaterThanOrEqual(1); + + // --- Cross-run: persist summary -> formatAgentMessages -> new run --- + const summaryBlock = completePayload.summary!; + const crossRunPayload: PayloadMessage[] = [ + { + role: 'assistant', + content: [ + { + type: 'summary', + content: [{ type: 'text', text: getSummaryText(summaryBlock) }], + tokenCount: summaryBlock.tokenCount, + }, + ], + }, + conversationPayload[conversationPayload.length - 2], + conversationPayload[conversationPayload.length - 1], + { + role: 'user', + content: 'What was the first calculation we did? Verify with calculator.', + }, + ]; + + spies.onSummarizeStart.mockClear(); + spies.onSummarizeComplete.mockClear(); + + const crossRun = await runFullTurn({ + payload: crossRunPayload, + agentProvider: Providers.ANTHROPIC, + summarizationProvider: Providers.ANTHROPIC, + summarizationModel: 'claude-haiku-4-5-20251001', + maxContextTokens: 2000, + instructions, + spies, + tokenCounter, + }); + + console.log( + ` Cross-run: messages=${crossRun.runMessages.length}, content=${crossRun.contentParts.length}, deltas=${spies.onMessageDelta.mock.calls.length}`, + ); + // Content aggregator should have received response deltas even if getRunMessages is empty + expect(crossRun.contentParts.length + spies.onMessageDelta.mock.calls.length).toBeGreaterThan( + 0, + ); + }); + + test('tight context (maxContextTokens=200) does not infinite-loop', async () => { + const spies = createSpies(); + const tokenCounter = await createTokenCounter(); + const conversationPayload: PayloadMessage[] = []; + + conversationPayload.push({ role: 'user', content: 'What is 42 * 58? Calculator.' }); + const t1 = await runFullTurn({ + payload: conversationPayload, + agentProvider: Providers.ANTHROPIC, + summarizationProvider: Providers.ANTHROPIC, + summarizationModel: 'claude-haiku-4-5-20251001', + maxContextTokens: 2000, + instructions, + spies, + tokenCounter, + }); + conversationPayload.push({ role: 'assistant', content: getLastContent(t1.runMessages) }); + + conversationPayload.push({ role: 'user', content: 'Now compute 2436 + 1337. Calculator.' }); + const t2 = await runFullTurn({ + payload: conversationPayload, + agentProvider: Providers.ANTHROPIC, + summarizationProvider: Providers.ANTHROPIC, + summarizationModel: 'claude-haiku-4-5-20251001', + maxContextTokens: 2000, + instructions, + spies, + tokenCounter, + }); + conversationPayload.push({ role: 'assistant', content: getLastContent(t2.runMessages) }); + + conversationPayload.push({ role: 'user', content: 'What is 100 / 4? Calculator.' }); + + let error: Error | undefined; + try { + await runFullTurn({ + payload: conversationPayload, + agentProvider: Providers.ANTHROPIC, + summarizationProvider: Providers.ANTHROPIC, + summarizationModel: 'claude-haiku-4-5-20251001', + maxContextTokens: 200, + instructions, + spies, + tokenCounter, + }); + } catch (err) { + error = err as Error; + } + + // The key guarantee: the system terminates — no true infinite loop. + // With very tight context, the graph may either: + // 1. Complete normally (model responds within budget) + // 2. Hit recursion limit (bounded tool-call cycles) + // 3. Error with empty_messages (context too small for any message) + // All are valid termination modes. + if (error) { + const isCleanTermination = + error.message.includes('Recursion limit') || error.message.includes('empty_messages'); + + expect(isCleanTermination).toBe(true); + } + + // Summarization may or may not fire depending on whether the budget + // allows any messages before the graph terminates. With 200 tokens + // and instructions at ~100 tokens, there may be no room for history, + // which correctly skips summarization. + + console.log( + ` Tight context: summarize=${spies.onSummarizeStart.mock.calls.length}, error=${error?.message?.substring(0, 80) ?? 'none'}`, + ); + }); +}); + +// --------------------------------------------------------------------------- +// OpenAI Tests +// --------------------------------------------------------------------------- + +const hasOpenAI = process.env.OPENAI_API_KEY != null && process.env.OPENAI_API_KEY !== 'test'; +(hasOpenAI ? describe : describe.skip)('OpenAI Summarization E2E (LibreChat)', () => { + jest.setTimeout(180_000); + + const instructions = + 'You are a helpful math tutor. Use the calculator tool for ALL computations. Keep responses concise.'; + + test('multi-turn with cross-run summary continuity', async () => { + const spies = createSpies(); + const tokenCounter = await createTokenCounter(); + const conversationPayload: PayloadMessage[] = []; + + const addTurn = async (userMsg: string, maxTokens: number) => { + conversationPayload.push({ role: 'user', content: userMsg }); + const result = await runFullTurn({ + payload: conversationPayload, + agentProvider: Providers.OPENAI, + summarizationProvider: Providers.OPENAI, + summarizationModel: 'gpt-4.1-mini', + maxContextTokens: maxTokens, + instructions, + spies, + tokenCounter, + }); + conversationPayload.push({ role: 'assistant', content: getLastContent(result.runMessages) }); + return result; + }; + + await addTurn('What is 1234 * 5678? Calculator.', 2000); + await addTurn('Compute sqrt(7006652) with calculator.', 1500); + await addTurn('Calculate 99*101 and 2^15. Calculator for each.', 1200); + await addTurn('What is 314159 * 271828? Calculator. Remind me of all prior results.', 800); + + if (spies.onSummarizeStart.mock.calls.length === 0) { + await addTurn('Calculate 999999 / 7. Calculator.', 600); + } + if (spies.onSummarizeStart.mock.calls.length === 0) { + await addTurn('What is 42 + 58? Calculator.', 400); + } + if (spies.onSummarizeStart.mock.calls.length === 0) { + await addTurn('Calculate 7 * 13. Calculator.', 300); + } + if (spies.onSummarizeStart.mock.calls.length === 0) { + await addTurn('What is 100 - 37? Calculator.', 200); + } + + expect(spies.onSummarizeStart.mock.calls.length).toBeGreaterThanOrEqual(1); + expect(spies.onSummarizeComplete.mock.calls.length).toBeGreaterThanOrEqual(1); + + const complete = spies.onSummarizeComplete.mock.calls[0][0] as SummarizeCompleteEvent; + expect(getSummaryText(complete.summary!).length).toBeGreaterThan(10); + expect(complete.summary!.tokenCount).toBeGreaterThan(0); + expect(complete.summary!.summaryVersion).toBeGreaterThanOrEqual(1); + expect(complete.summary!.provider).toBe(Providers.OPENAI); + + const summaryBlock = complete.summary!; + const crossRunPayload: PayloadMessage[] = [ + { + role: 'assistant', + content: [ + { + type: 'summary', + content: [{ type: 'text', text: getSummaryText(summaryBlock) }], + tokenCount: summaryBlock.tokenCount, + }, + ], + }, + conversationPayload[conversationPayload.length - 2], + conversationPayload[conversationPayload.length - 1], + { role: 'user', content: 'What was the first number we calculated? Verify with calculator.' }, + ]; + + spies.onSummarizeStart.mockClear(); + spies.onSummarizeComplete.mockClear(); + + const crossRun = await runFullTurn({ + payload: crossRunPayload, + agentProvider: Providers.OPENAI, + summarizationProvider: Providers.OPENAI, + summarizationModel: 'gpt-4.1-mini', + maxContextTokens: 2000, + instructions, + spies, + tokenCounter, + }); + + console.log( + ` Cross-run: messages=${crossRun.runMessages.length}, content=${crossRun.contentParts.length}, deltas=${spies.onMessageDelta.mock.calls.length}`, + ); + expect(crossRun.contentParts.length + spies.onMessageDelta.mock.calls.length).toBeGreaterThan( + 0, + ); + }); +}); + +// --------------------------------------------------------------------------- +// Cross-provider: Anthropic agent, OpenAI summarizer +// --------------------------------------------------------------------------- + +const hasBothProviders = hasAnthropic && hasOpenAI; +(hasBothProviders ? describe : describe.skip)( + 'Cross-provider Summarization E2E (LibreChat)', + () => { + jest.setTimeout(180_000); + + const instructions = + 'You are a math assistant. Use the calculator for every computation. Be brief.'; + + test('Anthropic agent with OpenAI summarizer', async () => { + const spies = createSpies(); + const tokenCounter = await createTokenCounter(); + const conversationPayload: PayloadMessage[] = []; + + const addTurn = async (userMsg: string, maxTokens: number) => { + conversationPayload.push({ role: 'user', content: userMsg }); + const result = await runFullTurn({ + payload: conversationPayload, + agentProvider: Providers.ANTHROPIC, + summarizationProvider: Providers.OPENAI, + summarizationModel: 'gpt-4.1-mini', + maxContextTokens: maxTokens, + instructions, + spies, + tokenCounter, + }); + conversationPayload.push({ + role: 'assistant', + content: getLastContent(result.runMessages), + }); + return result; + }; + + await addTurn('Compute 54321 * 12345 using calculator.', 2000); + await addTurn('Now calculate 670592745 / 99991. Calculator.', 1500); + await addTurn('What is sqrt(670592745)? Calculator.', 1000); + await addTurn('Compute 2^32 with calculator. List all prior results.', 600); + + if (spies.onSummarizeStart.mock.calls.length === 0) { + await addTurn('13 * 17 * 19 = ? Calculator.', 400); + } + + expect(spies.onSummarizeComplete.mock.calls.length).toBeGreaterThanOrEqual(1); + const complete = spies.onSummarizeComplete.mock.calls[0][0] as SummarizeCompleteEvent; + + expect(complete.summary!.provider).toBe(Providers.OPENAI); + expect(complete.summary!.model).toBe('gpt-4.1-mini'); + expect(getSummaryText(complete.summary!).length).toBeGreaterThan(10); + }); + }, +); diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index af604beb81..f1eb4a41da 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -32,6 +32,8 @@ import { generateArtifactsPrompt } from '~/prompts'; import { getProviderConfig } from '~/endpoints'; import { primeResources } from './resources'; +const DEFAULT_RESERVE_RATIO = 0.05; + /** * Extended agent type with additional fields needed after initialization */ @@ -40,6 +42,8 @@ export type InitializedAgent = Agent & { attachments: IMongoFile[]; toolContextMap: Record; maxContextTokens: number; + /** Pre-ratio context budget (agentMaxContextNum - maxOutputTokensNum). Used by createRun to apply a configurable reserve ratio. */ + baseContextTokens?: number; useLegacyContent: boolean; resendFiles: boolean; tool_resources?: AgentToolResources; @@ -52,6 +56,8 @@ export type InitializedAgent = Agent & { toolDefinitions?: LCTool[]; /** Precomputed flag indicating if any tools have defer_loading enabled (for efficient runtime checks) */ hasDeferredTools?: boolean; + /** Maximum characters allowed in a single tool result before truncation. */ + maxToolResultChars?: number; }; /** @@ -302,7 +308,7 @@ export async function initializeAgent( hasDeferredTools: false, }; - const { getOptions, overrideProvider } = getProviderConfig({ + const { getOptions, overrideProvider, customEndpointConfig } = getProviderConfig({ provider, appConfig: req.config, }); @@ -396,11 +402,25 @@ export async function initializeAgent( const agentMaxContextNum = Number(agentMaxContextTokens) || 18000; const maxOutputTokensNum = Number(maxOutputTokens) || 0; + const baseContextTokens = agentMaxContextNum - maxOutputTokensNum; const finalAttachments: IMongoFile[] = (primedAttachments ?? []) .filter((a): a is TFile => a != null) .map((a) => a as unknown as IMongoFile); + const endpointConfigs = req.config?.endpoints; + const providerConfig = + customEndpointConfig ?? endpointConfigs?.[agent.provider as keyof typeof endpointConfigs]; + const providerMaxToolResultChars = + providerConfig != null && + typeof providerConfig === 'object' && + !Array.isArray(providerConfig) && + 'maxToolResultChars' in providerConfig + ? (providerConfig.maxToolResultChars as number | undefined) + : undefined; + const maxToolResultCharsResolved = + providerMaxToolResultChars ?? endpointConfigs?.all?.maxToolResultChars; + const initializedAgent: InitializedAgent = { ...agent, resendFiles, @@ -409,14 +429,16 @@ export async function initializeAgent( userMCPAuthMap, toolDefinitions, hasDeferredTools, + baseContextTokens, attachments: finalAttachments, toolContextMap: toolContextMap ?? {}, useLegacyContent: !!options.useLegacyContent, tools: (tools ?? []) as GenericTool[] & string[], + maxToolResultChars: maxToolResultCharsResolved, maxContextTokens: maxContextTokens != null && maxContextTokens > 0 ? maxContextTokens - : Math.round((agentMaxContextNum - maxOutputTokensNum) * 0.9), + : Math.max(1024, Math.round(baseContextTokens * (1 - DEFAULT_RESERVE_RATIO))), }; return initializedAgent; diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts index 189ef59469..20064374a2 100644 --- a/packages/api/src/agents/run.ts +++ b/packages/api/src/agents/run.ts @@ -2,7 +2,9 @@ import { Run, Providers, Constants } from '@librechat/agents'; import { providerEndpointMap, KnownEndpoints } from 'librechat-data-provider'; import type { BaseMessage } from '@langchain/core/messages'; import type { + SummarizationConfig as AgentSummarizationConfig, MultiAgentGraphConfig, + ContextPruningConfig, OpenAIClientOptions, StandardGraphConfig, LCToolRegistry, @@ -13,7 +15,7 @@ import type { LCTool, } from '@librechat/agents'; import type { IUser } from '@librechat/data-schemas'; -import type { Agent } from 'librechat-data-provider'; +import type { Agent, SummarizationConfig } from 'librechat-data-provider'; import type * as t from '~/types'; import { resolveHeaders, createSafeUser } from '~/utils/env'; @@ -162,6 +164,8 @@ export function getReasoningKey( type RunAgent = Omit & { tools?: GenericTool[]; maxContextTokens?: number; + /** Pre-ratio context budget from initializeAgent. */ + baseContextTokens?: number; useLegacyContent?: boolean; toolContextMap?: Record; toolRegistry?: LCToolRegistry; @@ -169,6 +173,13 @@ type RunAgent = Omit & { toolDefinitions?: LCTool[]; /** Precomputed flag indicating if any tools have defer_loading enabled */ hasDeferredTools?: boolean; + /** Optional per-agent summarization overrides */ + summarization?: SummarizationConfig; + /** + * Maximum characters allowed in a single tool result before truncation. + * Overrides the default computed from maxContextTokens. + */ + maxToolResultChars?: number; }; /** @@ -196,6 +207,8 @@ export async function createRun({ tokenCounter, customHandlers, indexTokenCountMap, + summarizationConfig, + initialSummary, streaming = true, streamUsage = true, }: { @@ -208,6 +221,9 @@ export async function createRun({ user?: IUser; /** Message history for extracting previously discovered tools */ messages?: BaseMessage[]; + summarizationConfig?: SummarizationConfig; + /** Cross-run summary from formatAgentMessages, forwarded to AgentContext */ + initialSummary?: { text: string; tokenCount: number }; } & Pick): Promise< Run > { @@ -228,6 +244,28 @@ export async function createRun({ const agentInputs: AgentInputs[] = []; const buildAgentContext = (agent: RunAgent) => { + const selfProvider = + (providerEndpointMap[ + agent.provider as keyof typeof providerEndpointMap + ] as unknown as string) ?? agent.provider; + const selfModel = agent.model_parameters?.model ?? (agent.model as string | undefined); + + const resolvedSummarizationConfig: SummarizationConfig | undefined = agent.summarization ?? + summarizationConfig ?? { + enabled: true, + provider: selfProvider, + model: selfModel, + }; + + const resolvedSummarizationProvider = resolvedSummarizationConfig?.provider; + const resolvedSummarizationModel = resolvedSummarizationConfig?.model; + const hasSummarizationProvider = + typeof resolvedSummarizationProvider === 'string' && + resolvedSummarizationProvider.trim().length > 0; + const hasSummarizationModel = + typeof resolvedSummarizationModel === 'string' && + resolvedSummarizationModel.trim().length > 0; + const provider = (providerEndpointMap[ agent.provider as keyof typeof providerEndpointMap @@ -299,6 +337,21 @@ export async function createRun({ } } + // Apply configurable reserve ratio when available, falling back to the + // pre-computed maxContextTokens from initializeAgent. + const reserveRatio = resolvedSummarizationConfig?.reserveRatio; + const ratioComputed = + reserveRatio != null && + reserveRatio > 0 && + reserveRatio < 1 && + agent.baseContextTokens != null + ? Math.max(1024, Math.round(agent.baseContextTokens * (1 - reserveRatio))) + : null; + const effectiveMaxContextTokens = + ratioComputed != null + ? Math.min(agent.maxContextTokens ?? ratioComputed, ratioComputed) + : agent.maxContextTokens; + const reasoningKey = getReasoningKey(provider, llmConfig, agent.endpoint); const agentInput: AgentInputs = { provider, @@ -310,9 +363,38 @@ export async function createRun({ instructions: systemContent, name: agent.name ?? undefined, toolRegistry: agent.toolRegistry, - maxContextTokens: agent.maxContextTokens, + maxContextTokens: effectiveMaxContextTokens, useLegacyContent: agent.useLegacyContent ?? false, discoveredTools: discoveredTools.size > 0 ? Array.from(discoveredTools) : undefined, + summarizationEnabled: + resolvedSummarizationConfig != null && + resolvedSummarizationConfig.enabled !== false && + hasSummarizationProvider && + hasSummarizationModel, + summarizationConfig: resolvedSummarizationConfig + ? ({ + trigger: + resolvedSummarizationConfig.trigger?.type && + resolvedSummarizationConfig.trigger?.value + ? { + type: resolvedSummarizationConfig.trigger.type, + value: resolvedSummarizationConfig.trigger.value, + } + : undefined, + provider: resolvedSummarizationConfig.provider, + model: resolvedSummarizationConfig.model, + parameters: resolvedSummarizationConfig.parameters, + prompt: resolvedSummarizationConfig.prompt, + updatePrompt: resolvedSummarizationConfig.updatePrompt, + reserveRatio: resolvedSummarizationConfig.reserveRatio, + maxSummaryTokens: resolvedSummarizationConfig.maxSummaryTokens, + } satisfies AgentSummarizationConfig) + : undefined, + initialSummary, + contextPruningConfig: resolvedSummarizationConfig?.contextPruning as + | ContextPruningConfig + | undefined, + maxToolResultChars: agent.maxToolResultChars, }; agentInputs.push(agentInput); }; diff --git a/packages/api/src/agents/usage.spec.ts b/packages/api/src/agents/usage.spec.ts index d0b065b8ff..cf9248cfe7 100644 --- a/packages/api/src/agents/usage.spec.ts +++ b/packages/api/src/agents/usage.spec.ts @@ -108,6 +108,46 @@ describe('recordCollectedUsage', () => { }); }); + describe('summarization usage segregation', () => { + it('keeps summarization usage out of message token rollups while still spending it', async () => { + const collectedUsage: UsageMetadata[] = [ + { + usage_type: 'message', + input_tokens: 120, + output_tokens: 40, + model: 'gpt-4', + }, + { + usage_type: 'summarization', + input_tokens: 30, + output_tokens: 12, + model: 'gpt-4.1-mini', + }, + ]; + + const result = await recordCollectedUsage(deps, { + ...baseParams, + collectedUsage, + }); + + expect(result).toEqual({ input_tokens: 120, output_tokens: 40 }); + expect(mockSpendTokens).toHaveBeenCalledTimes(2); + expect(mockSpendTokens).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ context: 'message', model: 'gpt-4' }), + expect.any(Object), + ); + expect(mockSpendTokens).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + context: 'summarization', + model: 'gpt-4.1-mini', + }), + expect.any(Object), + ); + }); + }); + describe('parallel execution (multiple agents)', () => { it('should handle parallel agents with independent input tokens', async () => { const collectedUsage: UsageMetadata[] = [ diff --git a/packages/api/src/agents/usage.ts b/packages/api/src/agents/usage.ts index c092702730..bdb701a0cb 100644 --- a/packages/api/src/agents/usage.ts +++ b/packages/api/src/agents/usage.ts @@ -73,15 +73,26 @@ export async function recordCollectedUsage( return; } - const firstUsage = collectedUsage[0]; + const messageUsages: UsageMetadata[] = []; + const summarizationUsages: UsageMetadata[] = []; + for (const usage of collectedUsage) { + if (usage == null) { + continue; + } + (usage.usage_type === 'summarization' ? summarizationUsages : messageUsages).push(usage); + } + + const firstUsage = messageUsages[0]; const input_tokens = - (firstUsage?.input_tokens || 0) + - (Number(firstUsage?.input_token_details?.cache_creation) || - Number(firstUsage?.cache_creation_input_tokens) || - 0) + - (Number(firstUsage?.input_token_details?.cache_read) || - Number(firstUsage?.cache_read_input_tokens) || - 0); + firstUsage == null + ? 0 + : (firstUsage.input_tokens || 0) + + (Number(firstUsage.input_token_details?.cache_creation) || + Number(firstUsage.cache_creation_input_tokens) || + 0) + + (Number(firstUsage.input_token_details?.cache_read) || + Number(firstUsage.cache_read_input_tokens) || + 0); let total_output_tokens = 0; @@ -90,7 +101,7 @@ export async function recordCollectedUsage( const allDocs: PreparedEntry[] = []; - for (const usage of collectedUsage) { + for (const usage of messageUsages) { if (!usage) { continue; } @@ -179,6 +190,97 @@ export async function recordCollectedUsage( } } + const summarizationDocs: PreparedEntry[] = []; + + for (const usage of summarizationUsages) { + if (!usage) { + continue; + } + + const cache_creation = + Number(usage.input_token_details?.cache_creation) || + Number(usage.cache_creation_input_tokens) || + 0; + const cache_read = + Number(usage.input_token_details?.cache_read) || Number(usage.cache_read_input_tokens) || 0; + + const txMetadata: TxMetadata = { + context: 'summarization', + balance, + transactions, + conversationId, + user, + endpointTokenConfig, + model: usage.model ?? model, + }; + + if (useBulk) { + const entries = + cache_creation > 0 || cache_read > 0 + ? prepareStructuredTokenSpend( + txMetadata, + { + promptTokens: { + input: usage.input_tokens, + write: cache_creation, + read: cache_read, + }, + completionTokens: usage.output_tokens, + }, + pricing, + ) + : prepareTokenSpend( + txMetadata, + { + promptTokens: usage.input_tokens, + completionTokens: usage.output_tokens, + }, + pricing, + ); + summarizationDocs.push(...entries); + continue; + } + + if (cache_creation > 0 || cache_read > 0) { + deps + .spendStructuredTokens(txMetadata, { + promptTokens: { + input: usage.input_tokens, + write: cache_creation, + read: cache_read, + }, + completionTokens: usage.output_tokens, + }) + .catch((err) => { + logger.error( + '[packages/api #recordCollectedUsage] Error spending structured summarization tokens', + err, + ); + }); + continue; + } + + deps + .spendTokens(txMetadata, { + promptTokens: usage.input_tokens, + completionTokens: usage.output_tokens, + }) + .catch((err) => { + logger.error( + '[packages/api #recordCollectedUsage] Error spending summarization tokens', + err, + ); + }); + } + + if (useBulk && summarizationDocs.length > 0) { + try { + await bulkWriteTransactions({ user, docs: summarizationDocs }, bulkWriteOps); + } catch (err) { + logger.error('[packages/api #recordCollectedUsage] Error in summarization bulk write', err); + } + } + return { input_tokens, output_tokens: total_output_tokens, diff --git a/packages/api/src/app/AppService.spec.ts b/packages/api/src/app/AppService.spec.ts index a7b5a46054..df607d612b 100644 --- a/packages/api/src/app/AppService.spec.ts +++ b/packages/api/src/app/AppService.spec.ts @@ -181,6 +181,43 @@ describe('AppService', () => { ); }); + it('should enable summarization when it is configured without enabled flag', async () => { + const config = { + summarization: { + prompt: 'Summarize with emphasis on next actions', + }, + } as Partial & { summarization: Record }; + + const result = await AppService({ config }); + expect(result).toEqual( + expect.objectContaining({ + summarization: expect.objectContaining({ + enabled: true, + prompt: 'Summarize with emphasis on next actions', + }), + }), + ); + }); + + it('should preserve explicit summarization disable flag', async () => { + const config = { + summarization: { + enabled: false, + prompt: 'Ignored while disabled', + }, + } as Partial & { summarization: Record }; + + const result = await AppService({ config }); + expect(result).toEqual( + expect.objectContaining({ + summarization: expect.objectContaining({ + enabled: false, + prompt: 'Ignored while disabled', + }), + }), + ); + }); + it('should load and format tools accurately with defined structure', async () => { const config = {}; diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index 5486b941eb..fadddb840d 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -65,12 +65,18 @@ export interface SerializableJobData { * ``` */ export interface UsageMetadata { + /** Logical usage bucket for accounting/reporting. Defaults to model response usage. */ + usage_type?: 'message' | 'summarization'; /** Total input tokens (prompt tokens) */ input_tokens?: number; /** Total output tokens (completion tokens) */ output_tokens?: number; + /** Total billed tokens when provided by the model/runtime */ + total_tokens?: number; /** Model identifier that generated this usage */ model?: string; + /** Provider identifier that generated this usage */ + provider?: string; /** * OpenAI-style cache token details. * Present for OpenAI models (GPT-4, o1, etc.) diff --git a/packages/api/src/utils/index.ts b/packages/api/src/utils/index.ts index a13a49146f..b89706248b 100644 --- a/packages/api/src/utils/index.ts +++ b/packages/api/src/utils/index.ts @@ -21,6 +21,7 @@ export * from './text'; export * from './yaml'; export * from './http'; export * from './tokens'; +export * from './tokenMap'; export * from './url'; export * from './message'; export * from './tracing'; diff --git a/packages/api/src/utils/tokenMap.ts b/packages/api/src/utils/tokenMap.ts new file mode 100644 index 0000000000..4536bd0e85 --- /dev/null +++ b/packages/api/src/utils/tokenMap.ts @@ -0,0 +1,44 @@ +import type { BaseMessage } from '@langchain/core/messages'; + +/** Signature for a function that counts tokens in a LangChain message. */ +export type TokenCounter = (message: BaseMessage) => number; + +/** + * Lazily fills missing token counts for formatted LangChain messages. + * Preserves precomputed counts and only computes undefined indices. + * + * This is used after `formatAgentMessages` to ensure every message index + * has a token count before passing `indexTokenCountMap` to the agent run. + */ +export function hydrateMissingIndexTokenCounts({ + messages, + indexTokenCountMap, + tokenCounter, +}: { + messages: BaseMessage[]; + indexTokenCountMap: Record | undefined; + tokenCounter: TokenCounter; +}): Record { + const hydratedMap: Record = {}; + + if (indexTokenCountMap) { + for (const [index, tokenCount] of Object.entries(indexTokenCountMap)) { + if (typeof tokenCount === 'number' && Number.isFinite(tokenCount) && tokenCount > 0) { + hydratedMap[Number(index)] = tokenCount; + } + } + } + + for (let i = 0; i < messages.length; i++) { + if ( + typeof hydratedMap[i] === 'number' && + Number.isFinite(hydratedMap[i]) && + hydratedMap[i] > 0 + ) { + continue; + } + hydratedMap[i] = tokenCounter(messages[i]); + } + + return hydratedMap; +} diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index f6de096577..4fc3a9d824 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -205,6 +205,8 @@ export const baseEndpointSchema = z.object({ .optional(), titleEndpoint: z.string().optional(), titlePromptTemplate: z.string().optional(), + /** Maximum characters allowed in a single tool result before truncation. */ + maxToolResultChars: z.number().positive().optional(), }); export type TBaseEndpoint = z.infer; @@ -948,6 +950,34 @@ export const memorySchema = z.object({ export type TMemoryConfig = DeepPartial>; +export const summarizationTriggerSchema = z.object({ + type: z.string(), + value: z.number().positive(), +}); + +export const contextPruningSchema = z.object({ + enabled: z.boolean().optional(), + keepLastAssistants: z.number().min(0).max(10).optional(), + softTrimRatio: z.number().min(0).max(1).optional(), + hardClearRatio: z.number().min(0).max(1).optional(), + minPrunableToolChars: z.number().min(0).optional(), +}); + +export const summarizationConfigSchema = z.object({ + enabled: z.boolean().optional(), + provider: z.string().optional(), + model: z.string().optional(), + parameters: z.record(z.any()).optional(), + trigger: summarizationTriggerSchema.optional(), + prompt: z.string().optional(), + updatePrompt: z.string().optional(), + reserveRatio: z.number().min(0).max(1).optional(), + maxSummaryTokens: z.number().positive().optional(), + contextPruning: contextPruningSchema.optional(), +}); + +export type SummarizationConfig = DeepPartial>; + const customEndpointsSchema = z.array(endpointSchema.partial()).optional(); export const configSchema = z.object({ @@ -956,6 +986,7 @@ export const configSchema = z.object({ ocr: ocrSchema.optional(), webSearch: webSearchSchema.optional(), memory: memorySchema.optional(), + summarization: summarizationConfigSchema.optional(), secureImageLinks: z.boolean().optional(), imageOutputType: z.nativeEnum(EImageOutputType).default(EImageOutputType.PNG), includedTools: z.array(z.string()).optional(), diff --git a/packages/data-provider/src/types/agents.ts b/packages/data-provider/src/types/agents.ts index ac3f464019..08f01fa973 100644 --- a/packages/data-provider/src/types/agents.ts +++ b/packages/data-provider/src/types/agents.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-namespace */ import { StepTypes, ContentTypes, ToolCallTypes } from './runs'; import type { TAttachment, TPlugin } from 'src/schemas'; -import type { FunctionToolCall } from './assistants'; +import type { FunctionToolCall, SummaryContentPart } from './assistants'; export namespace Agents { export type MessageType = 'human' | 'ai' | 'generic' | 'system' | 'function' | 'tool' | 'remove'; @@ -53,6 +53,8 @@ export namespace Agents { | MessageContentImageUrl | MessageContentVideoUrl | MessageContentInputAudio + | SummaryContentPart + | ToolCallContent // eslint-disable-next-line @typescript-eslint/no-explicit-any | (Record & { type?: ContentTypes | string }) // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -187,6 +189,7 @@ export namespace Agents { /** Group ID for parallel content - parts with same groupId are displayed in columns */ groupId?: number; // #new stepDetails: StepDetails; + summary?: SummaryContentPart; usage: null | object; }; @@ -313,6 +316,28 @@ export namespace Agents { | ContentTypes.VIDEO_URL | ContentTypes.INPUT_AUDIO | string; + + export interface SummarizeStartEvent { + agentId: string; + provider: string; + model?: string; + messagesToRefineCount: number; + summaryVersion: number; + } + + export interface SummarizeDeltaEvent { + id: string; + delta: { + summary: SummaryContentPart; + }; + } + + export interface SummarizeCompleteEvent { + id: string; + agentId: string; + summary?: SummaryContentPart; + error?: string; + } } export type ToolCallResult = { diff --git a/packages/data-provider/src/types/assistants.ts b/packages/data-provider/src/types/assistants.ts index 22072403d3..3bfe05c59f 100644 --- a/packages/data-provider/src/types/assistants.ts +++ b/packages/data-provider/src/types/assistants.ts @@ -521,6 +521,21 @@ export type ContentPart = ( export type TextData = (Text & PartMetadata) | undefined; +export type SummaryContentPart = { + type: ContentTypes.SUMMARY; + content?: Agents.MessageContentComplex[]; + tokenCount?: number; + summarizing?: boolean; + summaryVersion?: number; + model?: string; + provider?: string; + createdAt?: string; + boundary?: { + messageId: string; + contentIndex: number; + }; +}; + export type TMessageContentParts = | ({ type: ContentTypes.ERROR; @@ -545,6 +560,7 @@ export type TMessageContentParts = PartMetadata; } & ContentMetadata) | ({ type: ContentTypes.IMAGE_FILE; image_file: ImageFile & PartMetadata } & ContentMetadata) + | (SummaryContentPart & ContentMetadata) | (Agents.AgentUpdate & ContentMetadata) | (Agents.MessageContentImageUrl & ContentMetadata) | (Agents.MessageContentVideoUrl & ContentMetadata) diff --git a/packages/data-provider/src/types/runs.ts b/packages/data-provider/src/types/runs.ts index de61357b92..b159f99daf 100644 --- a/packages/data-provider/src/types/runs.ts +++ b/packages/data-provider/src/types/runs.ts @@ -8,6 +8,7 @@ export enum ContentTypes { VIDEO_URL = 'video_url', INPUT_AUDIO = 'input_audio', AGENT_UPDATE = 'agent_update', + SUMMARY = 'summary', ERROR = 'error', } @@ -24,3 +25,16 @@ export enum ToolCallTypes { /* Agents Tool Call */ TOOL_CALL = 'tool_call', } + +/** Event names dispatched by the agent graph and consumed by step handlers. */ +export enum StepEvents { + ON_RUN_STEP = 'on_run_step', + ON_AGENT_UPDATE = 'on_agent_update', + ON_MESSAGE_DELTA = 'on_message_delta', + ON_REASONING_DELTA = 'on_reasoning_delta', + ON_RUN_STEP_DELTA = 'on_run_step_delta', + ON_RUN_STEP_COMPLETED = 'on_run_step_completed', + ON_SUMMARIZE_START = 'on_summarize_start', + ON_SUMMARIZE_DELTA = 'on_summarize_delta', + ON_SUMMARIZE_COMPLETE = 'on_summarize_complete', +} diff --git a/packages/data-schemas/src/app/service.ts b/packages/data-schemas/src/app/service.ts index 9f9f521f59..91407b06c4 100644 --- a/packages/data-schemas/src/app/service.ts +++ b/packages/data-schemas/src/app/service.ts @@ -1,4 +1,8 @@ -import { EModelEndpoint, getConfigDefaults } from 'librechat-data-provider'; +import { + EModelEndpoint, + getConfigDefaults, + summarizationConfigSchema, +} from 'librechat-data-provider'; import type { TCustomConfig, FileSources, DeepPartial } from 'librechat-data-provider'; import type { AppConfig, FunctionTool } from '~/types/app'; import { loadDefaultInterface } from './interface'; @@ -9,6 +13,25 @@ import { processModelSpecs } from './specs'; import { loadMemoryConfig } from './memory'; import { loadEndpoints } from './endpoints'; import { loadOCRConfig } from './ocr'; +import logger from '~/config/winston'; + +function loadSummarizationConfig(config: DeepPartial): AppConfig['summarization'] { + const raw = config.summarization; + if (!raw || typeof raw !== 'object') { + return undefined; + } + + const parsed = summarizationConfigSchema.safeParse(raw); + if (!parsed.success) { + logger.warn('[AppService] Invalid summarization config', parsed.error.flatten()); + return undefined; + } + + return { + ...parsed.data, + enabled: parsed.data.enabled !== false, + }; +} export type Paths = { root: string; @@ -41,6 +64,7 @@ export const AppService = async (params?: { const ocr = loadOCRConfig(config.ocr); const webSearch = loadWebSearchConfig(config.webSearch); const memory = loadMemoryConfig(config.memory); + const summarization = loadSummarizationConfig(config); const filteredTools = config.filteredTools; const includedTools = config.includedTools; const fileStrategy = (config.fileStrategy ?? configDefaults.fileStrategy) as @@ -76,18 +100,19 @@ export const AppService = async (params?: { speech, balance, actions, - transactions, - mcpConfig: mcpServersConfig, - mcpSettings, webSearch, + mcpSettings, + transactions, fileStrategy, registration, filteredTools, includedTools, + summarization, availableTools, imageOutputType, interfaceConfig, turnstileConfig, + mcpConfig: mcpServersConfig, fileStrategies: config.fileStrategies, }; diff --git a/packages/data-schemas/src/types/app.ts b/packages/data-schemas/src/types/app.ts index 36891bfaec..3b3132c35a 100644 --- a/packages/data-schemas/src/types/app.ts +++ b/packages/data-schemas/src/types/app.ts @@ -11,6 +11,7 @@ import type { TCustomEndpoints, TAssistantEndpoint, TAnthropicEndpoint, + SummarizationConfig, } from 'librechat-data-provider'; export type JsonSchemaType = { @@ -29,6 +30,10 @@ export type ConvertJsonSchemaToZodOptions = { transformOneOfAnyOf?: boolean; }; +export type AppSummarizationConfig = SummarizationConfig & { + enabled: boolean; +}; + export interface FunctionTool { type: 'function'; function: { @@ -56,6 +61,8 @@ export interface AppConfig { }; /** Memory configuration */ memory?: TMemoryConfig; + /** Summarization configuration */ + summarization?: AppSummarizationConfig; /** Web search configuration */ webSearch?: TCustomConfig['webSearch']; /** File storage strategy ('local', 's3', 'firebase', 'azure_blob') */