diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 24e5d0af3d..0054f5b436 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -24,7 +24,9 @@ const { Constants, FileSources, Tools, + ErrorTypes, ContentTypes, + isCompactedLeaf, excludedKeys, EModelEndpoint, mergeFileConfig, @@ -438,6 +440,10 @@ class BaseClient { const [overrideConvoId, overrideUserMessageId] = this.processOverideIds(); const { isEdited, isContinued } = opts; + if (opts.isCompaction === true) { + /** The leaf stands in for the user message and is already persisted. */ + this.skipSaveUserMessage = true; + } const user = opts.user ?? null; this.user = user; const saveOptions = this.getSaveOptions(); @@ -497,6 +503,64 @@ class BaseClient { }; } + /** + * The message a compaction turn hangs off: the branch's leaf, presented in + * the user-message slot so the response parents onto it and every consumer + * of `userMessage` (progress, job metadata, the abort path) keeps working. + * Identity fields only: the row stays in history untouched, and the object + * mirrored into job metadata must not carry the leaf's full content. + * @param {string} parentMessageId + * @returns {TMessage} + */ + getCompactionAnchor(parentMessageId) { + const leaf = this.currentMessages[this.currentMessages.length - 1]; + if (leaf == null || leaf.messageId !== parentMessageId) { + throw Object.assign(new Error('The message to compact up to was not found.'), { + statusCode: 404, + code: 'COMPACTION_ANCHOR_NOT_FOUND', + }); + } + if (isCompactedLeaf(leaf)) { + /** Typed so a stream that already started renders localized copy. */ + throw Object.assign( + new Error( + JSON.stringify({ + type: ErrorTypes.COMPACTION_SKIPPED, + reason: 'nothing_to_summarize', + }), + ), + { statusCode: 409, code: 'NOTHING_TO_COMPACT' }, + ); + } + return { + messageId: leaf.messageId, + parentMessageId: leaf.parentMessageId, + conversationId: leaf.conversationId, + isCreatedByUser: leaf.isCreatedByUser === true, + text: '', + }; + } + + /** + * The message the turn hangs off: a fresh user message, the edited message + * already in history, or (for a compaction) the branch's leaf. + * @returns {TMessage} + */ + resolveStartUserMessage({ opts, message, userMessageId, parentMessageId, conversationId }) { + if (opts.isCompaction) { + return this.getCompactionAnchor(parentMessageId); + } + if (opts.isEdited) { + return this.currentMessages[this.currentMessages.length - 2]; + } + return this.createUserMessage({ + messageId: userMessageId, + parentMessageId, + conversationId, + text: message, + }); + } + async handleStartMethods(message, opts) { const { user, @@ -510,14 +574,13 @@ class BaseClient { } = await this.setMessageOptions(opts); this.options.startupTelemetry?.mark('history_loaded'); - const userMessage = opts.isEdited - ? this.currentMessages[this.currentMessages.length - 2] - : this.createUserMessage({ - messageId: userMessageId, - parentMessageId, - conversationId, - text: message, - }); + const userMessage = this.resolveStartUserMessage({ + opts, + message, + userMessageId, + parentMessageId, + conversationId, + }); /** * Attach quoted excerpts (the "Add to chat" selections from `req.body.quotes`) @@ -527,7 +590,7 @@ class BaseClient { * merged into the model-facing text later, per message, in `buildMessages`, * keeping the stored `text` clean while the count stays consistent. */ - if (!opts.isEdited) { + if (!opts.isEdited && !opts.isCompaction) { const referencedQuotes = getReferencedQuotes(this.options.req?.body?.quotes); if (referencedQuotes != null) { userMessage.quotes = referencedQuotes; @@ -729,7 +792,7 @@ class BaseClient { } } this.continued = true; - } else { + } else if (opts.isCompaction !== true) { this.currentMessages.push(userMessage); } @@ -771,7 +834,8 @@ class BaseClient { this.assertBuiltModelBoundContent(payload); this.options.startupTelemetry?.mark('messages_built'); - if (tokenCountMap && tokenCountMap[userMessage.messageId]) { + /** A compaction anchor is the persisted leaf, whose own count must stay. */ + if (tokenCountMap && tokenCountMap[userMessage.messageId] && opts.isCompaction !== true) { userMessage.tokenCount = tokenCountMap[userMessage.messageId]; logger.debug('[BaseClient] userMessage', { messageId: userMessage.messageId, @@ -1092,6 +1156,7 @@ class BaseClient { } if ( + opts.isCompaction !== true && this.contextMeta?.calibrationRatio > 0 && this.contextMeta.calibrationRatio !== 1 && userMessage.tokenCount > 0 diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index ef1a17c066..2197e0e54d 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -3084,3 +3084,140 @@ describe('BaseClient', () => { }); }); }); + +describe('BaseClient compaction turns', () => { + const compactionOptions = { modelOptions: { model: 'gpt-4o-mini', temperature: 0 } }; + const compactionHistory = [ + { role: 'user', isCreatedByUser: true, text: 'Hello', messageId: 'u1' }, + { + role: 'assistant', + isCreatedByUser: false, + text: 'Hi', + messageId: 'a1', + parentMessageId: 'u1', + tokenCount: 7, + }, + ]; + let CompactClient; + + beforeEach(() => { + jest.clearAllMocks(); + CompactClient = initializeFakeClient(apiKey, compactionOptions, compactionHistory); + }); + + test('presents the leaf as the user message and never re-saves it', async () => { + const result = await CompactClient.handleStartMethods('', { + conversationId: 'convo-compact', + parentMessageId: 'a1', + preallocatedUserMessageId: 'a1', + isCompaction: true, + }); + + expect(result.userMessage).toEqual({ + messageId: 'a1', + parentMessageId: 'u1', + conversationId: undefined, + isCreatedByUser: false, + text: '', + }); + expect(CompactClient.skipSaveUserMessage).toBe(true); + /** History is loaded through the leaf, and nothing is appended to it. */ + expect(CompactClient.currentMessages.map((message) => message.messageId)).toEqual(['u1', 'a1']); + }); + + test('refuses a compaction whose anchor is not the loaded leaf', async () => { + await expect( + CompactClient.handleStartMethods('', { + conversationId: 'convo-compact', + parentMessageId: 'missing', + preallocatedUserMessageId: 'missing', + isCompaction: true, + }), + ).rejects.toMatchObject({ statusCode: 404, code: 'COMPACTION_ANCHOR_NOT_FOUND' }); + }); + + test('refuses to compact a branch whose leaf is already a finished compaction', async () => { + const compacted = [ + ...compactionHistory, + { + role: 'assistant', + isCreatedByUser: false, + text: '', + messageId: 's1', + parentMessageId: 'a1', + content: [ + { + type: ContentTypes.SUMMARY, + content: [{ type: ContentTypes.TEXT, text: 'checkpoint' }], + }, + ], + }, + ]; + CompactClient = initializeFakeClient(apiKey, compactionOptions, compacted); + + await expect( + CompactClient.handleStartMethods('', { + conversationId: 'convo-compact', + parentMessageId: 's1', + preallocatedUserMessageId: 's1', + isCompaction: true, + }), + ).rejects.toMatchObject({ + statusCode: 409, + code: 'NOTHING_TO_COMPACT', + message: JSON.stringify({ type: 'compaction_skipped', reason: 'nothing_to_summarize' }), + }); + }); + + test('lets an interrupted compaction be retried', async () => { + const interrupted = [ + ...compactionHistory, + { + role: 'assistant', + isCreatedByUser: false, + text: '', + messageId: 's1', + parentMessageId: 'a1', + content: [{ type: ContentTypes.SUMMARY, content: [], summarizing: true }], + }, + ]; + CompactClient = initializeFakeClient(apiKey, compactionOptions, interrupted); + + const result = await CompactClient.handleStartMethods('', { + conversationId: 'convo-compact', + parentMessageId: 's1', + preallocatedUserMessageId: 's1', + isCompaction: true, + }); + + expect(result.userMessage.messageId).toBe('s1'); + }); + + test('parents the response onto the leaf and persists only the response', async () => { + const saveSpy = jest.spyOn(CompactClient, 'saveMessageToDatabase').mockResolvedValue({}); + const updateSpy = jest.spyOn(CompactClient, 'updateMessageInDatabase').mockResolvedValue({}); + /** A calibration ratio and a counted anchor would, on an ordinary turn, + * rewrite the user message's persisted count; the leaf's must survive. */ + CompactClient.contextMeta = { calibrationRatio: 0.5 }; + CompactClient.buildMessages = jest.fn(async () => ({ + prompt: [], + tokenCountMap: { a1: 7 }, + promptTokens: 7, + })); + + const response = await CompactClient.sendMessage('', { + conversationId: 'convo-compact', + parentMessageId: 'a1', + preallocatedUserMessageId: 'a1', + isCompaction: true, + }); + + expect(response.parentMessageId).toBe('a1'); + expect(response.isCreatedByUser).toBe(false); + expect(saveSpy).toHaveBeenCalledTimes(1); + expect(saveSpy.mock.calls[0][0].messageId).toBe(response.messageId); + expect(updateSpy).not.toHaveBeenCalled(); + expect(CompactClient.currentMessages.map((message) => message.messageId)).toEqual(['u1', 'a1']); + expect(compactionHistory[1].tokenCount).toBe(7); + }); +}); diff --git a/api/package.json b/api/package.json index 7fa65a7ea7..1eb1194707 100644 --- a/api/package.json +++ b/api/package.json @@ -46,7 +46,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^3.8.0", + "@librechat/agents": "^3.8.1", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 451fb77cf5..42b44580fa 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -176,6 +176,7 @@ const { UsageEvents, Permissions, VisionModes, + ErrorTypes, ContentTypes, FileSources, ApprovalEvents, @@ -318,19 +319,48 @@ function captureRunContextMeta(client) { }); } +/** Text of a summary content part; empty for anything else. */ +function getSummaryPartText(part) { + if (part?.type !== ContentTypes.SUMMARY || !Array.isArray(part.content)) { + return ''; + } + return part.content + .map((block) => (typeof block?.text === 'string' ? block.text : '')) + .join('') + .trim(); +} + +/** + * A compaction turn's response is its summary. The run emits no text, so a + * completion without a usable summary part means the summarizer produced + * nothing. A run that already recorded why (an error part, e.g. a skipped + * compaction) persists with that explanation; one that ended with neither + * fails as a typed error instead of persisting an empty assistant message. + * @param {Array} contentParts + */ +function markCompactionSummary(contentParts) { + const summary = contentParts.find( + (part) => part?.failed !== true && getSummaryPartText(part).length > 0, + ); + if (summary != null) { + summary.initiatedBy = 'user'; + return; + } + if (contentParts.some((part) => part?.type === ContentTypes.ERROR)) { + return; + } + throw Object.assign(new Error(JSON.stringify({ type: ErrorTypes.COMPACTION_FAILED })), { + code: 'COMPACTION_FAILED', + }); +} + function getLatestEventActorSummary(contentParts) { if (!Array.isArray(contentParts)) { return undefined; } for (let index = contentParts.length - 1; index >= 0; index -= 1) { const part = contentParts[index]; - if (part?.type !== ContentTypes.SUMMARY || !Array.isArray(part.content)) { - continue; - } - const text = part.content - .map((block) => (typeof block?.text === 'string' ? block.text : '')) - .join('') - .trim(); + const text = getSummaryPartText(part); if (text.length === 0) { continue; } @@ -349,6 +379,10 @@ function getLatestEventActorSummary(contentParts) { * still reaches the logs through `getSafeErrorMetadata`. */ function getUserFacingRequestError(baseMessage, error, appConfig) { + /** Carries no model or user content, so it is safe under every filter. */ + if (error?.name === 'ManualSummarizationSkippedError') { + return JSON.stringify({ type: ErrorTypes.COMPACTION_SKIPPED, reason: error.reason }); + } const protectionEnabled = hasModelBoundContentProtection( appConfig?.filters, appConfig?.messageFilter?.pii, @@ -3502,10 +3536,18 @@ class AgentClient extends BaseClient { }); const completion = filterMalformedContentParts(this.contentParts); + if (this.isCompactionTurn()) { + markCompactionSummary(completion); + } const metadata = this.buildResponseMetadata(); return metadata ? { completion, metadata } : { completion }; } + /** A manual compaction runs the graph summarize-only: the summary is the response. */ + isCompactionTurn() { + return this.options?.req?.body?.compact === true; + } + /** * Assembles the response message `metadata`: Vertex thought signatures plus * the persisted context breakdown (Part A) and the usage/cost rollup (Part B), @@ -4466,8 +4508,16 @@ class AgentClient extends BaseClient { * synthetic prefix. Names NOT primed this turn still reconstruct from * history, preserving sticky manual re-priming across turns. */ - const manualSkillPrimes = this.options.agent?.manualSkillPrimes; - const alwaysApplySkillPrimes = this.options.agent?.alwaysApplySkillPrimes; + /** A compaction summarizes what was already said. No user turn was + * submitted, so it primes no skills into the transcript it is about + * to summarize and runs no memory pass over it. */ + const isCompactionTurn = this.isCompactionTurn(); + const manualSkillPrimes = isCompactionTurn + ? undefined + : this.options.agent?.manualSkillPrimes; + const alwaysApplySkillPrimes = isCompactionTurn + ? undefined + : this.options.agent?.alwaysApplySkillPrimes; const freshSkillPrimeNames = collectFreshSkillPrimeNames({ manualSkillPrimes, alwaysApplySkillPrimes, @@ -4602,7 +4652,7 @@ class AgentClient extends BaseClient { }); const memoryMessages = - this.processMemory && this.memoryPayload + this.processMemory && this.memoryPayload && !isCompactionTurn ? formatAgentMessages( stripActivityLabelParts(this.memoryPayload), undefined, @@ -4661,7 +4711,7 @@ class AgentClient extends BaseClient { // messages = addCacheControl(messages); // } - if (this.processMemory) { + if (this.processMemory && !isCompactionTurn) { memoryPromise = this.runMemory(memoryMessages); } @@ -4787,6 +4837,7 @@ class AgentClient extends BaseClient { traceContext: buildTraceContext(this.options), tenantId: resolveRequestTenantId(this.options.req ?? {}), summarizationConfig: appConfig?.summarization, + summarizeOnly: this.isCompactionTurn(), appConfig, tokenCounter, /** Bills subagent child-run model calls — foreground usage joins diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index a1bf9bdcc6..cfa0a383f2 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -149,6 +149,81 @@ function getPreliminaryResponseMessageId({ messageId, responseMessageId }) { return `${messageId.replace(/_+$/, '')}_`; } +/** + * Manual compaction runs as a summarize-only turn hung off the branch's leaf. + * It needs an existing branch to summarize, and it cannot be combined with the + * turn shapes that create or rewrite a user message. + * @returns {{ status: number, code: string, error: string } | null} + */ +function getCompactionRejection(req, { conversationId, parentMessageId }) { + if (req.config?.summarization?.enabled === false) { + return { + status: 400, + code: 'COMPACTION_DISABLED', + error: 'Context compaction is disabled for this deployment.', + }; + } + if (!conversationId || conversationId === Constants.NEW_CONVO) { + return { + status: 400, + code: 'INVALID_COMPACTION_REQUEST', + error: 'Compaction requires an existing conversation.', + }; + } + if ( + typeof parentMessageId !== 'string' || + parentMessageId.length === 0 || + parentMessageId === Constants.NO_PARENT + ) { + return { + status: 400, + code: 'INVALID_COMPACTION_REQUEST', + error: 'Compaction requires the message to compact up to.', + }; + } + const { isContinued, isRegenerate, editedContent, responseMessageId } = req.body ?? {}; + if (isContinued || isRegenerate || editedContent != null || responseMessageId) { + return { + status: 400, + code: 'INVALID_COMPACTION_REQUEST', + error: 'Compaction cannot be combined with an edit, regenerate, or continue.', + }; + } + return null; +} + +/** + * The leaf a compaction hangs off, in the user-message slot the job metadata + * and the abort path read before the branch is loaded. Identity only: the + * client validates the leaf against the history it loads anyway, and the job + * must not carry the leaf's content. + */ +function projectCompactionAnchor({ messageId, conversationId }) { + return { messageId, conversationId, text: '' }; +} + +/** + * The id the turn's user message is created under. A compaction creates no + * user message: its "user message" slot holds the leaf it summarizes up to, + * and a bound event turn keys the id off its task. + * @returns {string} + */ +function resolvePreallocatedUserMessageId({ + isCompaction, + parentMessageId, + eventTaskId, + overrideUserMessageId, + overrideParentMessageId, +}) { + if (isCompaction) { + return parentMessageId; + } + if (eventTaskId != null) { + return `${eventTaskId}:user`; + } + return overrideUserMessageId ?? overrideParentMessageId ?? crypto.randomUUID(); +} + function getPreliminaryUserMessage( { messageId, parentMessageId, text, quotes, files, manualSkills, alwaysAppliedSkills }, conversationId, @@ -292,7 +367,16 @@ async function saveErrorTurn( let userMessage = null; let errorMessageId = null; let errorParentMessageId = null; - if (isRegenerate) { + if (req.body?.compact === true) { + /** The anchor is the persisted leaf, never rewritten. Without the + * loaded anchor (the branch failed to load) there is nothing safe + * to parent an error row onto, so nothing is written. */ + if (liveUserMessage?.messageId == null) { + return; + } + errorMessageId = getPreliminaryResponseMessageId({ messageId: liveUserMessage.messageId }); + errorParentMessageId = liveUserMessage.messageId; + } else if (isRegenerate) { errorMessageId = typeof responseMessageId === 'string' && responseMessageId.length > 0 ? responseMessageId @@ -615,6 +699,22 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit const userId = req.user.id; const tenantId = req.user.tenantId; + const isCompaction = req.body?.compact === true; + if (isCompaction) { + const rejection = getCompactionRejection(req, { + conversationId: reqConversationId, + parentMessageId, + }); + if (rejection) { + startupTelemetry?.end('rejected'); + return sendGenerationJson( + res, + rejection.status, + { code: rejection.code, error: rejection.error }, + generationProtocolVersion, + ); + } + } const rawClientRequestId = req.body?.clientRequestId; if ( rawClientRequestId != null && @@ -1355,10 +1455,13 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit if (eventTaskId != null) { req._agentEventTaskId = eventTaskId; } - const preallocatedUserMessageId = - eventTaskId == null - ? (overrideUserMessageId ?? overrideParentMessageId ?? crypto.randomUUID()) - : `${eventTaskId}:user`; + const preallocatedUserMessageId = resolvePreallocatedUserMessageId({ + isCompaction, + parentMessageId, + eventTaskId, + overrideUserMessageId, + overrideParentMessageId, + }); const overrideConversationId = rawOverrideConversationId ? rawOverrideConversationId.split(Constants.COMMON_DIVIDER)[0] : undefined; @@ -1441,11 +1544,13 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit const endpointIconURL = getEndpointIconURL(req, endpointOption); const responseModel = getAgentResponseModel(req, endpointOption); - const preliminaryUserMessage = getPreliminaryUserMessage( - { ...req.body, messageId: preallocatedUserMessageId }, - conversationId, - req._agentEventTriggerProjection, - ); + const preliminaryUserMessage = isCompaction + ? projectCompactionAnchor({ messageId: parentMessageId, conversationId }) + : getPreliminaryUserMessage( + { ...req.body, messageId: preallocatedUserMessageId }, + conversationId, + req._agentEventTriggerProjection, + ); const job = await GenerationJobManager.createJob(streamId, userId, conversationId, { startupTelemetry, ...(recoveredSteerId && { recoveredSteerId }), @@ -1493,7 +1598,10 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit }), }), }), - ...(isRegenerate && { isRegenerate: true }), + /** A compaction is regenerate-shaped for every consumer of the job: + * no user message of its own, the response parented onto an + * existing message. A reconnecting client rebuilds it that way. */ + ...((isRegenerate || isCompaction) && { isRegenerate: true }), ...(scheduleId ? { scheduleId, @@ -2149,6 +2257,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit getReqData, isContinued, isRegenerate, + isCompaction, editedContent, conversationId, parentMessageId, diff --git a/api/server/routes/config.js b/api/server/routes/config.js index 7439ccec9d..4f584b40c5 100644 --- a/api/server/routes/config.js +++ b/api/server/routes/config.js @@ -308,6 +308,7 @@ router.get('/', async function (req, res) { langfuseFanoutEnabled, langfuseConnectionAccess, insightsEnabled: isEnabled(process.env.ENABLE_INSIGHTS), + compactionEnabled: appConfig?.summarization?.enabled !== false, ...(cloudFront ? { cloudFront } : {}), ...(rum ? { rum } : {}), fileUploadSseEnabled: isEnabled(process.env.FILE_UPLOAD_SSE_ENABLED), diff --git a/client/src/common/types.ts b/client/src/common/types.ts index eb3f0199ae..d7e2a81a24 100644 --- a/client/src/common/types.ts +++ b/client/src/common/types.ts @@ -334,6 +334,13 @@ export type TOptions = { isRegenerate?: boolean; isContinued?: boolean; isEdited?: boolean; + /** + * Manual context compaction: a summarize-only turn hung off the branch's + * leaf (`messageId`). Shaped like a regenerate on the client — no new user + * bubble, the response placeholder parents onto the leaf — and sent with + * `compact: true` so the server runs the graph summarize-only. + */ + compact?: boolean; overrideMessages?: t.TMessage[]; /** * Authoritative attachment list for this submission: a rerun replays the edited diff --git a/client/src/components/Chat/Input/TokenUsage/CompactAction.tsx b/client/src/components/Chat/Input/TokenUsage/CompactAction.tsx new file mode 100644 index 0000000000..df07a5b215 --- /dev/null +++ b/client/src/components/Chat/Input/TokenUsage/CompactAction.tsx @@ -0,0 +1,57 @@ +import { memo, useId } from 'react'; +import { ScrollText } from 'lucide-react'; +import { Button, Spinner, TooltipAnchor } from '@librechat/client'; +import { useLocalize } from '~/hooks'; + +interface CompactActionProps { + compact: () => void; + canCompact: boolean; + isCompacting: boolean; +} + +/** + * Manual context compaction, surfaced where the context usage is already read. + * Presentational on purpose: the popover unmounts on hide, so the hook that + * drives the submission lives in the always-mounted indicator above. + */ +function CompactAction({ compact, canCompact, isCompacting }: CompactActionProps) { + const localize = useLocalize(); + const descriptionId = useId(); + const description = localize('com_ui_context_compact_info'); + + return ( + <> + + {isCompacting ? ( + + ) : ( +