From 2b4e5b5acdde5e4170c45af1de52efaf8e9b22f4 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 3 Oct 2026 11:36:08 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=90=20perf:=20Coalesce=20Ready=20Backg?= =?UTF-8?q?round=20Completions=20by=20Default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- librechat.example.yaml | 3 + packages/api/src/agents/triggers/README.md | 29 +++++++-- .../triggers/delivery.integration.spec.ts | 61 ++++++++++++++++++- .../agents/triggers/service.delivery.spec.ts | 11 +++- packages/api/src/agents/triggers/service.ts | 6 +- packages/data-provider/src/config.spec.ts | 18 ++++-- packages/data-provider/src/config.ts | 10 ++- 7 files changed, 120 insertions(+), 18 deletions(-) diff --git a/librechat.example.yaml b/librechat.example.yaml index a338bf2b8a..e5bd5e4912 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -893,6 +893,9 @@ endpoints: # # Enabled by default; set false to keep background tools and subagents poll-only. # backgroundTasks: # completionWakeups: true + # # Deliver a conversation's ready results together in one wake-up. On by default; set false + # # during a rolling upgrade while replicas from releases without it still serve traffic. + # completionReceiptBatching: true # # Allow owners to cooperatively cancel ordinary background tools, including BYOM Bash. # ordinaryToolCancellation: false # # "run_in_background" makes Code Interpreter tools eligible by default and enables per-tool MCP opt-in. diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md index 3d266ce654..40f737f1a4 100644 --- a/packages/api/src/agents/triggers/README.md +++ b/packages/api/src/agents/triggers/README.md @@ -294,20 +294,41 @@ opportunistic, not one-turn-per-conversation election. injection, rolling-upgrade isolation, and an 8-conversation × 4-result storage stress harness. Its latency measures receipt admission, not model turns or deployment latency. -Receipt batching defaults off. Deploy compatible receipt and manual-poll consumers on -**every replica**, then enable: +Receipt batching is on by default. An explicit `completionReceiptBatching: false` stops +new v3 production; new receipts are then admitted as v2 with task-local delivery. + +**Rolling upgrades.** No fleet-wide gate decides when v3 is safe. Replicas from releases +before receipt batching never advertise what they can read, so a new replica cannot prove +they are gone. Queue capability fencing keeps them from claiming v3 rows, but they still +serve manual polls, pending-task listings, and conversation deletion for any +conversation, and those paths read only v2 rows. While old and new replicas serve traffic together: + +- An old replica's manual poll cannot see the v3 receipt, so its manual claim on the + message projection looks uncontested and it returns the result. Old polls never mark + receipt reconciliation, so the v3 collector treats that claim as speculative: once the + polling generation is no longer active, it releases the claim and delivers the result + again in a wake-up. The model sees one result twice. +- Old pending-task listings omit v3 deliveries. +- Conversation deletion on an old replica neither erases nor fences v3 receipts. Their + stored output remains until the delivery row expires. + +No result is lost on these paths; new replicas still deliver every v3 receipt. +Single-instance deployments and stop-then-start rollouts never mix versions. For a rolling +upgrade from a release without receipt batching, keep it off until every replica runs a +release that has it, then remove the override in a second rollout: ```yaml endpoints: agents: backgroundTasks: - completionReceiptBatching: true + completionReceiptBatching: false ``` Older manual-poll workers cannot read v3 ownership. Queue capability fencing alone does not protect that path. Disabling batching stops new v3 production, not existing ownership. Do not downgrade poll consumers while v3 receipts remain, including delivered -receipts. A successor lease resumes interrupted cleanup before dispatch. +receipts. Because batching is the default, this applies to any deployment that ran with +it unset. A successor lease resumes interrupted cleanup before dispatch. Every plan, receipt claim, and message claim carries a physical batch identity. Cleanup matches that identity; its final plan deletion also matches a release identity and the diff --git a/packages/api/src/agents/triggers/delivery.integration.spec.ts b/packages/api/src/agents/triggers/delivery.integration.spec.ts index 364edf0be9..a2afc1fac3 100644 --- a/packages/api/src/agents/triggers/delivery.integration.spec.ts +++ b/packages/api/src/agents/triggers/delivery.integration.spec.ts @@ -1,8 +1,14 @@ import mongoose from 'mongoose'; import { MongoMemoryServer } from 'mongodb-memory-server'; -import { createMethods, createModels } from '@librechat/data-schemas'; +import { + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, + createMethods, + createModels, +} from '@librechat/data-schemas'; import type { AgentTriggerDeliveryPersistence, AgentTriggerService } from './service'; import type { AgentTriggerFetch } from './host'; +import { createBackgroundToolCompletionWakeupHandler } from '../backgroundCompletionWakeup'; import { __resetShutdownStateForTests } from '../../app/shutdown'; import { prepareAgentTriggerDelivery } from './delivery'; import { createAgentTriggerEnvelope } from './envelope'; @@ -245,4 +251,57 @@ describe('durable trigger delivery integration', () => { expect(settled.every((delivery) => delivery.status === 'succeeded')).toBe(true); expect(new Set(receipts.map(({ deliveryKey }) => deliveryKey)).size).toBe(4); }); + + it.each([ + ['unset', undefined, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3], + ['true', true, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3], + ['false', false, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2], + ])( + 'admits background completions for receipt batching %s with the matching capability', + async (_label, completionReceiptBatching, requiredWorkerCapability) => { + const fetcher = jest.fn, Parameters>( + async () => new Response('{}', { status: 503 }), + ); + const active = createAgentTriggerService({ + methods: createMethods(mongoose) as ReturnType & + AgentTriggerDeliveryPersistence, + fetch: fetcher, + mintToken: () => 'trigger-token', + deliveryOptions: { concurrency: 1, tickMs: 60_000 }, + }); + service = active; + await active.initialize({ + address: { address: '127.0.0.1', family: 'IPv4', port: 3080 }, + ...(completionReceiptBatching != null && { completionReceiptBatching }), + }); + const preregister = createBackgroundToolCompletionWakeupHandler( + active.enqueue, + active.retire, + active.renewProducerLease, + undefined, + undefined, + active.getBackgroundCompletionReceiptBatching, + ); + + const admission = await preregister({ + taskId: 'task-1', + toolCallId: 'call-1', + toolName: 'slow_tool', + userId: new mongoose.Types.ObjectId().toString(), + tenantId: 'tenant-1', + conversationId: 'conversation-1', + parentMessageId: 'response-1', + parentAgentId: 'agent_parent_1', + createdAt: Date.now(), + }); + + expect(admission).not.toBe(false); + const rows = await mongoose.models.AgentTriggerDelivery.find({ + 'envelope.event.payload.taskId': 'task-1', + }) + .select('requiredWorkerCapability') + .lean<{ requiredWorkerCapability?: string }[]>(); + expect(rows).toEqual([expect.objectContaining({ requiredWorkerCapability })]); + }, + ); }); diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts index 2f0b9fb1d8..287340c1dd 100644 --- a/packages/api/src/agents/triggers/service.delivery.spec.ts +++ b/packages/api/src/agents/triggers/service.delivery.spec.ts @@ -213,12 +213,17 @@ describe('durable agent trigger service', () => { await service.stop(); }); - it('keeps receipt batching disabled until the deployment gate is explicitly enabled', async () => { + it('batches receipts by default and honors an explicit opt-out', async () => { const service = createAgentTriggerService({ mintToken: () => 'test' }); - expect(service.getBackgroundCompletionReceiptBatching()).toBe(false); - await service.initialize({ address: '127.0.0.1', completionReceiptBatching: true }); + expect(service.getBackgroundCompletionReceiptBatching()).toBe(true); + await service.initialize({ address: '127.0.0.1' }); expect(service.getBackgroundCompletionReceiptBatching()).toBe(true); await service.stop(); + + const optedOut = createAgentTriggerService({ mintToken: () => 'test' }); + await optedOut.initialize({ address: '127.0.0.1', completionReceiptBatching: false }); + expect(optedOut.getBackgroundCompletionReceiptBatching()).toBe(false); + await optedOut.stop(); }); describe('waiting completion deliveries', () => { diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts index 1dc249251c..86e86aacb1 100644 --- a/packages/api/src/agents/triggers/service.ts +++ b/packages/api/src/agents/triggers/service.ts @@ -1,3 +1,4 @@ +import { AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT } from 'librechat-data-provider'; import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, @@ -356,7 +357,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A } let boundOrigin: string | undefined; let backgroundCompletionResultBatchSize = 8; - let backgroundCompletionReceiptBatching = false; + let backgroundCompletionReceiptBatching = AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT; let completionWaitMaxIntervalMs = WAITING_RETRY_CAP_MS; let deliveryEngine: AgentTriggerDeliveryEngine | undefined; let initializePromise: Promise | undefined; @@ -630,7 +631,8 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A return { initialize: (options = {}) => { backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8; - backgroundCompletionReceiptBatching = options.completionReceiptBatching === true; + backgroundCompletionReceiptBatching = + options.completionReceiptBatching ?? AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT; completionWaitMaxIntervalMs = options.idlePolling?.completionWaitMaxIntervalMs ?? WAITING_RETRY_CAP_MS; boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin; diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index e8e2a15e5b..4081c33ffa 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -1089,13 +1089,19 @@ describe('agent background completion batch config', () => { endpoints: { agents: { backgroundTasks: {} } }, }); expect(defaults.endpoints?.agents?.backgroundTasks?.completionResultBatchSize).toBe(8); - expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(false); + expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(true); expect( configSchema.parse({ version: '1.0', - endpoints: { agents: { backgroundTasks: { completionReceiptBatching: true } } }, + endpoints: { agents: { backgroundTasks: { completionReceiptBatching: false } } }, }).endpoints?.agents?.backgroundTasks?.completionReceiptBatching, - ).toBe(true); + ).toBe(false); + expect( + configSchema.safeParse({ + version: '1.0', + endpoints: { agents: { backgroundTasks: { completionReceiptBatching: 'false' } } }, + }).success, + ).toBe(false); for (const completionResultBatchSize of [0, 17, 1.5]) { expect( @@ -1204,7 +1210,7 @@ describe('agent background task config', () => { } expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionResultBatchSize: 8, - completionReceiptBatching: false, + completionReceiptBatching: true, completionWakeups: true, completionResultMaxChars: 24 * 1024, ordinaryToolCancellation: false, @@ -1224,7 +1230,7 @@ describe('agent background task config', () => { } expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionResultBatchSize: 8, - completionReceiptBatching: false, + completionReceiptBatching: true, completionWakeups: false, completionResultMaxChars: 24 * 1024, ordinaryToolCancellation: false, @@ -1244,7 +1250,7 @@ describe('agent background task config', () => { } expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionResultBatchSize: 8, - completionReceiptBatching: false, + completionReceiptBatching: true, completionWakeups: true, completionResultMaxChars: 24 * 1024, ordinaryToolCancellation: true, diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 4e6d0342f7..44e6a247c7 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -37,6 +37,8 @@ import { scheduledMCPResourceBindingSchema } from './types/scheduleConsent'; export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT = 24 * 1024; export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX = 64 * 1024; +/** Coalesces a conversation's ready background results into one wake-up turn. */ +export const AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT: boolean = true; export const AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT = 5_000; import { DEFAULT_MCP_APP_CSP_LIMITS, @@ -1779,8 +1781,12 @@ export const agentsEndpointSchema = baseEndpointSchema .max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX) .optional() .default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT), - /** Enable only after every replica has compatible receipt/poll consumers. */ - completionReceiptBatching: z.boolean().optional().default(false), + /** Set `false` while replicas older than receipt batching still serve + * traffic; those replicas cannot read batched (v3) receipts. */ + completionReceiptBatching: z + .boolean() + .optional() + .default(AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT), /** Maximum compatible sibling results in one continuation. */ completionResultBatchSize: z.number().int().min(1).max(16).optional().default(8), /** Cooperative cancellation for process-local ordinary tools. Off