🎐 perf: Coalesce Ready Background Completions by Default

This commit is contained in:
Danny Avila 2026-10-03 11:36:08 -04:00 • committed by Danny Avila
parent f36fa7c15f
commit 2b4e5b5acd
7 changed files with 120 additions and 18 deletions

View file

@ -893,6 +893,9 @@ endpoints:
# # Enabled by default; set false to keep background tools and subagents poll-only.
# backgroundTasks:
# completionWakeups: true
# # Deliver a conversation's ready results together in one wake-up. On by default; set false
# # during a rolling upgrade while replicas from releases without it still serve traffic.
# completionReceiptBatching: true
# # Allow owners to cooperatively cancel ordinary background tools, including BYOM Bash.
# ordinaryToolCancellation: false
# # "run_in_background" makes Code Interpreter tools eligible by default and enables per-tool MCP opt-in.

View file

@ -294,20 +294,41 @@ opportunistic, not one-turn-per-conversation election.
injection, rolling-upgrade isolation, and an 8-conversation × 4-result storage stress
harness. Its latency measures receipt admission, not model turns or deployment latency.
Receipt batching defaults off. Deploy compatible receipt and manual-poll consumers on
**every replica**, then enable:
Receipt batching is on by default. An explicit `completionReceiptBatching: false` stops
new v3 production; new receipts are then admitted as v2 with task-local delivery.
**Rolling upgrades.** No fleet-wide gate decides when v3 is safe. Replicas from releases
before receipt batching never advertise what they can read, so a new replica cannot prove
they are gone. Queue capability fencing keeps them from claiming v3 rows, but they still
serve manual polls, pending-task listings, and conversation deletion for any
conversation, and those paths read only v2 rows. While old and new replicas serve traffic together:
- An old replica's manual poll cannot see the v3 receipt, so its manual claim on the
message projection looks uncontested and it returns the result. Old polls never mark
receipt reconciliation, so the v3 collector treats that claim as speculative: once the
polling generation is no longer active, it releases the claim and delivers the result
again in a wake-up. The model sees one result twice.
- Old pending-task listings omit v3 deliveries.
- Conversation deletion on an old replica neither erases nor fences v3 receipts. Their
stored output remains until the delivery row expires.
No result is lost on these paths; new replicas still deliver every v3 receipt.
Single-instance deployments and stop-then-start rollouts never mix versions. For a rolling
upgrade from a release without receipt batching, keep it off until every replica runs a
release that has it, then remove the override in a second rollout:
```yaml
endpoints:
agents:
backgroundTasks:
completionReceiptBatching: true
completionReceiptBatching: false
```
Older manual-poll workers cannot read v3 ownership. Queue capability fencing alone does
not protect that path. Disabling batching stops new v3 production, not existing
ownership. Do not downgrade poll consumers while v3 receipts remain, including delivered
receipts. A successor lease resumes interrupted cleanup before dispatch.
receipts. Because batching is the default, this applies to any deployment that ran with
it unset. A successor lease resumes interrupted cleanup before dispatch.
Every plan, receipt claim, and message claim carries a physical batch identity. Cleanup
matches that identity; its final plan deletion also matches a release identity and the

View file

@ -1,8 +1,14 @@
import mongoose from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { createMethods, createModels } from '@librechat/data-schemas';
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
createMethods,
createModels,
} from '@librechat/data-schemas';
import type { AgentTriggerDeliveryPersistence, AgentTriggerService } from './service';
import type { AgentTriggerFetch } from './host';
import { createBackgroundToolCompletionWakeupHandler } from '../backgroundCompletionWakeup';
import { __resetShutdownStateForTests } from '../../app/shutdown';
import { prepareAgentTriggerDelivery } from './delivery';
import { createAgentTriggerEnvelope } from './envelope';
@ -245,4 +251,57 @@ describe('durable trigger delivery integration', () => {
expect(settled.every((delivery) => delivery.status === 'succeeded')).toBe(true);
expect(new Set(receipts.map(({ deliveryKey }) => deliveryKey)).size).toBe(4);
});
it.each([
['unset', undefined, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3],
['true', true, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3],
['false', false, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
])(
'admits background completions for receipt batching %s with the matching capability',
async (_label, completionReceiptBatching, requiredWorkerCapability) => {
const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(
async () => new Response('{}', { status: 503 }),
);
const active = createAgentTriggerService({
methods: createMethods(mongoose) as ReturnType<typeof createMethods> &
AgentTriggerDeliveryPersistence,
fetch: fetcher,
mintToken: () => 'trigger-token',
deliveryOptions: { concurrency: 1, tickMs: 60_000 },
});
service = active;
await active.initialize({
address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
...(completionReceiptBatching != null && { completionReceiptBatching }),
});
const preregister = createBackgroundToolCompletionWakeupHandler(
active.enqueue,
active.retire,
active.renewProducerLease,
undefined,
undefined,
active.getBackgroundCompletionReceiptBatching,
);
const admission = await preregister({
taskId: 'task-1',
toolCallId: 'call-1',
toolName: 'slow_tool',
userId: new mongoose.Types.ObjectId().toString(),
tenantId: 'tenant-1',
conversationId: 'conversation-1',
parentMessageId: 'response-1',
parentAgentId: 'agent_parent_1',
createdAt: Date.now(),
});
expect(admission).not.toBe(false);
const rows = await mongoose.models.AgentTriggerDelivery.find({
'envelope.event.payload.taskId': 'task-1',
})
.select('requiredWorkerCapability')
.lean<{ requiredWorkerCapability?: string }[]>();
expect(rows).toEqual([expect.objectContaining({ requiredWorkerCapability })]);
},
);
});

View file

@ -213,12 +213,17 @@ describe('durable agent trigger service', () => {
await service.stop();
});
it('keeps receipt batching disabled until the deployment gate is explicitly enabled', async () => {
it('batches receipts by default and honors an explicit opt-out', async () => {
const service = createAgentTriggerService({ mintToken: () => 'test' });
expect(service.getBackgroundCompletionReceiptBatching()).toBe(false);
await service.initialize({ address: '127.0.0.1', completionReceiptBatching: true });
expect(service.getBackgroundCompletionReceiptBatching()).toBe(true);
await service.initialize({ address: '127.0.0.1' });
expect(service.getBackgroundCompletionReceiptBatching()).toBe(true);
await service.stop();
const optedOut = createAgentTriggerService({ mintToken: () => 'test' });
await optedOut.initialize({ address: '127.0.0.1', completionReceiptBatching: false });
expect(optedOut.getBackgroundCompletionReceiptBatching()).toBe(false);
await optedOut.stop();
});
describe('waiting completion deliveries', () => {

View file

@ -1,3 +1,4 @@
import { AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT } from 'librechat-data-provider';
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
@ -356,7 +357,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}
let boundOrigin: string | undefined;
let backgroundCompletionResultBatchSize = 8;
let backgroundCompletionReceiptBatching = false;
let backgroundCompletionReceiptBatching = AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT;
let completionWaitMaxIntervalMs = WAITING_RETRY_CAP_MS;
let deliveryEngine: AgentTriggerDeliveryEngine | undefined;
let initializePromise: Promise<void> | undefined;
@ -630,7 +631,8 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
return {
initialize: (options = {}) => {
backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8;
backgroundCompletionReceiptBatching = options.completionReceiptBatching === true;
backgroundCompletionReceiptBatching =
options.completionReceiptBatching ?? AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT;
completionWaitMaxIntervalMs =
options.idlePolling?.completionWaitMaxIntervalMs ?? WAITING_RETRY_CAP_MS;
boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;

View file

@ -1089,13 +1089,19 @@ describe('agent background completion batch config', () => {
endpoints: { agents: { backgroundTasks: {} } },
});
expect(defaults.endpoints?.agents?.backgroundTasks?.completionResultBatchSize).toBe(8);
expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(false);
expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(true);
expect(
configSchema.parse({
version: '1.0',
endpoints: { agents: { backgroundTasks: { completionReceiptBatching: true } } },
endpoints: { agents: { backgroundTasks: { completionReceiptBatching: false } } },
}).endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
).toBe(true);
).toBe(false);
expect(
configSchema.safeParse({
version: '1.0',
endpoints: { agents: { backgroundTasks: { completionReceiptBatching: 'false' } } },
}).success,
).toBe(false);
for (const completionResultBatchSize of [0, 17, 1.5]) {
expect(
@ -1204,7 +1210,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionResultBatchSize: 8,
completionReceiptBatching: false,
completionReceiptBatching: true,
completionWakeups: true,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
@ -1224,7 +1230,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionResultBatchSize: 8,
completionReceiptBatching: false,
completionReceiptBatching: true,
completionWakeups: false,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
@ -1244,7 +1250,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionResultBatchSize: 8,
completionReceiptBatching: false,
completionReceiptBatching: true,
completionWakeups: true,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: true,

View file

@ -37,6 +37,8 @@ import { scheduledMCPResourceBindingSchema } from './types/scheduleConsent';
export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT = 24 * 1024;
export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX = 64 * 1024;
/** Coalesces a conversation's ready background results into one wake-up turn. */
export const AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT: boolean = true;
export const AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT = 5_000;
import {
DEFAULT_MCP_APP_CSP_LIMITS,
@ -1779,8 +1781,12 @@ export const agentsEndpointSchema = baseEndpointSchema
.max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX)
.optional()
.default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT),
/** Enable only after every replica has compatible receipt/poll consumers. */
completionReceiptBatching: z.boolean().optional().default(false),
/** Set `false` while replicas older than receipt batching still serve
* traffic; those replicas cannot read batched (v3) receipts. */
completionReceiptBatching: z
.boolean()
.optional()
.default(AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT),
/** Maximum compatible sibling results in one continuation. */
completionResultBatchSize: z.number().int().min(1).max(16).optional().default(8),
/** Cooperative cancellation for process-local ordinary tools. Off