mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
🎐 perf: Coalesce Ready Background Completions by Default
This commit is contained in:
parent
f36fa7c15f
commit
2b4e5b5acd
7 changed files with 120 additions and 18 deletions
|
|
@ -893,6 +893,9 @@ endpoints:
|
|||
# # Enabled by default; set false to keep background tools and subagents poll-only.
|
||||
# backgroundTasks:
|
||||
# completionWakeups: true
|
||||
# # Deliver a conversation's ready results together in one wake-up. On by default; set false
|
||||
# # during a rolling upgrade while replicas from releases without it still serve traffic.
|
||||
# completionReceiptBatching: true
|
||||
# # Allow owners to cooperatively cancel ordinary background tools, including BYOM Bash.
|
||||
# ordinaryToolCancellation: false
|
||||
# # "run_in_background" makes Code Interpreter tools eligible by default and enables per-tool MCP opt-in.
|
||||
|
|
|
|||
|
|
@ -294,20 +294,41 @@ opportunistic, not one-turn-per-conversation election.
|
|||
injection, rolling-upgrade isolation, and an 8-conversation × 4-result storage stress
|
||||
harness. Its latency measures receipt admission, not model turns or deployment latency.
|
||||
|
||||
Receipt batching defaults off. Deploy compatible receipt and manual-poll consumers on
|
||||
**every replica**, then enable:
|
||||
Receipt batching is on by default. An explicit `completionReceiptBatching: false` stops
|
||||
new v3 production; new receipts are then admitted as v2 with task-local delivery.
|
||||
|
||||
**Rolling upgrades.** No fleet-wide gate decides when v3 is safe. Replicas from releases
|
||||
before receipt batching never advertise what they can read, so a new replica cannot prove
|
||||
they are gone. Queue capability fencing keeps them from claiming v3 rows, but they still
|
||||
serve manual polls, pending-task listings, and conversation deletion for any
|
||||
conversation, and those paths read only v2 rows. While old and new replicas serve traffic together:
|
||||
|
||||
- An old replica's manual poll cannot see the v3 receipt, so its manual claim on the
|
||||
message projection looks uncontested and it returns the result. Old polls never mark
|
||||
receipt reconciliation, so the v3 collector treats that claim as speculative: once the
|
||||
polling generation is no longer active, it releases the claim and delivers the result
|
||||
again in a wake-up. The model sees one result twice.
|
||||
- Old pending-task listings omit v3 deliveries.
|
||||
- Conversation deletion on an old replica neither erases nor fences v3 receipts. Their
|
||||
stored output remains until the delivery row expires.
|
||||
|
||||
No result is lost on these paths; new replicas still deliver every v3 receipt.
|
||||
Single-instance deployments and stop-then-start rollouts never mix versions. For a rolling
|
||||
upgrade from a release without receipt batching, keep it off until every replica runs a
|
||||
release that has it, then remove the override in a second rollout:
|
||||
|
||||
```yaml
|
||||
endpoints:
|
||||
agents:
|
||||
backgroundTasks:
|
||||
completionReceiptBatching: true
|
||||
completionReceiptBatching: false
|
||||
```
|
||||
|
||||
Older manual-poll workers cannot read v3 ownership. Queue capability fencing alone does
|
||||
not protect that path. Disabling batching stops new v3 production, not existing
|
||||
ownership. Do not downgrade poll consumers while v3 receipts remain, including delivered
|
||||
receipts. A successor lease resumes interrupted cleanup before dispatch.
|
||||
receipts. Because batching is the default, this applies to any deployment that ran with
|
||||
it unset. A successor lease resumes interrupted cleanup before dispatch.
|
||||
|
||||
Every plan, receipt claim, and message claim carries a physical batch identity. Cleanup
|
||||
matches that identity; its final plan deletion also matches a release identity and the
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { createMethods, createModels } from '@librechat/data-schemas';
|
||||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
createMethods,
|
||||
createModels,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { AgentTriggerDeliveryPersistence, AgentTriggerService } from './service';
|
||||
import type { AgentTriggerFetch } from './host';
|
||||
import { createBackgroundToolCompletionWakeupHandler } from '../backgroundCompletionWakeup';
|
||||
import { __resetShutdownStateForTests } from '../../app/shutdown';
|
||||
import { prepareAgentTriggerDelivery } from './delivery';
|
||||
import { createAgentTriggerEnvelope } from './envelope';
|
||||
|
|
@ -245,4 +251,57 @@ describe('durable trigger delivery integration', () => {
|
|||
expect(settled.every((delivery) => delivery.status === 'succeeded')).toBe(true);
|
||||
expect(new Set(receipts.map(({ deliveryKey }) => deliveryKey)).size).toBe(4);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['unset', undefined, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3],
|
||||
['true', true, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3],
|
||||
['false', false, AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
|
||||
])(
|
||||
'admits background completions for receipt batching %s with the matching capability',
|
||||
async (_label, completionReceiptBatching, requiredWorkerCapability) => {
|
||||
const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(
|
||||
async () => new Response('{}', { status: 503 }),
|
||||
);
|
||||
const active = createAgentTriggerService({
|
||||
methods: createMethods(mongoose) as ReturnType<typeof createMethods> &
|
||||
AgentTriggerDeliveryPersistence,
|
||||
fetch: fetcher,
|
||||
mintToken: () => 'trigger-token',
|
||||
deliveryOptions: { concurrency: 1, tickMs: 60_000 },
|
||||
});
|
||||
service = active;
|
||||
await active.initialize({
|
||||
address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
|
||||
...(completionReceiptBatching != null && { completionReceiptBatching }),
|
||||
});
|
||||
const preregister = createBackgroundToolCompletionWakeupHandler(
|
||||
active.enqueue,
|
||||
active.retire,
|
||||
active.renewProducerLease,
|
||||
undefined,
|
||||
undefined,
|
||||
active.getBackgroundCompletionReceiptBatching,
|
||||
);
|
||||
|
||||
const admission = await preregister({
|
||||
taskId: 'task-1',
|
||||
toolCallId: 'call-1',
|
||||
toolName: 'slow_tool',
|
||||
userId: new mongoose.Types.ObjectId().toString(),
|
||||
tenantId: 'tenant-1',
|
||||
conversationId: 'conversation-1',
|
||||
parentMessageId: 'response-1',
|
||||
parentAgentId: 'agent_parent_1',
|
||||
createdAt: Date.now(),
|
||||
});
|
||||
|
||||
expect(admission).not.toBe(false);
|
||||
const rows = await mongoose.models.AgentTriggerDelivery.find({
|
||||
'envelope.event.payload.taskId': 'task-1',
|
||||
})
|
||||
.select('requiredWorkerCapability')
|
||||
.lean<{ requiredWorkerCapability?: string }[]>();
|
||||
expect(rows).toEqual([expect.objectContaining({ requiredWorkerCapability })]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -213,12 +213,17 @@ describe('durable agent trigger service', () => {
|
|||
await service.stop();
|
||||
});
|
||||
|
||||
it('keeps receipt batching disabled until the deployment gate is explicitly enabled', async () => {
|
||||
it('batches receipts by default and honors an explicit opt-out', async () => {
|
||||
const service = createAgentTriggerService({ mintToken: () => 'test' });
|
||||
expect(service.getBackgroundCompletionReceiptBatching()).toBe(false);
|
||||
await service.initialize({ address: '127.0.0.1', completionReceiptBatching: true });
|
||||
expect(service.getBackgroundCompletionReceiptBatching()).toBe(true);
|
||||
await service.initialize({ address: '127.0.0.1' });
|
||||
expect(service.getBackgroundCompletionReceiptBatching()).toBe(true);
|
||||
await service.stop();
|
||||
|
||||
const optedOut = createAgentTriggerService({ mintToken: () => 'test' });
|
||||
await optedOut.initialize({ address: '127.0.0.1', completionReceiptBatching: false });
|
||||
expect(optedOut.getBackgroundCompletionReceiptBatching()).toBe(false);
|
||||
await optedOut.stop();
|
||||
});
|
||||
|
||||
describe('waiting completion deliveries', () => {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT } from 'librechat-data-provider';
|
||||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
|
|
@ -356,7 +357,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
}
|
||||
let boundOrigin: string | undefined;
|
||||
let backgroundCompletionResultBatchSize = 8;
|
||||
let backgroundCompletionReceiptBatching = false;
|
||||
let backgroundCompletionReceiptBatching = AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT;
|
||||
let completionWaitMaxIntervalMs = WAITING_RETRY_CAP_MS;
|
||||
let deliveryEngine: AgentTriggerDeliveryEngine | undefined;
|
||||
let initializePromise: Promise<void> | undefined;
|
||||
|
|
@ -630,7 +631,8 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
return {
|
||||
initialize: (options = {}) => {
|
||||
backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8;
|
||||
backgroundCompletionReceiptBatching = options.completionReceiptBatching === true;
|
||||
backgroundCompletionReceiptBatching =
|
||||
options.completionReceiptBatching ?? AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT;
|
||||
completionWaitMaxIntervalMs =
|
||||
options.idlePolling?.completionWaitMaxIntervalMs ?? WAITING_RETRY_CAP_MS;
|
||||
boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;
|
||||
|
|
|
|||
|
|
@ -1089,13 +1089,19 @@ describe('agent background completion batch config', () => {
|
|||
endpoints: { agents: { backgroundTasks: {} } },
|
||||
});
|
||||
expect(defaults.endpoints?.agents?.backgroundTasks?.completionResultBatchSize).toBe(8);
|
||||
expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(false);
|
||||
expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(true);
|
||||
expect(
|
||||
configSchema.parse({
|
||||
version: '1.0',
|
||||
endpoints: { agents: { backgroundTasks: { completionReceiptBatching: true } } },
|
||||
endpoints: { agents: { backgroundTasks: { completionReceiptBatching: false } } },
|
||||
}).endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
|
||||
).toBe(true);
|
||||
).toBe(false);
|
||||
expect(
|
||||
configSchema.safeParse({
|
||||
version: '1.0',
|
||||
endpoints: { agents: { backgroundTasks: { completionReceiptBatching: 'false' } } },
|
||||
}).success,
|
||||
).toBe(false);
|
||||
|
||||
for (const completionResultBatchSize of [0, 17, 1.5]) {
|
||||
expect(
|
||||
|
|
@ -1204,7 +1210,7 @@ describe('agent background task config', () => {
|
|||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionResultBatchSize: 8,
|
||||
completionReceiptBatching: false,
|
||||
completionReceiptBatching: true,
|
||||
completionWakeups: true,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: false,
|
||||
|
|
@ -1224,7 +1230,7 @@ describe('agent background task config', () => {
|
|||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionResultBatchSize: 8,
|
||||
completionReceiptBatching: false,
|
||||
completionReceiptBatching: true,
|
||||
completionWakeups: false,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: false,
|
||||
|
|
@ -1244,7 +1250,7 @@ describe('agent background task config', () => {
|
|||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionResultBatchSize: 8,
|
||||
completionReceiptBatching: false,
|
||||
completionReceiptBatching: true,
|
||||
completionWakeups: true,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: true,
|
||||
|
|
|
|||
|
|
@ -37,6 +37,8 @@ import { scheduledMCPResourceBindingSchema } from './types/scheduleConsent';
|
|||
|
||||
export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT = 24 * 1024;
|
||||
export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX = 64 * 1024;
|
||||
/** Coalesces a conversation's ready background results into one wake-up turn. */
|
||||
export const AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT: boolean = true;
|
||||
export const AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT = 5_000;
|
||||
import {
|
||||
DEFAULT_MCP_APP_CSP_LIMITS,
|
||||
|
|
@ -1779,8 +1781,12 @@ export const agentsEndpointSchema = baseEndpointSchema
|
|||
.max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX)
|
||||
.optional()
|
||||
.default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT),
|
||||
/** Enable only after every replica has compatible receipt/poll consumers. */
|
||||
completionReceiptBatching: z.boolean().optional().default(false),
|
||||
/** Set `false` while replicas older than receipt batching still serve
|
||||
* traffic; those replicas cannot read batched (v3) receipts. */
|
||||
completionReceiptBatching: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.default(AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT),
|
||||
/** Maximum compatible sibling results in one continuation. */
|
||||
completionResultBatchSize: z.number().int().min(1).max(16).optional().default(8),
|
||||
/** Cooperative cancellation for process-local ordinary tools. Off
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue