mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
🧺 fix: Bound Background Completion Batches (#16102)
* 🧺 fix: Coalesce Durable Background Wakeups
* fix: Bound replayed background result claims
* fix: Harden background result coalescing controls
* fix: Reconcile background result claims
* fix: close background batch ownership races
* fix: Keep completion batching atomic and bound serialized metadata
* refactor: Reuse canonical background claim contracts
* fix: Expose serialized claim budget in the persistence contract
* test: Preserve claimed V1 results across later background batches
This commit is contained in:
parent
8a3ab43965
commit
36efc7b4e2
13 changed files with 305 additions and 50 deletions
|
|
@ -705,7 +705,11 @@ if (cluster.isMaster) {
|
|||
await initializeMCPs();
|
||||
await initializeOAuthReconnectManager();
|
||||
await checkMigrations();
|
||||
await initializeAgentTriggerService({ address: server.address() });
|
||||
await initializeAgentTriggerService({
|
||||
address: server.address(),
|
||||
completionResultBatchSize:
|
||||
baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,
|
||||
});
|
||||
} catch (initErr) {
|
||||
logger.error(`Worker ${process.pid} post-listen initialization failed:`, initErr);
|
||||
process.exit(1);
|
||||
|
|
|
|||
|
|
@ -498,7 +498,11 @@ const startServer = async () => {
|
|||
if (inspectFlags || isEnabled(process.env.MEM_DIAG)) {
|
||||
memoryDiagnostics.start();
|
||||
}
|
||||
await initializeAgentTriggerService({ address: server.address() });
|
||||
await initializeAgentTriggerService({
|
||||
address: server.address(),
|
||||
completionResultBatchSize:
|
||||
appConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,
|
||||
});
|
||||
const scheduleEngineArmed = (await initializeScheduleEngine()) != null;
|
||||
scheduleEngineState = scheduleEngineArmed ? 'armed' : 'unavailable';
|
||||
if (!scheduleEngineArmed) {
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ const subagentCompletionAdapter = createSubagentCompletionWakeupResolver({
|
|||
const backgroundToolCompletionAdapter = createBackgroundToolCompletionWakeupResolver({
|
||||
methods,
|
||||
getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),
|
||||
getResultBatchSize: () => service.getBackgroundCompletionResultBatchSize(),
|
||||
});
|
||||
const eventActorAdapter = createAgentEventContinueResolver({
|
||||
methods,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
const mockCreateAgentTriggerService = jest.fn();
|
||||
const mockCreateBackgroundToolCompletionWakeupResolver = jest.fn(() => jest.fn());
|
||||
const mockGenerationJobManager = {
|
||||
supportsDetachedAgentEventActions: true,
|
||||
getJob: jest.fn(),
|
||||
|
|
@ -21,7 +22,8 @@ jest.mock('@librechat/api', () => ({
|
|||
createAgentTriggerService: (...args) => mockCreateAgentTriggerService(...args),
|
||||
createAgentContinuationResolver: jest.fn(() => jest.fn()),
|
||||
createAgentEventContinueResolver: jest.fn(() => jest.fn()),
|
||||
createBackgroundToolCompletionWakeupResolver: jest.fn(() => jest.fn()),
|
||||
createBackgroundToolCompletionWakeupResolver: (...args) =>
|
||||
mockCreateBackgroundToolCompletionWakeupResolver(...args),
|
||||
createSubagentCompletionWakeupResolver: jest.fn(() => jest.fn()),
|
||||
createAgentQueuedTurnLifecycle: jest.fn(() => mockQueuedTurnLifecycle),
|
||||
BACKGROUND_TOOL_COMPLETION_SOURCE: 'background-tool-completion',
|
||||
|
|
@ -39,8 +41,11 @@ describe('agent trigger service composition', () => {
|
|||
jest.resetModules();
|
||||
jest.clearAllMocks();
|
||||
mockGenerationJobManager.supportsDetachedAgentEventActions = true;
|
||||
let completionResultBatchSize = 8;
|
||||
mockCreateAgentTriggerService.mockReturnValue({
|
||||
initialize: jest.fn(),
|
||||
initialize: jest.fn(async (options) => {
|
||||
completionResultBatchSize = options.completionResultBatchSize ?? 8;
|
||||
}),
|
||||
stop: jest.fn(),
|
||||
dispatch: jest.fn(),
|
||||
enqueue: jest.fn(),
|
||||
|
|
@ -52,6 +57,7 @@ describe('agent trigger service composition', () => {
|
|||
prepareUserPurge: jest.fn(),
|
||||
cancelUserPurge: jest.fn(),
|
||||
purgeUser: jest.fn(),
|
||||
getBackgroundCompletionResultBatchSize: () => completionResultBatchSize,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -73,4 +79,16 @@ describe('agent trigger service composition', () => {
|
|||
mockGenerationJobManager.supportsDetachedAgentEventActions = false;
|
||||
expect(supportsDetachedActionCompletion()).toBe(false);
|
||||
});
|
||||
|
||||
it('injects the configured background completion batch size', async () => {
|
||||
const { initializeAgentTriggerService } = require('./triggers');
|
||||
await initializeAgentTriggerService({ address: 'local', completionResultBatchSize: 12 });
|
||||
|
||||
const resolverDeps = mockCreateBackgroundToolCompletionWakeupResolver.mock.calls[0][0];
|
||||
expect(resolverDeps.getResultBatchSize()).toBe(12);
|
||||
expect(mockCreateAgentTriggerService.mock.results[0].value.initialize).toHaveBeenCalledWith({
|
||||
address: 'local',
|
||||
completionResultBatchSize: 12,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -622,6 +622,54 @@ describe('background tool completion wakeups', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('passes the configured batch size and escaped metadata budget to the atomic claim', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
getResultBatchSize: () => 16,
|
||||
});
|
||||
await resolve(await envelope(), { idempotencyKey: 'delivery-1' });
|
||||
expect(methods.claimBackgroundToolResults).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
limit: 16,
|
||||
maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('releases projections acquired after receipt arbitration when admission fails', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'not_ready', results: [] });
|
||||
methods.claimAgentBackgroundToolResults.mockResolvedValueOnce({
|
||||
status: 'acquired',
|
||||
results: [
|
||||
{
|
||||
taskId: 'task-1',
|
||||
toolCallId: 'call-1',
|
||||
toolName: 'slow_tool',
|
||||
status: 'completed',
|
||||
output: 'done',
|
||||
},
|
||||
],
|
||||
} as never);
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
methods: methods as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' });
|
||||
if (prepared?.status !== 'ready') throw new Error('Expected ready');
|
||||
await prepared.releaseOnDefiniteFailure?.();
|
||||
expect(methods.releaseBackgroundToolResultClaims).toHaveBeenCalledWith({
|
||||
userId: 'user-1',
|
||||
conversationId: 'conversation-1',
|
||||
messageId: 'response-1',
|
||||
kind: 'wakeup',
|
||||
claimId: 'delivery-1',
|
||||
});
|
||||
expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('defers without claiming while the invoking generation is active', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
const resolve = createBackgroundToolCompletionWakeupResolver({
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { isEphemeralAgentId } from 'librechat-data-provider';
|
||||
import { backgroundResultMetadata, isEphemeralAgentId } from 'librechat-data-provider';
|
||||
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
|
||||
import type {
|
||||
AgentTriggerProducerLeaseStatus,
|
||||
AgentTriggerDeliveryMethods,
|
||||
BackgroundToolResultClaim,
|
||||
ConversationMethods,
|
||||
IMessage,
|
||||
MessageMethods,
|
||||
|
|
@ -63,25 +62,10 @@ export type PersistBackgroundToolCompletionResult = (
|
|||
) => Promise<boolean>;
|
||||
|
||||
type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
|
||||
Pick<MessageMethods, 'getMessages'> & {
|
||||
claimBackgroundToolResults(params: {
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
messageId: string;
|
||||
taskId: string;
|
||||
agentId?: string;
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
limit?: number;
|
||||
}): Promise<BackgroundToolResultClaim>;
|
||||
releaseBackgroundToolResultClaims(params: {
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
messageId: string;
|
||||
taskIds?: string[];
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
}): Promise<boolean>;
|
||||
Pick<
|
||||
MessageMethods,
|
||||
'getMessages' | 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'
|
||||
> & {
|
||||
getAgentTriggerDeliveryProducerLease(params: {
|
||||
deliveryKey: string;
|
||||
sourceId: string;
|
||||
|
|
@ -109,6 +93,7 @@ interface GenerationState {
|
|||
export interface BackgroundToolCompletionWakeupResolverDeps {
|
||||
methods: WakeupMethods;
|
||||
getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;
|
||||
getResultBatchSize?: () => number | undefined;
|
||||
}
|
||||
|
||||
function executionError(
|
||||
|
|
@ -245,13 +230,7 @@ function buildWakeupInput(
|
|||
results.length === 1
|
||||
? 'A background tool task has finished. Continue using its durable result below.'
|
||||
: `${results.length} background tool tasks have finished. Continue using their durable results below.`;
|
||||
const payload = results.map((result) => ({
|
||||
background_task_id: result.taskId,
|
||||
tool_call_id: result.toolCallId,
|
||||
tool: result.toolName,
|
||||
status: result.status,
|
||||
result: '',
|
||||
}));
|
||||
const payload = results.map(backgroundResultMetadata);
|
||||
let remaining = Math.max(
|
||||
0,
|
||||
BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - header.length - 1 - JSON.stringify(payload).length,
|
||||
|
|
@ -272,6 +251,7 @@ function buildWakeupInput(
|
|||
export function createBackgroundToolCompletionWakeupResolver({
|
||||
methods,
|
||||
getGenerationJob,
|
||||
getResultBatchSize,
|
||||
}: BackgroundToolCompletionWakeupResolverDeps): NonNullable<
|
||||
AgentTriggerExecutionHostDeps['prepareContinue']
|
||||
> {
|
||||
|
|
@ -345,7 +325,8 @@ export function createBackgroundToolCompletionWakeupResolver({
|
|||
agentId: envelope.target.agentId,
|
||||
kind: 'wakeup',
|
||||
claimId: context.idempotencyKey,
|
||||
limit: 1,
|
||||
limit: getResultBatchSize?.() ?? 8,
|
||||
maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,
|
||||
});
|
||||
if (claim.status === 'claimed') {
|
||||
const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ const DEFAULT_PURGE_RECOVERY_INTERVAL_MS = 30_000;
|
|||
const DEFAULT_PURGE_RECOVERY_LIMIT = 25;
|
||||
|
||||
export interface AgentTriggerServiceOptions {
|
||||
completionResultBatchSize?: number;
|
||||
address?: BoundAddress | string | null;
|
||||
}
|
||||
|
||||
|
|
@ -184,6 +185,7 @@ export interface AgentTriggerService {
|
|||
getBackgroundToolResultClaim: (
|
||||
input: Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>[0],
|
||||
) => ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>;
|
||||
getBackgroundCompletionResultBatchSize: () => number;
|
||||
releaseBackgroundToolResultClaims: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
|
||||
drainUser: (userId: string) => Promise<void>;
|
||||
prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;
|
||||
|
|
@ -272,6 +274,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
throw new TypeError('purgeRecoveryLimit must be a positive integer');
|
||||
}
|
||||
let boundOrigin: string | undefined;
|
||||
let backgroundCompletionResultBatchSize = 8;
|
||||
let deliveryEngine: AgentTriggerDeliveryEngine | undefined;
|
||||
let initializePromise: Promise<void> | undefined;
|
||||
let purgeRecoveryPromise: Promise<void> | undefined;
|
||||
|
|
@ -485,6 +488,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
|
||||
return {
|
||||
initialize: (options = {}) => {
|
||||
backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8;
|
||||
boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;
|
||||
if (deps.methods == null || deliveryReady) {
|
||||
return Promise.resolve();
|
||||
|
|
@ -628,6 +632,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
const getClaim = requireMethods().getAgentBackgroundToolResultClaim;
|
||||
return getClaim == null ? null : getClaim(input);
|
||||
}),
|
||||
getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize,
|
||||
releaseBackgroundToolResultClaims: (input) =>
|
||||
runAsSystem(async () => {
|
||||
const release = requireMethods().releaseAgentBackgroundToolResultClaims;
|
||||
|
|
|
|||
15
packages/data-provider/src/backgroundResults.ts
Normal file
15
packages/data-provider/src/backgroundResults.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
/** Shared serialized metadata shape for admission budgeting and continuation rendering. */
|
||||
export function backgroundResultMetadata(result: {
|
||||
taskId: string;
|
||||
toolCallId: string;
|
||||
toolName: string;
|
||||
status: 'completed' | 'error' | 'cancelled';
|
||||
}) {
|
||||
return {
|
||||
background_task_id: result.taskId,
|
||||
tool_call_id: result.toolCallId,
|
||||
tool: result.toolName,
|
||||
status: result.status,
|
||||
result: '',
|
||||
};
|
||||
}
|
||||
|
|
@ -509,6 +509,25 @@ describe('attached code environment user config schema', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('agent background completion batch config', () => {
|
||||
it('defaults and bounds automatic completion coalescing', () => {
|
||||
const defaults = configSchema.parse({
|
||||
version: '1.0',
|
||||
endpoints: { agents: { backgroundTasks: {} } },
|
||||
});
|
||||
expect(defaults.endpoints?.agents?.backgroundTasks?.completionResultBatchSize).toBe(8);
|
||||
|
||||
for (const completionResultBatchSize of [0, 17, 1.5]) {
|
||||
expect(
|
||||
configSchema.safeParse({
|
||||
version: '1.0',
|
||||
endpoints: { agents: { backgroundTasks: { completionResultBatchSize } } },
|
||||
}).success,
|
||||
).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('agent event runtime config', () => {
|
||||
it('accepts the routing choice and ignores removed rollout fields', () => {
|
||||
const result = configSchema.safeParse({
|
||||
|
|
@ -576,6 +595,7 @@ describe('agent background task config', () => {
|
|||
return;
|
||||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionResultBatchSize: 8,
|
||||
completionWakeups: true,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: false,
|
||||
|
|
@ -593,6 +613,7 @@ describe('agent background task config', () => {
|
|||
return;
|
||||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionResultBatchSize: 8,
|
||||
completionWakeups: false,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: false,
|
||||
|
|
@ -610,6 +631,7 @@ describe('agent background task config', () => {
|
|||
return;
|
||||
}
|
||||
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
|
||||
completionResultBatchSize: 8,
|
||||
completionWakeups: true,
|
||||
completionResultMaxChars: 24 * 1024,
|
||||
ordinaryToolCancellation: true,
|
||||
|
|
|
|||
|
|
@ -1477,6 +1477,9 @@ export const agentsEndpointSchema = baseEndpointSchema
|
|||
.max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX)
|
||||
.optional()
|
||||
.default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT),
|
||||
/** Maximum message-backed sibling results in one continuation.
|
||||
* Independent receipts retain task-local delivery ownership. */
|
||||
completionResultBatchSize: z.number().int().min(1).max(16).optional().default(8),
|
||||
/** Cooperative cancellation for process-local ordinary tools. Off
|
||||
* by default so existing deployments opt into the new control. */
|
||||
ordinaryToolCancellation: z.boolean().optional().default(false),
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ export { default as createPayload } from './createPayload';
|
|||
export * from './feedback';
|
||||
export * from './parameterSettings';
|
||||
export * from './agentToolOptions';
|
||||
export * from './backgroundResults';
|
||||
/* code-execution sandbox */
|
||||
export * from './codeEnvRef';
|
||||
export * from './code/worker';
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { Constants, RetentionMode } from 'librechat-data-provider';
|
||||
import { backgroundResultMetadata, Constants, RetentionMode } from 'librechat-data-provider';
|
||||
import type { AppConfig, IMessage } from '..';
|
||||
import {
|
||||
createMessageMethods,
|
||||
|
|
@ -1681,6 +1681,118 @@ describe('Message Operations', () => {
|
|||
status: 'claimed',
|
||||
claim: { kind: 'wakeup', claimId: 'delivery-1' },
|
||||
});
|
||||
/** V1 producers have no completionReceipt marker, but the V1 resolver
|
||||
* acquires resultClaim before delivery. A later batch must exclude it. */
|
||||
await expect(
|
||||
claimBackgroundToolResults({
|
||||
userId: 'user123',
|
||||
conversationId: mockMessageData.conversationId as string,
|
||||
messageId: 'msg123',
|
||||
taskId: 'task-3',
|
||||
kind: 'wakeup',
|
||||
claimId: 'delivery-2',
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
status: 'acquired',
|
||||
results: [{ taskId: 'task-3', output: 'three' }],
|
||||
});
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'keeps independent receipt ownership out of message batches (root=%s)',
|
||||
async (receiptRoot) => {
|
||||
const terminal = (taskId: string, receipt: boolean) => ({
|
||||
type: 'tool_call',
|
||||
tool_call: {
|
||||
id: taskId,
|
||||
output: taskId,
|
||||
backgroundTask: {
|
||||
taskId,
|
||||
toolName: 'tool',
|
||||
status: 'completed',
|
||||
completionWakeup: true,
|
||||
...(receipt ? { completionReceipt: true } : {}),
|
||||
},
|
||||
},
|
||||
});
|
||||
await saveMessage(mockCtx, {
|
||||
...mockMessageData,
|
||||
content: [
|
||||
terminal('root', receiptRoot),
|
||||
terminal('independent', true),
|
||||
terminal('message-only', false),
|
||||
],
|
||||
});
|
||||
const input = {
|
||||
userId: 'user123',
|
||||
conversationId: mockMessageData.conversationId as string,
|
||||
messageId: 'msg123',
|
||||
taskId: 'root',
|
||||
kind: 'wakeup' as const,
|
||||
claimId: 'delivery',
|
||||
limit: 16,
|
||||
};
|
||||
const claim = await claimBackgroundToolResults(input);
|
||||
expect(claim.status).toBe('acquired');
|
||||
if (claim.status !== 'acquired') throw new Error('Expected acquired claim');
|
||||
expect(claim.results.map((r) => r.taskId)).toEqual(
|
||||
receiptRoot ? ['root'] : ['root', 'message-only'],
|
||||
);
|
||||
await expect(claimBackgroundToolResults({ ...input, limit: 1 })).resolves.toEqual(claim);
|
||||
await expect(
|
||||
claimBackgroundToolResults({ ...input, taskId: 'independent', claimId: 'other' }),
|
||||
).resolves.toMatchObject({
|
||||
status: 'acquired',
|
||||
results: [{ taskId: 'independent' }],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it('bounds escaped metadata before claiming and freezes membership across competing same-owner preparations', async () => {
|
||||
const ids = Array.from({ length: 16 }, (_, i) => `${i}${'\u0001'.repeat(250)}`);
|
||||
const terminal = (id: string) => ({
|
||||
type: 'tool_call',
|
||||
tool_call: {
|
||||
id,
|
||||
output: 'done',
|
||||
backgroundTask: {
|
||||
taskId: id,
|
||||
toolName: '\u0001'.repeat(256),
|
||||
status: 'completed',
|
||||
completionWakeup: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
await saveMessage(mockCtx, { ...mockMessageData, content: ids.map(terminal) });
|
||||
const input = {
|
||||
userId: 'user123',
|
||||
conversationId: mockMessageData.conversationId as string,
|
||||
messageId: 'msg123',
|
||||
taskId: ids[15],
|
||||
kind: 'wakeup' as const,
|
||||
claimId: 'delivery',
|
||||
limit: 16,
|
||||
maxMetadataChars: 16 * 1024 - 256,
|
||||
};
|
||||
const attempts = await Promise.all([
|
||||
claimBackgroundToolResults(input),
|
||||
claimBackgroundToolResults(input),
|
||||
]);
|
||||
const claim = attempts.find((result) => result.status === 'acquired');
|
||||
if (claim?.status !== 'acquired') throw new Error('No claim elected');
|
||||
expect(claim.results.some((r) => r.taskId === ids[15])).toBe(true);
|
||||
expect(claim.results.length).toBeLessThan(16);
|
||||
expect(
|
||||
JSON.stringify(claim.results.map(backgroundResultMetadata)).length,
|
||||
).toBeLessThanOrEqual(input.maxMetadataChars);
|
||||
await Message.updateOne(
|
||||
{ messageId: 'msg123', user: 'user123' },
|
||||
{ $push: { content: terminal('late') } },
|
||||
);
|
||||
await expect(claimBackgroundToolResults({ ...input, limit: 1 })).resolves.toEqual(claim);
|
||||
await expect(
|
||||
claimBackgroundToolResults({ ...input, taskId: 'late', claimId: 'later' }),
|
||||
).resolves.toMatchObject({ status: 'acquired' });
|
||||
});
|
||||
|
||||
it('revalidates each sibling inside the atomic batch claim', async () => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
import { HITL_MESSAGE_FILTER_FIELDS, RetentionMode } from 'librechat-data-provider';
|
||||
import {
|
||||
backgroundResultMetadata,
|
||||
HITL_MESSAGE_FILTER_FIELDS,
|
||||
RetentionMode,
|
||||
} from 'librechat-data-provider';
|
||||
import type { DeleteResult, FilterQuery, Model, Types, UpdateQuery } from 'mongoose';
|
||||
import type { UserSubmittedMessageFieldPath } from 'librechat-data-provider';
|
||||
import type { SearchParams } from 'meilisearch';
|
||||
|
|
@ -749,6 +753,8 @@ export interface MessageMethods {
|
|||
/** Manual owner-process takeover after automatic delivery was retired. */
|
||||
allowUnfinished?: boolean;
|
||||
limit?: number;
|
||||
/** Includes JSON escaping, delimiters, and empty result fields. */
|
||||
maxMetadataChars?: number;
|
||||
}): Promise<BackgroundToolResultClaim>;
|
||||
releaseBackgroundToolResultClaims(params: {
|
||||
userId: string;
|
||||
|
|
@ -1423,7 +1429,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
|
|||
}
|
||||
}
|
||||
|
||||
const MAX_BACKGROUND_TOOL_RESULT_BATCH = 8;
|
||||
const MAX_BACKGROUND_TOOL_RESULT_BATCH = 16;
|
||||
/** Bounds the attachments-merge fence retries. Each lost round means a
|
||||
* concurrent writer changed the array, so re-reading converges. */
|
||||
const ATTACHMENT_MERGE_CAS_ATTEMPTS = 8;
|
||||
|
|
@ -1579,19 +1585,11 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
|
|||
claimId,
|
||||
generationId,
|
||||
allowUnfinished = false,
|
||||
limit = kind === 'wakeup' ? MAX_BACKGROUND_TOOL_RESULT_BATCH : 1,
|
||||
}: {
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
messageId?: string;
|
||||
taskId: string;
|
||||
agentId?: string;
|
||||
kind: 'manual' | 'wakeup';
|
||||
claimId: string;
|
||||
generationId?: string;
|
||||
allowUnfinished?: boolean;
|
||||
limit?: number;
|
||||
}): Promise<BackgroundToolResultClaim> {
|
||||
limit = kind === 'wakeup' ? 8 : 1,
|
||||
maxMetadataChars,
|
||||
}: Parameters<
|
||||
MessageMethods['claimBackgroundToolResults']
|
||||
>[0]): Promise<BackgroundToolResultClaim> {
|
||||
const requestedMessageId = messageId?.trim();
|
||||
const requestedGenerationId = generationId?.trim();
|
||||
if (
|
||||
|
|
@ -1605,7 +1603,11 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
|
|||
(requestedGenerationId.length === 0 || requestedGenerationId.length > 256)) ||
|
||||
(requestedGenerationId != null && kind !== 'manual') ||
|
||||
(allowUnfinished && kind !== 'manual') ||
|
||||
(kind !== 'manual' && kind !== 'wakeup')
|
||||
(kind !== 'manual' && kind !== 'wakeup') ||
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
(maxMetadataChars != null &&
|
||||
(!Number.isSafeInteger(maxMetadataChars) || maxMetadataChars < 1))
|
||||
) {
|
||||
throw new TypeError('Invalid background tool result claim');
|
||||
}
|
||||
|
|
@ -1665,6 +1667,8 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
|
|||
const requestedClaim = readBackgroundToolResultClaim(row, taskId);
|
||||
const replaying = requestedClaim?.kind === kind && requestedClaim.claimId === claimId;
|
||||
const candidates: string[] = [];
|
||||
const costs = new Map<string, number>();
|
||||
let receiptBacked = false;
|
||||
let requestedState: 'ready' | 'claimed' | 'missing' = 'missing';
|
||||
for (const part of row.content ?? []) {
|
||||
if (part == null || typeof part !== 'object' || Array.isArray(part)) {
|
||||
|
|
@ -1692,6 +1696,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
|
|||
const claimable =
|
||||
terminal && wakeupEligible && sameAgent && (replaying ? replay : resultClaim == null);
|
||||
if (candidateId === taskId) {
|
||||
receiptBacked = task?.completionReceipt === true;
|
||||
if (claimable) {
|
||||
requestedState = 'ready';
|
||||
} else if (resultClaim != null) {
|
||||
|
|
@ -1700,11 +1705,29 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
|
|||
}
|
||||
if (
|
||||
claimable &&
|
||||
(candidateId === taskId || task?.completionReceipt !== true) &&
|
||||
(candidateId === taskId || kind === 'wakeup') &&
|
||||
candidates.length < boundedLimit
|
||||
) {
|
||||
candidates.push(candidateId);
|
||||
}
|
||||
if (claimable && (candidateId === taskId || candidates.includes(candidateId))) {
|
||||
const call = (part as { tool_call?: { id?: string } }).tool_call;
|
||||
if (typeof call?.id === 'string' && typeof task?.toolName === 'string') {
|
||||
const terminalStatus = task.status === 'error' ? 'error' : 'completed';
|
||||
costs.set(
|
||||
candidateId,
|
||||
JSON.stringify(
|
||||
backgroundResultMetadata({
|
||||
taskId: candidateId,
|
||||
toolCallId: call.id,
|
||||
toolName: task.toolName,
|
||||
status: task.cancelled === true ? 'cancelled' : terminalStatus,
|
||||
}),
|
||||
).length + 1,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (requestedState === 'missing') {
|
||||
return { status: 'not_ready' };
|
||||
|
|
@ -1720,6 +1743,24 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
|
|||
candidates.unshift(taskId);
|
||||
candidates.splice(boundedLimit);
|
||||
}
|
||||
if (!replaying) {
|
||||
/** A receipt lives on another document, so it cannot join this atomic
|
||||
* message-row batch. Keep its delivery task-local in both directions. */
|
||||
if (receiptBacked) candidates.splice(0, candidates.length, taskId);
|
||||
if (maxMetadataChars != null) {
|
||||
let remaining = maxMetadataChars - 1 - (costs.get(taskId) ?? maxMetadataChars);
|
||||
if (remaining < 0)
|
||||
throw new TypeError('Background result metadata exceeds its input budget');
|
||||
const admitted = candidates.filter((id) => {
|
||||
if (id === taskId) return true;
|
||||
const cost = costs.get(id) ?? maxMetadataChars;
|
||||
if (cost > remaining) return false;
|
||||
remaining -= cost;
|
||||
return true;
|
||||
});
|
||||
candidates.splice(0, candidates.length, ...admitted);
|
||||
}
|
||||
}
|
||||
const claimedAt = new Date();
|
||||
const claimStamp = {
|
||||
kind,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue