diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.ts index c34d8d6826..65873abc2f 100644 --- a/packages/api/src/agents/background.spec.ts +++ b/packages/api/src/agents/background.spec.ts @@ -2310,7 +2310,7 @@ describe('runCheckBackgroundTask (singleton)', () => { ); }); - it('lets a later poll collect a receipt without inheriting an abandoned local claim', async () => { + it('lets a later-generation poll collect a receipt without inheriting an abandoned local claim', async () => { const created = backgroundTaskRegistry.create({ userId: 'claim_user', conversationId: 'claim_convo', @@ -2341,7 +2341,7 @@ describe('runCheckBackgroundTask (singleton)', () => { args: { background_task_id: created.task.id }, agentId: 'agent_parent_1', runId: 'poll-run', - generationId: 'response-claim', + generationId: 'response-later-poll', claimBackgroundToolResult, }; @@ -2359,7 +2359,7 @@ describe('runCheckBackgroundTask (singleton)', () => { expect(laterPoll).toMatchObject({ status: 'completed', result: 'CLAIMED RESULT' }); expect( backgroundTaskRegistry.get('claim_user', 'claim_convo', created.task.id)?.resultClaim, - ).toMatchObject({ kind: 'manual', generationId: 'response-claim' }); + ).toMatchObject({ kind: 'manual', generationId: 'response-later-poll' }); expect(retire).toHaveBeenCalledTimes(1); expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', { onlyIfUnclaimed: true, @@ -2369,12 +2369,63 @@ describe('runCheckBackgroundTask (singleton)', () => { messageId: 'response-claim', taskId: created.task.id, kind: 'manual', - generationId: 'response-claim', + generationId: 'response-later-poll', allowUnfinished: true, }), ); }); + it('delivers a completed result inside its originating generation before the response row finalizes', async () => { + const created = backgroundTaskRegistry.create({ + userId: 'originating_user', + conversationId: 'originating_convo', + toolCallId: 'call_originating', + toolName: 'search_mcp_docs', + messageId: 'response-originating', + }); + if ('atCapacity' in created) { + throw new Error('unexpected capacity'); + } + backgroundTaskRegistry.complete('originating_user', 'originating_convo', created.task.id, { + content: 'ORIGINATING RESULT', + }); + const retire = jest.fn(async () => true); + backgroundTaskRegistry.markCompletionWakeup( + 'originating_user', + 'originating_convo', + created.task.id, + { renew: jest.fn(async () => true), retire }, + ); + const claimBackgroundToolResult = jest.fn(async () => ({ status: 'not_ready' as const })); + const request = { + userId: 'originating_user', + conversationId: 'originating_convo', + args: { background_task_id: created.task.id }, + toolCallId: 'poll-originating', + runId: 'originating-run', + generationId: 'response-originating', + claimBackgroundToolResult, + }; + + const result = JSON.parse(await runCheckBackgroundTask(request)); + const replay = JSON.parse(await runCheckBackgroundTask(request)); + + expect(result).toMatchObject({ status: 'completed', result: 'ORIGINATING RESULT' }); + expect(replay).toMatchObject({ status: 'completed', result: 'ORIGINATING RESULT' }); + expect(retire).toHaveBeenCalledTimes(1); + expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', { + onlyIfUnclaimed: true, + }); + expect(claimBackgroundToolResult).toHaveBeenCalledTimes(2); + expect( + backgroundTaskRegistry.get('originating_user', 'originating_convo', created.task.id) + ?.resultClaim, + ).toMatchObject({ + kind: 'manual', + generationId: 'response-originating', + }); + }); + it('reclaims a dead manual delivery after its owning generation is gone', async () => { const claimBackgroundToolResult = jest .fn() diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.ts index 70e2429e0a..adc187943a 100644 --- a/packages/api/src/agents/background.ts +++ b/packages/api/src/agents/background.ts @@ -2210,8 +2210,11 @@ export async function runCheckBackgroundTask(params: { if (durableClaim.status === 'not_found' || durableClaim.status === 'not_ready') { const localReplay = task.resultClaim?.kind === 'manual' && task.resultClaim.claimId === invocationId; - let localClaimNeedsNoDurableConfirmation = - localReplay && task.liveArtifactPollRequired === true; + const pollOwnsOriginatingGeneration = + params.generationId != null && params.generationId === task.messageId; + const localClaimAllowed = + pollOwnsOriginatingGeneration || task.liveArtifactPollRequired === true; + let localClaimNeedsNoDurableConfirmation = localReplay && localClaimAllowed; if (!localReplay) { /** Retire the still-unclaimed delivery before creating local * ownership. A live resolver lease wins. Once that resolver is @@ -2259,7 +2262,7 @@ export async function runCheckBackgroundTask(params: { 'The task is finished and completion ownership is being settled. Retry this poll shortly.', }); } - if (task.liveArtifactPollRequired === true) { + if (localClaimAllowed) { const localClaim = backgroundTaskRegistry.claimResult( userId, conversationId, @@ -2285,21 +2288,21 @@ export async function runCheckBackgroundTask(params: { 'The task is finished and its result is being made durable. Retry this poll shortly.', }); } - /** The poll is executing inside the still-unfinished dispatch - * generation, so waiting for the durable row would require - * that generation to end before it can obey its mandatory - * live-artifact poll. The retired unclaimed wakeup plus this - * local manual claim is authoritative for this owner process; - * the persistence retry re-reads and copies the claim after - * the generation finalizes. */ + /** The poll is executing inside the unfinished dispatch + * generation, or it must deliver a live artifact. Waiting for + * the durable row would require that generation to end first. + * The retired unclaimed wakeup plus this local manual claim is + * authoritative for this owner process; the persistence retry + * re-reads and copies the claim after finalization. */ localClaimNeedsNoDurableConfirmation = true; } } /** Ordinary polls claim the durable terminal receipt directly. * They never reserve a process-local claim while the receipt is * absent: a later poll has a different provider tool-call id and - * could never take over that abandoned reservation. The live- - * artifact exception above cannot wait for its own generation. */ + * could never take over that abandoned reservation. A poll owned + * by the originating generation cannot wait for that same + * generation to finalize its durable response row. */ if (!localClaimNeedsNoDurableConfirmation) { const reconciledClaim = await params.claimBackgroundToolResult({ ...durableClaimInput, diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts index 2bac0dacf7..ef1202bf4a 100644 --- a/packages/api/src/agents/handlers.background.spec.ts +++ b/packages/api/src/agents/handlers.background.spec.ts @@ -4,12 +4,16 @@ import { tool as structuredTool } from '@librechat/agents/langchain/tools'; import type { StructuredToolInterface } from '@librechat/agents/langchain/tools'; import type { FiltersConfig } from 'librechat-data-provider'; import type { ToolExecuteOptions } from './handlers'; +import { + BACKGROUND_TASK_ABORT_GRACE_MS, + BACKGROUND_TASK_TIMEOUT_MS, + BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS, +} from './backgroundCompletion'; import { backgroundTaskRegistry, runCheckBackgroundTask, CHECK_BACKGROUND_TASK_NAME, } from './background'; -import { BACKGROUND_TASK_ABORT_GRACE_MS, BACKGROUND_TASK_TIMEOUT_MS } from './backgroundCompletion'; import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from './invocation'; import { ContentFilterError } from '../middleware/contentFilter'; import { createToolExecuteHandler } from './handlers'; @@ -826,6 +830,64 @@ describe('createToolExecuteHandler — background tool calls', () => { } }); + it('stops producer heartbeats after the completion lease is lost', async () => { + jest.useFakeTimers({ doNotFake: ['setImmediate'] }); + try { + let finishTool: ((value: { content: string }) => void) | undefined; + const tool = { + name: 'search_mcp_docs', + description: 'search docs', + schema: z.object({ q: z.string() }), + invoke: jest.fn( + () => + new Promise<{ content: string }>((resolve) => { + finishTool = resolve; + }), + ), + } as unknown as StructuredToolInterface; + const renew = jest.fn(async () => false); + const handler = createToolExecuteHandler({ + loadTools: async () => ({ loadedTools: [tool] }), + backgroundToolCompletion: { + preregister: jest.fn(async () => ({ + renew, + retire: jest.fn(async () => true), + })), + persist: jest.fn(async () => true), + claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })), + }, + }); + + await runBatch(handler, { + toolCalls: [ + { + id: 'call-lost-producer-lease', + name: tool.name, + args: { q: 'lease', run_in_background: true }, + stepId: 'step-lost-producer-lease', + }, + ], + agentId: 'agent_parent_1', + configurable: buildConfig([tool.name]), + metadata: { thread_id: 'exec_convo', run_id: 'response-lost-producer-lease' }, + }); + + jest.advanceTimersByTime(BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS); + await flushMicrotasks(); + expect(renew).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS * 6); + await flushMicrotasks(); + expect(renew).toHaveBeenCalledTimes(1); + + finishTool?.({ content: 'settled after lease loss' }); + await flushMicrotasks(); + await flushMicrotasks(); + } finally { + jest.useRealTimers(); + } + }); + it('persists an Event Actor launch before invoking and terminal evidence before wakeup', async () => { const events: string[] = []; let finishTool: ((value: { content: string }) => void) | undefined; diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index 1f7cb845d5..155f20bd18 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -6083,6 +6083,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand .renew() .then((renewed) => { if (!renewed) { + producerHeartbeatStopped = true; + if (producerHeartbeat != null) { + clearInterval(producerHeartbeat); + } logger.warn( `[background] Completion producer lease was not renewed for task ${task.id}.`, );