📬 fix: Deliver Same-Turn Background Results (#16097)

This commit is contained in:
Danny Avila 2026-09-19 18:14:41 -04:00 • committed by GitHub
parent a35745bf5f
commit 5d2fa85d1f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 137 additions and 17 deletions

View file

@ -2310,7 +2310,7 @@ describe('runCheckBackgroundTask (singleton)', () => {
);
});
it('lets a later poll collect a receipt without inheriting an abandoned local claim', async () => {
it('lets a later-generation poll collect a receipt without inheriting an abandoned local claim', async () => {
const created = backgroundTaskRegistry.create({
userId: 'claim_user',
conversationId: 'claim_convo',
@ -2341,7 +2341,7 @@ describe('runCheckBackgroundTask (singleton)', () => {
args: { background_task_id: created.task.id },
agentId: 'agent_parent_1',
runId: 'poll-run',
generationId: 'response-claim',
generationId: 'response-later-poll',
claimBackgroundToolResult,
};
@ -2359,7 +2359,7 @@ describe('runCheckBackgroundTask (singleton)', () => {
expect(laterPoll).toMatchObject({ status: 'completed', result: 'CLAIMED RESULT' });
expect(
backgroundTaskRegistry.get('claim_user', 'claim_convo', created.task.id)?.resultClaim,
).toMatchObject({ kind: 'manual', generationId: 'response-claim' });
).toMatchObject({ kind: 'manual', generationId: 'response-later-poll' });
expect(retire).toHaveBeenCalledTimes(1);
expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', {
onlyIfUnclaimed: true,
@ -2369,12 +2369,63 @@ describe('runCheckBackgroundTask (singleton)', () => {
messageId: 'response-claim',
taskId: created.task.id,
kind: 'manual',
generationId: 'response-claim',
generationId: 'response-later-poll',
allowUnfinished: true,
}),
);
});
it('delivers a completed result inside its originating generation before the response row finalizes', async () => {
const created = backgroundTaskRegistry.create({
userId: 'originating_user',
conversationId: 'originating_convo',
toolCallId: 'call_originating',
toolName: 'search_mcp_docs',
messageId: 'response-originating',
});
if ('atCapacity' in created) {
throw new Error('unexpected capacity');
}
backgroundTaskRegistry.complete('originating_user', 'originating_convo', created.task.id, {
content: 'ORIGINATING RESULT',
});
const retire = jest.fn(async () => true);
backgroundTaskRegistry.markCompletionWakeup(
'originating_user',
'originating_convo',
created.task.id,
{ renew: jest.fn(async () => true), retire },
);
const claimBackgroundToolResult = jest.fn(async () => ({ status: 'not_ready' as const }));
const request = {
userId: 'originating_user',
conversationId: 'originating_convo',
args: { background_task_id: created.task.id },
toolCallId: 'poll-originating',
runId: 'originating-run',
generationId: 'response-originating',
claimBackgroundToolResult,
};
const result = JSON.parse(await runCheckBackgroundTask(request));
const replay = JSON.parse(await runCheckBackgroundTask(request));
expect(result).toMatchObject({ status: 'completed', result: 'ORIGINATING RESULT' });
expect(replay).toMatchObject({ status: 'completed', result: 'ORIGINATING RESULT' });
expect(retire).toHaveBeenCalledTimes(1);
expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', {
onlyIfUnclaimed: true,
});
expect(claimBackgroundToolResult).toHaveBeenCalledTimes(2);
expect(
backgroundTaskRegistry.get('originating_user', 'originating_convo', created.task.id)
?.resultClaim,
).toMatchObject({
kind: 'manual',
generationId: 'response-originating',
});
});
it('reclaims a dead manual delivery after its owning generation is gone', async () => {
const claimBackgroundToolResult = jest
.fn()

View file

@ -2210,8 +2210,11 @@ export async function runCheckBackgroundTask(params: {
if (durableClaim.status === 'not_found' || durableClaim.status === 'not_ready') {
const localReplay =
task.resultClaim?.kind === 'manual' && task.resultClaim.claimId === invocationId;
let localClaimNeedsNoDurableConfirmation =
localReplay && task.liveArtifactPollRequired === true;
const pollOwnsOriginatingGeneration =
params.generationId != null && params.generationId === task.messageId;
const localClaimAllowed =
pollOwnsOriginatingGeneration || task.liveArtifactPollRequired === true;
let localClaimNeedsNoDurableConfirmation = localReplay && localClaimAllowed;
if (!localReplay) {
/** Retire the still-unclaimed delivery before creating local
* ownership. A live resolver lease wins. Once that resolver is
@ -2259,7 +2262,7 @@ export async function runCheckBackgroundTask(params: {
'The task is finished and completion ownership is being settled. Retry this poll shortly.',
});
}
if (task.liveArtifactPollRequired === true) {
if (localClaimAllowed) {
const localClaim = backgroundTaskRegistry.claimResult(
userId,
conversationId,
@ -2285,21 +2288,21 @@ export async function runCheckBackgroundTask(params: {
'The task is finished and its result is being made durable. Retry this poll shortly.',
});
}
/** The poll is executing inside the still-unfinished dispatch
* generation, so waiting for the durable row would require
* that generation to end before it can obey its mandatory
* live-artifact poll. The retired unclaimed wakeup plus this
* local manual claim is authoritative for this owner process;
* the persistence retry re-reads and copies the claim after
* the generation finalizes. */
/** The poll is executing inside the unfinished dispatch
* generation, or it must deliver a live artifact. Waiting for
* the durable row would require that generation to end first.
* The retired unclaimed wakeup plus this local manual claim is
* authoritative for this owner process; the persistence retry
* re-reads and copies the claim after finalization. */
localClaimNeedsNoDurableConfirmation = true;
}
}
/** Ordinary polls claim the durable terminal receipt directly.
* They never reserve a process-local claim while the receipt is
* absent: a later poll has a different provider tool-call id and
* could never take over that abandoned reservation. The live-
* artifact exception above cannot wait for its own generation. */
* could never take over that abandoned reservation. A poll owned
* by the originating generation cannot wait for that same
* generation to finalize its durable response row. */
if (!localClaimNeedsNoDurableConfirmation) {
const reconciledClaim = await params.claimBackgroundToolResult({
...durableClaimInput,

View file

@ -4,12 +4,16 @@ import { tool as structuredTool } from '@librechat/agents/langchain/tools';
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
import type { FiltersConfig } from 'librechat-data-provider';
import type { ToolExecuteOptions } from './handlers';
import {
BACKGROUND_TASK_ABORT_GRACE_MS,
BACKGROUND_TASK_TIMEOUT_MS,
BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS,
} from './backgroundCompletion';
import {
backgroundTaskRegistry,
runCheckBackgroundTask,
CHECK_BACKGROUND_TASK_NAME,
} from './background';
import { BACKGROUND_TASK_ABORT_GRACE_MS, BACKGROUND_TASK_TIMEOUT_MS } from './backgroundCompletion';
import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from './invocation';
import { ContentFilterError } from '../middleware/contentFilter';
import { createToolExecuteHandler } from './handlers';
@ -826,6 +830,64 @@ describe('createToolExecuteHandler — background tool calls', () => {
}
});
it('stops producer heartbeats after the completion lease is lost', async () => {
jest.useFakeTimers({ doNotFake: ['setImmediate'] });
try {
let finishTool: ((value: { content: string }) => void) | undefined;
const tool = {
name: 'search_mcp_docs',
description: 'search docs',
schema: z.object({ q: z.string() }),
invoke: jest.fn(
() =>
new Promise<{ content: string }>((resolve) => {
finishTool = resolve;
}),
),
} as unknown as StructuredToolInterface;
const renew = jest.fn(async () => false);
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [tool] }),
backgroundToolCompletion: {
preregister: jest.fn(async () => ({
renew,
retire: jest.fn(async () => true),
})),
persist: jest.fn(async () => true),
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
await runBatch(handler, {
toolCalls: [
{
id: 'call-lost-producer-lease',
name: tool.name,
args: { q: 'lease', run_in_background: true },
stepId: 'step-lost-producer-lease',
},
],
agentId: 'agent_parent_1',
configurable: buildConfig([tool.name]),
metadata: { thread_id: 'exec_convo', run_id: 'response-lost-producer-lease' },
});
jest.advanceTimersByTime(BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS);
await flushMicrotasks();
expect(renew).toHaveBeenCalledTimes(1);
jest.advanceTimersByTime(BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS * 6);
await flushMicrotasks();
expect(renew).toHaveBeenCalledTimes(1);
finishTool?.({ content: 'settled after lease loss' });
await flushMicrotasks();
await flushMicrotasks();
} finally {
jest.useRealTimers();
}
});
it('persists an Event Actor launch before invoking and terminal evidence before wakeup', async () => {
const events: string[] = [];
let finishTool: ((value: { content: string }) => void) | undefined;

View file

@ -6083,6 +6083,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
.renew()
.then((renewed) => {
if (!renewed) {
producerHeartbeatStopped = true;
if (producerHeartbeat != null) {
clearInterval(producerHeartbeat);
}
logger.warn(
`[background] Completion producer lease was not renewed for task ${task.id}.`,
);