mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-17 21:43:06 +00:00
📬 fix: Deliver Same-Turn Background Results (#16097)
This commit is contained in:
parent
a35745bf5f
commit
5d2fa85d1f
4 changed files with 137 additions and 17 deletions
|
|
@ -2310,7 +2310,7 @@ describe('runCheckBackgroundTask (singleton)', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('lets a later poll collect a receipt without inheriting an abandoned local claim', async () => {
|
||||
it('lets a later-generation poll collect a receipt without inheriting an abandoned local claim', async () => {
|
||||
const created = backgroundTaskRegistry.create({
|
||||
userId: 'claim_user',
|
||||
conversationId: 'claim_convo',
|
||||
|
|
@ -2341,7 +2341,7 @@ describe('runCheckBackgroundTask (singleton)', () => {
|
|||
args: { background_task_id: created.task.id },
|
||||
agentId: 'agent_parent_1',
|
||||
runId: 'poll-run',
|
||||
generationId: 'response-claim',
|
||||
generationId: 'response-later-poll',
|
||||
claimBackgroundToolResult,
|
||||
};
|
||||
|
||||
|
|
@ -2359,7 +2359,7 @@ describe('runCheckBackgroundTask (singleton)', () => {
|
|||
expect(laterPoll).toMatchObject({ status: 'completed', result: 'CLAIMED RESULT' });
|
||||
expect(
|
||||
backgroundTaskRegistry.get('claim_user', 'claim_convo', created.task.id)?.resultClaim,
|
||||
).toMatchObject({ kind: 'manual', generationId: 'response-claim' });
|
||||
).toMatchObject({ kind: 'manual', generationId: 'response-later-poll' });
|
||||
expect(retire).toHaveBeenCalledTimes(1);
|
||||
expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', {
|
||||
onlyIfUnclaimed: true,
|
||||
|
|
@ -2369,12 +2369,63 @@ describe('runCheckBackgroundTask (singleton)', () => {
|
|||
messageId: 'response-claim',
|
||||
taskId: created.task.id,
|
||||
kind: 'manual',
|
||||
generationId: 'response-claim',
|
||||
generationId: 'response-later-poll',
|
||||
allowUnfinished: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('delivers a completed result inside its originating generation before the response row finalizes', async () => {
|
||||
const created = backgroundTaskRegistry.create({
|
||||
userId: 'originating_user',
|
||||
conversationId: 'originating_convo',
|
||||
toolCallId: 'call_originating',
|
||||
toolName: 'search_mcp_docs',
|
||||
messageId: 'response-originating',
|
||||
});
|
||||
if ('atCapacity' in created) {
|
||||
throw new Error('unexpected capacity');
|
||||
}
|
||||
backgroundTaskRegistry.complete('originating_user', 'originating_convo', created.task.id, {
|
||||
content: 'ORIGINATING RESULT',
|
||||
});
|
||||
const retire = jest.fn(async () => true);
|
||||
backgroundTaskRegistry.markCompletionWakeup(
|
||||
'originating_user',
|
||||
'originating_convo',
|
||||
created.task.id,
|
||||
{ renew: jest.fn(async () => true), retire },
|
||||
);
|
||||
const claimBackgroundToolResult = jest.fn(async () => ({ status: 'not_ready' as const }));
|
||||
const request = {
|
||||
userId: 'originating_user',
|
||||
conversationId: 'originating_convo',
|
||||
args: { background_task_id: created.task.id },
|
||||
toolCallId: 'poll-originating',
|
||||
runId: 'originating-run',
|
||||
generationId: 'response-originating',
|
||||
claimBackgroundToolResult,
|
||||
};
|
||||
|
||||
const result = JSON.parse(await runCheckBackgroundTask(request));
|
||||
const replay = JSON.parse(await runCheckBackgroundTask(request));
|
||||
|
||||
expect(result).toMatchObject({ status: 'completed', result: 'ORIGINATING RESULT' });
|
||||
expect(replay).toMatchObject({ status: 'completed', result: 'ORIGINATING RESULT' });
|
||||
expect(retire).toHaveBeenCalledTimes(1);
|
||||
expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', {
|
||||
onlyIfUnclaimed: true,
|
||||
});
|
||||
expect(claimBackgroundToolResult).toHaveBeenCalledTimes(2);
|
||||
expect(
|
||||
backgroundTaskRegistry.get('originating_user', 'originating_convo', created.task.id)
|
||||
?.resultClaim,
|
||||
).toMatchObject({
|
||||
kind: 'manual',
|
||||
generationId: 'response-originating',
|
||||
});
|
||||
});
|
||||
|
||||
it('reclaims a dead manual delivery after its owning generation is gone', async () => {
|
||||
const claimBackgroundToolResult = jest
|
||||
.fn()
|
||||
|
|
|
|||
|
|
@ -2210,8 +2210,11 @@ export async function runCheckBackgroundTask(params: {
|
|||
if (durableClaim.status === 'not_found' || durableClaim.status === 'not_ready') {
|
||||
const localReplay =
|
||||
task.resultClaim?.kind === 'manual' && task.resultClaim.claimId === invocationId;
|
||||
let localClaimNeedsNoDurableConfirmation =
|
||||
localReplay && task.liveArtifactPollRequired === true;
|
||||
const pollOwnsOriginatingGeneration =
|
||||
params.generationId != null && params.generationId === task.messageId;
|
||||
const localClaimAllowed =
|
||||
pollOwnsOriginatingGeneration || task.liveArtifactPollRequired === true;
|
||||
let localClaimNeedsNoDurableConfirmation = localReplay && localClaimAllowed;
|
||||
if (!localReplay) {
|
||||
/** Retire the still-unclaimed delivery before creating local
|
||||
* ownership. A live resolver lease wins. Once that resolver is
|
||||
|
|
@ -2259,7 +2262,7 @@ export async function runCheckBackgroundTask(params: {
|
|||
'The task is finished and completion ownership is being settled. Retry this poll shortly.',
|
||||
});
|
||||
}
|
||||
if (task.liveArtifactPollRequired === true) {
|
||||
if (localClaimAllowed) {
|
||||
const localClaim = backgroundTaskRegistry.claimResult(
|
||||
userId,
|
||||
conversationId,
|
||||
|
|
@ -2285,21 +2288,21 @@ export async function runCheckBackgroundTask(params: {
|
|||
'The task is finished and its result is being made durable. Retry this poll shortly.',
|
||||
});
|
||||
}
|
||||
/** The poll is executing inside the still-unfinished dispatch
|
||||
* generation, so waiting for the durable row would require
|
||||
* that generation to end before it can obey its mandatory
|
||||
* live-artifact poll. The retired unclaimed wakeup plus this
|
||||
* local manual claim is authoritative for this owner process;
|
||||
* the persistence retry re-reads and copies the claim after
|
||||
* the generation finalizes. */
|
||||
/** The poll is executing inside the unfinished dispatch
|
||||
* generation, or it must deliver a live artifact. Waiting for
|
||||
* the durable row would require that generation to end first.
|
||||
* The retired unclaimed wakeup plus this local manual claim is
|
||||
* authoritative for this owner process; the persistence retry
|
||||
* re-reads and copies the claim after finalization. */
|
||||
localClaimNeedsNoDurableConfirmation = true;
|
||||
}
|
||||
}
|
||||
/** Ordinary polls claim the durable terminal receipt directly.
|
||||
* They never reserve a process-local claim while the receipt is
|
||||
* absent: a later poll has a different provider tool-call id and
|
||||
* could never take over that abandoned reservation. The live-
|
||||
* artifact exception above cannot wait for its own generation. */
|
||||
* could never take over that abandoned reservation. A poll owned
|
||||
* by the originating generation cannot wait for that same
|
||||
* generation to finalize its durable response row. */
|
||||
if (!localClaimNeedsNoDurableConfirmation) {
|
||||
const reconciledClaim = await params.claimBackgroundToolResult({
|
||||
...durableClaimInput,
|
||||
|
|
|
|||
|
|
@ -4,12 +4,16 @@ import { tool as structuredTool } from '@librechat/agents/langchain/tools';
|
|||
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
|
||||
import type { FiltersConfig } from 'librechat-data-provider';
|
||||
import type { ToolExecuteOptions } from './handlers';
|
||||
import {
|
||||
BACKGROUND_TASK_ABORT_GRACE_MS,
|
||||
BACKGROUND_TASK_TIMEOUT_MS,
|
||||
BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS,
|
||||
} from './backgroundCompletion';
|
||||
import {
|
||||
backgroundTaskRegistry,
|
||||
runCheckBackgroundTask,
|
||||
CHECK_BACKGROUND_TASK_NAME,
|
||||
} from './background';
|
||||
import { BACKGROUND_TASK_ABORT_GRACE_MS, BACKGROUND_TASK_TIMEOUT_MS } from './backgroundCompletion';
|
||||
import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from './invocation';
|
||||
import { ContentFilterError } from '../middleware/contentFilter';
|
||||
import { createToolExecuteHandler } from './handlers';
|
||||
|
|
@ -826,6 +830,64 @@ describe('createToolExecuteHandler — background tool calls', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('stops producer heartbeats after the completion lease is lost', async () => {
|
||||
jest.useFakeTimers({ doNotFake: ['setImmediate'] });
|
||||
try {
|
||||
let finishTool: ((value: { content: string }) => void) | undefined;
|
||||
const tool = {
|
||||
name: 'search_mcp_docs',
|
||||
description: 'search docs',
|
||||
schema: z.object({ q: z.string() }),
|
||||
invoke: jest.fn(
|
||||
() =>
|
||||
new Promise<{ content: string }>((resolve) => {
|
||||
finishTool = resolve;
|
||||
}),
|
||||
),
|
||||
} as unknown as StructuredToolInterface;
|
||||
const renew = jest.fn(async () => false);
|
||||
const handler = createToolExecuteHandler({
|
||||
loadTools: async () => ({ loadedTools: [tool] }),
|
||||
backgroundToolCompletion: {
|
||||
preregister: jest.fn(async () => ({
|
||||
renew,
|
||||
retire: jest.fn(async () => true),
|
||||
})),
|
||||
persist: jest.fn(async () => true),
|
||||
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
|
||||
},
|
||||
});
|
||||
|
||||
await runBatch(handler, {
|
||||
toolCalls: [
|
||||
{
|
||||
id: 'call-lost-producer-lease',
|
||||
name: tool.name,
|
||||
args: { q: 'lease', run_in_background: true },
|
||||
stepId: 'step-lost-producer-lease',
|
||||
},
|
||||
],
|
||||
agentId: 'agent_parent_1',
|
||||
configurable: buildConfig([tool.name]),
|
||||
metadata: { thread_id: 'exec_convo', run_id: 'response-lost-producer-lease' },
|
||||
});
|
||||
|
||||
jest.advanceTimersByTime(BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS);
|
||||
await flushMicrotasks();
|
||||
expect(renew).toHaveBeenCalledTimes(1);
|
||||
|
||||
jest.advanceTimersByTime(BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS * 6);
|
||||
await flushMicrotasks();
|
||||
expect(renew).toHaveBeenCalledTimes(1);
|
||||
|
||||
finishTool?.({ content: 'settled after lease loss' });
|
||||
await flushMicrotasks();
|
||||
await flushMicrotasks();
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('persists an Event Actor launch before invoking and terminal evidence before wakeup', async () => {
|
||||
const events: string[] = [];
|
||||
let finishTool: ((value: { content: string }) => void) | undefined;
|
||||
|
|
|
|||
|
|
@ -6083,6 +6083,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
.renew()
|
||||
.then((renewed) => {
|
||||
if (!renewed) {
|
||||
producerHeartbeatStopped = true;
|
||||
if (producerHeartbeat != null) {
|
||||
clearInterval(producerHeartbeat);
|
||||
}
|
||||
logger.warn(
|
||||
`[background] Completion producer lease was not renewed for task ${task.id}.`,
|
||||
);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue