mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
⏭️ fix: Let Queued User Turns Go Before Completion Wake-ups at Run End (#18027)
* fix: Let Queued User Turns Go Before Completion Wake-ups at Run End When a run ends with both a server-owned queued user turn and a background or subagent completion wake-up waiting on it, the wake-up was expedited at generation settlement while the queued turn sat out its five-second readiness re-check, so the wake-up claimed the predecessor fence first and the user's turn waited behind a whole automated turn. - Generation settlement now also expedites the conversation's queued-turn deliveries. - Background and subagent wake-ups defer without consuming an attempt while a queued turn holds an unexpired claim or is the due lane head, then run after it, chained to its response. - hasPendingAgentQueuedTurn answers false for rows that need recovery or a user decision (reservations, expired claims, indeterminate admissions, dead heads awaiting dismissal), so a stuck queued turn never holds a wake-up. The read runs in parallel with the existing conversation read and fails open. - Readiness deferral logs now carry the execution error code. * fix: Read Queued-Turn Precedence in Claim Order and Defer on Unknown State - Read the live claim after the head read settles, so a queued -> claimed transition between the two reads cannot answer false. - Take plain identifiers in hasPendingAgentQueuedTurn and convert the user id inside data-schemas, keeping Mongoose types out of packages/api. - A failed precedence read now defers the wake-up without consuming an attempt (QUEUED_TURN_STATE_UNAVAILABLE) instead of proceeding. * fix: Read the Active Queued-Turn Lane in One Query A claim released back to `queued` between the head read and the live-claim read could still answer false. Read the reserving, queued and claimed rows in one query over the capacity index, whose `activeSlot` key holds steady from reservation until a terminal state, so each row is seen exactly once in one state. Dead heads and indeterminate quarantines stay separate indexed reads in parallel. Falls back to a planner-chosen read when the capacity index has not been built yet. * fix: Fall Back to a Planner Read Whenever the Lane Hint Is Refused A hint can be refused for reasons other than a missing index on some engines. The hinted read only pins the stable key, so any refusal now retries unhinted; a genuine outage fails that read too and defers the wake-up. * fix: Read Queued-Turn Blockers After the Turn They Block The precedence read gathered the active lane, the dead lane head and the indeterminate quarantine in parallel, so a blocker could be older than the queued turn it blocked. A dead head dismissed while the reads were in flight still answered false and let a completion wake-up race the successor it had just unblocked. Read the active lane first, and the blockers (a reservation, an expired claim, an indeterminate admission, a dead row ahead of the head) only afterwards and only for a due head, in one existence query. A blocker that has already cleared no longer answers false, and a conversation with no due queued turn now costs one read instead of three.
This commit is contained in:
parent
c102e2d498
commit
6e00a9743a
11 changed files with 1010 additions and 19 deletions
|
|
@ -28,12 +28,14 @@ import type { ScheduleMCPCompletionLookup } from '~/schedules/authorization/cont
|
|||
import type { AgentContinueTriggerEnvelope } from './triggers/envelope';
|
||||
import type { AgentTriggerDispatchContext } from './triggers/dispatch';
|
||||
import type { AgentTriggerEnqueueOptions } from './triggers/delivery';
|
||||
import type { QueuedTurnPrecedenceMethods } from './precedence';
|
||||
import { resolveScheduleMCPCompletion } from '~/schedules/authorization/continuation';
|
||||
import { WAITING_RETRY_CAP_MS, waitingRetryAfter } from './triggers/backoff';
|
||||
import { BACKGROUND_TOOL_PRODUCER_LEASE_MS } from './backgroundCompletion';
|
||||
import { SUBAGENT_COMPLETION_SOURCE } from './subagentCompletionWakeup';
|
||||
import { createAgentTriggerEnvelope } from './triggers/envelope';
|
||||
import { AgentTriggerExecutionError } from './triggers/host';
|
||||
import { getQueuedTurnDeferral } from './precedence';
|
||||
import { truncateMiddle } from '~/utils';
|
||||
|
||||
const WAKEUP_ADMISSION_DELAY_MS = 250;
|
||||
|
|
@ -72,6 +74,7 @@ export type PersistBackgroundToolCompletionResult = (
|
|||
) => Promise<boolean>;
|
||||
|
||||
type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
|
||||
QueuedTurnPrecedenceMethods &
|
||||
Pick<
|
||||
MessageMethods,
|
||||
'getMessages' | 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'
|
||||
|
|
@ -316,10 +319,8 @@ export function createBackgroundToolCompletionWakeupResolver({
|
|||
{ code: 'PARENT_STATE_UNAVAILABLE', retryable: true },
|
||||
);
|
||||
}
|
||||
if (
|
||||
isParentActive(parentJob) &&
|
||||
parentJob?.metadata?.idempotencyClientRequestId !== context.idempotencyKey
|
||||
) {
|
||||
const ownsParent = parentJob?.metadata?.idempotencyClientRequestId === context.idempotencyKey;
|
||||
if (isParentActive(parentJob) && !ownsParent) {
|
||||
throw executionError('The parent generation has not settled yet.', {
|
||||
code: 'PARENT_NOT_READY',
|
||||
retryable: true,
|
||||
|
|
@ -329,7 +330,10 @@ export function createBackgroundToolCompletionWakeupResolver({
|
|||
});
|
||||
}
|
||||
const userId = envelope.principal.userId;
|
||||
const parent = await methods.getConvo(userId, envelope.target.conversationId);
|
||||
const [parent, queuedTurnDeferral] = await Promise.all([
|
||||
methods.getConvo(userId, envelope.target.conversationId),
|
||||
ownsParent ? null : getQueuedTurnDeferral(methods, envelope, Date.now()),
|
||||
]);
|
||||
if (parent == null || parent.tenantId !== envelope.principal.tenantId) {
|
||||
throw executionError('The parent conversation is no longer available.', {
|
||||
code: 'PARENT_NOT_FOUND',
|
||||
|
|
@ -337,6 +341,9 @@ export function createBackgroundToolCompletionWakeupResolver({
|
|||
status: 404,
|
||||
});
|
||||
}
|
||||
if (queuedTurnDeferral != null) {
|
||||
throw queuedTurnDeferral;
|
||||
}
|
||||
const payload = envelope.event.payload;
|
||||
const scheduleMCPIdentity = await resolveScheduleMCPCompletion(
|
||||
{
|
||||
|
|
|
|||
388
packages/api/src/agents/precedence.integration.spec.ts
Normal file
388
packages/api/src/agents/precedence.integration.spec.ts
Normal file
|
|
@ -0,0 +1,388 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import {
|
||||
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
createMethods,
|
||||
createModels,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { AllMethods } from '@librechat/data-schemas';
|
||||
import type { AgentTriggerDeliveryPersistence, AgentTriggerService } from './triggers/service';
|
||||
import type { AgentTriggerFetch } from './triggers/host';
|
||||
import {
|
||||
BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
createBackgroundToolCompletionWakeupResolver,
|
||||
} from './backgroundCompletionWakeup';
|
||||
import { createAgentTriggerEnvelope, getAgentTriggerIdempotencyKey } from './triggers/envelope';
|
||||
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
|
||||
import { AGENT_QUEUED_TURN_SOURCE, createAgentQueuedTurnLifecycle } from './queuedTurns';
|
||||
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
|
||||
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
|
||||
import { createAgentContinuationResolver } from './triggers/continuation';
|
||||
import { __resetShutdownStateForTests } from '../app/shutdown';
|
||||
import { createAgentTriggerService } from './triggers/service';
|
||||
|
||||
const DB_SETUP_TIMEOUT_MS = 60_000;
|
||||
const TENANT_ID = 'tenant-1';
|
||||
const AGENT_ID = 'agent_parent';
|
||||
const RUN_RESPONSE_ID = 'run-response';
|
||||
|
||||
/** Below the five-second readiness re-check a deferred delivery otherwise waits out. */
|
||||
const PROMPT_ADMISSION_MS = 2_500;
|
||||
|
||||
interface Dispatch {
|
||||
kind: 'queued-turn' | 'wake-up';
|
||||
parentMessageId: string;
|
||||
responseMessageId: string;
|
||||
generationCreatedAt: number;
|
||||
dispatchedAt: number;
|
||||
}
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
let methods: AllMethods;
|
||||
let userId: string;
|
||||
let conversationId: string;
|
||||
let manager: GenerationJobManagerClass;
|
||||
let service: AgentTriggerService | undefined;
|
||||
let dispatches: Dispatch[];
|
||||
|
||||
async function waitUntil<T>(read: () => Promise<T | undefined>, label: string): Promise<T> {
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (Date.now() < deadline) {
|
||||
const value = await read();
|
||||
if (value !== undefined) {
|
||||
return value;
|
||||
}
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${label}`);
|
||||
}
|
||||
|
||||
async function deliveryOf(source: string) {
|
||||
return mongoose.models.AgentTriggerDelivery.findOne({
|
||||
'envelope.event.source.id': source,
|
||||
}).lean<{ attempts: number; availableAt: Date; status: string }>();
|
||||
}
|
||||
|
||||
/** Readiness deferrals push `availableAt` at least five seconds out. */
|
||||
function waitForDeferral(source: string) {
|
||||
return waitUntil(async () => {
|
||||
const delivery = await deliveryOf(source);
|
||||
return delivery != null && delivery.availableAt.getTime() > Date.now() + 2_000
|
||||
? delivery
|
||||
: undefined;
|
||||
}, `${source} to defer`);
|
||||
}
|
||||
|
||||
/** Stands in for the agents controller: refuses to replace an active
|
||||
* generation, otherwise starts one and persists its user and response rows. */
|
||||
const controller: AgentTriggerFetch = async (_url, init) => {
|
||||
const body = JSON.parse(String(init?.body)) as {
|
||||
text: string;
|
||||
parentMessageId: string;
|
||||
clientRequestId: string;
|
||||
agentContinuationAdmission?: unknown;
|
||||
};
|
||||
let createdAt: number;
|
||||
try {
|
||||
({ createdAt } = await manager.createJob(conversationId, userId, conversationId, {
|
||||
rejectActivePredecessor: true,
|
||||
}));
|
||||
} catch (error) {
|
||||
if ((error as { code?: string }).code !== 'GENERATION_PREDECESSOR_MISMATCH') {
|
||||
throw error;
|
||||
}
|
||||
return new Response(JSON.stringify({ code: 'PARENT_NOT_READY' }), {
|
||||
status: 409,
|
||||
headers: { 'retry-after': '1' },
|
||||
});
|
||||
}
|
||||
const userMessageId = `${body.clientRequestId}:user`;
|
||||
const responseMessageId = `${body.clientRequestId}:assistant`;
|
||||
await mongoose.models.Message.create([
|
||||
{
|
||||
messageId: userMessageId,
|
||||
conversationId,
|
||||
user: userId,
|
||||
tenantId: TENANT_ID,
|
||||
parentMessageId: body.parentMessageId,
|
||||
isCreatedByUser: true,
|
||||
text: body.text,
|
||||
endpoint: 'agents',
|
||||
},
|
||||
{
|
||||
messageId: responseMessageId,
|
||||
conversationId,
|
||||
user: userId,
|
||||
tenantId: TENANT_ID,
|
||||
parentMessageId: userMessageId,
|
||||
isCreatedByUser: false,
|
||||
endpoint: 'agents',
|
||||
content: [],
|
||||
},
|
||||
]);
|
||||
dispatches.push({
|
||||
kind: body.agentContinuationAdmission != null ? 'queued-turn' : 'wake-up',
|
||||
parentMessageId: body.parentMessageId,
|
||||
responseMessageId,
|
||||
generationCreatedAt: createdAt,
|
||||
dispatchedAt: Date.now(),
|
||||
});
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
status: 'started',
|
||||
streamId: conversationId,
|
||||
conversationId,
|
||||
generationCreatedAt: createdAt,
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
};
|
||||
|
||||
async function startDelivery(): Promise<{
|
||||
service: AgentTriggerService;
|
||||
lifecycle: ReturnType<typeof createAgentQueuedTurnLifecycle>;
|
||||
}> {
|
||||
const getGenerationJob = async (id: string) => (await manager.getJob(id)) ?? null;
|
||||
const getGenerationAdmissionEvidence = (
|
||||
...args: Parameters<GenerationJobManagerClass['getGenerationAdmissionEvidence']>
|
||||
) => manager.getGenerationAdmissionEvidence(...args);
|
||||
const lifecycle = createAgentQueuedTurnLifecycle({
|
||||
methods,
|
||||
getGenerationJob,
|
||||
getGenerationAdmissionEvidence,
|
||||
enqueue: (envelope, options) => triggers.enqueue(envelope, options),
|
||||
});
|
||||
const triggers = createAgentTriggerService({
|
||||
methods: methods as AllMethods & AgentTriggerDeliveryPersistence,
|
||||
fetch: controller,
|
||||
mintToken: () => 'trigger-token',
|
||||
subscribeGenerationSettled: (listener) => manager.onGenerationSettled(listener),
|
||||
settleSourceBeforeDeadLetter: lifecycle.settleBeforeDeadLetter,
|
||||
prepareContinue: createAgentContinuationResolver({
|
||||
eventActor: async () => undefined,
|
||||
internalSources: new Map([
|
||||
[
|
||||
BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
createBackgroundToolCompletionWakeupResolver({
|
||||
methods,
|
||||
getGenerationJob,
|
||||
getGenerationAdmissionEvidence,
|
||||
}),
|
||||
],
|
||||
[AGENT_QUEUED_TURN_SOURCE, lifecycle.prepareContinue],
|
||||
]),
|
||||
}),
|
||||
deliveryOptions: { concurrency: 4, tickMs: 20, maxIdleTickMs: 50 },
|
||||
});
|
||||
await triggers.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
|
||||
service = triggers;
|
||||
return { service: triggers, lifecycle };
|
||||
}
|
||||
|
||||
async function enqueueWakeup(triggers: AgentTriggerService): Promise<string> {
|
||||
const taskId = 'background-task';
|
||||
const envelope = createAgentTriggerEnvelope({
|
||||
mode: 'continue',
|
||||
requestId: taskId,
|
||||
deliveryId: taskId,
|
||||
receivedAt: Date.now(),
|
||||
principal: { id: userId, tenantId: TENANT_ID },
|
||||
event: {
|
||||
id: taskId,
|
||||
type: 'background-tool.completion',
|
||||
occurredAt: Date.now(),
|
||||
source: { id: BACKGROUND_TOOL_COMPLETION_SOURCE, type: 'internal' },
|
||||
payload: { taskId, toolCallId: taskId, toolName: 'tool' },
|
||||
},
|
||||
target: { conversationId, parentMessageId: RUN_RESPONSE_ID, agentId: AGENT_ID },
|
||||
input: 'waiting',
|
||||
});
|
||||
const { deliveryKey } = await triggers.enqueue(envelope, {
|
||||
orderingKey: taskId,
|
||||
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
|
||||
});
|
||||
expect(deliveryKey).toBe(getAgentTriggerIdempotencyKey(envelope));
|
||||
await triggers.persistBackgroundToolResult({
|
||||
deliveryKey,
|
||||
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
|
||||
result: { status: 'completed', output: 'tool output', settledAt: new Date() },
|
||||
});
|
||||
return deliveryKey;
|
||||
}
|
||||
|
||||
async function enqueueQueuedTurn(
|
||||
lifecycle: ReturnType<typeof createAgentQueuedTurnLifecycle>,
|
||||
predecessorCreatedAt: number,
|
||||
) {
|
||||
const { turn } = await methods.enqueueAgentQueuedTurn({
|
||||
user: new mongoose.Types.ObjectId(userId),
|
||||
tenantId: TENANT_ID,
|
||||
conversationId,
|
||||
agentId: AGENT_ID,
|
||||
parentMessageId: RUN_RESPONSE_ID,
|
||||
clientRequestId: 'queued-follow-up',
|
||||
text: 'follow-up queued while the run was busy',
|
||||
expectedPredecessorCreatedAt: predecessorCreatedAt,
|
||||
});
|
||||
await lifecycle.schedule(turn);
|
||||
return turn;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
createModels(mongoose);
|
||||
methods = createMethods(mongoose);
|
||||
await mongoose.connect(mongod.getUri());
|
||||
await methods.ensureAgentQueuedTurnIndexes();
|
||||
}, DB_SETUP_TIMEOUT_MS);
|
||||
|
||||
beforeEach(async () => {
|
||||
__resetShutdownStateForTests();
|
||||
await Promise.all(
|
||||
[
|
||||
'AgentTriggerDelivery',
|
||||
'AgentTriggerLaneSequence',
|
||||
'AgentQueuedTurn',
|
||||
'AgentQueuedTurnSequence',
|
||||
'Conversation',
|
||||
'Message',
|
||||
].map((name) => mongoose.models[name].deleteMany({})),
|
||||
);
|
||||
dispatches = [];
|
||||
userId = new mongoose.Types.ObjectId().toString();
|
||||
conversationId = `conversation-${userId}`;
|
||||
manager = new GenerationJobManagerClass();
|
||||
manager.configure({
|
||||
jobStore: new InMemoryJobStore({ ttlAfterComplete: 0 }),
|
||||
eventTransport: new InMemoryEventTransport(),
|
||||
isRedis: false,
|
||||
});
|
||||
manager.initialize();
|
||||
await mongoose.models.Conversation.create({
|
||||
conversationId,
|
||||
user: userId,
|
||||
tenantId: TENANT_ID,
|
||||
endpoint: 'agents',
|
||||
agent_id: AGENT_ID,
|
||||
});
|
||||
await mongoose.models.Message.create({
|
||||
messageId: RUN_RESPONSE_ID,
|
||||
conversationId,
|
||||
user: userId,
|
||||
tenantId: TENANT_ID,
|
||||
parentMessageId: 'run-request',
|
||||
isCreatedByUser: false,
|
||||
endpoint: 'agents',
|
||||
content: [],
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await service?.stop();
|
||||
service = undefined;
|
||||
await manager.destroy();
|
||||
__resetShutdownStateForTests();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
}, DB_SETUP_TIMEOUT_MS);
|
||||
|
||||
describe('completion wake-ups and queued user turns at run end', () => {
|
||||
it('admits the queued turn first, then runs the wake-up chained to its response', async () => {
|
||||
const run = await manager.createJob(conversationId, userId, conversationId);
|
||||
const { service: triggers, lifecycle } = await startDelivery();
|
||||
const wakeupKey = await enqueueWakeup(triggers);
|
||||
const queued = await enqueueQueuedTurn(lifecycle, run.createdAt);
|
||||
await waitForDeferral(BACKGROUND_TOOL_COMPLETION_SOURCE);
|
||||
await waitForDeferral(AGENT_QUEUED_TURN_SOURCE);
|
||||
expect(dispatches).toEqual([]);
|
||||
|
||||
const runEndedAt = Date.now();
|
||||
await manager.completeJob(conversationId, undefined, run.createdAt);
|
||||
|
||||
const [userTurn] = await waitUntil(
|
||||
async () => (dispatches.length > 0 ? dispatches : undefined),
|
||||
'the first admission after the run ended',
|
||||
);
|
||||
expect(userTurn).toMatchObject({ kind: 'queued-turn', parentMessageId: RUN_RESPONSE_ID });
|
||||
expect(userTurn.dispatchedAt - runEndedAt).toBeLessThan(PROMPT_ADMISSION_MS);
|
||||
await waitUntil(async () => {
|
||||
const row = await mongoose.models.AgentQueuedTurn.findById(queued.queuedTurnId).lean<{
|
||||
status: string;
|
||||
}>();
|
||||
return row?.status === 'admitted' ? row : undefined;
|
||||
}, 'the queued turn admission receipt');
|
||||
const yielded = await waitForDeferral(BACKGROUND_TOOL_COMPLETION_SOURCE);
|
||||
expect(yielded.attempts).toBe(0);
|
||||
expect(dispatches).toHaveLength(1);
|
||||
|
||||
await manager.completeJob(conversationId, undefined, userTurn.generationCreatedAt);
|
||||
|
||||
const [, wakeup] = await waitUntil(
|
||||
async () => (dispatches.length > 1 ? dispatches : undefined),
|
||||
'the wake-up after the queued turn',
|
||||
);
|
||||
expect(wakeup).toMatchObject({
|
||||
kind: 'wake-up',
|
||||
parentMessageId: userTurn.responseMessageId,
|
||||
});
|
||||
await expect(
|
||||
mongoose.models.AgentTriggerDelivery.findOne({ deliveryKey: wakeupKey }).lean(),
|
||||
).resolves.toMatchObject({ attempts: 1 });
|
||||
});
|
||||
|
||||
it('runs the wake-up at run end when no queued turn is pending', async () => {
|
||||
const run = await manager.createJob(conversationId, userId, conversationId);
|
||||
const { service: triggers } = await startDelivery();
|
||||
await enqueueWakeup(triggers);
|
||||
await waitForDeferral(BACKGROUND_TOOL_COMPLETION_SOURCE);
|
||||
|
||||
await manager.completeJob(conversationId, undefined, run.createdAt);
|
||||
|
||||
const [wakeup] = await waitUntil(
|
||||
async () => (dispatches.length > 0 ? dispatches : undefined),
|
||||
'the wake-up after the run ended',
|
||||
);
|
||||
expect(wakeup).toMatchObject({ kind: 'wake-up', parentMessageId: RUN_RESPONSE_ID });
|
||||
});
|
||||
|
||||
it('does not hold a wake-up behind a queued turn whose claim owner died', async () => {
|
||||
const run = await manager.createJob(conversationId, userId, conversationId);
|
||||
const { service: triggers } = await startDelivery();
|
||||
const { turn } = await methods.enqueueAgentQueuedTurn({
|
||||
user: new mongoose.Types.ObjectId(userId),
|
||||
tenantId: TENANT_ID,
|
||||
conversationId,
|
||||
agentId: AGENT_ID,
|
||||
parentMessageId: RUN_RESPONSE_ID,
|
||||
clientRequestId: 'abandoned-follow-up',
|
||||
text: 'follow-up whose worker disappeared',
|
||||
availableAt: new Date(Date.now() - 10 * 60_000),
|
||||
});
|
||||
const claimedAt = new Date(Date.now() - 5 * 60_000);
|
||||
await expect(
|
||||
methods.claimNextAgentQueuedTurn({
|
||||
user: new mongoose.Types.ObjectId(userId),
|
||||
tenantId: TENANT_ID,
|
||||
conversationId,
|
||||
queuedTurnId: turn.queuedTurnId,
|
||||
claimId: 'lost-claim',
|
||||
claimBy: 'lost-worker',
|
||||
now: claimedAt,
|
||||
leaseUntil: new Date(claimedAt.getTime() + 2 * 60_000),
|
||||
}),
|
||||
).resolves.toMatchObject({ outcome: 'acquired' });
|
||||
await enqueueWakeup(triggers);
|
||||
await waitForDeferral(BACKGROUND_TOOL_COMPLETION_SOURCE);
|
||||
|
||||
await manager.completeJob(conversationId, undefined, run.createdAt);
|
||||
|
||||
const [wakeup] = await waitUntil(
|
||||
async () => (dispatches.length > 0 ? dispatches : undefined),
|
||||
'the wake-up past an expired queued-turn claim',
|
||||
);
|
||||
expect(wakeup).toMatchObject({ kind: 'wake-up', parentMessageId: RUN_RESPONSE_ID });
|
||||
});
|
||||
});
|
||||
60
packages/api/src/agents/precedence.ts
Normal file
60
packages/api/src/agents/precedence.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import type { AgentQueuedTurnMethods } from '@librechat/data-schemas';
|
||||
import type { AgentContinueTriggerEnvelope } from './triggers/envelope';
|
||||
import { AgentTriggerExecutionError } from './triggers/host';
|
||||
|
||||
/** Optional so partial method bundles keep the previous behavior. */
|
||||
export type QueuedTurnPrecedenceMethods = Partial<
|
||||
Pick<AgentQueuedTurnMethods, 'hasPendingAgentQueuedTurn'>
|
||||
>;
|
||||
|
||||
function deferral(code: string, message: string): AgentTriggerExecutionError {
|
||||
return new AgentTriggerExecutionError(message, {
|
||||
mode: 'continue',
|
||||
certainty: 'definite',
|
||||
code,
|
||||
retryable: true,
|
||||
status: 409,
|
||||
retryAfter: '1',
|
||||
deferWithoutAttempt: true,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The deferral a completion wake-up must take so the user's queued turn gets
|
||||
* the conversation first, or null when the wake-up may proceed. Read under the
|
||||
* delivery's own principal and tenant. A failed read is not proof of absence,
|
||||
* so it defers too; neither deferral consumes a delivery attempt. The queued
|
||||
* turn's own generation then holds the conversation, and the wake-up resumes
|
||||
* after it on that turn's response.
|
||||
*/
|
||||
export async function getQueuedTurnDeferral(
|
||||
methods: QueuedTurnPrecedenceMethods,
|
||||
envelope: AgentContinueTriggerEnvelope,
|
||||
now: number,
|
||||
): Promise<AgentTriggerExecutionError | null> {
|
||||
const hasPending = methods.hasPendingAgentQueuedTurn;
|
||||
if (hasPending == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const pending = await hasPending({
|
||||
userId: envelope.principal.userId,
|
||||
...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }),
|
||||
conversationId: envelope.target.conversationId,
|
||||
now: new Date(now),
|
||||
});
|
||||
return pending
|
||||
? deferral('QUEUED_TURN_PENDING', 'A queued user turn is admitted before this wake-up.')
|
||||
: null;
|
||||
} catch (error) {
|
||||
logger.warn('[agent-triggers] Queued turn precedence is unavailable; deferring wake-up', {
|
||||
conversationId: envelope.target.conversationId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return deferral(
|
||||
'QUEUED_TURN_STATE_UNAVAILABLE',
|
||||
'Queued user turn state is temporarily unavailable.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -309,6 +309,108 @@ describe('createSubagentCompletionWakeupResolver', () => {
|
|||
expect(methods.claimSubagentTaskResult).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe('queued user turn precedence', () => {
|
||||
const OWNER_ID = '507f1f77bcf86cd799439011';
|
||||
|
||||
function ownedEnvelope(): AgentContinueTriggerEnvelope {
|
||||
const envelope = wakeupEnvelope();
|
||||
return { ...envelope, principal: { ...envelope.principal, userId: OWNER_ID } };
|
||||
}
|
||||
|
||||
it('defers without consuming an attempt or claiming the child result', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
const hasPendingAgentQueuedTurn = jest.fn(async () => true);
|
||||
const resolve = createSubagentCompletionWakeupResolver({
|
||||
methods: { ...methods, hasPendingAgentQueuedTurn } as never,
|
||||
getGenerationJob: async () => null,
|
||||
now: () => NOW,
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolve(ownedEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),
|
||||
).rejects.toMatchObject({
|
||||
code: 'QUEUED_TURN_PENDING',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
});
|
||||
expect(hasPendingAgentQueuedTurn).toHaveBeenCalledTimes(1);
|
||||
expect(hasPendingAgentQueuedTurn).toHaveBeenCalledWith({
|
||||
userId: OWNER_ID,
|
||||
tenantId: 'tenant-1',
|
||||
conversationId: 'conversation-1',
|
||||
now: new Date(NOW),
|
||||
});
|
||||
expect(methods.claimSubagentTaskResult).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps conversation authorization ahead of the queued-turn deferral', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
methods.getConvo.mockImplementation(async (_userId: string, conversationId: string) => ({
|
||||
conversationId,
|
||||
tenantId: 'tenant-2',
|
||||
}));
|
||||
const resolve = createSubagentCompletionWakeupResolver({
|
||||
methods: { ...methods, hasPendingAgentQueuedTurn: async () => true } as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolve(ownedEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),
|
||||
).rejects.toMatchObject({ code: 'PARENT_NOT_FOUND', retryable: false });
|
||||
});
|
||||
|
||||
it('does not consult queued turns for a replay of its own admitted continuation', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
const hasPendingAgentQueuedTurn = jest.fn(async () => true);
|
||||
const resolve = createSubagentCompletionWakeupResolver({
|
||||
methods: { ...methods, hasPendingAgentQueuedTurn } as never,
|
||||
getGenerationJob: async () => ({
|
||||
status: 'running',
|
||||
metadata: { idempotencyClientRequestId: 'trigger_claim_1' },
|
||||
}),
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolve(ownedEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),
|
||||
).resolves.toMatchObject({ status: 'ready' });
|
||||
expect(hasPendingAgentQueuedTurn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('proceeds when no queued turn is pending', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
const resolve = createSubagentCompletionWakeupResolver({
|
||||
methods: { ...methods, hasPendingAgentQueuedTurn: async () => false } as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolve(ownedEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),
|
||||
).resolves.toMatchObject({ status: 'ready' });
|
||||
});
|
||||
|
||||
it('defers rather than proceeding when queued-turn state cannot be read', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
const resolve = createSubagentCompletionWakeupResolver({
|
||||
methods: {
|
||||
...methods,
|
||||
hasPendingAgentQueuedTurn: async () => {
|
||||
throw new Error('queued turn read failed');
|
||||
},
|
||||
} as never,
|
||||
getGenerationJob: async () => null,
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolve(ownedEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),
|
||||
).rejects.toMatchObject({
|
||||
code: 'QUEUED_TURN_STATE_UNAVAILABLE',
|
||||
retryable: true,
|
||||
deferWithoutAttempt: true,
|
||||
});
|
||||
expect(methods.claimSubagentTaskResult).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it('lets a lost-receipt retry reach HTTP dedup for its own active continuation', async () => {
|
||||
const { methods } = resolverMethods();
|
||||
const resolve = createSubagentCompletionWakeupResolver({
|
||||
|
|
|
|||
|
|
@ -10,11 +10,13 @@ import type { SubagentTaskWakeupRegistration } from './subagentThreads';
|
|||
import type { AgentContinueTriggerEnvelope } from './triggers/envelope';
|
||||
import type { AgentTriggerDispatchContext } from './triggers/dispatch';
|
||||
import type { AgentTriggerEnqueueOptions } from './triggers/delivery';
|
||||
import type { QueuedTurnPrecedenceMethods } from './precedence';
|
||||
import { resolveScheduleMCPCompletion } from '~/schedules/authorization/continuation';
|
||||
import { WAITING_RETRY_CAP_MS, waitingRetryAfter } from './triggers/backoff';
|
||||
import { boundedSubagentTaskResult } from './subagentTaskRouting';
|
||||
import { createAgentTriggerEnvelope } from './triggers/envelope';
|
||||
import { AgentTriggerExecutionError } from './triggers/host';
|
||||
import { getQueuedTurnDeferral } from './precedence';
|
||||
|
||||
const WAKEUP_ADMISSION_DELAY_MS = 250;
|
||||
/** SDK tasks time out after 30 minutes; this grace covers terminal persistence. */
|
||||
|
|
@ -39,6 +41,7 @@ export type EnqueueAgentTrigger = (
|
|||
|
||||
type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
|
||||
Pick<ConversationMethods, 'listActiveSubagentThreadLeases'> &
|
||||
QueuedTurnPrecedenceMethods &
|
||||
Pick<
|
||||
MessageMethods,
|
||||
'claimSubagentTaskResult' | 'getMessages' | 'releaseSubagentTaskResultClaim'
|
||||
|
|
@ -624,10 +627,8 @@ export function createSubagentCompletionWakeupResolver({
|
|||
{ code: 'PARENT_STATE_UNAVAILABLE', retryable: true },
|
||||
);
|
||||
}
|
||||
if (
|
||||
isParentActive(parentJob) &&
|
||||
parentJob?.metadata?.idempotencyClientRequestId !== context.idempotencyKey
|
||||
) {
|
||||
const ownsParent = parentJob?.metadata?.idempotencyClientRequestId === context.idempotencyKey;
|
||||
if (isParentActive(parentJob) && !ownsParent) {
|
||||
throw executionError('The parent generation has not settled yet.', {
|
||||
code: 'PARENT_NOT_READY',
|
||||
retryable: true,
|
||||
|
|
@ -639,7 +640,7 @@ export function createSubagentCompletionWakeupResolver({
|
|||
|
||||
const userId = envelope.principal.userId;
|
||||
const tenantId = envelope.principal.tenantId;
|
||||
const [parent, child, taskMessages] = await Promise.all([
|
||||
const [parent, child, taskMessages, queuedTurnDeferral] = await Promise.all([
|
||||
methods.getConvo(userId, envelope.target.conversationId),
|
||||
methods.getConvo(userId, registration.threadId),
|
||||
methods.getMessages(
|
||||
|
|
@ -651,6 +652,7 @@ export function createSubagentCompletionWakeupResolver({
|
|||
TASK_SELECT,
|
||||
{ sort: { createdAt: 1, _id: 1 } },
|
||||
),
|
||||
ownsParent ? null : getQueuedTurnDeferral(methods, envelope, now()),
|
||||
]);
|
||||
if (parent == null || !sameTenant(parent.tenantId, tenantId)) {
|
||||
throw executionError('The parent conversation is no longer available.', {
|
||||
|
|
@ -673,6 +675,9 @@ export function createSubagentCompletionWakeupResolver({
|
|||
status: 404,
|
||||
});
|
||||
}
|
||||
if (queuedTurnDeferral != null) {
|
||||
throw queuedTurnDeferral;
|
||||
}
|
||||
let resultTaskId = registration.taskId;
|
||||
let terminal = taskMessages.find(
|
||||
(message) =>
|
||||
|
|
|
|||
|
|
@ -111,6 +111,20 @@ child again. A held delivery consumes its wake marker in the same fenced write t
|
|||
for ordering or defers readiness. These writes use classic operators for DocumentDB compatibility;
|
||||
older workers can ignore the optional marker and fall back to the configured polling interval.
|
||||
|
||||
A user's server-owned queued turn goes before completion wake-ups. Parent settlement also
|
||||
expedites that conversation's queued-turn deliveries, and a background or subagent wake-up defers
|
||||
without consuming an attempt while a queued turn holds an unexpired claim or is the due lane head.
|
||||
The wake-up then resumes after the queued turn's generation, on that turn's response. Rows that
|
||||
need recovery or a user decision never hold a wake-up: a reservation, an expired claim, an
|
||||
indeterminate admission, or a dead head awaiting dismissal. The check reads the active lane first
|
||||
and those blockers only afterwards, for a due head, so a blocker that has just cleared (a dismissed
|
||||
dead head, for example) cannot let a wake-up past the turn it was blocking; a conversation with no
|
||||
due queued turn costs one read. The answer holds for the instant of that last read: a turn that
|
||||
becomes admissible later is ordered by the generation predecessor fence, like any later send. A
|
||||
failed precedence read is not treated as absence; the wake-up defers on it the same way. Older
|
||||
replicas skip the check, so mixed-version fleets keep the earlier first-claim race until every
|
||||
replica is upgraded.
|
||||
|
||||
Setting a recovery cap to `30000` restores its original fixed recovery frequency. No stored-data
|
||||
migration is needed; optional activity reporting preserves the existing numeric/boolean results.
|
||||
|
||||
|
|
|
|||
|
|
@ -546,6 +546,8 @@ export function createAgentTriggerDeliveryEngine(
|
|||
logger.info('[agent-triggers] delivery deferred without consuming an attempt', {
|
||||
deliveryKey: delivery.deliveryKey,
|
||||
reason,
|
||||
...(error instanceof AgentTriggerExecutionError &&
|
||||
error.code != null && { code: error.code }),
|
||||
availableAt: availableAt.toISOString(),
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -340,7 +340,7 @@ describe('durable agent trigger service', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("expedites a principal's completion deliveries when one of its generations settles", async () => {
|
||||
it("expedites a principal's completion deliveries and queued turns when one of its generations settles", async () => {
|
||||
const methods = deliveryMethods({
|
||||
expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 2, held: 0 })),
|
||||
});
|
||||
|
|
@ -363,7 +363,7 @@ describe('durable agent trigger service', () => {
|
|||
expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({
|
||||
user: '507f1f77bcf86cd799439011',
|
||||
conversationId: 'conversation-1',
|
||||
sourceIds: ['background-tool-completion', 'subagent-completion'],
|
||||
sourceIds: ['background-tool-completion', 'subagent-completion', 'agent-queued-turn'],
|
||||
now: expect.any(Date),
|
||||
});
|
||||
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import { BACKGROUND_TOOL_COMPLETION_SOURCE } from '../backgroundCompletionWakeup
|
|||
import { isShutdownInProgress, registerShutdownTask } from '../../app/shutdown';
|
||||
import { SUBAGENT_COMPLETION_SOURCE } from '../subagentCompletionWakeup';
|
||||
import { generateAgentTriggerToken } from '../../crypto/jwt';
|
||||
import { AGENT_QUEUED_TURN_SOURCE } from '../queuedTurns';
|
||||
import { prepareAgentTriggerDelivery } from './delivery';
|
||||
import { selfOriginFromAddress } from '../../app/origin';
|
||||
import { createAgentTriggerExecutionHost } from './host';
|
||||
|
|
@ -42,6 +43,8 @@ import { WAITING_RETRY_CAP_MS } from './backoff';
|
|||
|
||||
/** Internal sources whose deliveries wait on a result or on their parent generation. */
|
||||
const COMPLETION_WAKEUP_SOURCES = [BACKGROUND_TOOL_COMPLETION_SOURCE, SUBAGENT_COMPLETION_SOURCE];
|
||||
/** A settled generation also releases the user's queued turns, which wake-ups yield to. */
|
||||
const SETTLED_CONVERSATION_SOURCES = [...COMPLETION_WAKEUP_SOURCES, AGENT_QUEUED_TURN_SOURCE];
|
||||
|
||||
export const AGENT_TRIGGER_TOKEN_TTL = '60s';
|
||||
const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000;
|
||||
|
|
@ -582,7 +585,12 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
* claim pass always runs: a matching delivery may already be due here without
|
||||
* having moved. Best effort: a missed expedite only means the delivery
|
||||
* re-checks at its backoff instead of immediately. */
|
||||
const expediteCompletions = (input: AgentTriggerCompletionExpedite): void => {
|
||||
const expediteCompletions = (
|
||||
input: AgentTriggerCompletionExpedite,
|
||||
sourceIds: readonly string[] = 'user' in input && input.taskIds != null
|
||||
? [SUBAGENT_COMPLETION_SOURCE]
|
||||
: COMPLETION_WAKEUP_SOURCES,
|
||||
): void => {
|
||||
const expedite = deps.methods?.expediteAgentTriggerDeliveries;
|
||||
if (expedite == null || !deliveryReady || stopping) {
|
||||
return;
|
||||
|
|
@ -596,10 +604,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
...(input.taskIds != null && { taskIds: input.taskIds }),
|
||||
}
|
||||
: { deliveryKeys: input.deliveryKeys }),
|
||||
sourceIds:
|
||||
'user' in input && input.taskIds != null
|
||||
? [SUBAGENT_COMPLETION_SOURCE]
|
||||
: COMPLETION_WAKEUP_SOURCES,
|
||||
sourceIds,
|
||||
now: new Date(),
|
||||
}),
|
||||
)
|
||||
|
|
@ -691,9 +696,12 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
deliveryReady = true;
|
||||
deliveryEngine.start();
|
||||
/** Deliveries waiting on a parent resume that parent's conversation, so a
|
||||
* settled generation wakes only those, not every waiting task of the user. */
|
||||
* settled generation wakes only those, not every waiting task of the user.
|
||||
* Queued turns move with them: wake-ups defer to a pending queued turn,
|
||||
* which must not wait out its own readiness backoff behind them. */
|
||||
unsubscribeGenerationSettled ??= deps.subscribeGenerationSettled?.(
|
||||
({ userId, conversationId }) => expediteCompletions({ user: userId, conversationId }),
|
||||
({ userId, conversationId }) =>
|
||||
expediteCompletions({ user: userId, conversationId }, SETTLED_CONVERSATION_SOURCES),
|
||||
);
|
||||
startPurgeRecovery();
|
||||
logger.info('[agent-triggers] durable delivery engine started');
|
||||
|
|
|
|||
|
|
@ -3153,3 +3153,304 @@ describe('agent queued turn methods', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('queued turn precedence', () => {
|
||||
const scope = () => ({ user, tenantId: 'tenant-1', conversationId: 'conversation-1' });
|
||||
const precedence = () => ({
|
||||
userId: user.toString(),
|
||||
tenantId: 'tenant-1',
|
||||
conversationId: 'conversation-1',
|
||||
});
|
||||
const pending = (now = START) => methods.hasPendingAgentQueuedTurn({ ...precedence(), now });
|
||||
|
||||
it('answers false for a conversation without queued turns', async () => {
|
||||
await expect(pending()).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('reports a due queued head only inside its exact owner scope', async () => {
|
||||
await methods.enqueueAgentQueuedTurn(enqueueInput());
|
||||
|
||||
await expect(pending()).resolves.toBe(true);
|
||||
await expect(
|
||||
methods.hasPendingAgentQueuedTurn({ ...precedence(), tenantId: 'tenant-2', now: START }),
|
||||
).resolves.toBe(false);
|
||||
await expect(
|
||||
methods.hasPendingAgentQueuedTurn({ ...precedence(), tenantId: undefined, now: START }),
|
||||
).resolves.toBe(false);
|
||||
await expect(
|
||||
methods.hasPendingAgentQueuedTurn({
|
||||
...precedence(),
|
||||
userId: new mongoose.Types.ObjectId().toString(),
|
||||
now: START,
|
||||
}),
|
||||
).resolves.toBe(false);
|
||||
await expect(
|
||||
methods.hasPendingAgentQueuedTurn({ ...precedence(), userId: 'not-an-id', now: START }),
|
||||
).resolves.toBe(false);
|
||||
await expect(
|
||||
methods.hasPendingAgentQueuedTurn({ ...precedence(), conversationId: 'other', now: START }),
|
||||
).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('does not report a head that is not yet available', async () => {
|
||||
await methods.enqueueAgentQueuedTurn(enqueueInput({ availableAt: LATER }));
|
||||
|
||||
await expect(pending()).resolves.toBe(false);
|
||||
await expect(pending(LATER)).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it('bounds an admitting claim by its lease and stops at admission', async () => {
|
||||
const first = await methods.enqueueAgentQueuedTurn(enqueueInput());
|
||||
await methods.enqueueAgentQueuedTurn(enqueueInput({ clientRequestId: 'second' }));
|
||||
const claimed = await methods.claimNextAgentQueuedTurn(claimInput(first.turn.queuedTurnId));
|
||||
expect(claimed.outcome).toBe('acquired');
|
||||
|
||||
await expect(pending()).resolves.toBe(true);
|
||||
/** A lease that outlived its owner fences the lane for recovery, so the
|
||||
* successor behind it cannot be admitted next either. */
|
||||
await expect(pending(new Date(LATER.getTime() + 1))).resolves.toBe(false);
|
||||
|
||||
await Turn.updateOne(
|
||||
{ _id: first.turn.queuedTurnId },
|
||||
{
|
||||
$set: { status: 'admitted' },
|
||||
$unset: { claimId: 1, claimBy: 1, claimUntil: 1, admissionSlot: 1, activeSlot: 1 },
|
||||
},
|
||||
);
|
||||
await expect(pending()).resolves.toBe(true);
|
||||
await Turn.updateMany({}, { $set: { status: 'admitted' } });
|
||||
await expect(pending()).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('does not wait on a dead head awaiting dismissal or on a quarantined admission', async () => {
|
||||
const first = await methods.enqueueAgentQueuedTurn(enqueueInput());
|
||||
await methods.enqueueAgentQueuedTurn(enqueueInput({ clientRequestId: 'second' }));
|
||||
await methods.claimNextAgentQueuedTurn(claimInput(first.turn.queuedTurnId));
|
||||
await methods.releaseAgentQueuedTurn({
|
||||
...scope(),
|
||||
queuedTurnId: first.turn.queuedTurnId,
|
||||
claimId: 'claim-1',
|
||||
claimBy: 'worker-1',
|
||||
disposition: 'dead',
|
||||
settledAt: START,
|
||||
failure: { code: 'PREDECESSOR_ABORTED', message: 'review this turn' },
|
||||
});
|
||||
|
||||
await expect(pending()).resolves.toBe(false);
|
||||
|
||||
await methods.cancelAgentQueuedTurn({
|
||||
...scope(),
|
||||
queuedTurnId: first.turn.queuedTurnId,
|
||||
settledAt: START,
|
||||
});
|
||||
await expect(pending()).resolves.toBe(true);
|
||||
|
||||
await Turn.updateOne(
|
||||
{ _id: first.turn.queuedTurnId },
|
||||
{
|
||||
$set: {
|
||||
status: 'dead',
|
||||
terminalReceipt: {
|
||||
outcome: 'dead',
|
||||
settledAt: START,
|
||||
failure: { code: 'ADMISSION_INDETERMINATE', message: 'owner disappeared' },
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
await expect(pending()).resolves.toBe(false);
|
||||
});
|
||||
|
||||
/** Runs `transition` around the active-lane read: once it is issued and
|
||||
* `before` it executes, or `after` it has returned and before the blocker read. */
|
||||
function interleaveActiveLaneRead(
|
||||
transition: () => Promise<unknown>,
|
||||
when: 'before' | 'after' = 'before',
|
||||
) {
|
||||
const find = Turn.find.bind(Turn);
|
||||
jest.spyOn(Turn, 'find').mockImplementationOnce((...args: Parameters<typeof find>) => {
|
||||
const query = find(...args);
|
||||
const exec = query.exec.bind(query);
|
||||
query.exec = async () => {
|
||||
if (when === 'before') {
|
||||
await transition();
|
||||
return exec();
|
||||
}
|
||||
const rows = await exec();
|
||||
await transition();
|
||||
return rows;
|
||||
};
|
||||
return query;
|
||||
});
|
||||
}
|
||||
|
||||
async function deadHeadWithSuccessor() {
|
||||
const first = await methods.enqueueAgentQueuedTurn(enqueueInput());
|
||||
await methods.enqueueAgentQueuedTurn(enqueueInput({ clientRequestId: 'second' }));
|
||||
await methods.claimNextAgentQueuedTurn(claimInput(first.turn.queuedTurnId));
|
||||
await methods.releaseAgentQueuedTurn({
|
||||
...scope(),
|
||||
queuedTurnId: first.turn.queuedTurnId,
|
||||
claimId: 'claim-1',
|
||||
claimBy: 'worker-1',
|
||||
disposition: 'dead',
|
||||
settledAt: START,
|
||||
failure: { code: 'PREDECESSOR_ABORTED', message: 'review this turn' },
|
||||
});
|
||||
return first.turn;
|
||||
}
|
||||
|
||||
async function reservationBesideHead() {
|
||||
const first = await methods.enqueueAgentQueuedTurn(enqueueInput());
|
||||
return Turn.create({
|
||||
...scope(),
|
||||
agentId: 'agent-1',
|
||||
parentMessageId: 'parent-1',
|
||||
clientRequestId: 'interrupted-reservation',
|
||||
fingerprint: 'x'.repeat(43),
|
||||
laneId: first.turn.laneId,
|
||||
activeSlot: ((first.turn.activeSlot ?? 0) + 1) % 100,
|
||||
status: 'reserving',
|
||||
priority: false,
|
||||
text: 'repair me',
|
||||
attempts: 0,
|
||||
availableAt: START,
|
||||
deliveryState: 'pending',
|
||||
});
|
||||
}
|
||||
|
||||
it('sees a claim that lands while the lane is being read', async () => {
|
||||
const queued = await methods.enqueueAgentQueuedTurn(enqueueInput());
|
||||
interleaveActiveLaneRead(() =>
|
||||
methods.claimNextAgentQueuedTurn(claimInput(queued.turn.queuedTurnId)),
|
||||
);
|
||||
|
||||
await expect(pending()).resolves.toBe(true);
|
||||
await expect(Turn.findById(queued.turn.queuedTurnId).lean()).resolves.toMatchObject({
|
||||
status: 'claimed',
|
||||
});
|
||||
});
|
||||
|
||||
it('sees a released claim that is due again while the lane is being read', async () => {
|
||||
const queued = await methods.enqueueAgentQueuedTurn(enqueueInput());
|
||||
await methods.claimNextAgentQueuedTurn(claimInput(queued.turn.queuedTurnId));
|
||||
interleaveActiveLaneRead(() =>
|
||||
methods.releaseAgentQueuedTurn({
|
||||
...scope(),
|
||||
queuedTurnId: queued.turn.queuedTurnId,
|
||||
claimId: 'claim-1',
|
||||
claimBy: 'worker-1',
|
||||
disposition: 'retry',
|
||||
availableAt: START,
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(pending()).resolves.toBe(true);
|
||||
await expect(Turn.findById(queued.turn.queuedTurnId).lean()).resolves.toMatchObject({
|
||||
status: 'queued',
|
||||
});
|
||||
});
|
||||
|
||||
it('orders a priority dead row ahead of an earlier queued row', async () => {
|
||||
await methods.enqueueAgentQueuedTurn(enqueueInput());
|
||||
const interrupt = await methods.enqueueAgentQueuedTurn(
|
||||
enqueueInput({ clientRequestId: 'interrupt', priority: true }),
|
||||
);
|
||||
await methods.claimNextAgentQueuedTurn(claimInput(interrupt.turn.queuedTurnId));
|
||||
await methods.releaseAgentQueuedTurn({
|
||||
...scope(),
|
||||
queuedTurnId: interrupt.turn.queuedTurnId,
|
||||
claimId: 'claim-1',
|
||||
claimBy: 'worker-1',
|
||||
disposition: 'dead',
|
||||
settledAt: START,
|
||||
failure: { code: 'PREDECESSOR_FAILED', message: 'review this turn' },
|
||||
});
|
||||
|
||||
await expect(pending()).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('still answers before startup has built the capacity index', async () => {
|
||||
await methods.enqueueAgentQueuedTurn(enqueueInput());
|
||||
await Turn.collection.dropIndex('agent_queued_turn_active_capacity');
|
||||
try {
|
||||
await expect(pending()).resolves.toBe(true);
|
||||
} finally {
|
||||
await Turn.createIndexes();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not wait on an unfinished reservation', async () => {
|
||||
await reservationBesideHead();
|
||||
|
||||
await expect(pending()).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it.each(['before', 'after'] as const)(
|
||||
'sees a successor whose dead head is dismissed %s the lane read returns',
|
||||
async (when) => {
|
||||
const dead = await deadHeadWithSuccessor();
|
||||
await expect(pending()).resolves.toBe(false);
|
||||
interleaveActiveLaneRead(
|
||||
() =>
|
||||
methods.cancelAgentQueuedTurn({
|
||||
...scope(),
|
||||
queuedTurnId: dead.queuedTurnId,
|
||||
settledAt: START,
|
||||
}),
|
||||
when,
|
||||
);
|
||||
|
||||
await expect(pending()).resolves.toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it('sees a head whose reservation fence settles after the lane read returns', async () => {
|
||||
const reservation = await reservationBesideHead();
|
||||
interleaveActiveLaneRead(
|
||||
() =>
|
||||
Turn.updateOne(
|
||||
{ _id: reservation._id, status: 'reserving' },
|
||||
{ $set: { status: 'queued', sequence: 2 } },
|
||||
),
|
||||
'after',
|
||||
);
|
||||
|
||||
await expect(pending()).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it('sees a head whose expired claim is renewed after the lane read returns', async () => {
|
||||
const first = await methods.enqueueAgentQueuedTurn(enqueueInput());
|
||||
await methods.enqueueAgentQueuedTurn(enqueueInput({ clientRequestId: 'second' }));
|
||||
await methods.claimNextAgentQueuedTurn(claimInput(first.turn.queuedTurnId));
|
||||
const expired = new Date(LATER.getTime() + 1);
|
||||
await expect(pending(expired)).resolves.toBe(false);
|
||||
interleaveActiveLaneRead(
|
||||
() =>
|
||||
Turn.updateOne(
|
||||
{ _id: first.turn.queuedTurnId, status: 'claimed' },
|
||||
{ $set: { claimUntil: new Date(expired.getTime() + 60_000) } },
|
||||
),
|
||||
'after',
|
||||
);
|
||||
|
||||
await expect(pending(expired)).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it('reads blockers only for a due head', async () => {
|
||||
const find = jest.spyOn(Turn, 'find');
|
||||
const exists = jest.spyOn(Turn, 'exists');
|
||||
|
||||
await expect(pending()).resolves.toBe(false);
|
||||
await methods.enqueueAgentQueuedTurn(enqueueInput({ availableAt: LATER }));
|
||||
find.mockClear();
|
||||
exists.mockClear();
|
||||
await expect(pending()).resolves.toBe(false);
|
||||
expect(find).toHaveBeenCalledTimes(1);
|
||||
expect(exists).not.toHaveBeenCalled();
|
||||
|
||||
await expect(pending(LATER)).resolves.toBe(true);
|
||||
expect(find).toHaveBeenCalledTimes(2);
|
||||
expect(exists).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ const RETIRED_LANE_RETENTION_MS = 24 * 60 * 60_000;
|
|||
const ROOT_LINEAGE_PREDECESSOR_PREFIX = 'root:';
|
||||
const CLAIM_LANE_INDEX = 'agent_queued_turn_claim_lane';
|
||||
const ADMISSION_STARTED_LANE_INDEX = 'agent_queued_turn_admission_started_lane';
|
||||
const ACTIVE_CAPACITY_INDEX = 'agent_queued_turn_active_capacity';
|
||||
const NAMESPACE_EXISTS = 48;
|
||||
|
||||
interface DuplicateKeyError {
|
||||
|
|
@ -125,6 +126,14 @@ export interface EnqueueAgentQueuedTurnInput extends AgentQueuedTurnConversation
|
|||
availableAt?: Date;
|
||||
}
|
||||
|
||||
/** Plain identifiers, so callers outside data-schemas need no Mongoose types. */
|
||||
export interface AgentQueuedTurnPrecedenceInput {
|
||||
userId: string;
|
||||
tenantId?: string;
|
||||
conversationId: string;
|
||||
now: Date;
|
||||
}
|
||||
|
||||
export interface AgentQueuedTurnClaimFence extends AgentQueuedTurnConversationScope {
|
||||
queuedTurnId: string;
|
||||
claimId: string;
|
||||
|
|
@ -261,6 +270,8 @@ export interface AgentQueuedTurnMethods {
|
|||
leaseUntil: Date;
|
||||
},
|
||||
) => Promise<ClaimAgentQueuedTurnResult>;
|
||||
/** Whether a queued user turn owns the conversation's next admission. */
|
||||
hasPendingAgentQueuedTurn: (input: AgentQueuedTurnPrecedenceInput) => Promise<boolean>;
|
||||
releaseAgentQueuedTurn: (
|
||||
input: AgentQueuedTurnClaimFence &
|
||||
(
|
||||
|
|
@ -617,6 +628,17 @@ function effectiveDeliveryState(turn: IAgentQueuedTurn) {
|
|||
return 'published' as const;
|
||||
}
|
||||
|
||||
/** Claim order: priority rows first, then ascending sequence. */
|
||||
function precedes(
|
||||
row: Pick<IAgentQueuedTurn, 'priority' | 'sequence'>,
|
||||
other: Pick<IAgentQueuedTurn, 'priority' | 'sequence'>,
|
||||
): boolean {
|
||||
if (row.priority !== other.priority) {
|
||||
return row.priority === true;
|
||||
}
|
||||
return (row.sequence ?? Number.MAX_SAFE_INTEGER) < (other.sequence ?? Number.MAX_SAFE_INTEGER);
|
||||
}
|
||||
|
||||
function toRecord(turn: IAgentQueuedTurn): AgentQueuedTurnRecord {
|
||||
if (
|
||||
turn._id == null ||
|
||||
|
|
@ -1886,6 +1908,87 @@ export function createAgentQueuedTurnMethods(
|
|||
: { outcome: 'replayed', claim: racedClaim };
|
||||
}
|
||||
|
||||
/**
|
||||
* True while a claim is admitting a queued turn under an unexpired lease, or
|
||||
* while the lane head is a due `queued` row that `claimNextAgentQueuedTurn`
|
||||
* could acquire. Rows that need recovery or a user decision (a reservation,
|
||||
* an expired claim, an indeterminate admission, a dead row ahead of the
|
||||
* head) block that head and answer false, so callers never wait on them.
|
||||
*
|
||||
* The reads are ordered so that no blocker is older than the turn it blocks.
|
||||
* The active lane is read first, in one query over the capacity index. Its
|
||||
* key (`activeSlot`) is held unchanged from reservation until a terminal
|
||||
* state, so a row moving between `queued` and `claimed` in either direction
|
||||
* is read exactly once, in one state or the other. Blockers are read only
|
||||
* after that, and only for a due head. A blocker that has already cleared (a
|
||||
* dismissed dead row, a finished reservation, a renewed claim) is therefore
|
||||
* never what answers false, and a lane with no due head costs one read.
|
||||
*
|
||||
* The answer holds for the instant of the last read. A turn that becomes
|
||||
* admissible afterwards arrives after the caller and is ordered by the
|
||||
* generation predecessor fence, as any later send is.
|
||||
*/
|
||||
async function hasPendingAgentQueuedTurn(
|
||||
input: AgentQueuedTurnPrecedenceInput,
|
||||
): Promise<boolean> {
|
||||
if (!(input.now instanceof Date) || !Number.isFinite(input.now.getTime())) {
|
||||
throw new TypeError('Agent queued turn precedence requires a valid time');
|
||||
}
|
||||
if (!mongoose.Types.ObjectId.isValid(input.userId)) {
|
||||
return false;
|
||||
}
|
||||
const scope = conversationScope({
|
||||
user: new mongoose.Types.ObjectId(input.userId),
|
||||
...(input.tenantId != null && { tenantId: input.tenantId }),
|
||||
conversationId: input.conversationId,
|
||||
});
|
||||
type LaneRow = Pick<
|
||||
IAgentQueuedTurn,
|
||||
'status' | 'priority' | 'sequence' | 'availableAt' | 'claimUntil'
|
||||
>;
|
||||
const readActiveLane = () =>
|
||||
Turn()
|
||||
.find({ ...scope, activeSlot: { $exists: true } })
|
||||
.select('status priority sequence availableAt claimUntil')
|
||||
.lean<LaneRow[]>();
|
||||
/** The hint only pins the stable key. When it is refused (the index is
|
||||
* not built yet, or an engine rejects the hint), a planner-chosen read
|
||||
* is still correct apart from that narrow transition window, and a
|
||||
* genuine outage fails this second read too. */
|
||||
const active = await readActiveLane()
|
||||
.hint(ACTIVE_CAPACITY_INDEX)
|
||||
.catch(() => readActiveLane());
|
||||
const now = input.now.getTime();
|
||||
let head: LaneRow | undefined;
|
||||
for (const row of active) {
|
||||
if (row.status === 'claimed' && row.claimUntil != null && row.claimUntil.getTime() > now) {
|
||||
return true;
|
||||
}
|
||||
if (row.status === 'queued' && (head == null || precedes(row, head))) {
|
||||
head = row;
|
||||
}
|
||||
}
|
||||
if (head == null || head.availableAt.getTime() > now) {
|
||||
return false;
|
||||
}
|
||||
const earlier = { sequence: { $lt: head.sequence ?? Number.MAX_SAFE_INTEGER } };
|
||||
const blocker = await Turn().exists({
|
||||
...scope,
|
||||
$or: [
|
||||
{ status: 'reserving' },
|
||||
{ status: 'claimed', claimUntil: { $not: { $gt: input.now } } },
|
||||
{ status: 'dead', 'terminalReceipt.failure.code': 'ADMISSION_INDETERMINATE' },
|
||||
{
|
||||
status: 'dead',
|
||||
...(head.priority === true
|
||||
? { priority: true, ...earlier }
|
||||
: { $or: [{ priority: true }, earlier] }),
|
||||
},
|
||||
],
|
||||
});
|
||||
return blocker == null;
|
||||
}
|
||||
|
||||
async function releaseAgentQueuedTurn(
|
||||
input: AgentQueuedTurnClaimFence &
|
||||
(
|
||||
|
|
@ -2969,6 +3072,7 @@ export function createAgentQueuedTurnMethods(
|
|||
markQueuedTurnScheduled,
|
||||
cancelAgentQueuedTurn,
|
||||
claimNextAgentQueuedTurn,
|
||||
hasPendingAgentQueuedTurn,
|
||||
beginAgentQueuedTurnAdmission,
|
||||
releaseAgentQueuedTurn,
|
||||
markAgentQueuedTurnAdmitted,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue