diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts index efb96e171a..6264820954 100644 --- a/packages/api/src/agents/backgroundCompletionWakeup.ts +++ b/packages/api/src/agents/backgroundCompletionWakeup.ts @@ -28,12 +28,14 @@ import type { ScheduleMCPCompletionLookup } from '~/schedules/authorization/cont import type { AgentContinueTriggerEnvelope } from './triggers/envelope'; import type { AgentTriggerDispatchContext } from './triggers/dispatch'; import type { AgentTriggerEnqueueOptions } from './triggers/delivery'; +import type { QueuedTurnPrecedenceMethods } from './precedence'; import { resolveScheduleMCPCompletion } from '~/schedules/authorization/continuation'; import { WAITING_RETRY_CAP_MS, waitingRetryAfter } from './triggers/backoff'; import { BACKGROUND_TOOL_PRODUCER_LEASE_MS } from './backgroundCompletion'; import { SUBAGENT_COMPLETION_SOURCE } from './subagentCompletionWakeup'; import { createAgentTriggerEnvelope } from './triggers/envelope'; import { AgentTriggerExecutionError } from './triggers/host'; +import { getQueuedTurnDeferral } from './precedence'; import { truncateMiddle } from '~/utils'; const WAKEUP_ADMISSION_DELAY_MS = 250; @@ -72,6 +74,7 @@ export type PersistBackgroundToolCompletionResult = ( ) => Promise; type WakeupMethods = Pick & + QueuedTurnPrecedenceMethods & Pick< MessageMethods, 'getMessages' | 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims' @@ -316,10 +319,8 @@ export function createBackgroundToolCompletionWakeupResolver({ { code: 'PARENT_STATE_UNAVAILABLE', retryable: true }, ); } - if ( - isParentActive(parentJob) && - parentJob?.metadata?.idempotencyClientRequestId !== context.idempotencyKey - ) { + const ownsParent = parentJob?.metadata?.idempotencyClientRequestId === context.idempotencyKey; + if (isParentActive(parentJob) && !ownsParent) { throw executionError('The parent generation has not settled yet.', { code: 'PARENT_NOT_READY', retryable: true, @@ -329,7 +330,10 @@ export function createBackgroundToolCompletionWakeupResolver({ }); } const userId = envelope.principal.userId; - const parent = await methods.getConvo(userId, envelope.target.conversationId); + const [parent, queuedTurnDeferral] = await Promise.all([ + methods.getConvo(userId, envelope.target.conversationId), + ownsParent ? null : getQueuedTurnDeferral(methods, envelope, Date.now()), + ]); if (parent == null || parent.tenantId !== envelope.principal.tenantId) { throw executionError('The parent conversation is no longer available.', { code: 'PARENT_NOT_FOUND', @@ -337,6 +341,9 @@ export function createBackgroundToolCompletionWakeupResolver({ status: 404, }); } + if (queuedTurnDeferral != null) { + throw queuedTurnDeferral; + } const payload = envelope.event.payload; const scheduleMCPIdentity = await resolveScheduleMCPCompletion( { diff --git a/packages/api/src/agents/precedence.integration.spec.ts b/packages/api/src/agents/precedence.integration.spec.ts new file mode 100644 index 0000000000..f67f563be6 --- /dev/null +++ b/packages/api/src/agents/precedence.integration.spec.ts @@ -0,0 +1,388 @@ +import mongoose from 'mongoose'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import { + AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, + createMethods, + createModels, +} from '@librechat/data-schemas'; +import type { AllMethods } from '@librechat/data-schemas'; +import type { AgentTriggerDeliveryPersistence, AgentTriggerService } from './triggers/service'; +import type { AgentTriggerFetch } from './triggers/host'; +import { + BACKGROUND_TOOL_COMPLETION_SOURCE, + createBackgroundToolCompletionWakeupResolver, +} from './backgroundCompletionWakeup'; +import { createAgentTriggerEnvelope, getAgentTriggerIdempotencyKey } from './triggers/envelope'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { AGENT_QUEUED_TURN_SOURCE, createAgentQueuedTurnLifecycle } from './queuedTurns'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import { createAgentContinuationResolver } from './triggers/continuation'; +import { __resetShutdownStateForTests } from '../app/shutdown'; +import { createAgentTriggerService } from './triggers/service'; + +const DB_SETUP_TIMEOUT_MS = 60_000; +const TENANT_ID = 'tenant-1'; +const AGENT_ID = 'agent_parent'; +const RUN_RESPONSE_ID = 'run-response'; + +/** Below the five-second readiness re-check a deferred delivery otherwise waits out. */ +const PROMPT_ADMISSION_MS = 2_500; + +interface Dispatch { + kind: 'queued-turn' | 'wake-up'; + parentMessageId: string; + responseMessageId: string; + generationCreatedAt: number; + dispatchedAt: number; +} + +let mongod: MongoMemoryServer; +let methods: AllMethods; +let userId: string; +let conversationId: string; +let manager: GenerationJobManagerClass; +let service: AgentTriggerService | undefined; +let dispatches: Dispatch[]; + +async function waitUntil(read: () => Promise, label: string): Promise { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const value = await read(); + if (value !== undefined) { + return value; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`Timed out waiting for ${label}`); +} + +async function deliveryOf(source: string) { + return mongoose.models.AgentTriggerDelivery.findOne({ + 'envelope.event.source.id': source, + }).lean<{ attempts: number; availableAt: Date; status: string }>(); +} + +/** Readiness deferrals push `availableAt` at least five seconds out. */ +function waitForDeferral(source: string) { + return waitUntil(async () => { + const delivery = await deliveryOf(source); + return delivery != null && delivery.availableAt.getTime() > Date.now() + 2_000 + ? delivery + : undefined; + }, `${source} to defer`); +} + +/** Stands in for the agents controller: refuses to replace an active + * generation, otherwise starts one and persists its user and response rows. */ +const controller: AgentTriggerFetch = async (_url, init) => { + const body = JSON.parse(String(init?.body)) as { + text: string; + parentMessageId: string; + clientRequestId: string; + agentContinuationAdmission?: unknown; + }; + let createdAt: number; + try { + ({ createdAt } = await manager.createJob(conversationId, userId, conversationId, { + rejectActivePredecessor: true, + })); + } catch (error) { + if ((error as { code?: string }).code !== 'GENERATION_PREDECESSOR_MISMATCH') { + throw error; + } + return new Response(JSON.stringify({ code: 'PARENT_NOT_READY' }), { + status: 409, + headers: { 'retry-after': '1' }, + }); + } + const userMessageId = `${body.clientRequestId}:user`; + const responseMessageId = `${body.clientRequestId}:assistant`; + await mongoose.models.Message.create([ + { + messageId: userMessageId, + conversationId, + user: userId, + tenantId: TENANT_ID, + parentMessageId: body.parentMessageId, + isCreatedByUser: true, + text: body.text, + endpoint: 'agents', + }, + { + messageId: responseMessageId, + conversationId, + user: userId, + tenantId: TENANT_ID, + parentMessageId: userMessageId, + isCreatedByUser: false, + endpoint: 'agents', + content: [], + }, + ]); + dispatches.push({ + kind: body.agentContinuationAdmission != null ? 'queued-turn' : 'wake-up', + parentMessageId: body.parentMessageId, + responseMessageId, + generationCreatedAt: createdAt, + dispatchedAt: Date.now(), + }); + return new Response( + JSON.stringify({ + status: 'started', + streamId: conversationId, + conversationId, + generationCreatedAt: createdAt, + }), + { status: 200 }, + ); +}; + +async function startDelivery(): Promise<{ + service: AgentTriggerService; + lifecycle: ReturnType; +}> { + const getGenerationJob = async (id: string) => (await manager.getJob(id)) ?? null; + const getGenerationAdmissionEvidence = ( + ...args: Parameters + ) => manager.getGenerationAdmissionEvidence(...args); + const lifecycle = createAgentQueuedTurnLifecycle({ + methods, + getGenerationJob, + getGenerationAdmissionEvidence, + enqueue: (envelope, options) => triggers.enqueue(envelope, options), + }); + const triggers = createAgentTriggerService({ + methods: methods as AllMethods & AgentTriggerDeliveryPersistence, + fetch: controller, + mintToken: () => 'trigger-token', + subscribeGenerationSettled: (listener) => manager.onGenerationSettled(listener), + settleSourceBeforeDeadLetter: lifecycle.settleBeforeDeadLetter, + prepareContinue: createAgentContinuationResolver({ + eventActor: async () => undefined, + internalSources: new Map([ + [ + BACKGROUND_TOOL_COMPLETION_SOURCE, + createBackgroundToolCompletionWakeupResolver({ + methods, + getGenerationJob, + getGenerationAdmissionEvidence, + }), + ], + [AGENT_QUEUED_TURN_SOURCE, lifecycle.prepareContinue], + ]), + }), + deliveryOptions: { concurrency: 4, tickMs: 20, maxIdleTickMs: 50 }, + }); + await triggers.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } }); + service = triggers; + return { service: triggers, lifecycle }; +} + +async function enqueueWakeup(triggers: AgentTriggerService): Promise { + const taskId = 'background-task'; + const envelope = createAgentTriggerEnvelope({ + mode: 'continue', + requestId: taskId, + deliveryId: taskId, + receivedAt: Date.now(), + principal: { id: userId, tenantId: TENANT_ID }, + event: { + id: taskId, + type: 'background-tool.completion', + occurredAt: Date.now(), + source: { id: BACKGROUND_TOOL_COMPLETION_SOURCE, type: 'internal' }, + payload: { taskId, toolCallId: taskId, toolName: 'tool' }, + }, + target: { conversationId, parentMessageId: RUN_RESPONSE_ID, agentId: AGENT_ID }, + input: 'waiting', + }); + const { deliveryKey } = await triggers.enqueue(envelope, { + orderingKey: taskId, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3, + }); + expect(deliveryKey).toBe(getAgentTriggerIdempotencyKey(envelope)); + await triggers.persistBackgroundToolResult({ + deliveryKey, + sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE, + result: { status: 'completed', output: 'tool output', settledAt: new Date() }, + }); + return deliveryKey; +} + +async function enqueueQueuedTurn( + lifecycle: ReturnType, + predecessorCreatedAt: number, +) { + const { turn } = await methods.enqueueAgentQueuedTurn({ + user: new mongoose.Types.ObjectId(userId), + tenantId: TENANT_ID, + conversationId, + agentId: AGENT_ID, + parentMessageId: RUN_RESPONSE_ID, + clientRequestId: 'queued-follow-up', + text: 'follow-up queued while the run was busy', + expectedPredecessorCreatedAt: predecessorCreatedAt, + }); + await lifecycle.schedule(turn); + return turn; +} + +beforeAll(async () => { + mongod = await MongoMemoryServer.create(); + createModels(mongoose); + methods = createMethods(mongoose); + await mongoose.connect(mongod.getUri()); + await methods.ensureAgentQueuedTurnIndexes(); +}, DB_SETUP_TIMEOUT_MS); + +beforeEach(async () => { + __resetShutdownStateForTests(); + await Promise.all( + [ + 'AgentTriggerDelivery', + 'AgentTriggerLaneSequence', + 'AgentQueuedTurn', + 'AgentQueuedTurnSequence', + 'Conversation', + 'Message', + ].map((name) => mongoose.models[name].deleteMany({})), + ); + dispatches = []; + userId = new mongoose.Types.ObjectId().toString(); + conversationId = `conversation-${userId}`; + manager = new GenerationJobManagerClass(); + manager.configure({ + jobStore: new InMemoryJobStore({ ttlAfterComplete: 0 }), + eventTransport: new InMemoryEventTransport(), + isRedis: false, + }); + manager.initialize(); + await mongoose.models.Conversation.create({ + conversationId, + user: userId, + tenantId: TENANT_ID, + endpoint: 'agents', + agent_id: AGENT_ID, + }); + await mongoose.models.Message.create({ + messageId: RUN_RESPONSE_ID, + conversationId, + user: userId, + tenantId: TENANT_ID, + parentMessageId: 'run-request', + isCreatedByUser: false, + endpoint: 'agents', + content: [], + }); +}); + +afterEach(async () => { + await service?.stop(); + service = undefined; + await manager.destroy(); + __resetShutdownStateForTests(); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongod.stop(); +}, DB_SETUP_TIMEOUT_MS); + +describe('completion wake-ups and queued user turns at run end', () => { + it('admits the queued turn first, then runs the wake-up chained to its response', async () => { + const run = await manager.createJob(conversationId, userId, conversationId); + const { service: triggers, lifecycle } = await startDelivery(); + const wakeupKey = await enqueueWakeup(triggers); + const queued = await enqueueQueuedTurn(lifecycle, run.createdAt); + await waitForDeferral(BACKGROUND_TOOL_COMPLETION_SOURCE); + await waitForDeferral(AGENT_QUEUED_TURN_SOURCE); + expect(dispatches).toEqual([]); + + const runEndedAt = Date.now(); + await manager.completeJob(conversationId, undefined, run.createdAt); + + const [userTurn] = await waitUntil( + async () => (dispatches.length > 0 ? dispatches : undefined), + 'the first admission after the run ended', + ); + expect(userTurn).toMatchObject({ kind: 'queued-turn', parentMessageId: RUN_RESPONSE_ID }); + expect(userTurn.dispatchedAt - runEndedAt).toBeLessThan(PROMPT_ADMISSION_MS); + await waitUntil(async () => { + const row = await mongoose.models.AgentQueuedTurn.findById(queued.queuedTurnId).lean<{ + status: string; + }>(); + return row?.status === 'admitted' ? row : undefined; + }, 'the queued turn admission receipt'); + const yielded = await waitForDeferral(BACKGROUND_TOOL_COMPLETION_SOURCE); + expect(yielded.attempts).toBe(0); + expect(dispatches).toHaveLength(1); + + await manager.completeJob(conversationId, undefined, userTurn.generationCreatedAt); + + const [, wakeup] = await waitUntil( + async () => (dispatches.length > 1 ? dispatches : undefined), + 'the wake-up after the queued turn', + ); + expect(wakeup).toMatchObject({ + kind: 'wake-up', + parentMessageId: userTurn.responseMessageId, + }); + await expect( + mongoose.models.AgentTriggerDelivery.findOne({ deliveryKey: wakeupKey }).lean(), + ).resolves.toMatchObject({ attempts: 1 }); + }); + + it('runs the wake-up at run end when no queued turn is pending', async () => { + const run = await manager.createJob(conversationId, userId, conversationId); + const { service: triggers } = await startDelivery(); + await enqueueWakeup(triggers); + await waitForDeferral(BACKGROUND_TOOL_COMPLETION_SOURCE); + + await manager.completeJob(conversationId, undefined, run.createdAt); + + const [wakeup] = await waitUntil( + async () => (dispatches.length > 0 ? dispatches : undefined), + 'the wake-up after the run ended', + ); + expect(wakeup).toMatchObject({ kind: 'wake-up', parentMessageId: RUN_RESPONSE_ID }); + }); + + it('does not hold a wake-up behind a queued turn whose claim owner died', async () => { + const run = await manager.createJob(conversationId, userId, conversationId); + const { service: triggers } = await startDelivery(); + const { turn } = await methods.enqueueAgentQueuedTurn({ + user: new mongoose.Types.ObjectId(userId), + tenantId: TENANT_ID, + conversationId, + agentId: AGENT_ID, + parentMessageId: RUN_RESPONSE_ID, + clientRequestId: 'abandoned-follow-up', + text: 'follow-up whose worker disappeared', + availableAt: new Date(Date.now() - 10 * 60_000), + }); + const claimedAt = new Date(Date.now() - 5 * 60_000); + await expect( + methods.claimNextAgentQueuedTurn({ + user: new mongoose.Types.ObjectId(userId), + tenantId: TENANT_ID, + conversationId, + queuedTurnId: turn.queuedTurnId, + claimId: 'lost-claim', + claimBy: 'lost-worker', + now: claimedAt, + leaseUntil: new Date(claimedAt.getTime() + 2 * 60_000), + }), + ).resolves.toMatchObject({ outcome: 'acquired' }); + await enqueueWakeup(triggers); + await waitForDeferral(BACKGROUND_TOOL_COMPLETION_SOURCE); + + await manager.completeJob(conversationId, undefined, run.createdAt); + + const [wakeup] = await waitUntil( + async () => (dispatches.length > 0 ? dispatches : undefined), + 'the wake-up past an expired queued-turn claim', + ); + expect(wakeup).toMatchObject({ kind: 'wake-up', parentMessageId: RUN_RESPONSE_ID }); + }); +}); diff --git a/packages/api/src/agents/precedence.ts b/packages/api/src/agents/precedence.ts new file mode 100644 index 0000000000..e8076ac250 --- /dev/null +++ b/packages/api/src/agents/precedence.ts @@ -0,0 +1,60 @@ +import { logger } from '@librechat/data-schemas'; +import type { AgentQueuedTurnMethods } from '@librechat/data-schemas'; +import type { AgentContinueTriggerEnvelope } from './triggers/envelope'; +import { AgentTriggerExecutionError } from './triggers/host'; + +/** Optional so partial method bundles keep the previous behavior. */ +export type QueuedTurnPrecedenceMethods = Partial< + Pick +>; + +function deferral(code: string, message: string): AgentTriggerExecutionError { + return new AgentTriggerExecutionError(message, { + mode: 'continue', + certainty: 'definite', + code, + retryable: true, + status: 409, + retryAfter: '1', + deferWithoutAttempt: true, + }); +} + +/** + * The deferral a completion wake-up must take so the user's queued turn gets + * the conversation first, or null when the wake-up may proceed. Read under the + * delivery's own principal and tenant. A failed read is not proof of absence, + * so it defers too; neither deferral consumes a delivery attempt. The queued + * turn's own generation then holds the conversation, and the wake-up resumes + * after it on that turn's response. + */ +export async function getQueuedTurnDeferral( + methods: QueuedTurnPrecedenceMethods, + envelope: AgentContinueTriggerEnvelope, + now: number, +): Promise { + const hasPending = methods.hasPendingAgentQueuedTurn; + if (hasPending == null) { + return null; + } + try { + const pending = await hasPending({ + userId: envelope.principal.userId, + ...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }), + conversationId: envelope.target.conversationId, + now: new Date(now), + }); + return pending + ? deferral('QUEUED_TURN_PENDING', 'A queued user turn is admitted before this wake-up.') + : null; + } catch (error) { + logger.warn('[agent-triggers] Queued turn precedence is unavailable; deferring wake-up', { + conversationId: envelope.target.conversationId, + error: error instanceof Error ? error.message : String(error), + }); + return deferral( + 'QUEUED_TURN_STATE_UNAVAILABLE', + 'Queued user turn state is temporarily unavailable.', + ); + } +} diff --git a/packages/api/src/agents/subagentCompletionWakeup.spec.ts b/packages/api/src/agents/subagentCompletionWakeup.spec.ts index 460da1ea69..afc24488ac 100644 --- a/packages/api/src/agents/subagentCompletionWakeup.spec.ts +++ b/packages/api/src/agents/subagentCompletionWakeup.spec.ts @@ -309,6 +309,108 @@ describe('createSubagentCompletionWakeupResolver', () => { expect(methods.claimSubagentTaskResult).not.toHaveBeenCalled(); }); + describe('queued user turn precedence', () => { + const OWNER_ID = '507f1f77bcf86cd799439011'; + + function ownedEnvelope(): AgentContinueTriggerEnvelope { + const envelope = wakeupEnvelope(); + return { ...envelope, principal: { ...envelope.principal, userId: OWNER_ID } }; + } + + it('defers without consuming an attempt or claiming the child result', async () => { + const { methods } = resolverMethods(); + const hasPendingAgentQueuedTurn = jest.fn(async () => true); + const resolve = createSubagentCompletionWakeupResolver({ + methods: { ...methods, hasPendingAgentQueuedTurn } as never, + getGenerationJob: async () => null, + now: () => NOW, + }); + + await expect( + resolve(ownedEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never), + ).rejects.toMatchObject({ + code: 'QUEUED_TURN_PENDING', + retryable: true, + deferWithoutAttempt: true, + }); + expect(hasPendingAgentQueuedTurn).toHaveBeenCalledTimes(1); + expect(hasPendingAgentQueuedTurn).toHaveBeenCalledWith({ + userId: OWNER_ID, + tenantId: 'tenant-1', + conversationId: 'conversation-1', + now: new Date(NOW), + }); + expect(methods.claimSubagentTaskResult).not.toHaveBeenCalled(); + }); + + it('keeps conversation authorization ahead of the queued-turn deferral', async () => { + const { methods } = resolverMethods(); + methods.getConvo.mockImplementation(async (_userId: string, conversationId: string) => ({ + conversationId, + tenantId: 'tenant-2', + })); + const resolve = createSubagentCompletionWakeupResolver({ + methods: { ...methods, hasPendingAgentQueuedTurn: async () => true } as never, + getGenerationJob: async () => null, + }); + + await expect( + resolve(ownedEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never), + ).rejects.toMatchObject({ code: 'PARENT_NOT_FOUND', retryable: false }); + }); + + it('does not consult queued turns for a replay of its own admitted continuation', async () => { + const { methods } = resolverMethods(); + const hasPendingAgentQueuedTurn = jest.fn(async () => true); + const resolve = createSubagentCompletionWakeupResolver({ + methods: { ...methods, hasPendingAgentQueuedTurn } as never, + getGenerationJob: async () => ({ + status: 'running', + metadata: { idempotencyClientRequestId: 'trigger_claim_1' }, + }), + }); + + await expect( + resolve(ownedEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never), + ).resolves.toMatchObject({ status: 'ready' }); + expect(hasPendingAgentQueuedTurn).not.toHaveBeenCalled(); + }); + + it('proceeds when no queued turn is pending', async () => { + const { methods } = resolverMethods(); + const resolve = createSubagentCompletionWakeupResolver({ + methods: { ...methods, hasPendingAgentQueuedTurn: async () => false } as never, + getGenerationJob: async () => null, + }); + + await expect( + resolve(ownedEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never), + ).resolves.toMatchObject({ status: 'ready' }); + }); + + it('defers rather than proceeding when queued-turn state cannot be read', async () => { + const { methods } = resolverMethods(); + const resolve = createSubagentCompletionWakeupResolver({ + methods: { + ...methods, + hasPendingAgentQueuedTurn: async () => { + throw new Error('queued turn read failed'); + }, + } as never, + getGenerationJob: async () => null, + }); + + await expect( + resolve(ownedEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never), + ).rejects.toMatchObject({ + code: 'QUEUED_TURN_STATE_UNAVAILABLE', + retryable: true, + deferWithoutAttempt: true, + }); + expect(methods.claimSubagentTaskResult).not.toHaveBeenCalled(); + }); + }); + it('lets a lost-receipt retry reach HTTP dedup for its own active continuation', async () => { const { methods } = resolverMethods(); const resolve = createSubagentCompletionWakeupResolver({ diff --git a/packages/api/src/agents/subagentCompletionWakeup.ts b/packages/api/src/agents/subagentCompletionWakeup.ts index 357f4e847d..b7eb21c82e 100644 --- a/packages/api/src/agents/subagentCompletionWakeup.ts +++ b/packages/api/src/agents/subagentCompletionWakeup.ts @@ -10,11 +10,13 @@ import type { SubagentTaskWakeupRegistration } from './subagentThreads'; import type { AgentContinueTriggerEnvelope } from './triggers/envelope'; import type { AgentTriggerDispatchContext } from './triggers/dispatch'; import type { AgentTriggerEnqueueOptions } from './triggers/delivery'; +import type { QueuedTurnPrecedenceMethods } from './precedence'; import { resolveScheduleMCPCompletion } from '~/schedules/authorization/continuation'; import { WAITING_RETRY_CAP_MS, waitingRetryAfter } from './triggers/backoff'; import { boundedSubagentTaskResult } from './subagentTaskRouting'; import { createAgentTriggerEnvelope } from './triggers/envelope'; import { AgentTriggerExecutionError } from './triggers/host'; +import { getQueuedTurnDeferral } from './precedence'; const WAKEUP_ADMISSION_DELAY_MS = 250; /** SDK tasks time out after 30 minutes; this grace covers terminal persistence. */ @@ -39,6 +41,7 @@ export type EnqueueAgentTrigger = ( type WakeupMethods = Pick & Pick & + QueuedTurnPrecedenceMethods & Pick< MessageMethods, 'claimSubagentTaskResult' | 'getMessages' | 'releaseSubagentTaskResultClaim' @@ -624,10 +627,8 @@ export function createSubagentCompletionWakeupResolver({ { code: 'PARENT_STATE_UNAVAILABLE', retryable: true }, ); } - if ( - isParentActive(parentJob) && - parentJob?.metadata?.idempotencyClientRequestId !== context.idempotencyKey - ) { + const ownsParent = parentJob?.metadata?.idempotencyClientRequestId === context.idempotencyKey; + if (isParentActive(parentJob) && !ownsParent) { throw executionError('The parent generation has not settled yet.', { code: 'PARENT_NOT_READY', retryable: true, @@ -639,7 +640,7 @@ export function createSubagentCompletionWakeupResolver({ const userId = envelope.principal.userId; const tenantId = envelope.principal.tenantId; - const [parent, child, taskMessages] = await Promise.all([ + const [parent, child, taskMessages, queuedTurnDeferral] = await Promise.all([ methods.getConvo(userId, envelope.target.conversationId), methods.getConvo(userId, registration.threadId), methods.getMessages( @@ -651,6 +652,7 @@ export function createSubagentCompletionWakeupResolver({ TASK_SELECT, { sort: { createdAt: 1, _id: 1 } }, ), + ownsParent ? null : getQueuedTurnDeferral(methods, envelope, now()), ]); if (parent == null || !sameTenant(parent.tenantId, tenantId)) { throw executionError('The parent conversation is no longer available.', { @@ -673,6 +675,9 @@ export function createSubagentCompletionWakeupResolver({ status: 404, }); } + if (queuedTurnDeferral != null) { + throw queuedTurnDeferral; + } let resultTaskId = registration.taskId; let terminal = taskMessages.find( (message) => diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md index 40f737f1a4..8433de2a98 100644 --- a/packages/api/src/agents/triggers/README.md +++ b/packages/api/src/agents/triggers/README.md @@ -111,6 +111,20 @@ child again. A held delivery consumes its wake marker in the same fenced write t for ordering or defers readiness. These writes use classic operators for DocumentDB compatibility; older workers can ignore the optional marker and fall back to the configured polling interval. +A user's server-owned queued turn goes before completion wake-ups. Parent settlement also +expedites that conversation's queued-turn deliveries, and a background or subagent wake-up defers +without consuming an attempt while a queued turn holds an unexpired claim or is the due lane head. +The wake-up then resumes after the queued turn's generation, on that turn's response. Rows that +need recovery or a user decision never hold a wake-up: a reservation, an expired claim, an +indeterminate admission, or a dead head awaiting dismissal. The check reads the active lane first +and those blockers only afterwards, for a due head, so a blocker that has just cleared (a dismissed +dead head, for example) cannot let a wake-up past the turn it was blocking; a conversation with no +due queued turn costs one read. The answer holds for the instant of that last read: a turn that +becomes admissible later is ordered by the generation predecessor fence, like any later send. A +failed precedence read is not treated as absence; the wake-up defers on it the same way. Older +replicas skip the check, so mixed-version fleets keep the earlier first-claim race until every +replica is upgraded. + Setting a recovery cap to `30000` restores its original fixed recovery frequency. No stored-data migration is needed; optional activity reporting preserves the existing numeric/boolean results. diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.ts index 569288d6dd..e8d6c779ab 100644 --- a/packages/api/src/agents/triggers/engine.ts +++ b/packages/api/src/agents/triggers/engine.ts @@ -546,6 +546,8 @@ export function createAgentTriggerDeliveryEngine( logger.info('[agent-triggers] delivery deferred without consuming an attempt', { deliveryKey: delivery.deliveryKey, reason, + ...(error instanceof AgentTriggerExecutionError && + error.code != null && { code: error.code }), availableAt: availableAt.toISOString(), }); } diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts index 287340c1dd..5a80152b21 100644 --- a/packages/api/src/agents/triggers/service.delivery.spec.ts +++ b/packages/api/src/agents/triggers/service.delivery.spec.ts @@ -340,7 +340,7 @@ describe('durable agent trigger service', () => { ); }); - it("expedites a principal's completion deliveries when one of its generations settles", async () => { + it("expedites a principal's completion deliveries and queued turns when one of its generations settles", async () => { const methods = deliveryMethods({ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 2, held: 0 })), }); @@ -363,7 +363,7 @@ describe('durable agent trigger service', () => { expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({ user: '507f1f77bcf86cd799439011', conversationId: 'conversation-1', - sourceIds: ['background-tool-completion', 'subagent-completion'], + sourceIds: ['background-tool-completion', 'subagent-completion', 'agent-queued-turn'], now: expect.any(Date), }); expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled(); diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts index 5993d8819c..07f11300e9 100644 --- a/packages/api/src/agents/triggers/service.ts +++ b/packages/api/src/agents/triggers/service.ts @@ -33,6 +33,7 @@ import { BACKGROUND_TOOL_COMPLETION_SOURCE } from '../backgroundCompletionWakeup import { isShutdownInProgress, registerShutdownTask } from '../../app/shutdown'; import { SUBAGENT_COMPLETION_SOURCE } from '../subagentCompletionWakeup'; import { generateAgentTriggerToken } from '../../crypto/jwt'; +import { AGENT_QUEUED_TURN_SOURCE } from '../queuedTurns'; import { prepareAgentTriggerDelivery } from './delivery'; import { selfOriginFromAddress } from '../../app/origin'; import { createAgentTriggerExecutionHost } from './host'; @@ -42,6 +43,8 @@ import { WAITING_RETRY_CAP_MS } from './backoff'; /** Internal sources whose deliveries wait on a result or on their parent generation. */ const COMPLETION_WAKEUP_SOURCES = [BACKGROUND_TOOL_COMPLETION_SOURCE, SUBAGENT_COMPLETION_SOURCE]; +/** A settled generation also releases the user's queued turns, which wake-ups yield to. */ +const SETTLED_CONVERSATION_SOURCES = [...COMPLETION_WAKEUP_SOURCES, AGENT_QUEUED_TURN_SOURCE]; export const AGENT_TRIGGER_TOKEN_TTL = '60s'; const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000; @@ -582,7 +585,12 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A * claim pass always runs: a matching delivery may already be due here without * having moved. Best effort: a missed expedite only means the delivery * re-checks at its backoff instead of immediately. */ - const expediteCompletions = (input: AgentTriggerCompletionExpedite): void => { + const expediteCompletions = ( + input: AgentTriggerCompletionExpedite, + sourceIds: readonly string[] = 'user' in input && input.taskIds != null + ? [SUBAGENT_COMPLETION_SOURCE] + : COMPLETION_WAKEUP_SOURCES, + ): void => { const expedite = deps.methods?.expediteAgentTriggerDeliveries; if (expedite == null || !deliveryReady || stopping) { return; @@ -596,10 +604,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A ...(input.taskIds != null && { taskIds: input.taskIds }), } : { deliveryKeys: input.deliveryKeys }), - sourceIds: - 'user' in input && input.taskIds != null - ? [SUBAGENT_COMPLETION_SOURCE] - : COMPLETION_WAKEUP_SOURCES, + sourceIds, now: new Date(), }), ) @@ -691,9 +696,12 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A deliveryReady = true; deliveryEngine.start(); /** Deliveries waiting on a parent resume that parent's conversation, so a - * settled generation wakes only those, not every waiting task of the user. */ + * settled generation wakes only those, not every waiting task of the user. + * Queued turns move with them: wake-ups defer to a pending queued turn, + * which must not wait out its own readiness backoff behind them. */ unsubscribeGenerationSettled ??= deps.subscribeGenerationSettled?.( - ({ userId, conversationId }) => expediteCompletions({ user: userId, conversationId }), + ({ userId, conversationId }) => + expediteCompletions({ user: userId, conversationId }, SETTLED_CONVERSATION_SOURCES), ); startPurgeRecovery(); logger.info('[agent-triggers] durable delivery engine started'); diff --git a/packages/data-schemas/src/methods/queuedTurn.spec.ts b/packages/data-schemas/src/methods/queuedTurn.spec.ts index 7f0a8d1091..00bdb18854 100644 --- a/packages/data-schemas/src/methods/queuedTurn.spec.ts +++ b/packages/data-schemas/src/methods/queuedTurn.spec.ts @@ -3153,3 +3153,304 @@ describe('agent queued turn methods', () => { }); }); }); + +describe('queued turn precedence', () => { + const scope = () => ({ user, tenantId: 'tenant-1', conversationId: 'conversation-1' }); + const precedence = () => ({ + userId: user.toString(), + tenantId: 'tenant-1', + conversationId: 'conversation-1', + }); + const pending = (now = START) => methods.hasPendingAgentQueuedTurn({ ...precedence(), now }); + + it('answers false for a conversation without queued turns', async () => { + await expect(pending()).resolves.toBe(false); + }); + + it('reports a due queued head only inside its exact owner scope', async () => { + await methods.enqueueAgentQueuedTurn(enqueueInput()); + + await expect(pending()).resolves.toBe(true); + await expect( + methods.hasPendingAgentQueuedTurn({ ...precedence(), tenantId: 'tenant-2', now: START }), + ).resolves.toBe(false); + await expect( + methods.hasPendingAgentQueuedTurn({ ...precedence(), tenantId: undefined, now: START }), + ).resolves.toBe(false); + await expect( + methods.hasPendingAgentQueuedTurn({ + ...precedence(), + userId: new mongoose.Types.ObjectId().toString(), + now: START, + }), + ).resolves.toBe(false); + await expect( + methods.hasPendingAgentQueuedTurn({ ...precedence(), userId: 'not-an-id', now: START }), + ).resolves.toBe(false); + await expect( + methods.hasPendingAgentQueuedTurn({ ...precedence(), conversationId: 'other', now: START }), + ).resolves.toBe(false); + }); + + it('does not report a head that is not yet available', async () => { + await methods.enqueueAgentQueuedTurn(enqueueInput({ availableAt: LATER })); + + await expect(pending()).resolves.toBe(false); + await expect(pending(LATER)).resolves.toBe(true); + }); + + it('bounds an admitting claim by its lease and stops at admission', async () => { + const first = await methods.enqueueAgentQueuedTurn(enqueueInput()); + await methods.enqueueAgentQueuedTurn(enqueueInput({ clientRequestId: 'second' })); + const claimed = await methods.claimNextAgentQueuedTurn(claimInput(first.turn.queuedTurnId)); + expect(claimed.outcome).toBe('acquired'); + + await expect(pending()).resolves.toBe(true); + /** A lease that outlived its owner fences the lane for recovery, so the + * successor behind it cannot be admitted next either. */ + await expect(pending(new Date(LATER.getTime() + 1))).resolves.toBe(false); + + await Turn.updateOne( + { _id: first.turn.queuedTurnId }, + { + $set: { status: 'admitted' }, + $unset: { claimId: 1, claimBy: 1, claimUntil: 1, admissionSlot: 1, activeSlot: 1 }, + }, + ); + await expect(pending()).resolves.toBe(true); + await Turn.updateMany({}, { $set: { status: 'admitted' } }); + await expect(pending()).resolves.toBe(false); + }); + + it('does not wait on a dead head awaiting dismissal or on a quarantined admission', async () => { + const first = await methods.enqueueAgentQueuedTurn(enqueueInput()); + await methods.enqueueAgentQueuedTurn(enqueueInput({ clientRequestId: 'second' })); + await methods.claimNextAgentQueuedTurn(claimInput(first.turn.queuedTurnId)); + await methods.releaseAgentQueuedTurn({ + ...scope(), + queuedTurnId: first.turn.queuedTurnId, + claimId: 'claim-1', + claimBy: 'worker-1', + disposition: 'dead', + settledAt: START, + failure: { code: 'PREDECESSOR_ABORTED', message: 'review this turn' }, + }); + + await expect(pending()).resolves.toBe(false); + + await methods.cancelAgentQueuedTurn({ + ...scope(), + queuedTurnId: first.turn.queuedTurnId, + settledAt: START, + }); + await expect(pending()).resolves.toBe(true); + + await Turn.updateOne( + { _id: first.turn.queuedTurnId }, + { + $set: { + status: 'dead', + terminalReceipt: { + outcome: 'dead', + settledAt: START, + failure: { code: 'ADMISSION_INDETERMINATE', message: 'owner disappeared' }, + }, + }, + }, + ); + await expect(pending()).resolves.toBe(false); + }); + + /** Runs `transition` around the active-lane read: once it is issued and + * `before` it executes, or `after` it has returned and before the blocker read. */ + function interleaveActiveLaneRead( + transition: () => Promise, + when: 'before' | 'after' = 'before', + ) { + const find = Turn.find.bind(Turn); + jest.spyOn(Turn, 'find').mockImplementationOnce((...args: Parameters) => { + const query = find(...args); + const exec = query.exec.bind(query); + query.exec = async () => { + if (when === 'before') { + await transition(); + return exec(); + } + const rows = await exec(); + await transition(); + return rows; + }; + return query; + }); + } + + async function deadHeadWithSuccessor() { + const first = await methods.enqueueAgentQueuedTurn(enqueueInput()); + await methods.enqueueAgentQueuedTurn(enqueueInput({ clientRequestId: 'second' })); + await methods.claimNextAgentQueuedTurn(claimInput(first.turn.queuedTurnId)); + await methods.releaseAgentQueuedTurn({ + ...scope(), + queuedTurnId: first.turn.queuedTurnId, + claimId: 'claim-1', + claimBy: 'worker-1', + disposition: 'dead', + settledAt: START, + failure: { code: 'PREDECESSOR_ABORTED', message: 'review this turn' }, + }); + return first.turn; + } + + async function reservationBesideHead() { + const first = await methods.enqueueAgentQueuedTurn(enqueueInput()); + return Turn.create({ + ...scope(), + agentId: 'agent-1', + parentMessageId: 'parent-1', + clientRequestId: 'interrupted-reservation', + fingerprint: 'x'.repeat(43), + laneId: first.turn.laneId, + activeSlot: ((first.turn.activeSlot ?? 0) + 1) % 100, + status: 'reserving', + priority: false, + text: 'repair me', + attempts: 0, + availableAt: START, + deliveryState: 'pending', + }); + } + + it('sees a claim that lands while the lane is being read', async () => { + const queued = await methods.enqueueAgentQueuedTurn(enqueueInput()); + interleaveActiveLaneRead(() => + methods.claimNextAgentQueuedTurn(claimInput(queued.turn.queuedTurnId)), + ); + + await expect(pending()).resolves.toBe(true); + await expect(Turn.findById(queued.turn.queuedTurnId).lean()).resolves.toMatchObject({ + status: 'claimed', + }); + }); + + it('sees a released claim that is due again while the lane is being read', async () => { + const queued = await methods.enqueueAgentQueuedTurn(enqueueInput()); + await methods.claimNextAgentQueuedTurn(claimInput(queued.turn.queuedTurnId)); + interleaveActiveLaneRead(() => + methods.releaseAgentQueuedTurn({ + ...scope(), + queuedTurnId: queued.turn.queuedTurnId, + claimId: 'claim-1', + claimBy: 'worker-1', + disposition: 'retry', + availableAt: START, + }), + ); + + await expect(pending()).resolves.toBe(true); + await expect(Turn.findById(queued.turn.queuedTurnId).lean()).resolves.toMatchObject({ + status: 'queued', + }); + }); + + it('orders a priority dead row ahead of an earlier queued row', async () => { + await methods.enqueueAgentQueuedTurn(enqueueInput()); + const interrupt = await methods.enqueueAgentQueuedTurn( + enqueueInput({ clientRequestId: 'interrupt', priority: true }), + ); + await methods.claimNextAgentQueuedTurn(claimInput(interrupt.turn.queuedTurnId)); + await methods.releaseAgentQueuedTurn({ + ...scope(), + queuedTurnId: interrupt.turn.queuedTurnId, + claimId: 'claim-1', + claimBy: 'worker-1', + disposition: 'dead', + settledAt: START, + failure: { code: 'PREDECESSOR_FAILED', message: 'review this turn' }, + }); + + await expect(pending()).resolves.toBe(false); + }); + + it('still answers before startup has built the capacity index', async () => { + await methods.enqueueAgentQueuedTurn(enqueueInput()); + await Turn.collection.dropIndex('agent_queued_turn_active_capacity'); + try { + await expect(pending()).resolves.toBe(true); + } finally { + await Turn.createIndexes(); + } + }); + + it('does not wait on an unfinished reservation', async () => { + await reservationBesideHead(); + + await expect(pending()).resolves.toBe(false); + }); + + it.each(['before', 'after'] as const)( + 'sees a successor whose dead head is dismissed %s the lane read returns', + async (when) => { + const dead = await deadHeadWithSuccessor(); + await expect(pending()).resolves.toBe(false); + interleaveActiveLaneRead( + () => + methods.cancelAgentQueuedTurn({ + ...scope(), + queuedTurnId: dead.queuedTurnId, + settledAt: START, + }), + when, + ); + + await expect(pending()).resolves.toBe(true); + }, + ); + + it('sees a head whose reservation fence settles after the lane read returns', async () => { + const reservation = await reservationBesideHead(); + interleaveActiveLaneRead( + () => + Turn.updateOne( + { _id: reservation._id, status: 'reserving' }, + { $set: { status: 'queued', sequence: 2 } }, + ), + 'after', + ); + + await expect(pending()).resolves.toBe(true); + }); + + it('sees a head whose expired claim is renewed after the lane read returns', async () => { + const first = await methods.enqueueAgentQueuedTurn(enqueueInput()); + await methods.enqueueAgentQueuedTurn(enqueueInput({ clientRequestId: 'second' })); + await methods.claimNextAgentQueuedTurn(claimInput(first.turn.queuedTurnId)); + const expired = new Date(LATER.getTime() + 1); + await expect(pending(expired)).resolves.toBe(false); + interleaveActiveLaneRead( + () => + Turn.updateOne( + { _id: first.turn.queuedTurnId, status: 'claimed' }, + { $set: { claimUntil: new Date(expired.getTime() + 60_000) } }, + ), + 'after', + ); + + await expect(pending(expired)).resolves.toBe(true); + }); + + it('reads blockers only for a due head', async () => { + const find = jest.spyOn(Turn, 'find'); + const exists = jest.spyOn(Turn, 'exists'); + + await expect(pending()).resolves.toBe(false); + await methods.enqueueAgentQueuedTurn(enqueueInput({ availableAt: LATER })); + find.mockClear(); + exists.mockClear(); + await expect(pending()).resolves.toBe(false); + expect(find).toHaveBeenCalledTimes(1); + expect(exists).not.toHaveBeenCalled(); + + await expect(pending(LATER)).resolves.toBe(true); + expect(find).toHaveBeenCalledTimes(2); + expect(exists).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/data-schemas/src/methods/queuedTurn.ts b/packages/data-schemas/src/methods/queuedTurn.ts index cfefee4421..640bd5c3f6 100644 --- a/packages/data-schemas/src/methods/queuedTurn.ts +++ b/packages/data-schemas/src/methods/queuedTurn.ts @@ -35,6 +35,7 @@ const RETIRED_LANE_RETENTION_MS = 24 * 60 * 60_000; const ROOT_LINEAGE_PREDECESSOR_PREFIX = 'root:'; const CLAIM_LANE_INDEX = 'agent_queued_turn_claim_lane'; const ADMISSION_STARTED_LANE_INDEX = 'agent_queued_turn_admission_started_lane'; +const ACTIVE_CAPACITY_INDEX = 'agent_queued_turn_active_capacity'; const NAMESPACE_EXISTS = 48; interface DuplicateKeyError { @@ -125,6 +126,14 @@ export interface EnqueueAgentQueuedTurnInput extends AgentQueuedTurnConversation availableAt?: Date; } +/** Plain identifiers, so callers outside data-schemas need no Mongoose types. */ +export interface AgentQueuedTurnPrecedenceInput { + userId: string; + tenantId?: string; + conversationId: string; + now: Date; +} + export interface AgentQueuedTurnClaimFence extends AgentQueuedTurnConversationScope { queuedTurnId: string; claimId: string; @@ -261,6 +270,8 @@ export interface AgentQueuedTurnMethods { leaseUntil: Date; }, ) => Promise; + /** Whether a queued user turn owns the conversation's next admission. */ + hasPendingAgentQueuedTurn: (input: AgentQueuedTurnPrecedenceInput) => Promise; releaseAgentQueuedTurn: ( input: AgentQueuedTurnClaimFence & ( @@ -617,6 +628,17 @@ function effectiveDeliveryState(turn: IAgentQueuedTurn) { return 'published' as const; } +/** Claim order: priority rows first, then ascending sequence. */ +function precedes( + row: Pick, + other: Pick, +): boolean { + if (row.priority !== other.priority) { + return row.priority === true; + } + return (row.sequence ?? Number.MAX_SAFE_INTEGER) < (other.sequence ?? Number.MAX_SAFE_INTEGER); +} + function toRecord(turn: IAgentQueuedTurn): AgentQueuedTurnRecord { if ( turn._id == null || @@ -1886,6 +1908,87 @@ export function createAgentQueuedTurnMethods( : { outcome: 'replayed', claim: racedClaim }; } + /** + * True while a claim is admitting a queued turn under an unexpired lease, or + * while the lane head is a due `queued` row that `claimNextAgentQueuedTurn` + * could acquire. Rows that need recovery or a user decision (a reservation, + * an expired claim, an indeterminate admission, a dead row ahead of the + * head) block that head and answer false, so callers never wait on them. + * + * The reads are ordered so that no blocker is older than the turn it blocks. + * The active lane is read first, in one query over the capacity index. Its + * key (`activeSlot`) is held unchanged from reservation until a terminal + * state, so a row moving between `queued` and `claimed` in either direction + * is read exactly once, in one state or the other. Blockers are read only + * after that, and only for a due head. A blocker that has already cleared (a + * dismissed dead row, a finished reservation, a renewed claim) is therefore + * never what answers false, and a lane with no due head costs one read. + * + * The answer holds for the instant of the last read. A turn that becomes + * admissible afterwards arrives after the caller and is ordered by the + * generation predecessor fence, as any later send is. + */ + async function hasPendingAgentQueuedTurn( + input: AgentQueuedTurnPrecedenceInput, + ): Promise { + if (!(input.now instanceof Date) || !Number.isFinite(input.now.getTime())) { + throw new TypeError('Agent queued turn precedence requires a valid time'); + } + if (!mongoose.Types.ObjectId.isValid(input.userId)) { + return false; + } + const scope = conversationScope({ + user: new mongoose.Types.ObjectId(input.userId), + ...(input.tenantId != null && { tenantId: input.tenantId }), + conversationId: input.conversationId, + }); + type LaneRow = Pick< + IAgentQueuedTurn, + 'status' | 'priority' | 'sequence' | 'availableAt' | 'claimUntil' + >; + const readActiveLane = () => + Turn() + .find({ ...scope, activeSlot: { $exists: true } }) + .select('status priority sequence availableAt claimUntil') + .lean(); + /** The hint only pins the stable key. When it is refused (the index is + * not built yet, or an engine rejects the hint), a planner-chosen read + * is still correct apart from that narrow transition window, and a + * genuine outage fails this second read too. */ + const active = await readActiveLane() + .hint(ACTIVE_CAPACITY_INDEX) + .catch(() => readActiveLane()); + const now = input.now.getTime(); + let head: LaneRow | undefined; + for (const row of active) { + if (row.status === 'claimed' && row.claimUntil != null && row.claimUntil.getTime() > now) { + return true; + } + if (row.status === 'queued' && (head == null || precedes(row, head))) { + head = row; + } + } + if (head == null || head.availableAt.getTime() > now) { + return false; + } + const earlier = { sequence: { $lt: head.sequence ?? Number.MAX_SAFE_INTEGER } }; + const blocker = await Turn().exists({ + ...scope, + $or: [ + { status: 'reserving' }, + { status: 'claimed', claimUntil: { $not: { $gt: input.now } } }, + { status: 'dead', 'terminalReceipt.failure.code': 'ADMISSION_INDETERMINATE' }, + { + status: 'dead', + ...(head.priority === true + ? { priority: true, ...earlier } + : { $or: [{ priority: true }, earlier] }), + }, + ], + }); + return blocker == null; + } + async function releaseAgentQueuedTurn( input: AgentQueuedTurnClaimFence & ( @@ -2969,6 +3072,7 @@ export function createAgentQueuedTurnMethods( markQueuedTurnScheduled, cancelAgentQueuedTurn, claimNextAgentQueuedTurn, + hasPendingAgentQueuedTurn, beginAgentQueuedTurnAdmission, releaseAgentQueuedTurn, markAgentQueuedTurnAdmitted,