diff --git a/CONTEXT.md b/CONTEXT.md index f45bb9f1c1..6bac598b1d 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -12,7 +12,7 @@ - **Agent event handling outcome**: the durable, generation-fenced result of a previously accepted event delivery. `started` proves generation admission; terminal states distinguish verified tool application, clean completion without action, failure, and cancellation. Transport success remains separate so an accepted event cannot masquerade as completed work. - **Agent event expected action**: an optional source-declared tool name and bounded argument subset evaluated against host-observed completed run steps. It is evidence policy, not authorization and not a model-authored success claim. - **Event actor head**: the private, durable pointer on an event-bound child conversation to its latest committed LangGraph checkpoint, plus one previous checkpoint for safe cleanup. Only a qualifying applied action advances it through compare-and-swap; failed, cancelled, or no-action invocations leave it unchanged. A legacy-path event marks the head for a cold rebuild from durable message history before fork mode can resume. Every applied commit conflict, unverified commit, or post-commit persistence failure is retained in a private reconciliation journal that blocks later actor turns instead of continuing from stale state; an exact marker can be cleared only after its checkpoint is verified authoritative, its history is repaired, or its external action is explicitly compensated. -- **Event actor invocation fork**: a delivery-owned checkpoint namespace copied from the event actor head. A warm invocation receives only the new trusted event, then commits its terminal checkpoint when the expected action is observed or deletes the fork otherwise. Pause-capable actors remain on the existing resumable path until forked HITL has an explicit contract. +- **Event actor invocation fork**: a delivery-owned checkpoint namespace copied from the event actor head. A warm invocation receives only the new trusted event, then commits its terminal checkpoint when the expected action is observed or deletes the fork otherwise. When the invocation pauses for approval or Ask User, the SDK emits signed, versioned suspension evidence. The child Conversation is the canonical one-shot suspension authority; the generation job carries only a versioned projection for UI, rolling-deploy routing, and the existing resume endpoint. Shared-store deployments publish new suspensions only under generation protocol v2, after the homogeneous-fleet cutover; protocol v1 keeps pause-capable actors on the legacy history path so an old resume consumer cannot consume unknown evidence. A resume shares one identity between its Conversation claim and provider-owner CAS, clears the predecessor projection, and publishes a successor only after a re-pause is canonical. A pending interrupt takes precedence over expected-action evidence from the same segment; if that segment already applied the expected action, publishing its successor pause cold-marks the prior head until a later applied commit replaces it. An ambiguous projection write is accepted only after reading back the exact generation, action, and suspension. The provider-start CAS is written only after client reconstruction and immediately before the continuation gate opens, then retains its exact execution identity after drain, so terminal recovery can compensate a projected claim only when that identity proves execution never began. Durable approval projection is exposed before the persistence barrier opens, preventing a resolved action from being announced afterward. Terminal no-action retirement cancels or settles the exact suspension and releases its delivery-side action admission before public settlement; if retention already removed the child Conversation, the delivery remains authoritative for its exact admission identity. Resume, re-pause, cancellation, and expiry claim or replace that exact suspension before touching its job projection, so later mailbox deliveries stay blocked until terminal history and handling evidence settle. - **Event actor receipt**: the private, terminal proof stored on the authoritative `AgentTriggerDelivery` row for one bound actor invocation. Its unique delivery identity, terminal resolution, exact checkpoint, and bounded action identity provide replay and recovery for the retention window without storing prompts, events, tool arguments, tool output, or conversation history. It does not own the actor checkpoint; the conversation keeps only the actor head and any active unresolved reconciliation until this receipt is durable. - **Agent event actor mailbox**: the automatic durable delivery-ordering lane for one authenticated source binding. It keeps later deliveries queued after transport admission until the current child turn records an authoritative terminal handling outcome. It serializes existing coalesced batches and individual events without becoming a second execution controller or actor checkpoint store. - **Theme definition**: a versioned, data-only description of LibreChat semantic colors and shared appearance roles, optionally specialized by light or dark mode. The theme module validates and resolves partial definitions against bundled defaults before adapters apply them. A theme definition does not contain arbitrary CSS, application behavior, or alternate feature layouts. diff --git a/api/package.json b/api/package.json index a8600fa1b0..fb6bd9c93b 100644 --- a/api/package.json +++ b/api/package.json @@ -46,7 +46,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^3.7.6", + "@librechat/agents": "^3.7.7", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", diff --git a/api/server/controllers/agents/__tests__/client.eventActorPause.spec.js b/api/server/controllers/agents/__tests__/client.eventActorPause.spec.js new file mode 100644 index 0000000000..39320172f3 --- /dev/null +++ b/api/server/controllers/agents/__tests__/client.eventActorPause.spec.js @@ -0,0 +1,74 @@ +const mockPause = jest.fn(); +const mockGetJob = jest.fn(); + +jest.mock('@librechat/api', () => ({ + ...jest.requireActual('@librechat/api'), + GenerationJobManager: { + approvals: { pause: (...args) => mockPause(...args) }, + getJob: (...args) => mockGetJob(...args), + }, +})); + +const AgentClient = require('../client'); + +function clientForProjection() { + const pendingAction = { actionId: 'action-1', expiresAt: Date.now() + 60_000 }; + return { + stagedApproval: { + streamId: 'conversation-1', + pendingAction, + discoveredTools: [], + activityPhaseSnapshot: null, + }, + pendingApproval: null, + jobCreatedAt: 123, + }; +} + +describe('AgentClient Event Actor pause projection', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('confirms the exact durable projection when Redis loses the pause reply', async () => { + const self = clientForProjection(); + const suspension = { version: 1, suspensionId: 'suspension-1', attempt: 2 }; + mockPause.mockRejectedValue(new Error('reply lost')); + mockGetJob.mockResolvedValue({ + createdAt: 123, + status: 'requires_action', + metadata: { + pendingAction: self.stagedApproval.pendingAction, + agentEventSuspension: suspension, + }, + }); + + await expect(AgentClient.prototype.publishStagedApproval.call(self, suspension)).resolves.toBe( + true, + ); + expect(self.pendingApproval).toBe(self.stagedApproval.pendingAction); + }); + + it('propagates an ambiguous failure when the durable projection does not match', async () => { + const self = clientForProjection(); + const error = new Error('reply lost'); + mockPause.mockRejectedValue(error); + mockGetJob.mockResolvedValue({ + createdAt: 123, + status: 'requires_action', + metadata: { + pendingAction: self.stagedApproval.pendingAction, + agentEventSuspension: { version: 1, suspensionId: 'different', attempt: 2 }, + }, + }); + + await expect( + AgentClient.prototype.publishStagedApproval.call(self, { + version: 1, + suspensionId: 'suspension-1', + attempt: 2, + }), + ).rejects.toBe(error); + expect(self.pendingApproval).toBeNull(); + }); +}); diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js index 3c4db501cb..2b37fd53c8 100644 --- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js +++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js @@ -85,8 +85,8 @@ const mockResolveAgentTurnExecutionPlan = jest.fn((input) => { input.event?.binding != null && input.event?.expectedAction != null && input.checkpointerType !== 'memory' && - !input.canPause && - !input.expectedActionMayDetach; + !input.expectedActionMayDetach && + (!input.canPause || input.durableEventActorSuspensions); let strategy = 'history'; if (input.isNewConversation) { strategy = 'fresh'; @@ -3042,10 +3042,12 @@ describe('ResumableAgentController resume metadata', () => { }); let observedHookResult; const completedResponseWrite = jest.fn(); + const exposePendingApproval = jest.fn().mockResolvedValue(undefined); const client = { options: {}, jobCreatedAt: 1000, pendingApproval: { actionId: 'action-pause-barrier' }, + exposePendingApproval, skipSaveUserMessage: false, skipSaveConvo: false, getSaveOptions: jest.fn(() => ({ endpoint: 'agents' })), @@ -3125,6 +3127,9 @@ describe('ResumableAgentController resume metadata', () => { expect(mockSaveMessage.mock.invocationCallOrder[0]).toBeLessThan( mockGenerationJobManager.approvals.finishPausePersistence.mock.invocationCallOrder[0], ); + expect(exposePendingApproval.mock.invocationCallOrder[0]).toBeLessThan( + mockGenerationJobManager.approvals.finishPausePersistence.mock.invocationCallOrder[0], + ); expect(mockGenerationJobManager.claimTerminalJob).not.toHaveBeenCalled(); }); @@ -4482,15 +4487,36 @@ describe('ResumableAgentController resume metadata', () => { undefined, undefined, ], + [ + 'pre-cutover pause-capable fleet', + { toolDefinitions: [] }, + { toolApproval: { enabled: true } }, + undefined, + undefined, + 1, + ], ])( - 'keeps %s event actors on the existing resumable path', - async (_label, agent, config, agentConfigs, clientOptions) => { + 'routes %s event actors through the compatible continuation path', + async (_label, agent, config, agentConfigs, clientOptions, generationProtocolVersion = 2) => { mockGenerationJobManager.claimGeneration.mockResolvedValue( wonGenerationClaim({ streamId: 'child-conversation', conversationId: 'child-conversation', + generationProtocolVersion, }), ); + mockGenerationJobManager.createJob.mockResolvedValueOnce({ + createdAt: 1000, + metadata: { + checkpointNamespace: '1000', + providerExecutionId: 'provider-segment-1', + providerDrained: true, + generationProtocolVersion, + }, + readyPromise: Promise.resolve(), + abortController: new AbortController(), + emitter: { on: jest.fn() }, + }); mockGetConvo.mockResolvedValue({ conversationId: 'parent-conversation', agent_id: 'parent-agent', @@ -4503,12 +4529,28 @@ describe('ResumableAgentController resume metadata', () => { throw new Error('stop after legacy event invocation started'); }), }; + const shouldCheckpoint = + _label !== 'memory-checkpointer' && + _label !== 'background-capable expected action' && + _label !== 'pre-cutover pause-capable fleet'; + if (shouldCheckpoint) { + mockExecuteAgentEventActor.mockImplementationOnce(async (input) => { + await input.invoke({ + checkpointNamespace: 'event-actor/pause-capable', + checkpointId: 'checkpoint-pause-capable', + invocationId: 'req-event-hitl', + continuation: 'warm', + signal: input.signal, + }); + }); + } const req = { user: { id: 'user-123', tenantId: 'tenant-1' }, body: { text: 'Continue with a pause-capable actor.', clientRequestId: 'req-event-hitl', conversationId: 'child-conversation', + generationProtocolVersion, endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } }, agentEventDelivery: { deliveryKey: 'req-event-hitl', @@ -4541,6 +4583,18 @@ describe('ResumableAgentController resume metadata', () => { ); await nextTick(); + if (shouldCheckpoint) { + expect(mockExecuteAgentEventActor).toHaveBeenCalled(); + expect(mockGetMessages).not.toHaveBeenCalled(); + expect(mockBeginAgentEventActorLegacyTurn).not.toHaveBeenCalled(); + expect(mockGenerationJobManager.updateMetadata).not.toHaveBeenCalledWith( + 'child-conversation', + expect.objectContaining({ agentEventLegacyTurnToken: expect.any(String) }), + 1000, + ); + expect(client.sendMessage).toHaveBeenCalledTimes(1); + return; + } expect(mockExecuteAgentEventActor).not.toHaveBeenCalled(); expect(mockBeginAgentEventActorLegacyTurn).toHaveBeenCalledWith({ user: 'user-123', diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js index dddf10c4cd..8b73032b25 100644 --- a/api/server/controllers/agents/__tests__/resume.spec.js +++ b/api/server/controllers/agents/__tests__/resume.spec.js @@ -103,6 +103,13 @@ const mockAcquireEventChildGenerationLease = jest.fn(); const mockReleaseEventChildLease = jest.fn(); const mockIsSubagentOwnerAdmissible = jest.fn(); const mockCompleteAgentEventActorLegacyTurn = jest.fn(); +const mockGetAgentEventActorSnapshot = jest.fn(); +const mockCommitAgentEventActorState = jest.fn(); +const mockStoreAgentEventActorSuspension = jest.fn(); +const mockClaimAgentEventActorSuspension = jest.fn(); +const mockSettleAgentEventActorSuspension = jest.fn(); +const mockRecordAgentEventActorReconciliation = jest.fn(); +const mockResumeAgentEventActor = jest.fn(); jest.mock('@librechat/data-schemas', () => ({ ...jest.requireActual('@librechat/data-schemas'), @@ -124,6 +131,7 @@ jest.mock('@librechat/api', () => ({ }), getAgentCheckpointer: (...args) => mockGetAgentCheckpointer(...args), checkAccess: (...args) => mockCheckAccess(...args), + resumeAgentEventActor: (...args) => mockResumeAgentEventActor(...args), })); jest.mock('~/models', () => ({ @@ -137,6 +145,13 @@ jest.mock('~/models', () => ({ getRoleByName: (...args) => mockGetRoleByName(...args), isSubagentOwnerAdmissible: (...args) => mockIsSubagentOwnerAdmissible(...args), completeAgentEventActorLegacyTurn: (...args) => mockCompleteAgentEventActorLegacyTurn(...args), + getAgentEventActorSnapshot: (...args) => mockGetAgentEventActorSnapshot(...args), + commitAgentEventActorState: (...args) => mockCommitAgentEventActorState(...args), + storeAgentEventActorSuspension: (...args) => mockStoreAgentEventActorSuspension(...args), + claimAgentEventActorSuspension: (...args) => mockClaimAgentEventActorSuspension(...args), + settleAgentEventActorSuspension: (...args) => mockSettleAgentEventActorSuspension(...args), + recordAgentEventActorReconciliation: (...args) => + mockRecordAgentEventActorReconciliation(...args), })); jest.mock('~/server/services/Endpoints/agents/eventChildLease', () => ({ @@ -354,6 +369,12 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { mockReleaseEventChildLease.mockResolvedValue(undefined); mockIsSubagentOwnerAdmissible.mockResolvedValue(true); mockCompleteAgentEventActorLegacyTurn.mockResolvedValue(true); + mockGetAgentEventActorSnapshot.mockResolvedValue(undefined); + mockCommitAgentEventActorState.mockResolvedValue({ status: 'committed' }); + mockStoreAgentEventActorSuspension.mockResolvedValue({ status: 'stored' }); + mockClaimAgentEventActorSuspension.mockResolvedValue({ status: 'claimed' }); + mockSettleAgentEventActorSuspension.mockResolvedValue({ status: 'settled' }); + mockRecordAgentEventActorReconciliation.mockResolvedValue(true); endpointAgent = { _id: 'mongo-agent-abc', id: AGENT_ID, @@ -441,6 +462,307 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { }; describe('event-bound actor resume lifecycle', () => { + it('claims a versioned Conversation suspension and recovers an ambiguous job projection ACK', async () => { + configureEventActorResume(); + requestStateOverrides._agentEventBindingId = 'binding-1'; + const expectedAction = { toolName: 'lookup' }; + const suspension = { + version: 1, + suspensionId: 'suspension-1', + attempt: 0, + issuedAt: Date.now(), + expiresAt: Date.now() + 60_000, + invocation: { + invocationId: 'trigger_event_delivery', + continuation: 'warm', + base: { actorThreadId: CONVO_ID, generation: 1 }, + fork: { + threadId: CONVO_ID, + checkpointNs: 'event-actor', + checkpointId: 'checkpoint-paused', + invocationId: 'trigger_event_delivery', + }, + }, + checkpoint: { + threadId: CONVO_ID, + checkpointNs: 'event-actor', + checkpointId: 'checkpoint-paused', + invocationId: 'trigger_event_delivery', + }, + interrupt: { + id: 'interrupt-1', + payload: { + type: 'tool_approval', + _librechatEventActor: { expectedAction }, + }, + }, + suspensionDigest: 'signed-digest', + }; + const pausedJob = makeToolApprovalJob({ + metadata: { + idempotencyClientRequestId: 'trigger_event_delivery', + agentEventExpectedAction: expectedAction, + agentEventSuspension: { + version: 1, + suspensionId: suspension.suspensionId, + attempt: suspension.attempt, + }, + }, + }); + pausedJob.metadata.pendingAction.payload.review_configs = [ + { tool_call_id: 'tc1', allowed_decisions: ['respond'] }, + ]; + mockGenerationJobManager.getJob.mockResolvedValue(pausedJob); + let projectedProviderExecutionId; + mockGenerationJobManager.approvals.resolve.mockImplementation( + async (_streamId, _actionId, resumePatch) => { + projectedProviderExecutionId = resumePatch.providerExecutionId; + mockGenerationJobManager.getJob.mockResolvedValue({ + ...pausedJob, + status: 'running', + metadata: { + ...pausedJob.metadata, + providerExecutionId: resumePatch.providerExecutionId, + }, + }); + throw new Error('redis committed the CAS but lost its reply'); + }, + ); + mockGetAgentEventActorSnapshot.mockResolvedValue({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: ACTION_ID, + jobCreatedAt: 1000, + status: 'pending', + }, + }); + const resumedClient = makeClient({ + contentParts: [makeToolCallContent({ output: 'human supplied output' })], + run: { + getRunSteps: () => [ + { + type: 'tool_calls', + status: 'completed', + stepDetails: { + type: 'tool_calls', + tool_calls: [ + { + id: 'tc1', + name: 'lookup', + args: {}, + output: 'human supplied output', + }, + ], + }, + }, + ], + }, + }); + mockInitializeClient.mockResolvedValue({ client: resumedClient, userMCPAuthMap: {} }); + mockResumeAgentEventActor.mockImplementation(async (input, dependencies) => { + await dependencies.claimSuspension({ + user: USER_ID, + tenantId: TENANT_ID, + conversationId: CONVO_ID, + suspensionId: suspension.suspensionId, + attempt: suspension.attempt, + actionId: ACTION_ID, + jobCreatedAt: 1000, + resumeAttemptId: input.resumeAttemptId, + }); + expect(await input.claimProjection()).toBe(true); + expect(input.resumeAttemptId).toBe(projectedProviderExecutionId); + const value = await input.resume({ + checkpointNamespace: 'event-actor', + checkpointId: 'checkpoint-paused', + invocationId: 'trigger_event_delivery', + continuation: 'warm', + signal: input.signal, + }); + expect(input.readAppliedAction()).toBeUndefined(); + return { + value, + execution: { status: 'completed_no_action' }, + }; + }); + + const res = await post( + approveBody({ + decisions: [ + { tool_call_id: 'tc1', decision: 'respond', responseText: 'human supplied output' }, + ], + }), + ); + expect(res.status).toBe(200); + await settled; + await flush(); + + expect(mockGetAgentEventActorSnapshot).toHaveBeenCalledWith({ + user: USER_ID, + tenantId: TENANT_ID, + conversationId: CONVO_ID, + }); + expect(mockClaimAgentEventActorSuspension.mock.invocationCallOrder[0]).toBeLessThan( + mockGenerationJobManager.approvals.resolve.mock.invocationCallOrder[0], + ); + expect(mockGenerationJobManager.getJob).toHaveBeenCalledTimes(2); + expect(resumedClient.resumeCompletion).toHaveBeenCalledTimes(1); + expect(mockRecordAgentEventActorReconciliation).not.toHaveBeenCalled(); + }); + + it('does not record provider execution when client reconstruction fails before continuation', async () => { + configureEventActorResume(); + const suspension = { + version: 1, + suspensionId: 'suspension-init-failure', + attempt: 0, + issuedAt: Date.now(), + expiresAt: Date.now() + 60_000, + invocation: { + invocationId: 'trigger_event_delivery', + continuation: 'warm', + base: { actorThreadId: CONVO_ID, generation: 0 }, + fork: { + threadId: CONVO_ID, + checkpointNs: 'event-actor', + checkpointId: 'checkpoint-paused', + invocationId: 'trigger_event_delivery', + }, + }, + checkpoint: { + threadId: CONVO_ID, + checkpointNs: 'event-actor', + checkpointId: 'checkpoint-paused', + invocationId: 'trigger_event_delivery', + }, + interrupt: { + id: 'interrupt-init-failure', + payload: { + type: 'tool_approval', + _librechatEventActor: { expectedAction: { toolName: 'lookup' } }, + }, + }, + suspensionDigest: 'signed-digest', + }; + mockGenerationJobManager.getJob.mockResolvedValue( + makeToolApprovalJob({ + metadata: { + idempotencyClientRequestId: 'trigger_event_delivery', + agentEventSuspension: { + version: 1, + suspensionId: suspension.suspensionId, + attempt: suspension.attempt, + }, + }, + }), + ); + mockGetAgentEventActorSnapshot.mockResolvedValue({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: ACTION_ID, + jobCreatedAt: 1000, + status: 'pending', + }, + }); + mockInitializeClient.mockRejectedValue(new Error('client reconstruction failed')); + mockResumeAgentEventActor.mockImplementation(async (input) => { + expect(await input.claimProjection()).toBe(true); + return input.resume({ + checkpointNamespace: 'event-actor', + checkpointId: 'checkpoint-paused', + invocationId: 'trigger_event_delivery', + continuation: 'warm', + signal: input.signal, + }); + }); + + const res = await post(approveBody()); + expect(res.status).toBe(200); + await settled; + await flush(); + + expect(mockInitializeClient).toHaveBeenCalledTimes(1); + expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled(); + expect(mockGenerationJobManager.completeJob).toHaveBeenCalled(); + }); + + it('fails closed when a versioned job marker no longer matches canonical suspension', async () => { + configureEventActorResume(); + mockGenerationJobManager.getJob.mockResolvedValue( + makeToolApprovalJob({ + metadata: { + agentEventSuspension: { version: 1, suspensionId: 'stale', attempt: 0 }, + }, + }), + ); + mockGetAgentEventActorSnapshot.mockResolvedValue({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: null, + }); + + const res = await post(approveBody()); + + expect(res.status).toBe(409); + expect(res.body).toMatchObject({ code: 'EVENT_ACTOR_SUSPENSION_STALE' }); + expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled(); + expect(mockResumeAgentEventActor).not.toHaveBeenCalled(); + }); + + it('fails promptly when suspension validation rejects before the job claim callback', async () => { + configureEventActorResume(); + const suspension = { + version: 1, + suspensionId: 'suspension-invalid-signature', + attempt: 0, + invocation: { invocationId: 'trigger_event_delivery' }, + interrupt: { + payload: { _librechatEventActor: { expectedAction: { toolName: 'lookup' } } }, + }, + }; + mockGenerationJobManager.getJob.mockResolvedValue( + makeToolApprovalJob({ + metadata: { + idempotencyClientRequestId: 'trigger_event_delivery', + agentEventSuspension: { + version: 1, + suspensionId: suspension.suspensionId, + attempt: 0, + }, + }, + }), + ); + mockGetAgentEventActorSnapshot.mockResolvedValue({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: ACTION_ID, + jobCreatedAt: 1000, + status: 'pending', + }, + }); + mockResumeAgentEventActor.mockRejectedValue(new Error('invalid signed suspension')); + + const res = await post(approveBody()); + + expect(res.status).toBe(500); + expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled(); + expect(mockDecrementPendingRequest).toHaveBeenCalled(); + }); + it('leaves the approval pending when the previous segment still owns the lease', async () => { configureEventActorResume(); mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); @@ -1674,7 +1996,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { ); expect( mockGenerationJobManager.beginProviderExecution.mock.invocationCallOrder[0], - ).toBeLessThan(mockInitializeClient.mock.invocationCallOrder[0]); + ).toBeGreaterThan(mockInitializeClient.mock.invocationCallOrder[0]); }); }); @@ -3134,8 +3456,12 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { describe('non-finalizing outcomes', () => { it('re-pause: does not finalize when the run pauses again', async () => { mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); + const exposePendingApproval = jest.fn().mockResolvedValue(undefined); mockInitializeClient.mockResolvedValue({ - client: makeClient({ pendingApproval: { actionId: NEXT_ACTION_ID } }), + client: makeClient({ + pendingApproval: { actionId: NEXT_ACTION_ID }, + exposePendingApproval, + }), userMCPAuthMap: {}, }); @@ -3165,6 +3491,9 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { 1000, ); expect(mockGenerationJobManager.failPausePersistence).not.toHaveBeenCalled(); + expect(exposePendingApproval.mock.invocationCallOrder[0]).toBeLessThan( + mockGenerationJobManager.approvals.finishPausePersistence.mock.invocationCallOrder[0], + ); // The slot is still released and the client disposed. expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID); expect(mockDisposeClient).toHaveBeenCalledTimes(1); diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index d5c442e169..fbbdef00d5 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -3410,12 +3410,135 @@ class AgentClient extends BaseClient { activityPhase?.complete?.(); } + /** Returns the exact staged approval envelope the SDK signs into a suspension. */ + readEventActorSuspension() { + const staged = this.stagedApproval; + if (staged == null || this.eventActorInvocationId == null) { + return undefined; + } + return { + actionId: staged.pendingAction.actionId, + jobCreatedAt: this.jobCreatedAt, + interrupt: { + id: staged.interruptId, + payload: { ...staged.pendingAction, type: staged.interruptType }, + }, + }; + } + + /** Projects an already-staged pause into the shared job store. Event Actors + * call this only after their signed Conversation suspension is durable. */ + async publishStagedApproval(eventActorSuspension) { + const staged = this.stagedApproval; + if (staged == null) { + return false; + } + if (this.pendingApproval?.actionId === staged.pendingAction.actionId) { + return true; + } + const pauseProjection = { + expectedCreatedAt: this.jobCreatedAt, + ...(staged.discoveredTools.length > 0 ? { discoveredTools: staged.discoveredTools } : {}), + ...(staged.activityPhaseSnapshot == null + ? {} + : { activityPhaseSnapshot: staged.activityPhaseSnapshot }), + persistencePending: true, + ...(eventActorSuspension == null + ? {} + : { + agentEventSuspension: { + version: eventActorSuspension.version, + suspensionId: eventActorSuspension.suspensionId, + attempt: eventActorSuspension.attempt, + }, + }), + }; + let paused; + try { + paused = await GenerationJobManager.approvals.pause( + staged.streamId, + staged.pendingAction, + pauseProjection, + ); + } catch (error) { + /** Redis may commit running -> requires_action and lose only its reply. + * The Conversation suspension is already canonical at this point, so + * confirm this exact generation/action/projection before declaring the + * publication failed and driving terminal compensation. */ + const currentJob = await GenerationJobManager.getJob(staged.streamId).catch(() => null); + const projected = currentJob?.metadata?.agentEventSuspension; + const expectedProjection = pauseProjection.agentEventSuspension; + if ( + currentJob?.createdAt === this.jobCreatedAt && + currentJob.status === 'requires_action' && + currentJob.metadata?.pendingAction?.actionId === staged.pendingAction.actionId && + expectedProjection != null && + projected?.version === expectedProjection.version && + projected.suspensionId === expectedProjection.suspensionId && + projected.attempt === expectedProjection.attempt + ) { + paused = true; + } else { + throw error; + } + } + if (!paused) { + logger.debug( + `[AgentClient] Interrupt fired but job ${staged.streamId} was not running; not pausing`, + ); + return false; + } + this.pendingApproval = staged.pendingAction; + return true; + } + + /** Exposes a durable pause after its controller-owned history barrier clears. */ + async exposePendingApproval() { + const staged = this.stagedApproval; + if ( + staged == null || + this.pendingApproval?.actionId !== staged.pendingAction.actionId || + this.exposedApprovalActionId === staged.pendingAction.actionId + ) { + return false; + } + if (!this.pendingRequestReleased) { + try { + if (this.options.req?._scheduleConcurrencyExempt !== true) { + await decrementPendingRequest(this.options.req?.user?.id); + } + this.pendingRequestReleased = true; + } catch (err) { + logger.error( + `[AgentClient] Failed to release request slot on pause ${staged.streamId}`, + getSafeErrorMetadata(err), + ); + } + } + // Steers accepted before the pause remain in the shared store throughout + // review. The resumed run rehydrates them; exposing the action never moves + // their only copy into this replica's ephemeral client state. + await GenerationJobManager.emitChunk( + staged.streamId, + { + event: ApprovalEvents.ON_PENDING_ACTION, + data: toClientPendingAction(staged.pendingAction), + }, + { expectedCreatedAt: this.jobCreatedAt }, + ); + this.exposedApprovalActionId = staged.pendingAction.actionId; + logger.debug( + `[AgentClient] Paused ${staged.streamId} for ${staged.interruptType} (action ${staged.pendingAction.actionId})`, + ); + return true; + } + /** * Surface any human-in-the-loop interrupt the SDK captured during the most * recent `processStream` / `resume`. When the run paused for tool approval (or - * an ask-user question), mark the job `requires_action`, persist the pending - * review record, and emit it to live clients — then set `this.pendingApproval` - * so the controller leaves the turn unfinalized for the resume route to continue. + * an ask-user question), stage its exact envelope. Ordinary turns immediately + * publish and expose it; Event Actors let the SDK persist signed suspension + * evidence first, then publish under the same history barrier. * * No-op when the run completed without an interrupt, or when the job was aborted * between the interrupt firing and this mark (a late interrupt must not pause a @@ -3554,59 +3677,20 @@ class AgentClient extends BaseClient { ); } - const paused = await GenerationJobManager.approvals.pause(streamId, pendingAction, { - expectedCreatedAt: this.jobCreatedAt, - ...(discoveredTools.length > 0 ? { discoveredTools } : {}), - ...(this.activityPhaseWiring?.snapshot != null && { - activityPhaseSnapshot: this.activityPhaseWiring.snapshot(), - }), - persistencePending: true, - }); - if (!paused) { - logger.debug( - `[AgentClient] Interrupt fired but job ${streamId} was not running; not pausing`, - ); + this.stagedApproval = { + streamId, + pendingAction, + interruptId: interrupt.interruptId, + interruptType: interrupt.payload.type, + discoveredTools, + activityPhaseSnapshot: this.activityPhaseWiring?.snapshot?.(), + }; + if (this.eventActorInvocationId != null) { return; } - - this.pendingApproval = pendingAction; - // Release the concurrency slot this request held the MOMENT the turn is durably - // paused — before the approval card is emitted — so the user's `/resume` can - // re-acquire one immediately. Otherwise a fast Approve races the HTTP-driver - // teardown (request.js pause branch / resume.js finally) that would otherwise - // release it, and `/resume` 429s under LIMIT_CONCURRENT_MESSAGES. Idempotent via - // the flag; if it fails here, the teardown still releases (it checks the flag). - if (!this.pendingRequestReleased) { - try { - if (this.options.req?._scheduleConcurrencyExempt !== true) { - await decrementPendingRequest(this.options.req?.user?.id); - } - this.pendingRequestReleased = true; - } catch (err) { - logger.error( - `[AgentClient] Failed to release request slot on pause ${streamId}`, - getSafeErrorMetadata(err), - ); - } + if (await this.publishStagedApproval()) { + await this.exposePendingApproval(); } - await GenerationJobManager.emitChunk( - streamId, - { - event: ApprovalEvents.ON_PENDING_ACTION, - data: toClientPendingAction(pendingAction), - }, - { expectedCreatedAt: this.jobCreatedAt }, - ); - // Steers queued before this pause stay IN the store for the whole approval - // window: `resumeState.pendingSteers` re-seeds the client's chips on - // reload, and the resumed run drains them at its first tool boundary. - // Draining here would leave the only copy in ephemeral client state — a - // reload during the pause would silently lose the user's message. New - // steers are rejected while paused (enqueue is status-guarded), and the - // requires_action TTL extension keeps the queue key alive. - logger.debug( - `[AgentClient] Paused ${streamId} for ${interrupt.payload.type} (action ${pendingAction.actionId})`, - ); } async chatCompletion({ payload, userMCPAuthMap, abortController = null }) { diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index e2b920a06b..00e04e5c89 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -754,6 +754,80 @@ describe('AgentClient - interrupt discovery persistence', () => { await GenerationJobManager.destroy(); }); + it('stages an event-actor interrupt until its signed suspension is durable', async () => { + const streamId = 'conversation-event-actor-staged-pause'; + const job = await GenerationJobManager.createJob(streamId, 'user-123', streamId); + const client = new AgentClient({ + req: { + user: { id: 'user-123' }, + body: { endpoint: EModelEndpoint.agents, agent_id: 'agent-123' }, + config: { endpoints: { [EModelEndpoint.agents]: {} } }, + }, + res: {}, + agent: { + id: 'agent-123', + endpoint: EModelEndpoint.openAI, + provider: EModelEndpoint.openAI, + model_parameters: { model: 'gpt-4' }, + }, + contentParts: [], + collectedUsage: [], + artifactPromises: [], + }); + client.conversationId = streamId; + client.responseMessageId = 'response-event-actor-pause'; + client.jobCreatedAt = job.createdAt; + client.eventActorInvocationId = 'event-pause'; + + await client.handleRunInterrupt( + { + getInterrupt: () => ({ + interruptId: 'interrupt-event-actor', + threadId: streamId, + payload: { + type: 'ask_user_question', + question: { question: 'Proceed?' }, + }, + }), + getDiscoveredTools: () => ['save_issue_mcp_linear'], + }, + streamId, + ); + + await expect(GenerationJobManager.getJobStatus(streamId)).resolves.toBe('running'); + expect(client.pendingApproval).toBeUndefined(); + expect(client.readEventActorSuspension()).toMatchObject({ + actionId: expect.any(String), + jobCreatedAt: job.createdAt, + interrupt: { + id: 'interrupt-event-actor', + payload: { + type: 'ask_user_question', + actionId: expect.any(String), + }, + }, + }); + + await expect( + client.publishStagedApproval({ version: 1, suspensionId: 'signed-suspension', attempt: 0 }), + ).resolves.toBe(true); + await expect(GenerationJobManager.getJobStatus(streamId)).resolves.toBe('requires_action'); + await expect(GenerationJobManager.getJob(streamId)).resolves.toMatchObject({ + metadata: { + agentEventSuspension: { + version: 1, + suspensionId: 'signed-suspension', + attempt: 0, + }, + }, + }); + expect(client.pendingApproval).toMatchObject({ actionId: expect.any(String) }); + expect(client.pendingRequestReleased).toBeFalsy(); + + await client.exposePendingApproval(); + expect(client.pendingRequestReleased).toBe(true); + }); + it('makes the run discovery snapshot durable when the run pauses', async () => { const streamId = 'conversation-discovered-pause'; const job = await GenerationJobManager.createJob(streamId, 'user-123', streamId); diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index 06a2c446a7..87ddfbde76 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -51,6 +51,7 @@ const { getConvo, getAgentEventActorSnapshot, commitAgentEventActorState, + storeAgentEventActorSuspension, beginAgentEventActorLegacyTurn, completeAgentEventActorLegacyTurn, recordAgentEventActorReconciliation, @@ -1725,6 +1726,10 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit } : undefined, canPause: eventActorMayPause, + /** Protocol v2 is the existing homogeneous-fleet cutover. Keeping + * pause-capable producers on the legacy path under v1 prevents an old + * `/resume` replica from consuming a signed suspension it cannot claim. */ + durableEventActorSuspensions: generationProtocolVersion >= GENERATION_PROTOCOL_V2, checkpointerType: agentsConfig?.checkpointer?.type, expectedActionMayDetach: eventActorActionMayDetach, }); @@ -2087,10 +2092,12 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit client?.run?.getRunSteps?.() ?? [], client?.contentParts ?? [], ), + readSuspension: () => client.readEventActorSuspension(), }, { getSnapshot: getAgentEventActorSnapshot, commitState: commitAgentEventActorState, + storeSuspension: storeAgentEventActorSuspension, recordReconciliation: recordAgentEventActorReconciliation, resolveReconciliation: resolveAgentEventActorReconciliation, admitAction: admitAgentEventActorAction, @@ -2099,7 +2106,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit getReceipt: getAgentEventActorReceipt, clearReconciliation: clearAgentEventActorReconciliation, }, - ).then(({ value, execution }) => { + ).then(async ({ value, execution }) => { if (execution.status === 'applied') { appliedEventActor = { invocationId: eventTaskId, @@ -2107,6 +2114,10 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit checkpoint: execution.head.checkpoint, action: execution.result.action, }; + } else if (execution.status === 'suspended') { + if (!(await client.publishStagedApproval(execution.suspension))) { + throw new Error('Event actor suspension could not be projected to its job'); + } } logger.info('[event-actor] Bound child event completed', { conversationId, @@ -2313,6 +2324,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit } throw pausePersistenceError; } + await client.exposePendingApproval?.(); const released = await GenerationJobManager.approvals.finishPausePersistence( streamId, pauseActionId, diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.js index 7b8801d5ae..f6644de811 100644 --- a/api/server/controllers/agents/resume.js +++ b/api/server/controllers/agents/resume.js @@ -1,4 +1,5 @@ const { randomUUID } = require('crypto'); +const { isDeepStrictEqual } = require('util'); const { logger } = require('@librechat/data-schemas'); const { Constants, @@ -39,6 +40,9 @@ const { createMCPRuntimeRequestBody, getSafeErrorMetadata, isAgentEventRetentionActive, + resumeAgentEventActor, + createAgentEventActionRecorder, + findAgentEventAppliedAction, } = require('@librechat/api'); const { disposeClient } = require('~/server/cleanup'); const { decryptMetadata } = require('~/server/services/ActionService'); @@ -57,6 +61,12 @@ const { getUserMemories, getRoleByName, isSubagentOwnerAdmissible, + getAgentEventActorSnapshot, + commitAgentEventActorState, + storeAgentEventActorSuspension, + claimAgentEventActorSuspension, + settleAgentEventActorSuspension, + recordAgentEventActorReconciliation, completeAgentEventActorLegacyTurn, } = require('~/models'); const { @@ -92,6 +102,25 @@ function sendGenerationJson(res, status, body, generationProtocolVersion) { */ const STEER_RESUME_SETUP_TIMEOUT_MS = 1000; +function deferred() { + let resolve; + let reject; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +function getSuspendedEventActorExpectedAction(suspension) { + const payload = suspension?.interrupt?.payload; + const expectedAction = + payload != null && typeof payload === 'object' && !Array.isArray(payload) + ? payload._librechatEventActor?.expectedAction + : undefined; + return expectedAction != null && typeof expectedAction === 'object' ? expectedAction : undefined; +} + /** * New jobs are physically isolated by an immutable saver namespace, so a * terminal owner deletes the whole namespace and catches writes that landed @@ -347,6 +376,7 @@ async function finalizeResumedTurn({ conversationId, addTitle, checkpointGeneration, + appliedEventActor, }) { const userId = req.user.id; const checkpointerCfg = req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer; @@ -498,6 +528,26 @@ async function finalizeResumedTurn({ if (!savedResponseMessage) { throw new Error('Resumed response could not be persisted before terminal publication'); } + if (appliedEventActor != null) { + const recorded = await recordAgentEventActorReconciliation({ + user: userId, + conversationId, + ...(req._agentEventBindingTenantId == null + ? {} + : { tenantId: req._agentEventBindingTenantId }), + reconciliation: { + invocationId: appliedEventActor.invocationId, + actionAdmitted: true, + status: 'history_persisted', + checkpoint: appliedEventActor.checkpoint, + action: appliedEventActor.action, + observedAt: new Date(), + }, + }); + if (!recorded) { + throw new Error('Resumed event actor history barrier could not be durably recorded'); + } + } /** The response row is now the durable history barrier for the resumed * legacy turn. Seal its exact pre-pause token before publishing FINAL; a * failed seal remains fail-closed and is recovered by the bounded path. */ @@ -1145,6 +1195,12 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) let releaseEventChildLease; let eventLeaseTransferredToRun = false; + let durableEventActorSuspension; + let eventActorResumePromise; + let eventActorStartGate; + let eventActorContinuationStarted = false; + let eventActorActionRecorder; + let appliedEventActor; const providerExecutionId = randomUUID(); try { if (req._agentEventBindingParentConversationId != null) { @@ -1243,6 +1299,74 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) await decrementPendingRequest(userId); return sendGenerationJson(res, 409, eventActorRejection, generationProtocolVersion); } + /** Missing means this pause was produced by a pre-durable-suspension + * replica and must retain the legacy resume path during rolling deploys. + * Presence opts the job into the fail-closed, Conversation-authoritative + * protocol; malformed or stale markers never downgrade to legacy. */ + const suspensionProjection = job.metadata?.agentEventSuspension; + if (suspensionProjection != null) { + const projectionValid = + suspensionProjection.version === 1 && + typeof suspensionProjection.suspensionId === 'string' && + suspensionProjection.suspensionId.length > 0 && + Number.isSafeInteger(suspensionProjection.attempt) && + suspensionProjection.attempt >= 0; + const actorSnapshot = projectionValid + ? await getAgentEventActorSnapshot({ + user: userId, + conversationId, + ...(req._agentEventBindingTenantId == null + ? {} + : { tenantId: req._agentEventBindingTenantId }), + }) + : undefined; + const suspensionRecord = actorSnapshot?.suspension; + if ( + projectionValid && + suspensionRecord?.status === 'pending' && + suspensionRecord.actionId === pendingAction.actionId && + suspensionRecord.jobCreatedAt === job.createdAt && + suspensionRecord.suspension.suspensionId === suspensionProjection.suspensionId && + suspensionRecord.suspension.attempt === suspensionProjection.attempt + ) { + durableEventActorSuspension = suspensionRecord.suspension; + const signedExpectedAction = getSuspendedEventActorExpectedAction( + durableEventActorSuspension, + ); + if ( + job.metadata.agentEventExpectedAction != null && + !isDeepStrictEqual(signedExpectedAction, job.metadata.agentEventExpectedAction) + ) { + const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); + await rollbackUnconsumedScheduleClaim(currentJob); + await releaseScheduleFence(); + await decrementPendingRequest(userId); + return sendGenerationJson( + res, + 409, + { + code: 'EVENT_ACTOR_SUSPENSION_STALE', + error: 'This event actor action is no longer current', + }, + generationProtocolVersion, + ); + } + } else { + const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); + await rollbackUnconsumedScheduleClaim(currentJob); + await releaseScheduleFence(); + await decrementPendingRequest(userId); + return sendGenerationJson( + res, + 409, + { + code: 'EVENT_ACTOR_SUSPENSION_STALE', + error: 'This event actor action is no longer current', + }, + generationProtocolVersion, + ); + } + } } // Atomically claim the resume. The single winner drives the run; a racing second @@ -1253,18 +1377,12 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) // that releases it, so a store/Redis error here (unlike the clean `!claimed` branch) // would leak the concurrency slot until the counter TTL expires — spuriously 429'ing // the user when they retry the still-paused approval. Release the slot on that path too. - let claimed; - try { - /** The CAS that reopens steering must also publish THIS owner's seal - * capability. A separate write after status=`running` leaves a window in - * which steer/arm requests read the previous replica's capability. */ - claimed = await GenerationJobManager.approvals.resolve( + const claimJobApproval = () => + GenerationJobManager.approvals.resolve( streamId, pendingAction.actionId, { preemptCapable: isSteerPreemptSupported(), - // The handover owner's quote handling replaces the previous - // replica's flag, mirroring `preemptCapable` above. steerQuotesCapable: true, providerExecutionId, providerDrained: true, @@ -1272,6 +1390,90 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) }, job.createdAt, ); + let claimed; + try { + /** The CAS that reopens steering must also publish THIS owner's seal + * capability. A separate write after status=`running` leaves a window in + * which steer/arm requests read the previous replica's capability. */ + if (durableEventActorSuspension == null) { + claimed = await claimJobApproval(); + } else { + const claimGate = deferred(); + eventActorStartGate = deferred(); + const expectedAction = getSuspendedEventActorExpectedAction(durableEventActorSuspension); + eventActorActionRecorder = createAgentEventActionRecorder(expectedAction); + req._agentEventActionObserver = eventActorActionRecorder.observeToolEnd; + eventActorResumePromise = resumeAgentEventActor( + { + user: userId, + conversationId, + ...(req._agentEventBindingTenantId == null + ? {} + : { tenantId: req._agentEventBindingTenantId }), + bindingId: req._agentEventBindingId, + suspension: durableEventActorSuspension, + /** One identity spans the Conversation claim and the job's + * provider-owner CAS. A terminal hook can therefore prove whether + * an abort won before or after the resume projection. */ + resumeAttemptId: providerExecutionId, + resumeValue: mapped.resumeValue, + signal: job.abortController.signal, + checkpointer: checkpointerCfg, + expectedAction, + claimProjection: async () => { + try { + const projected = await claimJobApproval(); + claimGate.resolve(projected); + return projected; + } catch (error) { + /** Redis can commit its CAS and lose only the reply. Read back + * this exact resume capability before declaring the earlier + * Conversation claim orphaned. */ + const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); + if ( + currentJob?.createdAt === job.createdAt && + currentJob.status === 'running' && + currentJob.metadata?.providerExecutionId === providerExecutionId + ) { + claimGate.resolve(true); + return true; + } + claimGate.reject(error); + throw error; + } + }, + resume: async (actorContext) => { + const start = await eventActorStartGate.promise; + return start(actorContext); + }, + readAppliedAction: () => + eventActorActionRecorder.read() ?? + findAgentEventAppliedAction( + expectedAction, + client?.run?.getRunSteps?.() ?? [], + client?.contentParts ?? [], + { userSubmittedMessageFieldPaths }, + ), + readSuspension: () => client?.readEventActorSuspension(), + readResultContext: () => client?.getEventActorContext(), + }, + { + getSnapshot: getAgentEventActorSnapshot, + commitState: commitAgentEventActorState, + storeSuspension: storeAgentEventActorSuspension, + claimSuspension: claimAgentEventActorSuspension, + settleSuspension: settleAgentEventActorSuspension, + recordReconciliation: recordAgentEventActorReconciliation, + }, + ); + eventActorResumePromise.catch(() => {}); + claimed = await Promise.race([ + claimGate.promise, + eventActorResumePromise.then(() => { + throw new Error('Event actor suspension completed before claiming its job projection'); + }), + ]); + } } catch (err) { const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); await rollbackUnconsumedScheduleClaim(currentJob); @@ -1460,17 +1662,6 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) let pausePersistenceFailed = false; let pausePersistenceFailureFinalized = false; try { - if ( - !(await GenerationJobManager.beginProviderExecution( - streamId, - job.createdAt, - providerExecutionId, - )) - ) { - throw Object.assign(new Error('Generation stopped before provider resume'), { - code: 'RUN_REPLACED', - }); - } if (userSubmittedPaths.length > 0) { job.metadata.userSubmittedPaths = userSubmittedPaths; } @@ -1520,22 +1711,58 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) GenerationJobManager.setContentParts(streamId, client.contentParts, job.createdAt); } - await client.resumeCompletion({ - resumeValue: mapped.resumeValue, - seedContent, - runSteps: resumeState?.runSteps ?? [], - storedMessages, - abortController: job.abortController, - // Carry the user's MCP auth so approved MCP tools run with their credentials. - userMCPAuthMap: result.userMCPAuthMap, - // Replay deferred tools discovered before the pause (captured at pause). The rebuilt - // graph passes `messages: []`, so without these the model would lose their schemas. - discoveredToolNames: job.metadata?.discoveredTools, - activityPhaseSnapshot: job.metadata?.activityPhaseSnapshot, - }); + const resumeClient = () => + client.resumeCompletion({ + resumeValue: mapped.resumeValue, + seedContent, + runSteps: resumeState?.runSteps ?? [], + storedMessages, + abortController: job.abortController, + // Carry the user's MCP auth so approved MCP tools run with their credentials. + userMCPAuthMap: result.userMCPAuthMap, + // Replay deferred tools discovered before the pause (captured at pause). The rebuilt + // graph passes `messages: []`, so without these the model would lose their schemas. + discoveredToolNames: job.metadata?.discoveredTools, + activityPhaseSnapshot: job.metadata?.activityPhaseSnapshot, + }); + if ( + !(await GenerationJobManager.beginProviderExecution( + streamId, + job.createdAt, + providerExecutionId, + )) + ) { + throw Object.assign(new Error('Generation stopped before provider resume'), { + code: 'RUN_REPLACED', + }); + } + if (eventActorResumePromise == null) { + await resumeClient(); + } else { + eventActorContinuationStarted = true; + eventActorStartGate.resolve(async (actorContext) => { + client.checkpointNamespace = actorContext.checkpointNamespace; + client.eventActorCheckpointId = actorContext.checkpointId; + client.eventActorInvocationId = actorContext.invocationId; + client.eventActorContinuation = actorContext.continuation; + return resumeClient(); + }); + const actorResult = await eventActorResumePromise; + if (actorResult.execution.status === 'suspended') { + if (!(await client.publishStagedApproval(actorResult.execution.suspension))) { + throw new Error('Re-paused event actor suspension could not be projected to its job'); + } + } else if (actorResult.execution.status === 'applied') { + appliedEventActor = { + invocationId: durableEventActorSuspension.invocation.invocationId, + checkpoint: actorResult.execution.head.checkpoint, + action: actorResult.execution.result.action, + }; + } + } // The model may pause AGAIN (another tool, or a follow-up question). The pending - // action is already persisted + emitted; leave the job `requires_action`. + // action is durably projected; persist progress before exposing it to clients. if (client.pendingApproval) { logger.debug(`[ResumeAgentController] Re-paused for approval: ${streamId}`); const pauseActionId = client.pendingApproval.actionId; @@ -1576,6 +1803,7 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) } throw pausePersistenceError; } + await client.exposePendingApproval?.(); const released = await GenerationJobManager.approvals.finishPausePersistence( streamId, pauseActionId, @@ -1635,8 +1863,17 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) conversationId, addTitle, checkpointGeneration, + appliedEventActor, }); } catch (err) { + if ( + eventActorResumePromise != null && + eventActorStartGate != null && + !eventActorContinuationStarted + ) { + eventActorStartGate.reject(err); + await eventActorResumePromise.catch(() => {}); + } logger.error('[ResumeAgentController] Resume failed', getSafeErrorMetadata(err)); if (pausePersistenceFailed) { // failPausePersistence already performed the exact requires_action -> diff --git a/e2e/bombadil/hitl-lifecycle.specification.ts b/e2e/bombadil/hitl-lifecycle.specification.ts index aa7c90c898..5c2923c84c 100644 --- a/e2e/bombadil/hitl-lifecycle.specification.ts +++ b/e2e/bombadil/hitl-lifecycle.specification.ts @@ -159,6 +159,10 @@ const ui = extract((state: State) => { modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'), hitlModelSpec: target(state, '[role="option"]', HITL_MODEL_SPEC, HITL_MODEL_SPEC), stagingOption: target(state, 'button', 'Answer Staging', HITL_OPTION, true), + stagingSelected: Array.from( + state.document.querySelectorAll('button[aria-pressed="true"]'), + ).some((element) => element.textContent?.trim() === HITL_OPTION), + answerSubmit: target(state, 'button:not([disabled])', 'Submit answer', 'Submit'), }; }); @@ -190,6 +194,9 @@ export const hitlLifecycleActions = actions((): Action[] => { pausedReloadIssued = true; return ['Reload']; } + if (state.stagingSelected) { + return clickOrWait(state.answerSubmit); + } return clickOrWait(state.stagingOption); } diff --git a/package-lock.json b/package-lock.json index 9c407b901c..e3d95633c0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,7 +63,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^3.7.6", + "@librechat/agents": "^3.7.7", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", @@ -10633,9 +10633,9 @@ } }, "node_modules/@librechat/agents": { - "version": "3.7.6", - "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.7.6.tgz", - "integrity": "sha512-v+Oe3RUn4J5CG7+7Y3uGZ+MlC5zReDp0BnvGkyZd9oqlkypR8dbK+lsV3s41kOJ6VEL5rxerZ36b+5TmV/2mww==", + "version": "3.7.7", + "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.7.7.tgz", + "integrity": "sha512-3gDBoZodm2CZnwCjiYMI3KzYYwvP2M8k/pumj72eAd6u777oydV1NTXa6gQx1MvwQVaWyXztMWeg64TT31XQVA==", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.115.0", @@ -42838,7 +42838,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^3.7.6", + "@librechat/agents": "^3.7.7", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.30.0", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/package.json b/packages/api/package.json index 4b651b88b6..63d36cbc53 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -113,7 +113,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^3.7.6", + "@librechat/agents": "^3.7.7", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.30.0", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/src/agents/plan.spec.ts b/packages/api/src/agents/plan.spec.ts index cab33760a0..e98317ff26 100644 --- a/packages/api/src/agents/plan.spec.ts +++ b/packages/api/src/agents/plan.spec.ts @@ -6,6 +6,7 @@ const baseInput = (): ResolveAgentTurnExecutionPlanInput => ({ parentMessageId: 'message-1', isNewConversation: false, canPause: false, + durableEventActorSuspensions: false, expectedActionMayDetach: false, }); @@ -51,12 +52,26 @@ describe('resolveAgentTurnExecutionPlan', () => { }); }); + it('keeps a pause-capable bound event on checkpoint continuation', () => { + expect( + resolveAgentTurnExecutionPlan({ + ...baseInput(), + event: boundEvent(), + canPause: true, + durableEventActorSuspensions: true, + }).strategy, + ).toBe('checkpoint'); + }); + it.each([ ['no binding', { event: { ...boundEvent(), binding: undefined } }], ['no expected action', { event: { ...boundEvent(), expectedAction: undefined } }], ['memory checkpointer', { event: boundEvent(), checkpointerType: 'memory' }], - ['pause-capable run', { event: boundEvent(), canPause: true }], ['detachable action', { event: boundEvent(), expectedActionMayDetach: true }], + [ + 'pre-cutover pause consumer fleet', + { event: boundEvent(), canPause: true, durableEventActorSuspensions: false }, + ], ] as const)('falls back to history for %s', (_label, overrides) => { expect(resolveAgentTurnExecutionPlan({ ...baseInput(), ...overrides }).strategy).toBe( 'history', diff --git a/packages/api/src/agents/plan.ts b/packages/api/src/agents/plan.ts index 61d38733de..c15c88d69f 100644 --- a/packages/api/src/agents/plan.ts +++ b/packages/api/src/agents/plan.ts @@ -34,6 +34,9 @@ export interface ResolveAgentTurnExecutionPlanInput { expectedAction?: AgentTriggerExpectedAction; }; canPause: boolean; + /** Fleet-wide producer gate. Shared-store deployments enable this only + * after every resume consumer understands signed Event Actor suspensions. */ + durableEventActorSuspensions: boolean; checkpointerType?: TCheckpointerType; expectedActionMayDetach: boolean; } @@ -70,8 +73,8 @@ export function resolveAgentTurnExecutionPlan( binding != null && expectedAction != null && input.checkpointerType !== 'memory' && - !input.canPause && - !input.expectedActionMayDetach; + !input.expectedActionMayDetach && + (!input.canPause || input.durableEventActorSuspensions); let strategy: AgentTurnContinuationStrategy = 'history'; if (input.isNewConversation) { strategy = 'fresh'; diff --git a/packages/api/src/agents/triggers/actor.spec.ts b/packages/api/src/agents/triggers/actor.spec.ts index bf391fd350..4a429ff61d 100644 --- a/packages/api/src/agents/triggers/actor.spec.ts +++ b/packages/api/src/agents/triggers/actor.spec.ts @@ -1,16 +1,18 @@ import type { IAgentEventActorReconciliation, IAgentEventActorState, + IAgentEventActorSuspension, } from '@librechat/data-schemas'; +import type { EventActorInterrupt } from '@librechat/agents'; import { captureAgentEventCheckpoint, deleteAgentCheckpoint, forkAgentEventCheckpoint, getAgentCheckpointer, } from '../checkpointer'; +import { cancelAgentEventActor, executeAgentEventActor, resumeAgentEventActor } from './actor'; import { createAgentEventActionRecorder, findAgentEventAppliedAction } from './outcome'; import { createAgentContextFingerprint } from '../compatibility'; -import { executeAgentEventActor } from './actor'; jest.mock('../checkpointer', () => ({ ...jest.requireActual('../checkpointer'), @@ -27,12 +29,14 @@ const mockedGetCheckpointer = jest.mocked(getAgentCheckpointer); describe('event actor host adapter', () => { const conversationId = 'actor-thread'; + const originalCredsKey = process.env.CREDS_KEY; let state: IAgentEventActorState | null; let epoch = 0; let legacyTurn: { token: string; startedAt: Date } | null = null; let nextCheckpoint = 1; beforeEach(() => { + process.env.CREDS_KEY = 'event-actor-test-credentials-key'; state = null; epoch = 0; legacyTurn = null; @@ -52,11 +56,20 @@ describe('event actor host adapter', () => { mockedDelete.mockResolvedValue(); }); + afterAll(() => { + if (originalCredsKey == null) { + delete process.env.CREDS_KEY; + } else { + process.env.CREDS_KEY = originalCredsKey; + } + }); + const deps = () => ({ getSnapshot: jest.fn(async () => ({ state, reconciliations: [] as IAgentEventActorReconciliation[], legacyTurn, + suspension: null as IAgentEventActorSuspension | null, epoch, })), commitState: jest.fn( @@ -107,6 +120,381 @@ describe('event actor host adapter', () => { hasActionAdmission: jest.fn(async () => false), }); + it('publishes a signed durable suspension instead of discarding a paused fork', async () => { + const dependencies = { + ...deps(), + storeSuspension: jest.fn(async () => ({ status: 'stored' as const })), + }; + + const result = await executeAgentEventActor( + { + user: 'user-1', + conversationId, + invocationId: 'event-paused', + event: { id: 'event-paused', type: 'turn' }, + signal: new AbortController().signal, + invoke: async () => 'paused-response', + readAppliedAction: () => undefined, + readSuspension: () => ({ + actionId: 'action-paused', + jobCreatedAt: 123, + interrupt: { + id: 'interrupt-paused', + payload: { type: 'ask_user_question', question: 'Continue?' }, + }, + }), + }, + dependencies, + ); + + expect(result.value).toBe('paused-response'); + expect(result.execution).toMatchObject({ + status: 'suspended', + suspension: { + version: 1, + attempt: 0, + invocation: { invocationId: 'event-paused' }, + checkpoint: { checkpointId: 'checkpoint-1' }, + interrupt: { + id: 'interrupt-paused', + payload: { type: 'ask_user_question', question: 'Continue?' }, + }, + }, + }); + expect(dependencies.storeSuspension).toHaveBeenCalledWith( + expect.objectContaining({ + actionId: 'action-paused', + jobCreatedAt: 123, + suspension: expect.objectContaining({ suspensionId: expect.any(String) }), + }), + ); + expect(dependencies.storeSuspension).toHaveBeenCalledTimes(1); + expect(dependencies.commitState).not.toHaveBeenCalled(); + expect(mockedDelete).not.toHaveBeenCalled(); + }); + + it('preserves a pause reached after the expected action in the same fresh segment', async () => { + const dependencies = { + ...deps(), + storeSuspension: jest.fn(async () => ({ status: 'stored' as const })), + }; + + const result = await executeAgentEventActor( + { + user: 'user-1', + conversationId, + invocationId: 'event-action-then-pause', + event: { id: 'event-action-then-pause' }, + signal: new AbortController().signal, + invoke: async () => 'paused-after-action', + readAppliedAction: () => ({ toolName: 'submit_move', toolCallId: 'call-before-pause' }), + readSuspension: () => ({ + actionId: 'action-after-tool', + jobCreatedAt: 456, + interrupt: { id: 'interrupt-after-tool', payload: { type: 'tool_approval' } }, + }), + }, + dependencies, + ); + + expect(result.execution).toMatchObject({ + status: 'suspended', + suspension: { interrupt: { id: 'interrupt-after-tool' } }, + }); + expect(dependencies.storeSuspension).toHaveBeenCalledTimes(1); + expect(dependencies.commitState).not.toHaveBeenCalled(); + }); + + it('validates and cancels the exact signed suspension before deleting its fork', async () => { + const dependencies = { + ...deps(), + storeSuspension: jest.fn(async () => ({ status: 'stored' as const })), + cancelSuspension: jest.fn(async () => ({ status: 'cancelled' as const })), + }; + const paused = await executeAgentEventActor( + { + user: 'user-1', + conversationId, + invocationId: 'event-cancelled', + event: { id: 'event-cancelled' }, + signal: new AbortController().signal, + invoke: async () => 'paused-response', + readAppliedAction: () => undefined, + readSuspension: () => ({ + actionId: 'action-cancelled', + jobCreatedAt: 456, + interrupt: { id: 'interrupt-cancelled', payload: { type: 'tool_approval' } }, + }), + }, + dependencies, + ); + if (paused.execution.status !== 'suspended') { + throw new Error('test setup did not suspend'); + } + + await expect( + cancelAgentEventActor( + { + user: 'user-1', + conversationId, + suspension: JSON.parse(JSON.stringify(paused.execution.suspension)), + cancelAttemptId: 'cancel-attempt-1', + reason: 'cancelled', + }, + dependencies, + ), + ).resolves.toEqual({ status: 'cancelled' }); + expect(dependencies.cancelSuspension).toHaveBeenCalledWith( + expect.objectContaining({ + suspensionId: paused.execution.suspension.suspensionId, + invocationId: 'event-cancelled', + }), + ); + expect(mockedDelete).toHaveBeenCalledWith( + conversationId, + undefined, + undefined, + expect.objectContaining({ + throwOnError: true, + checkpointNamespace: paused.execution.suspension.checkpoint.checkpointNs, + }), + ); + }); + + it('resumes signed evidence on a new executor and consumes its claim with the head CAS', async () => { + let storedSuspension: IAgentEventActorSuspension | undefined; + let action: { toolName: string; toolCallId?: string } | undefined; + const dependencies = { + ...deps(), + storeSuspension: jest.fn(async (input) => { + storedSuspension = { + suspension: input.suspension, + actionId: input.actionId, + jobCreatedAt: input.jobCreatedAt, + status: 'pending', + observedAt: new Date(), + }; + return { status: 'stored' as const }; + }), + claimSuspension: jest.fn(async ({ resumeAttemptId }) => { + if (storedSuspension == null) { + throw new Error('test setup did not store a suspension'); + } + storedSuspension = { ...storedSuspension, status: 'claimed', resumeAttemptId }; + return { status: 'claimed' as const }; + }), + settleSuspension: jest.fn(async () => ({ status: 'settled' as const })), + }; + dependencies.getSnapshot.mockImplementation(async () => ({ + state, + reconciliations: [], + legacyTurn: null, + suspension: storedSuspension ?? null, + epoch, + })); + + const paused = await executeAgentEventActor( + { + user: 'user-1', + conversationId, + invocationId: 'event-cross-executor', + event: { id: 'event-cross-executor' }, + signal: new AbortController().signal, + invoke: async () => 'paused-response', + readAppliedAction: () => action, + readSuspension: () => ({ + actionId: 'action-cross-executor', + jobCreatedAt: 321, + interrupt: { + id: 'interrupt-cross-executor', + payload: { type: 'tool_approval', actionId: 'action-cross-executor' }, + }, + }), + }, + dependencies, + ); + if (paused.execution.status !== 'suspended') { + throw new Error('test setup did not suspend'); + } + const evidence = JSON.parse(JSON.stringify(paused.execution.suspension)); + dependencies.getSnapshot.mockClear(); + dependencies.claimSuspension.mockClear(); + dependencies.commitState.mockClear(); + + const resumed = await resumeAgentEventActor( + { + user: 'user-1', + conversationId, + bindingId: 'binding-1', + suspension: evidence, + resumeAttemptId: 'resume-cross-executor', + resumeValue: { approved: true }, + signal: new AbortController().signal, + resume: async () => { + action = { toolName: 'submit_move', toolCallId: 'call-resumed' }; + return 'resumed-response'; + }, + readAppliedAction: () => action, + }, + dependencies, + ); + + expect(resumed).toMatchObject({ + value: 'resumed-response', + execution: { + status: 'applied', + result: { action: { toolName: 'submit_move', toolCallId: 'call-resumed' } }, + }, + }); + expect(dependencies.claimSuspension).toHaveBeenCalledWith( + expect.objectContaining({ + suspensionId: evidence.suspensionId, + resumeAttemptId: 'resume-cross-executor', + actionId: 'action-cross-executor', + }), + ); + expect(dependencies.getSnapshot).toHaveBeenCalledTimes(1); + expect(dependencies.claimSuspension).toHaveBeenCalledTimes(1); + expect(dependencies.commitState).toHaveBeenCalledTimes(1); + expect(dependencies.commitState).toHaveBeenCalledWith( + expect.objectContaining({ + settlementAuthority: expect.objectContaining({ + suspensionId: evidence.suspensionId, + resumeAttemptId: 'resume-cross-executor', + }), + }), + ); + }); + + it('atomically re-pauses after an action and settles a later no-action reply', async () => { + let storedSuspension: IAgentEventActorSuspension | undefined; + let pendingPause: + | { actionId: string; jobCreatedAt: number; interrupt: EventActorInterrupt } + | undefined; + const dependencies = { + ...deps(), + storeSuspension: jest.fn(async (input) => { + storedSuspension = { + suspension: input.suspension, + actionId: input.actionId, + jobCreatedAt: input.jobCreatedAt, + status: 'pending', + observedAt: new Date(), + }; + return { status: 'stored' as const }; + }), + claimSuspension: jest.fn(async ({ resumeAttemptId }) => { + if (storedSuspension == null) { + throw new Error('test setup did not store a suspension'); + } + storedSuspension = { ...storedSuspension, status: 'claimed', resumeAttemptId }; + return { status: 'claimed' as const }; + }), + settleSuspension: jest.fn(async () => ({ status: 'settled' as const })), + }; + dependencies.getSnapshot.mockImplementation(async () => ({ + state, + reconciliations: [], + legacyTurn: null, + suspension: storedSuspension ?? null, + epoch, + })); + + const initial = await executeAgentEventActor( + { + user: 'user-1', + conversationId, + invocationId: 'event-repause', + event: { id: 'event-repause' }, + signal: new AbortController().signal, + invoke: async () => 'initial-pause', + readAppliedAction: () => undefined, + readSuspension: () => ({ + actionId: 'action-first', + jobCreatedAt: 789, + interrupt: { id: 'interrupt-first', payload: { type: 'tool_approval' } }, + }), + }, + dependencies, + ); + if (initial.execution.status !== 'suspended') { + throw new Error('test setup did not suspend'); + } + pendingPause = { + actionId: 'action-second', + jobCreatedAt: 789, + interrupt: { id: 'interrupt-second', payload: { type: 'ask_user_question' } }, + }; + const repaused = await resumeAgentEventActor( + { + user: 'user-1', + conversationId, + suspension: initial.execution.suspension, + resumeAttemptId: 'resume-first', + resumeValue: { approved: true }, + signal: new AbortController().signal, + resume: async () => 'second-pause', + readAppliedAction: () => ({ toolName: 'submit_move', toolCallId: 'call-before-repause' }), + readSuspension: () => pendingPause, + }, + dependencies, + ); + expect(repaused.execution).toMatchObject({ + status: 'suspended', + suspension: { attempt: 1, interrupt: { id: 'interrupt-second' } }, + }); + expect(dependencies.storeSuspension).toHaveBeenLastCalledWith( + expect.objectContaining({ + actionId: 'action-second', + invalidateHead: true, + previous: { + suspensionId: initial.execution.suspension.suspensionId, + attempt: 0, + resumeAttemptId: 'resume-first', + }, + }), + ); + if (repaused.execution.status !== 'suspended') { + throw new Error('test setup did not re-pause'); + } + pendingPause = undefined; + const rejected = await resumeAgentEventActor( + { + user: 'user-1', + conversationId, + suspension: repaused.execution.suspension, + resumeAttemptId: 'resume-second', + resumeValue: { rejected: true }, + signal: new AbortController().signal, + resume: async () => 'rejected-response', + readAppliedAction: () => undefined, + readSuspension: () => pendingPause, + }, + dependencies, + ); + expect(rejected).toMatchObject({ + value: 'rejected-response', + execution: { status: 'completed_no_action' }, + }); + expect(dependencies.settleSuspension).toHaveBeenCalledWith( + expect.objectContaining({ + suspensionId: repaused.execution.suspension.suspensionId, + attempt: 1, + resumeAttemptId: 'resume-second', + }), + ); + expect(mockedDelete).toHaveBeenCalledWith( + conversationId, + undefined, + undefined, + expect.objectContaining({ + throwOnError: true, + checkpointNamespace: repaused.execution.suspension.checkpoint.checkpointNs, + }), + ); + expect(dependencies.commitState).not.toHaveBeenCalled(); + }); + it('cold-starts once, then forks and warm-continues only the next event', async () => { const dependencies = deps(); const invocations: Array<{ continuation: string; checkpointId?: string }> = []; @@ -724,6 +1112,7 @@ describe('event actor host adapter', () => { state: baseState, reconciliations: [], legacyTurn: null, + suspension: null, epoch: 0, }) .mockRejectedValueOnce(new Error('readback unavailable')), @@ -781,6 +1170,7 @@ describe('event actor host adapter', () => { state: authoritative, reconciliations: [marker], legacyTurn: null, + suspension: null, epoch: 0, }); @@ -826,6 +1216,7 @@ describe('event actor host adapter', () => { }, ], legacyTurn: null, + suspension: null, epoch: 0, })), }; @@ -865,6 +1256,7 @@ describe('event actor host adapter', () => { }, ], legacyTurn: null, + suspension: null, epoch: 0, })), commitState: jest.fn(), @@ -938,6 +1330,7 @@ describe('event actor host adapter', () => { }, ], legacyTurn: null, + suspension: null, epoch: 0, })), getReceipt: jest.fn(async () => ({ @@ -1080,6 +1473,7 @@ describe('event actor host adapter', () => { }, ], legacyTurn: null, + suspension: null, epoch: 0, }); @@ -1207,6 +1601,7 @@ describe('event actor host adapter', () => { }, ], legacyTurn: null, + suspension: null, epoch: 0, }); diff --git a/packages/api/src/agents/triggers/actor.ts b/packages/api/src/agents/triggers/actor.ts index 9dd6e28b76..985fc0ca8c 100644 --- a/packages/api/src/agents/triggers/actor.ts +++ b/packages/api/src/agents/triggers/actor.ts @@ -6,18 +6,22 @@ import { type EventActorExecutionResult, type EventActorHead, type EventActorHostAdapter, + type EventActorInterrupt, + type EventActorSuspension, + type EventActorCancelSuspensionResult, } from '@librechat/agents'; import type { AgentTriggerDeliveryMethods, ConversationMethods, IAgentEventActorState, IAgentEventActorSkillIdentity, + IAgentEventActorSuspensionEvidence, } from '@librechat/data-schemas'; import type { TCheckpointerConfig } from 'librechat-data-provider'; import type { AgentEventCheckpointMessageOverlay } from '../checkpointer'; import type { AgentContextFingerprint } from '../compatibility'; import type { AgentTriggerExpectedAction } from './envelope'; -import type { AgentEventAppliedAction } from './outcome'; +import type { AgentEventAppliedAction } from './types'; import { captureAgentEventCheckpoint, deleteAgentCheckpoint, @@ -28,7 +32,7 @@ import { import { agentContextFingerprintsMatch } from '../compatibility'; interface EventActorResult extends Record { - action: AgentEventAppliedAction; + action: AgentEventAppliedAction & EventActorEvent; checkpointCaptureError: string | null; } @@ -58,6 +62,13 @@ export interface ExecuteAgentEventActorInput { legacyTurnStaleMs?: number; invoke(context: AgentEventActorInvocationContext): Promise; readAppliedAction(): AgentEventAppliedAction | undefined; + readSuspension?(): + | { + actionId: string; + jobCreatedAt: number; + interrupt: EventActorInterrupt; + } + | undefined; } export interface AgentEventActorContext { @@ -77,6 +88,10 @@ export interface ExecuteAgentEventActorResult { export interface AgentEventActorDependencies { getSnapshot: ConversationMethods['getAgentEventActorSnapshot']; commitState: ConversationMethods['commitAgentEventActorState']; + storeSuspension?: ConversationMethods['storeAgentEventActorSuspension']; + claimSuspension?: ConversationMethods['claimAgentEventActorSuspension']; + settleSuspension?: ConversationMethods['settleAgentEventActorSuspension']; + cancelSuspension?: ConversationMethods['cancelAgentEventActorSuspension']; recordReconciliation: ConversationMethods['recordAgentEventActorReconciliation']; resolveReconciliation: ConversationMethods['resolveAgentEventActorReconciliation']; admitAction?: AgentTriggerDeliveryMethods['admitAgentEventActorAction']; @@ -86,6 +101,77 @@ export interface AgentEventActorDependencies { clearReconciliation?: ConversationMethods['clearAgentEventActorReconciliation']; } +export interface ResumeAgentEventActorInput { + user: string; + tenantId?: string; + conversationId: string; + bindingId?: string; + suspension: EventActorSuspension; + resumeAttemptId: string; + resumeValue: EventActorEvent; + signal: AbortSignal; + checkpointer?: TCheckpointerConfig; + expectedAction?: AgentTriggerExpectedAction; + /** Projects the claimed Conversation fence into the exact job/action CAS. */ + claimProjection?(): Promise; + resume(context: AgentEventActorInvocationContext): Promise; + readAppliedAction(): AgentEventAppliedAction | undefined; + readSuspension?(): + | { + actionId: string; + jobCreatedAt: number; + interrupt: EventActorInterrupt; + } + | undefined; + readResultContext?(): Promise; +} + +export interface CancelAgentEventActorInput { + user: string; + tenantId?: string; + conversationId: string; + suspension: EventActorSuspension; + cancelAttemptId: string; + reason: 'cancelled' | 'expired'; + signal?: AbortSignal; + checkpointer?: TCheckpointerConfig; + /** Exact orphaned resume claim whose job never entered provider execution. */ + claimedResumeAttemptId?: string; +} + +function bindInterruptToExpectedAction( + interrupt: EventActorInterrupt, + expectedAction: AgentTriggerExpectedAction | undefined, +): EventActorInterrupt { + if ( + expectedAction == null || + interrupt.payload == null || + typeof interrupt.payload !== 'object' || + Array.isArray(interrupt.payload) + ) { + return interrupt; + } + return { + ...interrupt, + payload: { + ...interrupt.payload, + _librechatEventActor: { expectedAction: expectedAction as unknown as EventActorEvent }, + }, + }; +} + +function getEventActorSigningKey(): Buffer { + const credentialsKey = process.env.CREDS_KEY; + if (typeof credentialsKey !== 'string' || credentialsKey.length === 0) { + throw new Error('CREDS_KEY is required for durable event actor execution'); + } + return createHash('sha256') + .update('librechat:event-actor:suspension:v1') + .update('\0') + .update(credentialsKey) + .digest(); +} + function toHead(actorThreadId: string, state: IAgentEventActorState | null): EventActorHead { return state == null ? { actorThreadId, generation: 0 } @@ -96,6 +182,17 @@ function asError(value: unknown): Error { return value instanceof Error ? value : new Error(String(value)); } +/** The host action receipt contains strings only, so this copy is also a + * valid SDK event value without weakening the application-facing type. */ +function toEventActorAppliedAction( + action: AgentEventAppliedAction, +): AgentEventAppliedAction & EventActorEvent { + return { + toolName: action.toolName, + ...(action.toolCallId == null ? {} : { toolCallId: action.toolCallId }), + }; +} + function checkpointMatches( state: IAgentEventActorState, checkpoint: { threadId: string; checkpointId?: string; checkpointNs: string }, @@ -108,7 +205,7 @@ function checkpointMatches( ); } -function actionAdmissionId( +export function createAgentEventActorActionAdmissionId( invocationId: string, checkpoint: { threadId: string; checkpointId?: string; checkpointNs: string }, ): string { @@ -139,6 +236,10 @@ export async function executeAgentEventActor( let observedEpoch: number | undefined; let preparedContext: AgentEventActorContext | undefined; let resultContext: AgentEventActorContext | undefined; + let pendingSuspension: + | { actionId: string; jobCreatedAt: number; interrupt: EventActorInterrupt } + | undefined; + let actionAppliedBeforePause = false; const adapter: EventActorHostAdapter = { async prepare(request, context) { if (context.signal.aborted) { @@ -152,6 +253,12 @@ export async function executeAgentEventActor( if (snapshot === undefined) { throw new Error('Event actor binding is no longer active'); } + if ( + snapshot.suspension?.status === 'closed' && + snapshot.suspension.suspension.invocation.invocationId === input.invocationId + ) { + throw new Error('Event actor invocation already has terminal suspension proof'); + } if (input.bindingId != null && deps.getReceipt != null) { const receipt = await deps.getReceipt({ deliveryKey: input.invocationId, @@ -195,7 +302,7 @@ export async function executeAgentEventActor( (item) => item.invocationId === input.invocationId && item.status === 'invocation_pending', ); if (pendingInvocation != null && input.bindingId != null && deps.hasActionAdmission != null) { - const pendingAdmissionId = actionAdmissionId( + const pendingAdmissionId = createAgentEventActorActionAdmissionId( input.invocationId, pendingInvocation.checkpoint, ); @@ -340,7 +447,10 @@ export async function executeAgentEventActor( * and terminal settlement. A plain receipt read cannot close the final * read-before-invoke race across two Mongo documents. */ if (input.bindingId != null && deps.admitAction != null) { - const admissionId = actionAdmissionId(input.invocationId, invocation.fork); + const admissionId = createAgentEventActorActionAdmissionId( + input.invocationId, + invocation.fork, + ); const admitted = await deps.admitAction({ deliveryKey: input.invocationId, user: input.user, @@ -421,7 +531,34 @@ export async function executeAgentEventActor( } catch (error) { invocationError = error; } - const action = input.readAppliedAction(); + /** A graph can execute the expected action and then pause again in the + * same segment. The pause is nonterminal authority: committing its + * checkpoint as an applied terminal head would strand the staged HITL + * action. Preserve the suspension first; the expected-action evidence + * remains in the checkpoint and is classified after the pause resumes. */ + pendingSuspension = input.readSuspension?.(); + if (pendingSuspension != null) { + actionAppliedBeforePause = input.readAppliedAction() != null; + const checkpoint = await captureAgentEventCheckpoint( + input.conversationId, + invocation.fork.checkpointNs, + invocation.invocationId, + input.checkpointer, + ); + if (checkpoint?.checkpointId == null) { + throw new Error('Paused event actor has no observable interrupt checkpoint'); + } + return { + status: 'suspended', + checkpoint: { ...checkpoint, invocationId: invocation.invocationId }, + interrupt: bindInterruptToExpectedAction( + pendingSuspension.interrupt, + input.expectedAction, + ), + }; + } + const observedAction = input.readAppliedAction(); + const action = observedAction == null ? undefined : toEventActorAppliedAction(observedAction); if (action == null) { if (invocationError != null) { throw invocationError; @@ -471,6 +608,21 @@ export async function executeAgentEventActor( checkpoint: { ...checkpoint, invocationId: invocation.invocationId }, }; }, + async suspend(request) { + if (pendingSuspension == null || deps.storeSuspension == null) { + throw new Error('Event actor suspension storage is unavailable'); + } + return deps.storeSuspension({ + user: input.user, + conversationId: input.conversationId, + ...(input.tenantId == null ? {} : { tenantId: input.tenantId }), + suspension: request.suspension as IAgentEventActorSuspensionEvidence, + actionId: pendingSuspension.actionId, + jobCreatedAt: pendingSuspension.jobCreatedAt, + ...(actionAppliedBeforePause ? { invalidateHead: true } : {}), + ...(request.previous == null ? {} : { previous: request.previous }), + }); + }, async commit(request) { if (request.result.checkpointCaptureError != null) { throw new Error(request.result.checkpointCaptureError); @@ -642,6 +794,7 @@ export async function executeAgentEventActor( const executor = createEventActorExecutor(adapter, { maxDepth: 1, dormantCheckpointTtlMs: getApprovalTtlMs(input.checkpointer), + preparationSigningKey: getEventActorSigningKey(), }); let execution: EventActorExecutionResult = await executor.execute({ actorThreadId: input.conversationId, @@ -735,3 +888,447 @@ export async function executeAgentEventActor( } return { value: value as T, execution }; } + +/** Resumes one signed suspended fork on any replica using the Conversation as authority. */ +export async function resumeAgentEventActor( + input: ResumeAgentEventActorInput, + deps: AgentEventActorDependencies, +): Promise> { + let value: T | undefined; + let invocationError: unknown; + let observedState: IAgentEventActorState | null | undefined; + let observedEpoch: number | undefined; + let resultContext: AgentEventActorContext | undefined; + let pendingSuspension: + | { actionId: string; jobCreatedAt: number; interrupt: EventActorInterrupt } + | undefined; + let actionAppliedBeforePause = false; + + const owner = { + user: input.user, + conversationId: input.conversationId, + ...(input.tenantId == null ? {} : { tenantId: input.tenantId }), + }; + const adapter: EventActorHostAdapter = { + async prepare() { + throw new Error('A suspended event actor cannot prepare a fresh invocation'); + }, + async coldContinue() { + throw new Error('A suspended event actor cannot cold-start during resume'); + }, + async invoke() { + throw new Error('A suspended event actor must enter through resume'); + }, + async resume(request, context) { + if (deps.claimSuspension == null) { + throw new Error('Event actor suspension claim storage is unavailable'); + } + const snapshot = await deps.getSnapshot(owner); + const hostSuspension = snapshot?.suspension; + if ( + snapshot == null || + hostSuspension == null || + hostSuspension.status !== 'pending' || + hostSuspension.suspension.suspensionId !== request.suspension.suspensionId || + hostSuspension.suspension.attempt !== request.suspension.attempt || + hostSuspension.suspension.suspensionDigest !== request.suspension.suspensionDigest + ) { + return { status: 'stale' }; + } + observedState = snapshot.state; + observedEpoch = snapshot.epoch; + const base = request.suspension.invocation.base; + if ( + (observedState == null && base.generation !== 0) || + (observedState != null && + (observedState.generation !== base.generation || + observedState.checkpoint.threadId !== base.checkpoint?.threadId || + observedState.checkpoint.checkpointId !== base.checkpoint?.checkpointId || + observedState.checkpoint.checkpointNs !== base.checkpoint?.checkpointNs)) + ) { + return { status: 'stale' }; + } + const claimed = await deps.claimSuspension({ + ...owner, + suspensionId: request.suspension.suspensionId, + attempt: request.suspension.attempt, + actionId: hostSuspension.actionId, + jobCreatedAt: hostSuspension.jobCreatedAt, + resumeAttemptId: request.resumeAttemptId, + }); + if (claimed.status !== 'claimed') { + return { status: 'stale' }; + } + if (input.claimProjection != null && !(await input.claimProjection())) { + throw new Error('Event actor suspension claim could not be projected to its job'); + } + try { + value = await input.resume({ + checkpointNamespace: request.suspension.checkpoint.checkpointNs, + ...(request.suspension.checkpoint.checkpointId == null + ? {} + : { checkpointId: request.suspension.checkpoint.checkpointId }), + invocationId: request.suspension.invocation.invocationId, + continuation: request.suspension.invocation.continuation, + signal: context.signal, + }); + } catch (error) { + invocationError = error; + } + /** A resumed segment may both satisfy the delivery and reach its next + * human boundary. Publish the successor suspension before considering + * the segment terminal; otherwise the successor checkpoint is committed + * without any resumable host action. */ + pendingSuspension = input.readSuspension?.(); + if (pendingSuspension != null) { + actionAppliedBeforePause = input.readAppliedAction() != null; + const checkpoint = await captureAgentEventCheckpoint( + input.conversationId, + request.suspension.checkpoint.checkpointNs, + request.suspension.invocation.invocationId, + input.checkpointer, + ); + if (checkpoint?.checkpointId == null) { + throw new Error('Re-paused event actor has no observable interrupt checkpoint'); + } + return { + status: 'claimed', + result: { + status: 'suspended', + checkpoint: { + ...checkpoint, + invocationId: request.suspension.invocation.invocationId, + }, + interrupt: bindInterruptToExpectedAction( + pendingSuspension.interrupt, + input.expectedAction, + ), + }, + }; + } + const observedAction = input.readAppliedAction(); + const action = observedAction == null ? undefined : toEventActorAppliedAction(observedAction); + if (action == null) { + if (invocationError != null) { + return { status: 'claimed_failed', error: asError(invocationError) }; + } + return { status: 'claimed', result: { status: 'completed_no_action' } }; + } + try { + resultContext = input.readResultContext ? await input.readResultContext() : undefined; + } catch (error) { + return { + status: 'claimed', + result: { + status: 'applied', + result: { + action, + checkpointCaptureError: `Applied resumed context could not be captured: ${asError(error).message}`, + }, + checkpoint: request.suspension.checkpoint, + }, + }; + } + let checkpoint: Awaited>; + try { + checkpoint = await captureAgentEventCheckpoint( + input.conversationId, + request.suspension.checkpoint.checkpointNs, + request.suspension.invocation.invocationId, + input.checkpointer, + ); + } catch (error) { + return { + status: 'claimed', + result: { + status: 'applied', + result: { action, checkpointCaptureError: asError(error).message }, + checkpoint: request.suspension.checkpoint, + }, + }; + } + if (checkpoint?.checkpointId == null) { + return { + status: 'claimed', + result: { + status: 'applied', + result: { + action, + checkpointCaptureError: 'Applied resumed turn has no observable terminal checkpoint', + }, + checkpoint: request.suspension.checkpoint, + }, + }; + } + return { + status: 'claimed', + result: { + status: 'applied', + result: { action, checkpointCaptureError: null }, + checkpoint: { + ...checkpoint, + invocationId: request.suspension.invocation.invocationId, + }, + }, + }; + }, + async suspend(request) { + if (pendingSuspension == null || deps.storeSuspension == null) { + throw new Error('Event actor re-pause storage is unavailable'); + } + return deps.storeSuspension({ + ...owner, + suspension: request.suspension as IAgentEventActorSuspensionEvidence, + actionId: pendingSuspension.actionId, + jobCreatedAt: pendingSuspension.jobCreatedAt, + ...(actionAppliedBeforePause ? { invalidateHead: true } : {}), + ...(request.previous == null ? {} : { previous: request.previous }), + }); + }, + async settleSuspension(request) { + if (deps.settleSuspension == null) { + throw new Error('Event actor suspension settlement storage is unavailable'); + } + const settled = await deps.settleSuspension({ + ...owner, + ...request, + invocationId: input.suspension.invocation.invocationId, + checkpoint: input.suspension.invocation.fork, + }); + if (settled.status !== 'settled') { + return settled; + } + await deleteAgentCheckpoint( + input.suspension.checkpoint.threadId, + input.checkpointer, + undefined, + { + throwOnError: true, + checkpointNamespace: input.suspension.checkpoint.checkpointNs, + }, + ); + return settled; + }, + async commit(request) { + if (request.result.checkpointCaptureError != null) { + throw new Error(request.result.checkpointCaptureError); + } + if (observedState === undefined || observedEpoch === undefined) { + throw new Error('Resumed event actor commit is missing its claimed host state'); + } + const checkpointId = request.checkpoint.checkpointId; + if (checkpointId == null) { + throw new Error('Applied resumed event actor checkpoint is missing its id'); + } + const expected = + observedState == null + ? undefined + : { + generation: observedState.generation, + checkpoint: observedState.checkpoint, + ...(observedState.contextFingerprint == null + ? {} + : { contextFingerprint: observedState.contextFingerprint }), + ...(observedState.skillManifest == null + ? {} + : { skillManifest: observedState.skillManifest }), + ...(observedState.discoveredToolNames == null + ? {} + : { discoveredToolNames: observedState.discoveredToolNames }), + ...(observedState.summary == null ? {} : { summary: observedState.summary }), + ...(observedState.contextMeta == null + ? {} + : { contextMeta: observedState.contextMeta }), + ...(observedState.requiresColdStart === true ? { requiresColdStart: true } : {}), + }; + const committed = await deps.commitState({ + ...owner, + invocationId: request.invocation.invocationId, + action: request.result.action, + ...(expected == null ? {} : { expected }), + expectedEpoch: observedEpoch, + checkpoint: { + threadId: request.checkpoint.threadId, + checkpointId, + checkpointNs: request.checkpoint.checkpointNs, + }, + settlementAuthority: request.settlementAuthority!, + ...(resultContext == null + ? {} + : { + contextFingerprint: resultContext.fingerprint, + skillManifest: resultContext.skillManifest, + discoveredToolNames: resultContext.discoveredToolNames ?? [], + ...(resultContext.summary == null ? {} : { summary: resultContext.summary }), + ...(resultContext.contextMeta == null + ? {} + : { contextMeta: resultContext.contextMeta }), + }), + }); + if (committed.status === 'stale') { + return { + status: 'stale', + ...(committed.state == null + ? {} + : { head: toHead(input.conversationId, committed.state) }), + }; + } + if (committed.prunableCheckpoint != null) { + await deleteAgentCheckpoint( + committed.prunableCheckpoint.threadId, + input.checkpointer, + undefined, + { + throwOnError: true, + checkpointNamespace: committed.prunableCheckpoint.checkpointNs, + }, + ); + } + return { status: 'committed', head: toHead(input.conversationId, committed.state) }; + }, + async discard() { + throw new Error('Resumed event actor cleanup must use suspension settlement'); + }, + }; + + const executor = createEventActorExecutor(adapter, { + maxDepth: 1, + dormantCheckpointTtlMs: getApprovalTtlMs(input.checkpointer), + preparationSigningKey: getEventActorSigningKey(), + }); + const resumed = await executor.resume({ + suspension: input.suspension, + resumeAttemptId: input.resumeAttemptId, + value: input.resumeValue, + signal: input.signal, + }); + const continuation = input.suspension.invocation.continuation; + if (resumed.status === 'suspended') { + return { value: value as T, execution: { ...resumed, continuation } }; + } + if (resumed.status === 'completed_no_action') { + if (invocationError != null) { + throw invocationError; + } + return { value: value as T, execution: { ...resumed, continuation } }; + } + if (resumed.status === 'commit_indeterminate') { + throw new Error('Event actor resumed action requires commit_indeterminate reconciliation'); + } + const settlement = await executor.commit(resumed); + if (settlement.status === 'commit_indeterminate') { + const recorded = await deps.recordReconciliation({ + ...owner, + reconciliation: { + invocationId: input.suspension.invocation.invocationId, + ...(input.bindingId == null ? {} : { actionAdmitted: true }), + status: 'commit_indeterminate', + checkpoint: resumed.checkpoint, + action: resumed.result.action, + error: settlement.error.message.slice(0, 1024), + observedAt: new Date(), + }, + }); + if (!recorded) { + throw new Error('Resumed event actor indeterminate commit could not be reconciled'); + } + throw new Error('Event actor resumed action requires commit_indeterminate reconciliation'); + } + if (settlement.status === 'stale') { + const recorded = await deps.recordReconciliation({ + ...owner, + reconciliation: { + invocationId: input.suspension.invocation.invocationId, + ...(input.bindingId == null ? {} : { actionAdmitted: true }), + status: 'commit_conflict', + checkpoint: resumed.checkpoint, + action: resumed.result.action, + error: 'A competing checkpoint advanced the actor head', + observedAt: new Date(), + }, + }); + if (!recorded) { + throw new Error('Resumed event actor checkpoint conflict could not be reconciled'); + } + throw new Error('Event actor resumed action requires commit_conflict reconciliation'); + } + if (invocationError != null) { + throw invocationError; + } + return { + value: value as T, + execution: { + status: 'applied', + result: resumed.result, + head: settlement.head, + continuation, + }, + }; +} + +/** Cancels one exact current suspension through the SDK evidence validator. + * The Conversation CAS is the logical winner; checkpoint deletion follows + * idempotently so an ambiguous cleanup can safely retry the same proof. */ +export async function cancelAgentEventActor( + input: CancelAgentEventActorInput, + deps: Pick, +): Promise { + if (deps.cancelSuspension == null) { + throw new Error('Event actor suspension cancellation storage is unavailable'); + } + const adapter: EventActorHostAdapter = { + async prepare() { + throw new Error('A suspended event actor cannot prepare during cancellation'); + }, + async coldContinue() { + throw new Error('A suspended event actor cannot cold-start during cancellation'); + }, + async invoke() { + throw new Error('A suspended event actor cannot invoke during cancellation'); + }, + async cancelSuspension(request) { + const cancelled = await deps.cancelSuspension!({ + user: input.user, + conversationId: input.conversationId, + ...(input.tenantId == null ? {} : { tenantId: input.tenantId }), + suspensionId: request.suspension.suspensionId, + attempt: request.suspension.attempt, + invocationId: request.suspension.invocation.invocationId, + checkpoint: request.suspension.invocation.fork, + ...(input.claimedResumeAttemptId == null + ? {} + : { claimedResumeAttemptId: input.claimedResumeAttemptId }), + }); + if (cancelled.status !== 'cancelled') { + return cancelled; + } + await deleteAgentCheckpoint( + request.suspension.checkpoint.threadId, + input.checkpointer, + undefined, + { + throwOnError: true, + checkpointNamespace: request.suspension.checkpoint.checkpointNs, + }, + ); + return cancelled; + }, + async commit() { + throw new Error('A cancelled event actor cannot commit'); + }, + async discard() { + throw new Error('A cancelled event actor cleanup must use suspension cancellation'); + }, + }; + const executor = createEventActorExecutor(adapter, { + maxDepth: 1, + dormantCheckpointTtlMs: getApprovalTtlMs(input.checkpointer), + preparationSigningKey: getEventActorSigningKey(), + }); + return executor.cancelSuspension({ + suspension: input.suspension, + cancelAttemptId: input.cancelAttemptId, + reason: input.reason, + signal: input.signal, + }); +} diff --git a/packages/api/src/agents/triggers/outcome.spec.ts b/packages/api/src/agents/triggers/outcome.spec.ts index ef0f2999fb..e30e4e6890 100644 --- a/packages/api/src/agents/triggers/outcome.spec.ts +++ b/packages/api/src/agents/triggers/outcome.spec.ts @@ -5,6 +5,13 @@ import { createAgentEventTerminalHandler as createAgentEventTerminalHandlerImpl, createAgentEventActionRecorder, } from './outcome'; +import { cancelAgentEventActor } from './actor'; + +jest.mock('./actor', () => ({ + ...jest.requireActual('./actor'), + cancelAgentEventActor: jest.fn(), +})); +const mockedCancelAgentEventActor = jest.mocked(cancelAgentEventActor); const createAgentEventTerminalHandler = ( methods: Pick< @@ -22,6 +29,10 @@ const createAgentEventTerminalHandler = ( getAgentEventActorReceipt: jest.fn().mockResolvedValue(null), backfillAgentEventActorReceipt: jest.fn().mockResolvedValue(true), completeAgentEventActorLegacyTurn: jest.fn().mockResolvedValue(true), + cancelAgentEventActorSuspension: jest.fn().mockResolvedValue({ status: 'cancelled' }), + releaseAgentEventActorAction: jest.fn().mockResolvedValue(true), + getAgentEventActorActionAdmission: jest.fn().mockResolvedValue(null), + hasAgentEventActorActionAdmission: jest.fn().mockResolvedValue(false), ...methods, }); @@ -59,7 +70,525 @@ function completedToolStep(): Agents.RunStep { }; } +function suspensionEvidence(suspensionId: string, attempt = 0) { + return { + version: 1 as const, + suspensionId, + attempt, + issuedAt: 1, + expiresAt: 2, + invocation: { + actorThreadId: 'conversation-1', + invocationId: 'trigger_1', + depth: 1, + continuation: 'warm' as const, + base: { actorThreadId: 'conversation-1', generation: 1 }, + fork: { + threadId: 'conversation-1', + checkpointNs: 'event-actor/trigger-1', + checkpointId: `checkpoint-${attempt}`, + invocationId: 'trigger_1', + }, + }, + checkpoint: { + threadId: 'conversation-1', + checkpointNs: 'event-actor/trigger-1', + checkpointId: `checkpoint-${attempt}`, + invocationId: 'trigger_1', + }, + interrupt: { id: `interrupt-${attempt}`, payload: { type: 'tool_approval' } }, + suspensionDigest: `signed-digest-${attempt}`, + }; +} + describe('agent event terminal outcomes', () => { + beforeEach(() => { + mockedCancelAgentEventActor.mockReset(); + mockedCancelAgentEventActor.mockResolvedValue({ status: 'cancelled' }); + }); + + it('cancels a versioned paused actor before settling its expired delivery', async () => { + const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true); + const suspension = { + version: 1 as const, + suspensionId: 'suspension-1', + attempt: 0, + issuedAt: 1, + expiresAt: 2, + invocation: { + invocationId: 'trigger_1', + continuation: 'warm' as const, + base: { actorThreadId: 'conversation-1', generation: 1 }, + fork: { + threadId: 'conversation-1', + checkpointNs: 'event-actor/trigger-1', + checkpointId: 'checkpoint-1', + invocationId: 'trigger_1', + }, + }, + checkpoint: { + threadId: 'conversation-1', + checkpointNs: 'event-actor/trigger-1', + checkpointId: 'checkpoint-1', + invocationId: 'trigger_1', + }, + interrupt: { id: 'interrupt-1', payload: { type: 'tool_approval' } }, + suspensionDigest: 'signed-digest', + }; + const getAgentEventActorSnapshot = jest + .fn() + .mockResolvedValueOnce({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: 'action-1', + jobCreatedAt: 1_787_000_000_000, + status: 'pending', + observedAt: new Date(), + }, + }) + .mockResolvedValueOnce({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { suspension, status: 'closed', outcome: 'cancelled' }, + }); + const handler = createAgentEventTerminalHandler({ + settleAgentTriggerHandlingOutcome, + getAgentEventActorSnapshot, + }); + + await handler( + 'conversation-1', + job({ + status: 'aborted', + error: 'Approval expired before a decision was made', + agentEventBindingId: 'binding-1', + agentEventSuspension: { version: 1, suspensionId: 'suspension-1', attempt: 0 }, + }), + [], + ); + + expect(mockedCancelAgentEventActor).toHaveBeenCalledWith( + expect.objectContaining({ suspension, reason: 'expired' }), + expect.objectContaining({ cancelSuspension: expect.any(Function) }), + ); + expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledWith( + expect.objectContaining({ status: 'cancelled' }), + ); + }); + + it('compensates the exact claimed resume when approval expiry proves execution never began', async () => { + const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true); + const suspension = { + version: 1 as const, + suspensionId: 'suspension-claimed', + attempt: 0, + issuedAt: 1, + expiresAt: 2, + invocation: { + invocationId: 'trigger_1', + continuation: 'warm' as const, + base: { actorThreadId: 'conversation-1', generation: 1 }, + fork: { + threadId: 'conversation-1', + checkpointNs: 'event-actor/trigger-1', + checkpointId: 'checkpoint-1', + invocationId: 'trigger_1', + }, + }, + checkpoint: { + threadId: 'conversation-1', + checkpointNs: 'event-actor/trigger-1', + checkpointId: 'checkpoint-1', + invocationId: 'trigger_1', + }, + interrupt: { id: 'interrupt-1', payload: { type: 'tool_approval' } }, + suspensionDigest: 'signed-digest', + }; + const getAgentEventActorSnapshot = jest + .fn() + .mockResolvedValueOnce({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: 'action-1', + jobCreatedAt: 1_787_000_000_000, + status: 'claimed', + resumeAttemptId: 'resume-attempt-1', + observedAt: new Date(), + }, + }) + .mockResolvedValueOnce({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { suspension, status: 'closed', outcome: 'cancelled' }, + }); + const handler = createAgentEventTerminalHandler({ + settleAgentTriggerHandlingOutcome, + getAgentEventActorSnapshot, + }); + + await handler( + 'conversation-1', + job({ + status: 'aborted', + error: 'Approval expired before a decision was made', + agentEventBindingId: 'binding-1', + providerExecutionId: 'provider-paused', + agentEventSuspension: { + version: 1, + suspensionId: suspension.suspensionId, + attempt: suspension.attempt, + }, + }), + [], + ); + + expect(mockedCancelAgentEventActor).toHaveBeenCalledWith( + expect.objectContaining({ + suspension, + reason: 'expired', + claimedResumeAttemptId: 'resume-attempt-1', + }), + expect.objectContaining({ cancelSuspension: expect.any(Function) }), + ); + expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledWith( + expect.objectContaining({ status: 'cancelled' }), + ); + }); + + it('settles a resumed no-action turn without cancelling its already-closed suspension', async () => { + const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true); + const suspension = { + version: 1 as const, + suspensionId: 'suspension-closed', + attempt: 0, + invocation: { invocationId: 'trigger_1' }, + }; + const handler = createAgentEventTerminalHandler({ + settleAgentTriggerHandlingOutcome, + getAgentEventActorSnapshot: jest.fn().mockResolvedValue({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { suspension, status: 'closed', outcome: 'settled' }, + }), + }); + + await handler( + 'conversation-1', + job({ + agentEventSuspension: { + version: 1, + suspensionId: suspension.suspensionId, + attempt: suspension.attempt, + }, + }), + [], + ); + + expect(mockedCancelAgentEventActor).not.toHaveBeenCalled(); + expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledWith( + expect.objectContaining({ status: 'completed_no_action' }), + ); + }); + + it('cancels a pending suspension when paused-history persistence terminalizes the job', async () => { + const suspension = suspensionEvidence('suspension-persistence-error'); + const releaseAgentEventActorAction = jest.fn().mockResolvedValue(true); + const handler = createAgentEventTerminalHandler({ + settleAgentTriggerHandlingOutcome: jest.fn().mockResolvedValue(true), + releaseAgentEventActorAction, + getAgentEventActorSnapshot: jest + .fn() + .mockResolvedValueOnce({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: 'action-1', + jobCreatedAt: 1_787_000_000_000, + status: 'pending', + observedAt: new Date(), + }, + }) + .mockResolvedValueOnce({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: 'action-1', + jobCreatedAt: 1_787_000_000_000, + status: 'closed', + outcome: 'cancelled', + observedAt: new Date(), + }, + }), + }); + + await handler( + 'conversation-1', + job({ + status: 'error', + error: 'Failed to persist the paused response', + agentEventBindingId: 'binding-1', + agentEventSuspension: { + version: 1, + suspensionId: suspension.suspensionId, + attempt: suspension.attempt, + }, + }), + [], + ); + + expect(mockedCancelAgentEventActor).toHaveBeenCalledWith( + expect.objectContaining({ suspension, reason: 'cancelled' }), + expect.any(Object), + ); + expect(releaseAgentEventActorAction).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryKey: 'trigger_1', + bindingId: 'binding-1', + admissionId: expect.any(String), + }), + ); + }); + + it('compensates a claimed resume when termination wins before provider start', async () => { + const suspension = suspensionEvidence('suspension-pre-projection'); + const handler = createAgentEventTerminalHandler({ + settleAgentTriggerHandlingOutcome: jest.fn().mockResolvedValue(true), + getAgentEventActorSnapshot: jest + .fn() + .mockResolvedValueOnce({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: 'action-1', + jobCreatedAt: 1_787_000_000_000, + status: 'claimed', + resumeAttemptId: 'provider-new', + observedAt: new Date(), + }, + }) + .mockResolvedValueOnce({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: null, + }), + }); + + await handler( + 'conversation-1', + job({ + status: 'aborted', + providerExecutionId: 'provider-old', + agentEventBindingId: 'binding-1', + agentEventSuspension: { + version: 1, + suspensionId: suspension.suspensionId, + attempt: suspension.attempt, + }, + }), + [], + ); + + expect(mockedCancelAgentEventActor).toHaveBeenCalledWith( + expect.objectContaining({ + suspension, + claimedResumeAttemptId: 'provider-new', + }), + expect.any(Object), + ); + }); + + it('cancels an unprojected successor re-pause after its predecessor marker was cleared', async () => { + const suspension = suspensionEvidence('suspension-repause', 1); + const handler = createAgentEventTerminalHandler({ + settleAgentTriggerHandlingOutcome: jest.fn().mockResolvedValue(true), + getAgentEventActorSnapshot: jest + .fn() + .mockResolvedValueOnce({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: 'action-repause', + jobCreatedAt: 1_787_000_000_000, + status: 'pending', + observedAt: new Date(), + }, + }) + .mockResolvedValueOnce({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: null, + }), + }); + + await handler( + 'conversation-1', + job({ + status: 'aborted', + providerExecutionId: 'provider-resume', + agentEventBindingId: 'binding-1', + agentEventSuspension: undefined, + }), + [], + ); + + expect(mockedCancelAgentEventActor).toHaveBeenCalledWith( + expect.objectContaining({ suspension, reason: 'cancelled' }), + expect.any(Object), + ); + }); + + it('does not compensate a claimed resume after its provider start succeeded', async () => { + const suspension = suspensionEvidence('suspension-projected'); + const handler = createAgentEventTerminalHandler({ + settleAgentTriggerHandlingOutcome: jest.fn().mockResolvedValue(true), + getAgentEventActorSnapshot: jest.fn().mockResolvedValue({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: 'action-1', + jobCreatedAt: 1_787_000_000_000, + status: 'claimed', + resumeAttemptId: 'provider-new', + observedAt: new Date(), + }, + }), + }); + + await expect( + handler( + 'conversation-1', + job({ + status: 'aborted', + providerExecutionId: 'provider-new', + providerExecutionStartedId: 'provider-new', + agentEventSuspension: { + version: 1, + suspensionId: suspension.suspensionId, + attempt: suspension.attempt, + }, + }), + [], + ), + ).rejects.toThrow('claim is still in flight'); + expect(mockedCancelAgentEventActor).not.toHaveBeenCalled(); + }); + + it('releases the delivery-owned admission after the child Conversation disappears', async () => { + const releaseAgentEventActorAction = jest.fn().mockResolvedValue(true); + const getAgentEventActorActionAdmission = jest + .fn() + .mockResolvedValue('admission-deleted-child'); + const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true); + const handler = createAgentEventTerminalHandler({ + settleAgentTriggerHandlingOutcome, + releaseAgentEventActorAction, + getAgentEventActorActionAdmission, + getAgentEventActorSnapshot: jest.fn().mockResolvedValue(null), + }); + + await handler( + 'conversation-1', + job({ + status: 'aborted', + agentEventBindingId: 'binding-1', + providerExecutionId: 'provider-resume', + }), + [], + ); + + expect(getAgentEventActorActionAdmission).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryKey: 'trigger_1', + bindingId: 'binding-1', + conversationId: 'conversation-1', + }), + ); + expect(releaseAgentEventActorAction).toHaveBeenCalledWith( + expect.objectContaining({ admissionId: 'admission-deleted-child' }), + ); + expect(releaseAgentEventActorAction.mock.invocationCallOrder[0]).toBeLessThan( + settleAgentTriggerHandlingOutcome.mock.invocationCallOrder[0], + ); + }); + + it('releases the exact action admission after a resumed no-action settlement', async () => { + const suspension = suspensionEvidence('suspension-no-action'); + const releaseAgentEventActorAction = jest.fn().mockResolvedValue(true); + const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true); + const handler = createAgentEventTerminalHandler({ + settleAgentTriggerHandlingOutcome, + releaseAgentEventActorAction, + getAgentEventActorSnapshot: jest.fn().mockResolvedValue({ + state: null, + epoch: 1, + legacyTurn: null, + reconciliations: [], + suspension: { + suspension, + actionId: 'action-1', + jobCreatedAt: 1_787_000_000_000, + status: 'closed', + resumeAttemptId: 'provider-resume', + outcome: 'settled', + observedAt: new Date(), + }, + }), + }); + + await handler( + 'conversation-1', + job({ + agentEventBindingId: 'binding-1', + providerExecutionId: 'provider-resume', + }), + [], + ); + + expect(releaseAgentEventActorAction).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryKey: 'trigger_1', + bindingId: 'binding-1', + admissionId: expect.any(String), + }), + ); + expect(releaseAgentEventActorAction.mock.invocationCallOrder[0]).toBeLessThan( + settleAgentTriggerHandlingOutcome.mock.invocationCallOrder[0], + ); + }); + it('records applied only from completed tool evidence matching the expected fence', async () => { const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true); const handler = createAgentEventTerminalHandler({ settleAgentTriggerHandlingOutcome }); diff --git a/packages/api/src/agents/triggers/outcome.ts b/packages/api/src/agents/triggers/outcome.ts index 2aa953c133..199625765c 100644 --- a/packages/api/src/agents/triggers/outcome.ts +++ b/packages/api/src/agents/triggers/outcome.ts @@ -1,11 +1,16 @@ import type { AgentTriggerDeliveryMethods, ConversationMethods, + IAgentEventActorSuspension, MessageMethods, } from '@librechat/data-schemas'; import type { Agents } from 'librechat-data-provider'; import type { AgentTriggerExpectedAction } from './envelope'; +import type { AgentEventAppliedAction } from './types'; import type { SerializableJobData } from '~/stream'; +import { cancelAgentEventActor, createAgentEventActorActionAdmissionId } from './actor'; + +export type { AgentEventAppliedAction } from './types'; interface SettleAgentTriggerHandlingOutcomeInput { deliveryKey: string; @@ -28,8 +33,6 @@ export interface AgentEventRunOutcome { action?: { toolName: string; toolCallId?: string }; } -export type AgentEventAppliedAction = NonNullable; - const MAX_RECEIPT_ID_LENGTH = 256; async function hasDurableAgentEventHistory(input: { @@ -340,6 +343,10 @@ export function createAgentEventTerminalHandler(methods: { getAgentEventActorReceipt: AgentTriggerDeliveryMethods['getAgentEventActorReceipt']; backfillAgentEventActorReceipt: AgentTriggerDeliveryMethods['backfillAgentEventActorReceipt']; completeAgentEventActorLegacyTurn: ConversationMethods['completeAgentEventActorLegacyTurn']; + cancelAgentEventActorSuspension: ConversationMethods['cancelAgentEventActorSuspension']; + releaseAgentEventActorAction: AgentTriggerDeliveryMethods['releaseAgentEventActorAction']; + getAgentEventActorActionAdmission: AgentTriggerDeliveryMethods['getAgentEventActorActionAdmission']; + hasAgentEventActorActionAdmission: AgentTriggerDeliveryMethods['hasAgentEventActorActionAdmission']; getMessage: MessageMethods['getMessage']; }): ( streamId: string, @@ -362,11 +369,183 @@ export function createAgentEventTerminalHandler(methods: { let committedAction: AgentEventAppliedAction | undefined; let compensated = false; let actorReceiptSettled = false; - const snapshot = await methods.getAgentEventActorSnapshot({ + const owner = { user: job.userId, conversationId, ...(job.tenantId == null ? {} : { tenantId: job.tenantId }), - }); + }; + let snapshot = await methods.getAgentEventActorSnapshot(owner); + let retiredWithoutAction: IAgentEventActorSuspension | undefined; + const isIrrecoverablyTerminal = job.status === 'aborted' || job.status === 'error'; + const unprojectedSuspension = snapshot?.suspension; + if ( + job.agentEventSuspension == null && + isIrrecoverablyTerminal && + unprojectedSuspension?.status === 'pending' && + unprojectedSuspension.jobCreatedAt === job.createdAt && + unprojectedSuspension.suspension.invocation.invocationId === job.agentEventDeliveryKey + ) { + /** Recovery for a crash after the canonical suspension write but before + * its version marker reached the job store, including a re-pause after + * the predecessor marker was cleared by resume. A terminal exact + * generation proves the unpublished pause can no longer be exposed. */ + retiredWithoutAction = unprojectedSuspension; + const cancellation = await cancelAgentEventActor( + { + ...owner, + suspension: unprojectedSuspension.suspension, + cancelAttemptId: `terminal:${job.createdAt}`, + reason: + job.error === 'Approval expired before a decision was made' ? 'expired' : 'cancelled', + }, + { cancelSuspension: methods.cancelAgentEventActorSuspension }, + ); + if (cancellation.status !== 'cancelled') { + throw new Error( + `Agent event actor ${job.agentEventDeliveryKey} unpublished suspension cancellation is indeterminate`, + ); + } + snapshot = await methods.getAgentEventActorSnapshot(owner); + } + /** A retention/deletion winner may remove the private child before its + * already-aborted job hook replays. With no canonical owner or checkpoint + * left, cancellation is already physically complete; only the public + * delivery outcome remains. A successful generation still requires its + * actor proof and therefore fails closed here. */ + if (job.agentEventSuspension != null && snapshot == null && !isIrrecoverablyTerminal) { + throw new Error( + `Agent event actor ${job.agentEventDeliveryKey} terminal suspension owner is unavailable`, + ); + } + if (job.agentEventSuspension != null && snapshot != null) { + const current = snapshot?.suspension; + const currentMatches = + job.agentEventSuspension.version === 1 && + current != null && + current.suspension.suspensionId === job.agentEventSuspension.suspensionId && + current.suspension.attempt === job.agentEventSuspension.attempt; + if (!currentMatches || current == null) { + throw new Error( + `Agent event actor ${job.agentEventDeliveryKey} terminal suspension is stale`, + ); + } + if (current.status === 'pending') { + if (!isIrrecoverablyTerminal) { + throw new Error( + `Agent event actor ${job.agentEventDeliveryKey} terminated while its suspension remained pending`, + ); + } + retiredWithoutAction = current; + const cancellation = await cancelAgentEventActor( + { + ...owner, + suspension: current.suspension, + cancelAttemptId: `terminal:${job.createdAt}`, + reason: + job.error === 'Approval expired before a decision was made' ? 'expired' : 'cancelled', + }, + { cancelSuspension: methods.cancelAgentEventActorSuspension }, + ); + if (cancellation.status !== 'cancelled') { + throw new Error( + `Agent event actor ${job.agentEventDeliveryKey} suspension cancellation is indeterminate`, + ); + } + snapshot = await methods.getAgentEventActorSnapshot(owner); + } else if (current.status === 'claimed') { + /** The provider-start CAS retains its exact execution identity after + * drain. A missing/different identity proves this claimed resume never + * crossed provider start (including schedule invalidation after claim + * projection); equality means execution began and must fail closed. */ + const projectionNeverStarted = + isIrrecoverablyTerminal && + current.resumeAttemptId != null && + current.resumeAttemptId !== job.providerExecutionStartedId; + if (!projectionNeverStarted) { + throw new Error( + `Agent event actor ${job.agentEventDeliveryKey} terminal suspension claim is still in flight`, + ); + } + retiredWithoutAction = current; + const cancellation = await cancelAgentEventActor( + { + ...owner, + suspension: current.suspension, + cancelAttemptId: `terminal:${job.createdAt}`, + reason: + job.error === 'Approval expired before a decision was made' ? 'expired' : 'cancelled', + claimedResumeAttemptId: current.resumeAttemptId, + }, + { cancelSuspension: methods.cancelAgentEventActorSuspension }, + ); + if (cancellation.status !== 'cancelled') { + throw new Error( + `Agent event actor ${job.agentEventDeliveryKey} orphaned suspension claim is indeterminate`, + ); + } + snapshot = await methods.getAgentEventActorSnapshot(owner); + } + } + /** Replay recovery after the Conversation CAS succeeded but the delivery + * admission was not yet released. Only exact closed no-action evidence is + * eligible; a committed suspension represents an applied action. */ + const closed = snapshot?.suspension; + if ( + retiredWithoutAction == null && + closed?.status === 'closed' && + (closed.outcome === 'settled' || closed.outcome === 'cancelled') && + closed.jobCreatedAt === job.createdAt && + closed.suspension.invocation.invocationId === job.agentEventDeliveryKey && + (closed.outcome === 'settled' + ? closed.resumeAttemptId != null && closed.resumeAttemptId === job.providerExecutionId + : isIrrecoverablyTerminal && + (closed.resumeAttemptId == null || + closed.resumeAttemptId !== job.providerExecutionStartedId)) + ) { + retiredWithoutAction = closed; + } + let retiredAdmissionId = + retiredWithoutAction == null + ? null + : createAgentEventActorActionAdmissionId( + retiredWithoutAction.suspension.invocation.invocationId, + retiredWithoutAction.suspension.invocation.fork, + ); + if ( + retiredAdmissionId == null && + snapshot == null && + isIrrecoverablyTerminal && + job.agentEventBindingId != null + ) { + retiredAdmissionId = await methods.getAgentEventActorActionAdmission({ + deliveryKey: job.agentEventDeliveryKey, + user: job.userId, + ...(job.tenantId == null ? {} : { tenantId: job.tenantId }), + bindingId: job.agentEventBindingId, + conversationId, + }); + } + if (retiredAdmissionId != null) { + if (job.agentEventBindingId == null) { + throw new Error( + `Agent event actor ${job.agentEventDeliveryKey} retired without binding identity`, + ); + } + const admission = { + deliveryKey: job.agentEventDeliveryKey, + user: job.userId, + ...(job.tenantId == null ? {} : { tenantId: job.tenantId }), + bindingId: job.agentEventBindingId, + conversationId, + admissionId: retiredAdmissionId, + }; + const released = await methods.releaseAgentEventActorAction(admission); + if (!released && (await methods.hasAgentEventActorActionAdmission(admission))) { + throw new Error( + `Agent event actor ${job.agentEventDeliveryKey} action admission could not be released`, + ); + } + } const lifecycle = snapshot?.reconciliations.find( (item) => item.invocationId === job.agentEventDeliveryKey, ); diff --git a/packages/api/src/agents/triggers/types.ts b/packages/api/src/agents/triggers/types.ts index eada583069..ea44518061 100644 --- a/packages/api/src/agents/triggers/types.ts +++ b/packages/api/src/agents/triggers/types.ts @@ -7,3 +7,18 @@ export interface AgentTriggerExpectedAction { toolName: string; argumentSubset?: Record; } + +/** Minimal job-store projection of the canonical Conversation suspension. + * The signed suspension stays private in Mongo; this marker only routes a + * paused job through the durable resume protocol during rolling deploys. */ +export interface AgentEventSuspensionProjection { + version: 1; + suspensionId: string; + attempt: number; +} + +/** Durable host evidence for the exact external action an Event Actor applied. */ +export interface AgentEventAppliedAction { + toolName: string; + toolCallId?: string; +} diff --git a/packages/api/src/stream/ApprovalLifecycle.ts b/packages/api/src/stream/ApprovalLifecycle.ts index b9bae96e01..95b10b576c 100644 --- a/packages/api/src/stream/ApprovalLifecycle.ts +++ b/packages/api/src/stream/ApprovalLifecycle.ts @@ -34,6 +34,8 @@ export interface ApprovalPauseOptions { expectedCreatedAt?: number; /** Hold Stop/resume until the paused assistant row is durably unfinished. */ persistencePending?: boolean; + /** Versioned pointer to the canonical signed Conversation suspension. */ + agentEventSuspension?: import('~/agents/triggers/types').AgentEventSuspensionProjection; } export const PENDING_ACTION_EXPIRED_CODE = 'HITL_ACTION_EXPIRED'; @@ -142,6 +144,9 @@ export class ApprovalLifecycle { ? { discoveredTools: [...discoveredTools] } : {}), ...(activityPhaseSnapshot != null ? { activityPhaseSnapshot } : {}), + ...(options.agentEventSuspension != null + ? { agentEventSuspension: options.agentEventSuspension } + : {}), }, expectCreatedAt: expectedCreatedAt, notAfterMs: pendingAction.expiresAt, @@ -388,7 +393,16 @@ export class ApprovalLifecycle { const resumed = await this.store.transitionStatus(streamId, { from: 'requires_action', to: 'running', - clear: ['pendingAction', 'pendingActionId'], + /** The old suspension marker must not survive into the resumed provider + * segment. If that segment re-pauses, its canonical successor is stored + * before a new marker is published; clearing here makes a terminal job + * in that gap unambiguously recoverable as an unpublished re-pause. */ + clear: [ + 'pendingAction', + 'pendingActionId', + 'agentEventSuspension', + 'providerExecutionStartedId', + ], // Refresh the liveness basis so a long-paused run isn't reaped as stale // immediately after resuming (cleanup keys off lastActiveAt). /** Ownership can move across replicas on resume. Owner-specific fields diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index d6ab0d62c5..9c563497fb 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -2605,6 +2605,10 @@ class GenerationJobManagerClass { agent_id: jobData.agent_id, // Surface whether the turn was temporary so a resume keeps it non-persisted. isTemporary: jobData.isTemporary, + agentEventDeliveryKey: jobData.agentEventDeliveryKey, + agentEventBindingId: jobData.agentEventBindingId, + agentEventExpectedAction: jobData.agentEventExpectedAction, + agentEventSuspension: jobData.agentEventSuspension, scheduleId: jobData.scheduleId, scheduledFor: jobData.scheduledFor, scheduleConfigRevision: jobData.scheduleConfigRevision, @@ -2623,6 +2627,7 @@ class GenerationJobManagerClass { // legacy resume's execution rewrite invalidates a stale assertion. steerQuotesExecutionId: jobData.steerQuotesExecutionId, providerExecutionId: jobData.providerExecutionId, + providerExecutionStartedId: jobData.providerExecutionStartedId, providerDrained: jobData.providerDrained, steersClosed: jobData.steersClosed, idempotencyClientRequestId: jobData.idempotencyClientRequestId, diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts index 865f60a6dd..3ec4efa182 100644 --- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts +++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts @@ -87,6 +87,7 @@ describe('RedisJobStore', () => { expect(script).toContain('HGET", KEYS[1], "status") ~= "running"'); expect(script).toContain('HGET", KEYS[1], "providerDrained") ~= "1"'); expect(script).toContain('HSET", KEYS[1], "providerDrained", "0"'); + expect(script).toContain('"providerExecutionStartedId", ARGV[2]'); expect([keyCount, jobKey, createdAt, providerExecutionId]).toEqual([ 1, 'stream:{stream-provider-begin}:job', diff --git a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts index 70d62e738d..03f9e03fad 100644 --- a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts +++ b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts @@ -1332,6 +1332,11 @@ describe('RedisJobStore Integration Tests', () => { toolName: 'submit_move', argumentSubset: { gameId: 'game-1', expectedPly: 7 }, }, + agentEventSuspension: { + version: 1, + suspensionId: 'suspension-1', + attempt: 0, + }, agentEventLegacyTurnToken: 'legacy-hitl-token', discoveredTools: ['deep_tool'], userSubmittedPaths: ['/content/0/tool_call/args'], @@ -1346,6 +1351,11 @@ describe('RedisJobStore Integration Tests', () => { toolName: 'submit_move', argumentSubset: { gameId: 'game-1', expectedPly: 7 }, }); + expect(turn1?.agentEventSuspension).toEqual({ + version: 1, + suspensionId: 'suspension-1', + attempt: 0, + }); expect(turn1?.agentEventLegacyTurnToken).toBe('legacy-hitl-token'); expect(turn1?.discoveredTools).toEqual(['deep_tool']); expect(turn1?.userSubmittedPaths).toEqual(['/content/0/tool_call/args']); @@ -1365,6 +1375,7 @@ describe('RedisJobStore Integration Tests', () => { expect(turn2?.agentEventDeliveryKey).toBeUndefined(); expect(turn2?.agentEventBindingId).toBeUndefined(); expect(turn2?.agentEventExpectedAction).toBeUndefined(); + expect(turn2?.agentEventSuspension).toBeUndefined(); expect(turn2?.agentEventLegacyTurnToken).toBeUndefined(); expect(turn2?.discoveredTools).toBeUndefined(); expect(turn2?.userSubmittedPaths).toBeUndefined(); diff --git a/packages/api/src/stream/__tests__/pendingAction.spec.ts b/packages/api/src/stream/__tests__/pendingAction.spec.ts index 84592dbf0c..2c12148cab 100644 --- a/packages/api/src/stream/__tests__/pendingAction.spec.ts +++ b/packages/api/src/stream/__tests__/pendingAction.spec.ts @@ -669,6 +669,43 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', () expect(await manager.approvals.peek(streamId)).toBeNull(); }); + test('clears the predecessor Event Actor suspension projection on resume', async () => { + const streamId = 'stream-resolve-event-actor-suspension'; + const job = await manager.createJob(streamId, 'user-1', streamId, { + initialMetadata: { providerExecutionId: 'provider-paused' }, + }); + const pausedProviderExecutionId = job.metadata.providerExecutionId!; + const action = buildAction(streamId); + expect( + await manager.beginProviderExecution(streamId, job.createdAt, pausedProviderExecutionId), + ).toBe(true); + await manager.approvals.pause(streamId, action, { + expectedCreatedAt: job.createdAt, + agentEventSuspension: { version: 1, suspensionId: 'suspension-1', attempt: 0 }, + }); + + expect( + await manager.approvals.resolve( + streamId, + action.actionId, + { providerExecutionId: 'provider-resume', providerDrained: true }, + job.createdAt, + ), + ).toBe(true); + await expect(manager.getJob(streamId)).resolves.toMatchObject({ + status: 'running', + metadata: { providerExecutionId: 'provider-resume' }, + }); + expect((await manager.getJob(streamId))?.metadata.agentEventSuspension).toBeUndefined(); + expect((await manager.getJob(streamId))?.metadata.providerExecutionStartedId).toBeUndefined(); + expect(await manager.beginProviderExecution(streamId, job.createdAt, 'provider-resume')).toBe( + true, + ); + expect((await manager.getJob(streamId))?.metadata.providerExecutionStartedId).toBe( + 'provider-resume', + ); + }); + test('a concurrent double-resolve wins exactly once (race-safe)', async () => { const streamId = 'stream-double-resolve'; await manager.createJob(streamId, 'user-1'); diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.ts index a2e67b2be7..b886b05058 100644 --- a/packages/api/src/stream/implementations/InMemoryJobStore.ts +++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts @@ -737,6 +737,7 @@ export class InMemoryJobStore implements IJobStoreV2 { return false; } job.providerDrained = false; + job.providerExecutionStartedId = providerExecutionId; return true; } diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts index 63769e9568..787a8c8831 100644 --- a/packages/api/src/stream/implementations/RedisJobStore.ts +++ b/packages/api/src/stream/implementations/RedisJobStore.ts @@ -637,7 +637,7 @@ const PROVIDER_BEGIN_LUA = 'if redis.call("HGET", KEYS[1], "providerExecutionId") ~= ARGV[2] then return 0 end ' + 'if redis.call("HGET", KEYS[1], "status") ~= "running" then return 0 end ' + 'if redis.call("HGET", KEYS[1], "providerDrained") ~= "1" then return 0 end ' + - 'redis.call("HSET", KEYS[1], "providerDrained", "0") return 1'; + 'redis.call("HSET", KEYS[1], "providerDrained", "0", "providerExecutionStartedId", ARGV[2]) return 1'; /** Single-winner promotion from abort-persistence pending to a consumable * terminal payload. Owner success/failure and stale-owner recovery share this @@ -4742,6 +4742,9 @@ export class RedisJobStore implements IJobStoreV2 { agentEventExpectedAction: data.agentEventExpectedAction ? JSON.parse(data.agentEventExpectedAction) : undefined, + agentEventSuspension: data.agentEventSuspension + ? JSON.parse(data.agentEventSuspension) + : undefined, agentEventLegacyTurnToken: data.agentEventLegacyTurnToken || undefined, scheduleId: data.scheduleId || undefined, scheduledFor: data.scheduledFor || undefined, @@ -4781,6 +4784,7 @@ export class RedisJobStore implements IJobStoreV2 { providerAbortReady: data.providerAbortReady != null ? data.providerAbortReady === '1' : undefined, providerExecutionId: data.providerExecutionId || undefined, + providerExecutionStartedId: data.providerExecutionStartedId || undefined, providerDrained: data.providerDrained != null ? data.providerDrained === '1' : undefined, titleEvent: data.titleEvent || undefined, replayEvents: data.replayEvents || undefined, diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index 41aa3eaf23..fde92ad465 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -199,6 +199,10 @@ export interface SerializableJobData { /** Opaque identity of the currently executing provider segment. A HITL resume * replaces it so an earlier paused segment cannot acknowledge the new run. */ providerExecutionId?: string; + /** Durable evidence that the current provider owner crossed its start CAS. + * Unlike `providerDrained`, this identity survives terminal drain so host + * compensation can distinguish a projected-but-never-started resume. */ + providerExecutionStartedId?: string; /** False while the identified provider segment can still mutate user data; * true before provider startup and after the owner has fully unwound. */ providerDrained?: boolean; @@ -282,6 +286,8 @@ export interface SerializableJobData { /** Trusted actor binding copied from the authenticated delivery envelope. */ agentEventBindingId?: string; agentEventExpectedAction?: import('~/agents/triggers/types').AgentTriggerExpectedAction; + /** Versioned pointer to the canonical signed Conversation suspension. */ + agentEventSuspension?: import('~/agents/triggers/types').AgentEventSuspensionProjection; /** Exact durable legacy-turn fence carried across a HITL pause/resume. */ agentEventLegacyTurnToken?: string; @@ -404,6 +410,7 @@ export type JobMetadataPatch = Partial< | 'agentEventDeliveryKey' | 'agentEventBindingId' | 'agentEventExpectedAction' + | 'agentEventSuspension' | 'agentEventLegacyTurnToken' | 'scheduleId' | 'scheduledFor' diff --git a/packages/api/src/stream/metadata.ts b/packages/api/src/stream/metadata.ts index 745b9c898a..997d29b7ef 100644 --- a/packages/api/src/stream/metadata.ts +++ b/packages/api/src/stream/metadata.ts @@ -45,6 +45,9 @@ export function sanitizeJobMetadata(metadata: Partial): J if (metadata.agentEventExpectedAction) { patch.agentEventExpectedAction = metadata.agentEventExpectedAction; } + if (metadata.agentEventSuspension) { + patch.agentEventSuspension = metadata.agentEventSuspension; + } if (metadata.agentEventLegacyTurnToken) { patch.agentEventLegacyTurnToken = metadata.agentEventLegacyTurnToken; } diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.ts index d52711640f..b549aa4824 100644 --- a/packages/api/src/types/stream.ts +++ b/packages/api/src/types/stream.ts @@ -1,7 +1,10 @@ import type { Agents, UserSubmittedMessageFieldPath } from 'librechat-data-provider'; import type { EventEmitter } from 'events'; +import type { + AgentEventSuspensionProjection, + AgentTriggerExpectedAction, +} from '../agents/triggers/types'; import type { ActivityPhaseSnapshot } from '~/agents/activityPhases/runtime'; -import type { AgentTriggerExpectedAction } from '../agents/triggers/types'; import type { ResolvedAskUserQuestion } from '../agents/hitl/resume'; import type { MCPRuntimeRequestBody } from '../mcp/types'; import type { ServerSentEvent } from './events'; @@ -48,6 +51,8 @@ export interface GenerationJobMetadata { agentEventBindingId?: string; /** Optional action evidence contract declared by the authenticated event source. */ agentEventExpectedAction?: AgentTriggerExpectedAction; + /** Versioned pointer to the canonical signed suspension stored on the Conversation. */ + agentEventSuspension?: AgentEventSuspensionProjection; /** Exact durable legacy-turn fence carried across a HITL pause/resume. */ agentEventLegacyTurnToken?: string; /** Trusted scheduled-occurrence identity. These fields are accepted only from a @@ -80,6 +85,8 @@ export interface GenerationJobMetadata { steerQuotesExecutionId?: string; /** Exact provider segment whose completion gates destructive user cleanup. */ providerExecutionId?: string; + /** Exact provider owner that crossed its start fence. */ + providerExecutionStartedId?: string; /** False only while that exact provider segment can still mutate user data. */ providerDrained?: boolean; /** Terminal close has atomically stopped new steer acceptance, even if the diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index bf753d9494..4a4a2e1fb2 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -9,7 +9,12 @@ import type { UpdateFilter, UpdateResult, } from 'mongodb'; -import type { IAgentEventActorReconciliation, IChatProject, IConversation } from '../types'; +import type { + IAgentEventActorReconciliation, + IAgentEventActorSuspensionEvidence, + IChatProject, + IConversation, +} from '../types'; import { ConversationMethods, createConversationMethods } from './conversation'; import { tenantStorage, runAsSystem } from '~/config/tenantContext'; import { createModels } from '../models'; @@ -3926,6 +3931,394 @@ describe('Conversation Operations', () => { ); }); + it('serializes suspension ownership through claim, re-pause, and resumed commit', async () => { + const conversationId = uuidv4(); + const owner = { + user: 'suspended-actor-user', + tenantId: 'tenant-a', + conversationId, + }; + await Conversation.create({ + conversationId, + user: owner.user, + tenantId: owner.tenantId, + endpoint: EModelEndpoint.agents, + agent_id: 'agent-player', + agentEventBinding: { + bindingId: `evtbind_${'s'.repeat(48)}`, + sourceKeyId: 'key-a', + actorId: 'player-a', + }, + subagentThread: { + rootConversationId: 'parent', + parentConversationId: 'parent', + parentMessageId: 'parent-message', + parentToolCallId: 'event-binding', + parentAgentId: 'agent-director', + subagentType: 'agent-player', + subagentKind: 'agent', + depth: 1, + }, + }); + const checkpoint = (suffix: string) => ({ + threadId: conversationId, + checkpointId: `checkpoint-${suffix}`, + checkpointNs: `event-actor/${suffix}`, + }); + const suspension = (suffix: string, attempt: number): IAgentEventActorSuspensionEvidence => ({ + version: 1, + suspensionId: `suspension-${suffix}`, + attempt, + issuedAt: 1_000 + attempt, + expiresAt: 100_000 + attempt, + invocation: { + actorThreadId: conversationId, + invocationId: 'event-pause', + depth: 1, + continuation: 'cold', + base: { actorThreadId: conversationId, generation: 0 }, + fork: { ...checkpoint('fork'), invocationId: 'event-pause' }, + }, + checkpoint: { + ...checkpoint('fork'), + checkpointId: `checkpoint-${suffix}`, + invocationId: 'event-pause', + }, + interrupt: { + id: `interrupt-${suffix}`, + payload: { type: 'ask_user_question', actionId: `action-${suffix}` }, + }, + suspensionDigest: `digest-${suffix}`, + }); + + await expect( + methods.recordAgentEventActorReconciliation({ + ...owner, + reconciliation: { + invocationId: 'event-pause', + actionAdmitted: true, + status: 'invocation_pending', + checkpoint: checkpoint('fork'), + action: { toolName: 'submit_move' }, + observedAt: new Date(), + }, + }), + ).resolves.toBe(true); + const first = suspension('first', 0); + await expect( + methods.storeAgentEventActorSuspension({ + ...owner, + suspension: { + ...first, + interrupt: { + ...first.interrupt, + payload: { type: 'ask_user_question', question: 'x'.repeat(65 * 1_024) }, + }, + }, + actionId: 'action-oversized', + jobCreatedAt: 123, + }), + ).rejects.toThrow('Event actor suspension exceeds maximum payload size'); + await expect( + methods.storeAgentEventActorSuspension({ + ...owner, + suspension: first, + actionId: 'action-first', + jobCreatedAt: 123, + }), + ).resolves.toEqual({ status: 'stored' }); + await expect( + methods.storeAgentEventActorSuspension({ + ...owner, + suspension: { ...first, suspensionId: 'competing-suspension' }, + actionId: 'action-first', + jobCreatedAt: 123, + }), + ).resolves.toEqual({ status: 'stale' }); + + const claim = { + ...owner, + suspensionId: first.suspensionId, + attempt: first.attempt, + actionId: 'action-first', + jobCreatedAt: 123, + resumeAttemptId: 'resume-one', + }; + const claims = await Promise.all([ + methods.claimAgentEventActorSuspension(claim), + methods.claimAgentEventActorSuspension({ ...claim, resumeAttemptId: 'resume-two' }), + ]); + expect(claims).toEqual(expect.arrayContaining([{ status: 'claimed' }, { status: 'stale' }])); + const winningResumeAttemptId = claims[0].status === 'claimed' ? 'resume-one' : 'resume-two'; + + const second = suspension('second', 1); + await expect( + methods.storeAgentEventActorSuspension({ + ...owner, + suspension: second, + actionId: 'action-second', + jobCreatedAt: 123, + previous: { + suspensionId: first.suspensionId, + attempt: first.attempt, + resumeAttemptId: winningResumeAttemptId, + }, + }), + ).resolves.toEqual({ status: 'stored' }); + await expect( + methods.claimAgentEventActorSuspension({ + ...owner, + suspensionId: second.suspensionId, + attempt: second.attempt, + actionId: 'action-second', + jobCreatedAt: 123, + resumeAttemptId: 'resume-three', + }), + ).resolves.toEqual({ status: 'claimed' }); + + await expect( + methods.commitAgentEventActorState({ + ...owner, + invocationId: 'event-pause', + expectedEpoch: 0, + action: { toolName: 'submit_move' }, + checkpoint: checkpoint('committed'), + settlementAuthority: { + suspensionId: second.suspensionId, + attempt: second.attempt, + resumeAttemptId: 'resume-three', + }, + }), + ).resolves.toMatchObject({ status: 'committed' }); + await expect(methods.getAgentEventActorSnapshot(owner)).resolves.toMatchObject({ + suspension: { + status: 'closed', + outcome: 'committed', + resumeAttemptId: 'resume-three', + }, + state: { generation: 1, checkpoint: checkpoint('committed') }, + }); + await expect( + methods.recordAgentEventActorReconciliation({ + ...owner, + reconciliation: { + invocationId: 'event-pause', + actionAdmitted: true, + status: 'history_persisted', + checkpoint: checkpoint('committed'), + action: { toolName: 'submit_move' }, + observedAt: new Date(), + }, + }), + ).resolves.toBe(true); + await expect( + methods.resolveAgentEventActorReconciliation({ + ...owner, + invocationId: 'event-pause', + checkpoint: checkpoint('committed'), + resolution: 'checkpoint_verified', + }), + ).resolves.toBe(true); + + const cancellationCheckpoint = checkpoint('cancel'); + const cancellationSuspension: IAgentEventActorSuspensionEvidence = { + ...suspension('cancel', 0), + invocation: { + ...suspension('cancel', 0).invocation, + invocationId: 'event-cancel', + fork: { + ...cancellationCheckpoint, + invocationId: 'event-cancel', + }, + }, + checkpoint: { + ...cancellationCheckpoint, + invocationId: 'event-cancel', + }, + }; + await expect( + methods.recordAgentEventActorReconciliation({ + ...owner, + reconciliation: { + invocationId: 'event-cancel', + status: 'invocation_pending', + checkpoint: cancellationCheckpoint, + action: { toolName: 'submit_move' }, + observedAt: new Date(), + }, + }), + ).resolves.toBe(true); + await expect( + methods.storeAgentEventActorSuspension({ + ...owner, + suspension: cancellationSuspension, + actionId: 'action-cancel', + jobCreatedAt: 124, + }), + ).resolves.toEqual({ status: 'stored' }); + const [resumeRace, cancelRace] = await Promise.all([ + methods.claimAgentEventActorSuspension({ + ...owner, + suspensionId: cancellationSuspension.suspensionId, + attempt: 0, + actionId: 'action-cancel', + jobCreatedAt: 124, + resumeAttemptId: 'resume-race', + }), + methods.cancelAgentEventActorSuspension({ + ...owner, + suspensionId: cancellationSuspension.suspensionId, + attempt: 0, + invocationId: 'event-cancel', + checkpoint: cancellationCheckpoint, + }), + ]); + const raceStatuses = [resumeRace.status, cancelRace.status]; + expect(raceStatuses.filter((status) => status === 'stale')).toHaveLength(1); + expect(raceStatuses).toEqual( + expect.arrayContaining([expect.stringMatching(/^(claimed|cancelled)$/), 'stale']), + ); + if (resumeRace.status === 'claimed') { + await expect( + methods.cancelAgentEventActorSuspension({ + ...owner, + suspensionId: cancellationSuspension.suspensionId, + attempt: 0, + invocationId: 'event-cancel', + checkpoint: cancellationCheckpoint, + claimedResumeAttemptId: 'resume-race', + }), + ).resolves.toEqual({ status: 'cancelled' }); + } + await expect(methods.getAgentEventActorSnapshot(owner)).resolves.toMatchObject({ + suspension: { status: 'closed', outcome: 'cancelled' }, + }); + }); + + it('closes a resumed suspension when the actor-head commit is stale', async () => { + const conversationId = uuidv4(); + const owner = { user: 'stale-resume-user', tenantId: 'tenant-a', conversationId }; + const baseCheckpoint = { + threadId: conversationId, + checkpointId: 'checkpoint-base', + checkpointNs: 'event-actor/base', + }; + const baseState = { generation: 1, checkpoint: baseCheckpoint }; + const evidence: IAgentEventActorSuspensionEvidence = { + version: 1, + suspensionId: 'suspension-stale', + attempt: 0, + issuedAt: 1_000, + expiresAt: 100_000, + invocation: { + actorThreadId: conversationId, + invocationId: 'event-stale', + depth: 1, + continuation: 'warm', + base: { actorThreadId: conversationId, ...baseState }, + fork: { ...baseCheckpoint, invocationId: 'event-stale' }, + }, + checkpoint: { + ...baseCheckpoint, + checkpointId: 'checkpoint-paused', + invocationId: 'event-stale', + }, + interrupt: { id: 'interrupt-stale', payload: { type: 'tool_approval' } }, + suspensionDigest: 'digest-stale', + }; + await Conversation.create({ + conversationId, + user: owner.user, + tenantId: owner.tenantId, + endpoint: EModelEndpoint.agents, + agent_id: 'agent-player', + agentEventBinding: { + bindingId: `evtbind_${'t'.repeat(48)}`, + sourceKeyId: 'key-a', + actorId: 'player-a', + }, + subagentThread: { + rootConversationId: 'parent', + parentConversationId: 'parent', + parentMessageId: 'parent-message', + parentToolCallId: 'event-binding', + subagentType: 'agent-player', + subagentKind: 'agent', + depth: 1, + }, + agentEventActor: baseState, + agentEventActorReconciliations: [ + { + invocationId: 'event-stale', + actionAdmitted: true, + status: 'invocation_pending', + checkpoint: baseCheckpoint, + action: { toolName: 'submit_move' }, + observedAt: new Date(), + }, + ], + }); + await expect( + methods.storeAgentEventActorSuspension({ + ...owner, + suspension: evidence, + actionId: 'action-stale', + jobCreatedAt: 456, + invalidateHead: true, + }), + ).resolves.toEqual({ status: 'stored' }); + await expect(methods.getAgentEventActorSnapshot(owner)).resolves.toMatchObject({ + state: { ...baseState, requiresColdStart: true }, + suspension: { status: 'pending' }, + }); + await expect( + methods.claimAgentEventActorSuspension({ + ...owner, + suspensionId: evidence.suspensionId, + attempt: 0, + actionId: 'action-stale', + jobCreatedAt: 456, + resumeAttemptId: 'resume-stale', + }), + ).resolves.toEqual({ status: 'claimed' }); + const competingState = { + generation: 2, + checkpoint: { + threadId: conversationId, + checkpointId: 'checkpoint-competing', + checkpointNs: 'event-actor/competing', + }, + }; + await Conversation.updateOne( + { conversationId }, + { $set: { agentEventActor: competingState } }, + ); + + await expect( + methods.commitAgentEventActorState({ + ...owner, + invocationId: 'event-stale', + expectedEpoch: 0, + expected: baseState, + action: { toolName: 'submit_move' }, + checkpoint: { + threadId: conversationId, + checkpointId: 'checkpoint-resumed', + checkpointNs: 'event-actor/base', + }, + settlementAuthority: { + suspensionId: evidence.suspensionId, + attempt: 0, + resumeAttemptId: 'resume-stale', + }, + }), + ).resolves.toEqual({ status: 'stale', state: competingState }); + await expect(methods.getAgentEventActorSnapshot(owner)).resolves.toMatchObject({ + state: competingState, + suspension: { status: 'closed', outcome: 'stale' }, + }); + }); + it('commits event actor heads with full checkpoint CAS and two-checkpoint retention', async () => { const conversationId = uuidv4(); await Conversation.create({ @@ -4024,6 +4417,7 @@ describe('Conversation Operations', () => { state: first.state, epoch: 0, legacyTurn: null, + suspension: null, reconciliations: [ { invocationId: 'one', @@ -4187,6 +4581,7 @@ describe('Conversation Operations', () => { state: { ...third.state, requiresColdStart: true }, epoch: 1, legacyTurn: null, + suspension: null, reconciliations: expect.arrayContaining([ expect.objectContaining({ invocationId: 'one', status: 'settled' }), expect.objectContaining({ invocationId: 'two', status: 'settled' }), @@ -4477,6 +4872,7 @@ describe('Conversation Operations', () => { state: first.state, epoch: 0, legacyTurn: null, + suspension: null, reconciliations: [ expect.objectContaining({ invocationId: 'event-one', status: 'settled' }), reconciliation, @@ -4595,6 +4991,7 @@ describe('Conversation Operations', () => { state: { ...first.state!, requiresColdStart: true }, epoch: 0, legacyTurn: null, + suspension: null, reconciliations: [ expect.objectContaining({ invocationId: 'event-one', @@ -4640,6 +5037,7 @@ describe('Conversation Operations', () => { state: { ...first.state!, requiresColdStart: true }, epoch: 0, legacyTurn: null, + suspension: null, reconciliations: [ expect.objectContaining({ invocationId: 'event-one', status: 'settled' }), expect.objectContaining({ invocationId: 'event-conflict', status: 'settled' }), @@ -5036,6 +5434,7 @@ describe('Conversation Operations', () => { state: null, epoch: 0, legacyTurn: null, + suspension: null, reconciliations: [ expect.objectContaining({ invocationId: 'event-old', status: 'settled' }), expect.objectContaining({ invocationId: 'event-recent', status: 'settled' }), diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index f23a722aa6..de9fb429af 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -1,3 +1,4 @@ +import { Buffer } from 'node:buffer'; import { RetentionMode } from 'librechat-data-provider'; import type { AnyBulkWriteOperation, FilterQuery, Model, SortOrder, Types } from 'mongoose'; import type { DeleteResult } from 'mongoose'; @@ -6,6 +7,7 @@ import type { IAgentEventActorReconciliation, IAgentEventActorSnapshot, IAgentEventActorState, + IAgentEventActorSuspensionEvidence, IAgentEventBindingRecord, IAgentTriggerDeliveryDocument, AppConfig, @@ -39,6 +41,55 @@ import { decrementTagCounts } from './conversationTag'; import logger from '~/config/winston'; const AGENT_EVENT_ACTOR_RECEIPT_RETENTION_MS = 90 * 24 * 60 * 60_000; +const MAX_AGENT_EVENT_ACTOR_SUSPENSION_BYTES = 64 * 1_024; + +function validateAgentEventActorSuspension( + conversationId: string, + suspension: IAgentEventActorSuspensionEvidence, + actionId: string, + jobCreatedAt: number, + previous?: AgentEventActorSettlementAuthority, +): void { + const invocation = suspension?.invocation; + const fork = invocation?.fork; + const checkpoint = suspension?.checkpoint; + if ( + suspension?.version !== 1 || + typeof suspension.suspensionId !== 'string' || + suspension.suspensionId.length === 0 || + !Number.isSafeInteger(suspension.attempt) || + suspension.attempt < 0 || + !Number.isSafeInteger(suspension.issuedAt) || + !Number.isSafeInteger(suspension.expiresAt) || + suspension.expiresAt <= suspension.issuedAt || + invocation?.actorThreadId !== conversationId || + fork?.threadId !== conversationId || + checkpoint?.threadId !== conversationId || + fork?.invocationId !== invocation?.invocationId || + checkpoint?.invocationId !== invocation?.invocationId || + checkpoint?.checkpointNs !== fork?.checkpointNs || + typeof suspension.interrupt?.id !== 'string' || + suspension.interrupt.id.length === 0 || + typeof suspension.suspensionDigest !== 'string' || + suspension.suspensionDigest.length === 0 || + typeof actionId !== 'string' || + actionId.length === 0 || + !Number.isSafeInteger(jobCreatedAt) || + jobCreatedAt < 0 || + (previous == null ? suspension.attempt !== 0 : suspension.attempt !== previous.attempt + 1) + ) { + throw new Error('Event actor suspension evidence is invalid'); + } + let serialized: string; + try { + serialized = JSON.stringify(suspension); + } catch { + throw new Error('Event actor suspension evidence is not JSON-safe'); + } + if (Buffer.byteLength(serialized, 'utf8') > MAX_AGENT_EVENT_ACTOR_SUSPENSION_BYTES) { + throw new RangeError('Event actor suspension exceeds maximum payload size'); + } +} type ConversationUpdateResult = { value: @@ -76,6 +127,12 @@ export type AgentEventActorCommitResult = } | { status: 'stale'; state?: IAgentEventActorState }; +export interface AgentEventActorSettlementAuthority { + suspensionId: string; + attempt: number; + resumeAttemptId: string; +} + export interface AgentEventActorReconciliationStorageMetrics { pending: number; oldestPendingAgeSeconds: number; @@ -257,7 +314,50 @@ export interface ConversationMethods { discoveredToolNames?: IAgentEventActorState['discoveredToolNames']; summary?: IAgentEventActorState['summary']; contextMeta?: IAgentEventActorState['contextMeta']; + settlementAuthority?: AgentEventActorSettlementAuthority; }): Promise; + storeAgentEventActorSuspension(input: { + user: string; + conversationId: string; + tenantId?: string; + suspension: IAgentEventActorSuspensionEvidence; + actionId: string; + jobCreatedAt: number; + /** The segment applied its expected action before publishing this successor pause. */ + invalidateHead?: boolean; + previous?: AgentEventActorSettlementAuthority; + }): Promise<{ status: 'stored' | 'stale' }>; + claimAgentEventActorSuspension(input: { + user: string; + conversationId: string; + tenantId?: string; + suspensionId: string; + attempt: number; + actionId: string; + jobCreatedAt: number; + resumeAttemptId: string; + }): Promise<{ status: 'claimed' | 'stale' }>; + settleAgentEventActorSuspension(input: { + user: string; + conversationId: string; + tenantId?: string; + suspensionId: string; + attempt: number; + resumeAttemptId: string; + invocationId: string; + checkpoint: IAgentEventActorReconciliation['checkpoint']; + }): Promise<{ status: 'settled' | 'stale' }>; + cancelAgentEventActorSuspension(input: { + user: string; + conversationId: string; + tenantId?: string; + suspensionId: string; + attempt: number; + invocationId: string; + checkpoint: IAgentEventActorReconciliation['checkpoint']; + /** Exact orphaned resume claim proven not to have entered provider execution. */ + claimedResumeAttemptId?: string; + }): Promise<{ status: 'cancelled' | 'stale' }>; beginAgentEventActorLegacyTurn(input: { user: string; conversationId: string; @@ -522,7 +622,7 @@ export function createConversationMethods( ...activeExpirationFilter(), }) .select( - '+agentEventActor +agentEventActorReconciliations +agentEventActorEpoch +agentEventActorLegacyTurn', + '+agentEventActor +agentEventActorReconciliations +agentEventActorEpoch +agentEventActorLegacyTurn +agentEventActorSuspension', ) .lean(); return conversation == null @@ -531,10 +631,303 @@ export function createConversationMethods( state: conversation.agentEventActor ?? null, reconciliations: conversation.agentEventActorReconciliations ?? [], legacyTurn: conversation.agentEventActorLegacyTurn ?? null, + suspension: conversation.agentEventActorSuspension ?? null, epoch: conversation.agentEventActorEpoch ?? 0, }; } + /** Publishes one SDK-issued suspension, or atomically replaces its exact claimed predecessor. */ + async function storeAgentEventActorSuspension(input: { + user: string; + conversationId: string; + tenantId?: string; + suspension: IAgentEventActorSuspensionEvidence; + actionId: string; + jobCreatedAt: number; + invalidateHead?: boolean; + previous?: AgentEventActorSettlementAuthority; + }): Promise<{ status: 'stored' | 'stale' }> { + validateAgentEventActorSuspension( + input.conversationId, + input.suspension, + input.actionId, + input.jobCreatedAt, + input.previous, + ); + const Conversation = mongoose.models.Conversation as Model; + const predecessor: FilterQuery = + input.previous == null + ? { + $or: [ + { agentEventActorSuspension: { $exists: false } }, + { 'agentEventActorSuspension.status': 'closed' }, + ], + } + : { + 'agentEventActorSuspension.status': 'claimed', + 'agentEventActorSuspension.suspension.suspensionId': input.previous.suspensionId, + 'agentEventActorSuspension.suspension.attempt': input.previous.attempt, + 'agentEventActorSuspension.resumeAttemptId': input.previous.resumeAttemptId, + }; + const ownership = { + user: input.user, + conversationId: input.conversationId, + subagentThread: { $exists: true }, + agentEventBinding: { $exists: true }, + agentEventActorReconciliations: { + $elemMatch: { + invocationId: input.suspension.invocation.invocationId, + status: 'invocation_pending', + }, + }, + ...subagentLeaseTenantFilter(input.tenantId), + ...activeExpirationFilter(), + ...predecessor, + }; + const storeUpdate = { + $set: { + agentEventActorSuspension: { + suspension: input.suspension, + actionId: input.actionId, + jobCreatedAt: input.jobCreatedAt, + status: 'pending' as const, + observedAt: new Date(), + }, + ...(input.invalidateHead === true ? { 'agentEventActor.requiresColdStart': true } : {}), + }, + }; + let stored = await Conversation.findOneAndUpdate( + { + ...ownership, + ...(input.invalidateHead === true ? { agentEventActor: { $exists: true } } : {}), + }, + storeUpdate, + { new: false, timestamps: false }, + ) + .select('_id') + .lean(); + /** A headless actor is already guaranteed to cold-start; publish the + * successor without manufacturing a partial canonical state. */ + if (stored == null && input.invalidateHead === true) { + stored = await Conversation.findOneAndUpdate( + { ...ownership, agentEventActor: { $exists: false } }, + { + $set: { + agentEventActorSuspension: { + suspension: input.suspension, + actionId: input.actionId, + jobCreatedAt: input.jobCreatedAt, + status: 'pending', + observedAt: new Date(), + }, + }, + }, + { new: false, timestamps: false }, + ) + .select('_id') + .lean(); + } + return { status: stored == null ? 'stale' : 'stored' }; + } + + /** One resume attempt wins the canonical Conversation-side suspension fence. */ + async function claimAgentEventActorSuspension(input: { + user: string; + conversationId: string; + tenantId?: string; + suspensionId: string; + attempt: number; + actionId: string; + jobCreatedAt: number; + resumeAttemptId: string; + }): Promise<{ status: 'claimed' | 'stale' }> { + if ( + input.suspensionId.length === 0 || + !Number.isSafeInteger(input.attempt) || + input.attempt < 0 || + input.actionId.length === 0 || + !Number.isSafeInteger(input.jobCreatedAt) || + input.jobCreatedAt < 0 || + input.resumeAttemptId.length === 0 + ) { + throw new Error('Event actor suspension claim is invalid'); + } + const Conversation = mongoose.models.Conversation as Model; + const claimed = await Conversation.findOneAndUpdate( + { + user: input.user, + conversationId: input.conversationId, + subagentThread: { $exists: true }, + agentEventBinding: { $exists: true }, + 'agentEventActorSuspension.status': 'pending', + 'agentEventActorSuspension.suspension.suspensionId': input.suspensionId, + 'agentEventActorSuspension.suspension.attempt': input.attempt, + 'agentEventActorSuspension.actionId': input.actionId, + 'agentEventActorSuspension.jobCreatedAt': input.jobCreatedAt, + ...subagentLeaseTenantFilter(input.tenantId), + ...activeExpirationFilter(), + }, + { + $set: { + 'agentEventActorSuspension.status': 'claimed', + 'agentEventActorSuspension.resumeAttemptId': input.resumeAttemptId, + 'agentEventActorSuspension.observedAt': new Date(), + }, + $unset: { + 'agentEventActorSuspension.outcome': 1, + 'agentEventActorSuspension.closedAt': 1, + }, + }, + { new: false, timestamps: false }, + ) + .select('_id') + .lean(); + return { status: claimed == null ? 'stale' : 'claimed' }; + } + + /** Closes a claimed no-action suspension while retaining one bounded retry receipt. */ + async function settleAgentEventActorSuspension(input: { + user: string; + conversationId: string; + tenantId?: string; + suspensionId: string; + attempt: number; + resumeAttemptId: string; + invocationId: string; + checkpoint: IAgentEventActorReconciliation['checkpoint']; + }): Promise<{ status: 'settled' | 'stale' }> { + const Conversation = mongoose.models.Conversation as Model; + const settled = await Conversation.findOneAndUpdate( + { + user: input.user, + conversationId: input.conversationId, + 'agentEventActorSuspension.status': 'claimed', + 'agentEventActorSuspension.suspension.suspensionId': input.suspensionId, + 'agentEventActorSuspension.suspension.attempt': input.attempt, + 'agentEventActorSuspension.resumeAttemptId': input.resumeAttemptId, + agentEventActorReconciliations: { + $elemMatch: { + invocationId: input.invocationId, + status: 'invocation_pending', + 'checkpoint.threadId': input.checkpoint.threadId, + 'checkpoint.checkpointNs': input.checkpoint.checkpointNs, + ...(input.checkpoint.checkpointId == null + ? { 'checkpoint.checkpointId': { $exists: false } } + : { 'checkpoint.checkpointId': input.checkpoint.checkpointId }), + }, + }, + ...subagentLeaseTenantFilter(input.tenantId), + }, + { + $set: { + 'agentEventActorSuspension.status': 'closed', + 'agentEventActorSuspension.outcome': 'settled', + 'agentEventActorSuspension.closedAt': new Date(), + 'agentEventActorSuspension.observedAt': new Date(), + }, + $pull: { + agentEventActorReconciliations: { + invocationId: input.invocationId, + status: 'invocation_pending', + 'checkpoint.threadId': input.checkpoint.threadId, + 'checkpoint.checkpointNs': input.checkpoint.checkpointNs, + ...(input.checkpoint.checkpointId == null + ? { 'checkpoint.checkpointId': { $exists: false } } + : { 'checkpoint.checkpointId': input.checkpoint.checkpointId }), + }, + }, + }, + { new: false, timestamps: false }, + ) + .select('_id') + .lean(); + if (settled != null) { + return { status: 'settled' }; + } + const snapshot = await getAgentEventActorSnapshot(input); + return snapshot?.suspension?.status === 'closed' && + snapshot.suspension.outcome === 'settled' && + snapshot.suspension.suspension.suspensionId === input.suspensionId && + snapshot.suspension.suspension.attempt === input.attempt && + snapshot.suspension.resumeAttemptId === input.resumeAttemptId + ? { status: 'settled' } + : { status: 'stale' }; + } + + /** Cancellation races resume through the same current-suspension predicate. */ + async function cancelAgentEventActorSuspension(input: { + user: string; + conversationId: string; + tenantId?: string; + suspensionId: string; + attempt: number; + invocationId: string; + checkpoint: IAgentEventActorReconciliation['checkpoint']; + claimedResumeAttemptId?: string; + }): Promise<{ status: 'cancelled' | 'stale' }> { + const Conversation = mongoose.models.Conversation as Model; + const suspensionOwner = + input.claimedResumeAttemptId == null + ? { 'agentEventActorSuspension.status': 'pending' } + : { + 'agentEventActorSuspension.status': 'claimed', + 'agentEventActorSuspension.resumeAttemptId': input.claimedResumeAttemptId, + }; + const cancelled = await Conversation.findOneAndUpdate( + { + user: input.user, + conversationId: input.conversationId, + ...suspensionOwner, + 'agentEventActorSuspension.suspension.suspensionId': input.suspensionId, + 'agentEventActorSuspension.suspension.attempt': input.attempt, + agentEventActorReconciliations: { + $elemMatch: { + invocationId: input.invocationId, + status: 'invocation_pending', + 'checkpoint.threadId': input.checkpoint.threadId, + 'checkpoint.checkpointNs': input.checkpoint.checkpointNs, + ...(input.checkpoint.checkpointId == null + ? { 'checkpoint.checkpointId': { $exists: false } } + : { 'checkpoint.checkpointId': input.checkpoint.checkpointId }), + }, + }, + ...subagentLeaseTenantFilter(input.tenantId), + }, + { + $set: { + 'agentEventActorSuspension.status': 'closed', + 'agentEventActorSuspension.outcome': 'cancelled', + 'agentEventActorSuspension.closedAt': new Date(), + 'agentEventActorSuspension.observedAt': new Date(), + }, + $pull: { + agentEventActorReconciliations: { + invocationId: input.invocationId, + status: 'invocation_pending', + 'checkpoint.threadId': input.checkpoint.threadId, + 'checkpoint.checkpointNs': input.checkpoint.checkpointNs, + ...(input.checkpoint.checkpointId == null + ? { 'checkpoint.checkpointId': { $exists: false } } + : { 'checkpoint.checkpointId': input.checkpoint.checkpointId }), + }, + }, + }, + { new: false, timestamps: false }, + ) + .select('_id') + .lean(); + if (cancelled != null) { + return { status: 'cancelled' }; + } + const snapshot = await getAgentEventActorSnapshot(input); + return snapshot?.suspension?.status === 'closed' && + snapshot.suspension.outcome === 'cancelled' && + snapshot.suspension.suspension.suspensionId === input.suspensionId && + snapshot.suspension.suspension.attempt === input.attempt + ? { status: 'cancelled' } + : { status: 'stale' }; + } + /** Advances one actor head only when its complete prior identity still matches. */ async function commitAgentEventActorState(input: { user: string; @@ -550,6 +943,7 @@ export function createConversationMethods( discoveredToolNames?: IAgentEventActorState['discoveredToolNames']; summary?: IAgentEventActorState['summary']; contextMeta?: IAgentEventActorState['contextMeta']; + settlementAuthority?: AgentEventActorSettlementAuthority; }): Promise { if (input.checkpoint.threadId !== input.conversationId) { throw new Error('Event actor checkpoint changed its logical thread'); @@ -629,6 +1023,16 @@ export function createConversationMethods( input.expected.requiresColdStart === true ? true : { $ne: true }, }), }; + const settlementFilter: FilterQuery = + input.settlementAuthority == null + ? {} + : { + 'agentEventActorSuspension.status': 'claimed', + 'agentEventActorSuspension.suspension.suspensionId': + input.settlementAuthority.suspensionId, + 'agentEventActorSuspension.suspension.attempt': input.settlementAuthority.attempt, + 'agentEventActorSuspension.resumeAttemptId': input.settlementAuthority.resumeAttemptId, + }; const nextState: IAgentEventActorState = { generation: (input.expected?.generation ?? 0) + 1, checkpoint: input.checkpoint, @@ -656,6 +1060,7 @@ export function createConversationMethods( ...subagentLeaseTenantFilter(input.tenantId), ...activeExpirationFilter(), ...expectedFilter, + ...settlementFilter, }, { $set: { @@ -664,12 +1069,20 @@ export function createConversationMethods( 'agentEventActorReconciliations.$.checkpoint': input.checkpoint, 'agentEventActorReconciliations.$.action': input.action, 'agentEventActorReconciliations.$.observedAt': new Date(), + ...(input.settlementAuthority == null + ? {} + : { + 'agentEventActorSuspension.status': 'closed', + 'agentEventActorSuspension.outcome': 'committed', + 'agentEventActorSuspension.closedAt': new Date(), + 'agentEventActorSuspension.observedAt': new Date(), + }), }, $unset: { 'agentEventActorReconciliations.$.error': 1 }, }, { new: false, timestamps: false }, ) - .select('+agentEventActor') + .select('+agentEventActor +agentEventActorSuspension') .lean(); if (previous != null) { return { @@ -680,6 +1093,70 @@ export function createConversationMethods( : { prunableCheckpoint: previous.agentEventActor.previousCheckpoint }), }; } + if (input.settlementAuthority != null) { + /** A resumed action must close its claim even when another head won. + * The negative full-head predicate makes this mutually exclusive with + * the commit CAS above; the retained closure is the ambiguous-reply receipt. */ + const closedStale = await Conversation.findOneAndUpdate( + { + user: input.user, + conversationId: input.conversationId, + subagentThread: { $exists: true }, + agentEventBinding: { $exists: true }, + agentEventActorReconciliations: { + $elemMatch: { + invocationId: input.invocationId, + status: 'invocation_pending', + }, + }, + ...subagentLeaseTenantFilter(input.tenantId), + ...activeExpirationFilter(), + ...settlementFilter, + $nor: [expectedFilter], + }, + { + $set: { + 'agentEventActorSuspension.status': 'closed', + 'agentEventActorSuspension.outcome': 'stale', + 'agentEventActorSuspension.closedAt': new Date(), + 'agentEventActorSuspension.observedAt': new Date(), + }, + }, + { new: false, timestamps: false }, + ) + .select('+agentEventActor +agentEventActorSuspension') + .lean(); + if (closedStale != null) { + return { + status: 'stale', + ...(closedStale.agentEventActor == null ? {} : { state: closedStale.agentEventActor }), + }; + } + const current = await getAgentEventActorSnapshot(input); + const receipt = current?.suspension; + const matchesAuthority = + receipt?.suspension.suspensionId === input.settlementAuthority.suspensionId && + receipt?.suspension.attempt === input.settlementAuthority.attempt && + receipt?.resumeAttemptId === input.settlementAuthority.resumeAttemptId; + if (matchesAuthority && receipt?.status === 'closed') { + if (receipt.outcome === 'committed' && current?.state != null) { + return { status: 'committed', state: current.state }; + } + if (receipt.outcome === 'stale') { + return { + status: 'stale', + ...(current?.state == null ? {} : { state: current.state }), + }; + } + } + if (matchesAuthority) { + throw new Error('Resumed event actor commit could not close its suspension fence'); + } + return { + status: 'stale', + ...(current?.state == null ? {} : { state: current.state }), + }; + } const current = await getAgentEventActorSnapshot(input); return { status: 'stale', @@ -2651,6 +3128,10 @@ export function createConversationMethods( getAgentEventBinding, getAgentEventActorSnapshot, commitAgentEventActorState, + storeAgentEventActorSuspension, + claimAgentEventActorSuspension, + settleAgentEventActorSuspension, + cancelAgentEventActorSuspension, beginAgentEventActorLegacyTurn, completeAgentEventActorLegacyTurn, recordAgentEventActorReconciliation, diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts index 5ddb7b2f79..98fa95da7a 100644 --- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts +++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts @@ -1665,6 +1665,9 @@ describe('agent trigger delivery methods', () => { * the exact transport attempt is still leased. */ await expect(methods.admitAgentEventActorAction(actionAdmission)).resolves.toBe(true); await expect(methods.hasAgentEventActorActionAdmission(actionAdmission)).resolves.toBe(true); + await expect(methods.getAgentEventActorActionAdmission(actionAdmission)).resolves.toBe( + 'admission-1', + ); await expect(methods.admitAgentEventActorAction(actionAdmission)).resolves.toBe(false); const successorAdmission = { ...actionAdmission, admissionId: 'admission-2' }; await expect(methods.admitAgentEventActorAction(successorAdmission)).resolves.toBe(false); @@ -1675,6 +1678,7 @@ describe('agent trigger delivery methods', () => { await expect(methods.hasAgentEventActorActionAdmission(successorAdmission)).resolves.toBe(true); await expect(methods.releaseAgentEventActorAction(successorAdmission)).resolves.toBe(true); await expect(methods.hasAgentEventActorActionAdmission(actionAdmission)).resolves.toBe(false); + await expect(methods.getAgentEventActorActionAdmission(actionAdmission)).resolves.toBeNull(); /** A pre-token worker's live admission must remain opaque but protected * throughout a rolling upgrade. New workers may observe the fence, but * cannot replace or release it without its (unavailable) owner token. */ diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts index e9454271ec..d300d62191 100644 --- a/packages/data-schemas/src/methods/triggerDelivery.ts +++ b/packages/data-schemas/src/methods/triggerDelivery.ts @@ -175,6 +175,9 @@ export interface AgentTriggerDeliveryMethods { ) => Promise; admitAgentEventActorAction: (input: AdmitAgentEventActorActionInput) => Promise; releaseAgentEventActorAction: (input: AgentEventActorActionAdmissionInput) => Promise; + getAgentEventActorActionAdmission: ( + input: GetAgentEventActorReceiptInput, + ) => Promise; hasAgentEventActorActionAdmission: ( input: AgentEventActorActionAdmissionInput, ) => Promise; @@ -1658,6 +1661,34 @@ export function createAgentTriggerDeliveryMethods( ); } + /** Reads the delivery-owned action fence when the private child Conversation + * has already been removed. The terminal owner can then release the exact + * admission id without guessing from missing actor state. */ + async function getAgentEventActorActionAdmission( + input: GetAgentEventActorReceiptInput, + ): Promise { + const tenantScope = + input.tenantId == null ? { tenantId: { $exists: false } } : { tenantId: input.tenantId }; + const delivery = await Delivery() + .findOne({ + deliveryKey: input.deliveryKey, + user: input.user, + ...tenantScope, + 'envelope.target.bindingId': input.bindingId, + $or: [ + { handling: { $exists: false } }, + { 'handling.conversationId': input.conversationId }, + ], + actorReceipt: { $exists: false }, + actorActionAdmittedAt: { $exists: true }, + }) + .select('+actorActionAdmissionId') + .lean>(); + return typeof delivery?.actorActionAdmissionId === 'string' + ? delivery.actorActionAdmissionId + : null; + } + async function getAgentEventActorReceipt( input: GetAgentEventActorReceiptInput, ): Promise { @@ -2198,6 +2229,7 @@ export function createAgentTriggerDeliveryMethods( settleAgentTriggerHandlingOutcome, admitAgentEventActorAction, releaseAgentEventActorAction, + getAgentEventActorActionAdmission, hasAgentEventActorActionAdmission, settleAgentEventActorReceipt, getAgentEventActorReceipt, diff --git a/packages/data-schemas/src/schema/convo.ts b/packages/data-schemas/src/schema/convo.ts index c08fcf8d46..186fa8fa5b 100644 --- a/packages/data-schemas/src/schema/convo.ts +++ b/packages/data-schemas/src/schema/convo.ts @@ -228,6 +228,28 @@ const convoSchema: Schema = new Schema( default: undefined, select: false, }, + /** Current SDK-issued suspended invocation. The signed evidence remains + * opaque/Mixed so its exact versioned JSON shape survives round trips; + * mirrored host fields provide bounded CAS predicates. */ + agentEventActorSuspension: { + type: { + suspension: { type: Schema.Types.Mixed, required: true }, + actionId: { type: String, required: true }, + jobCreatedAt: { type: Number, required: true }, + status: { type: String, enum: ['pending', 'claimed', 'closed'], required: true }, + resumeAttemptId: { type: String, default: undefined }, + outcome: { + type: String, + enum: ['committed', 'stale', 'settled', 'cancelled'], + default: undefined, + }, + closedAt: { type: Date, default: undefined }, + observedAt: { type: Date, required: true }, + }, + _id: false, + default: undefined, + select: false, + }, tags: { type: [String], default: [], diff --git a/packages/data-schemas/src/types/convo.ts b/packages/data-schemas/src/types/convo.ts index 45c86586c3..0b8a94a009 100644 --- a/packages/data-schemas/src/types/convo.ts +++ b/packages/data-schemas/src/types/convo.ts @@ -99,10 +99,67 @@ export interface IAgentEventActorLegacyTurn { startedAt: Date; } +/** JSON-safe value retained inside SDK-issued event-actor evidence. */ +export type TAgentEventActorEvent = + | null + | boolean + | number + | string + | TAgentEventActorEvent[] + | { [key: string]: TAgentEventActorEvent }; + +export interface IAgentEventActorInvocationReference { + actorThreadId: string; + invocationId: string; + depth: number; + continuation: 'warm' | 'cold'; + base: { + actorThreadId: string; + generation: number; + checkpoint?: Omit & { checkpointId?: string }; + }; + fork: Omit & { + checkpointId?: string; + invocationId: string; + }; +} + +/** Exact, signed SDK evidence for a paused invocation fork. */ +export interface IAgentEventActorSuspensionEvidence { + version: 1; + suspensionId: string; + attempt: number; + issuedAt: number; + expiresAt: number; + invocation: IAgentEventActorInvocationReference; + checkpoint: IAgentEventActorInvocationReference['fork']; + interrupt: { + id: string; + payload: TAgentEventActorEvent; + }; + suspensionDigest: string; +} + +/** + * Host-owned current suspension fence. SDK evidence authenticates the fork; + * the mirrored action/job identity binds it to LibreChat's approval CAS. + */ +export interface IAgentEventActorSuspension { + suspension: IAgentEventActorSuspensionEvidence; + actionId: string; + jobCreatedAt: number; + status: 'pending' | 'claimed' | 'closed'; + resumeAttemptId?: string; + outcome?: 'committed' | 'stale' | 'settled' | 'cancelled'; + closedAt?: Date; + observedAt: Date; +} + export interface IAgentEventActorSnapshot { state: IAgentEventActorState | null; reconciliations: IAgentEventActorReconciliation[]; legacyTurn: IAgentEventActorLegacyTurn | null; + suspension: IAgentEventActorSuspension | null; /** Durable invalidation epoch. Every legacy-path event bumps it — including * for headless or already cold-marked actors, where the marker alone leaves * no CAS-visible trace — and the commit CAS requires the epoch observed at @@ -180,6 +237,8 @@ export interface IConversation extends Document { agentEventActorEpoch?: number; /** Private in-flight legacy-turn fence; see {@link IAgentEventActorLegacyTurn}. */ agentEventActorLegacyTurn?: IAgentEventActorLegacyTurn; + /** Private current suspended invocation; see {@link IAgentEventActorSuspension}. */ + agentEventActorSuspension?: IAgentEventActorSuspension; assistant_id?: string; instructions?: string; stop?: string[];