diff --git a/.env.example b/.env.example index 3b525166f9..188b1efb49 100644 --- a/.env.example +++ b/.env.example @@ -729,6 +729,10 @@ MESSAGE_USER_WINDOW=1 AGENT_EVENT_USER_MAX=40 AGENT_EVENT_USER_WINDOW=1 +# Enable only after every API replica runs a release that understands +# source-bound child actor continuations. +ENABLE_AGENT_EVENT_CHILD_TURNS=false + ILLEGAL_MODEL_REQ_SCORE=5 #========================# diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 99b76f0b38..5da4b34da4 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -1230,7 +1230,9 @@ class BaseClient { const hasAddedConvo = options?.req?.body?.addedConvo != null; const reqCtx = { userId: options?.req?.user?.id, - isTemporary: options?.req?.body?.isTemporary, + isTemporary: + options?.req?._agentEventBindingRetention?.isTemporary ?? options?.req?.body?.isTemporary, + expiredAt: options?.req?._agentEventBindingRetention?.expiredAt, interfaceConfig: options?.req?.config?.interfaceConfig, }; const savedMessage = await db.saveMessage( @@ -1306,6 +1308,7 @@ class BaseClient { const conversation = await db.saveConvo(reqCtx, fieldsToKeep, { context: 'api/app/clients/BaseClient.js - saveMessageToDatabase #saveConvo', unsetFields, + noUpsert: req?._agentEventBindingParentConversationId != null, createdAtOnInsert: shouldSetCreatedAtOnInsert ? validCreatedAtOnInsert : undefined, }); diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js index 3668c564d3..f223f2fd91 100644 --- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js +++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js @@ -64,6 +64,8 @@ const mockGetConvo = jest.fn(); const mockGetMessages = jest.fn(); const mockSaveMessage = jest.fn(); const mockIsAgentTriggerPrincipalActive = jest.fn(); +const mockIsSubagentOwnerAdmissible = jest.fn(); +const mockAcquireEventChildGenerationLease = jest.fn(); const mockIsScheduleFireRequest = jest.fn(); const mockExemptFromConcurrencyLimiter = jest.fn(); const mockRecordScheduleOutcome = jest.fn(); @@ -195,6 +197,8 @@ jest.mock('@librechat/api', () => ({ return messages.length === 0; }, deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args), + isAgentEventRetentionActive: (expiredAt) => + expiredAt == null || new Date(expiredAt).getTime() > Date.now(), createMCPRuntimeRequestBody: ({ messageId, conversationId, parentMessageId }) => ({ messageId, conversationId, @@ -223,6 +227,11 @@ jest.mock('~/models', () => ({ getMessages: (...args) => mockGetMessages(...args), getConvo: (...args) => mockGetConvo(...args), isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args), + isSubagentOwnerAdmissible: (...args) => mockIsSubagentOwnerAdmissible(...args), +})); + +jest.mock('~/server/services/Endpoints/agents/eventChildLease', () => ({ + acquireEventChildGenerationLease: (...args) => mockAcquireEventChildGenerationLease(...args), })); jest.mock('~/server/services/Schedules', () => ({ @@ -266,6 +275,8 @@ describe('ResumableAgentController resume metadata', () => { mockGetConvo.mockResolvedValue({ createdAt: '2026-06-07T00:00:00.000Z' }); mockGetMessages.mockResolvedValue([]); mockIsAgentTriggerPrincipalActive.mockResolvedValue(true); + mockIsSubagentOwnerAdmissible.mockResolvedValue(true); + mockAcquireEventChildGenerationLease.mockResolvedValue(jest.fn()); mockIsScheduleFireRequest.mockImplementation((req) => req?._isScheduledFire === true); mockExemptFromConcurrencyLimiter.mockImplementation( (req) => req?._isScheduledFire === true && req?._isManualScheduledFire !== true, @@ -3599,6 +3610,120 @@ describe('ResumableAgentController resume metadata', () => { ); }); + it('reports a temporary event-actor fence as retryable rather than ending the binding', async () => { + const expiredAt = new Date(Date.now() + 60_000); + mockGenerationJobManager.claimGeneration.mockResolvedValue( + wonGenerationClaim({ + streamId: 'child-conversation', + conversationId: 'child-conversation', + }), + ); + mockGetConvo.mockResolvedValue({ + conversationId: 'parent-conversation', + agent_id: 'parent-agent', + tenantId: 'tenant-1', + }); + mockIsSubagentOwnerAdmissible.mockResolvedValue(false); + const req = { + user: { id: 'user-123', tenantId: 'tenant-1' }, + body: { + text: 'Continue from event.', + messageId: 'user-msg', + clientRequestId: 'req-event', + conversationId: 'child-conversation', + endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } }, + }, + config: {}, + _agentEventBindingParentConversationId: 'parent-conversation', + _agentEventBindingParentAgentId: 'parent-agent', + _agentEventBindingTenantId: 'tenant-1', + _agentEventBindingRetention: { isTemporary: true, expiredAt }, + }; + const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() }; + + await AgentController(req, res, jest.fn(), jest.fn(), null); + + expect(res.status).toHaveBeenCalledWith(409); + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ code: 'EVENT_ACTOR_NOT_READY' }), + ); + expect(mockGenerationJobManager.createJob).toHaveBeenCalledTimes(1); + expect(mockAcquireEventChildGenerationLease).toHaveBeenCalledWith( + expect.objectContaining({ retentionExpiresAt: expiredAt }), + ); + }); + + it('uses the guard-normalized tenant for a legacy untenanted event actor', async () => { + const expiredAt = new Date(Date.now() + 60_000); + mockGenerationJobManager.claimGeneration.mockResolvedValue( + wonGenerationClaim({ + streamId: 'child-conversation', + conversationId: 'child-conversation', + }), + ); + mockGetConvo.mockResolvedValue({ + conversationId: 'parent-conversation', + agent_id: 'parent-agent', + }); + const req = { + user: { id: 'user-123', tenantId: '' }, + body: { + text: 'Continue from an old untenanted binding.', + messageId: 'user-msg', + clientRequestId: 'req-event-legacy', + conversationId: 'child-conversation', + endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } }, + }, + config: {}, + _agentEventBindingParentConversationId: 'parent-conversation', + _agentEventBindingParentAgentId: 'parent-agent', + _agentEventBindingTenantId: undefined, + _agentEventBindingRetention: { isTemporary: true, expiredAt }, + }; + const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() }; + + await AgentController(req, res, jest.fn(), jest.fn(), null); + + expect(mockAcquireEventChildGenerationLease).toHaveBeenCalledWith( + expect.objectContaining({ tenantId: undefined, retentionExpiresAt: expiredAt }), + ); + }); + + it('does not start an event actor whose inherited binding expired after the guard', async () => { + const expiredAt = new Date(Date.now() - 1); + mockGenerationJobManager.claimGeneration.mockResolvedValue( + wonGenerationClaim({ + streamId: 'child-conversation', + conversationId: 'child-conversation', + }), + ); + mockAcquireEventChildGenerationLease.mockResolvedValue(null); + const req = { + user: { id: 'user-123', tenantId: 'tenant-1' }, + body: { + text: 'This event arrived too late.', + messageId: 'user-msg', + clientRequestId: 'req-event-expired', + conversationId: 'child-conversation', + endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } }, + }, + config: {}, + _agentEventBindingParentConversationId: 'parent-conversation', + _agentEventBindingParentAgentId: 'parent-agent', + _agentEventBindingTenantId: 'tenant-1', + _agentEventBindingRetention: { isTemporary: true, expiredAt }, + }; + const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() }; + + await AgentController(req, res, jest.fn(), jest.fn(), null); + + expect(res.status).toHaveBeenCalledWith(409); + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ code: 'EVENT_BINDING_PARENT_ENDED' }), + ); + expect(mockIsSubagentOwnerAdmissible).not.toHaveBeenCalled(); + }); + it('releases the idempotency claim on a 429 only when it won the claim', async () => { mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim()); mockCheckAndIncrementPendingRequest.mockResolvedValue({ diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js index 6db510f9f1..be562830ad 100644 --- a/api/server/controllers/agents/__tests__/resume.spec.js +++ b/api/server/controllers/agents/__tests__/resume.spec.js @@ -99,6 +99,9 @@ const mockClaimScheduleResume = jest.fn(); const mockReleaseScheduleResumeClaim = jest.fn(); const mockFinalizeScheduleResumeClaim = jest.fn(); const mockReleaseScheduleResumeFence = jest.fn(); +const mockAcquireEventChildGenerationLease = jest.fn(); +const mockReleaseEventChildLease = jest.fn(); +const mockIsSubagentOwnerAdmissible = jest.fn(); jest.mock('@librechat/data-schemas', () => ({ ...jest.requireActual('@librechat/data-schemas'), @@ -131,6 +134,11 @@ jest.mock('~/models', () => ({ getActions: (...args) => mockGetActions(...args), getUserMemories: (...args) => mockGetUserMemories(...args), getRoleByName: (...args) => mockGetRoleByName(...args), + isSubagentOwnerAdmissible: (...args) => mockIsSubagentOwnerAdmissible(...args), +})); + +jest.mock('~/server/services/Endpoints/agents/eventChildLease', () => ({ + acquireEventChildGenerationLease: (...args) => mockAcquireEventChildGenerationLease(...args), })); jest.mock('~/server/services/ActionService', () => ({ @@ -259,6 +267,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { let mockAddTitle; let capturedInit; let requestConfigOverrides; + let requestStateOverrides; let endpointAgent; let settle; let settled; @@ -268,6 +277,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { capturedInit = null; requestConfigOverrides = {}; + requestStateOverrides = {}; mockCheckAndIncrementPendingRequest.mockResolvedValue({ allowed: true }); mockDecrementPendingRequest.mockResolvedValue(undefined); mockDeleteAgentCheckpoint.mockResolvedValue(undefined); @@ -338,6 +348,9 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { mockReleaseScheduleResumeClaim.mockResolvedValue(true); mockFinalizeScheduleResumeClaim.mockResolvedValue(true); mockReleaseScheduleResumeFence.mockResolvedValue(undefined); + mockAcquireEventChildGenerationLease.mockResolvedValue(mockReleaseEventChildLease); + mockReleaseEventChildLease.mockResolvedValue(undefined); + mockIsSubagentOwnerAdmissible.mockResolvedValue(true); endpointAgent = { _id: 'mongo-agent-abc', id: AGENT_ID, @@ -388,6 +401,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { model_parameters: {}, agent: Promise.resolve(endpointAgent), }; + Object.assign(req, requestStateOverrides); next(); }); app.post('/api/agents/chat/resume', (req, res, next) => @@ -406,6 +420,149 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { ...extra, }); + const configureEventActorResume = (expiredAt = new Date(Date.now() + 60_000)) => { + requestStateOverrides = { + _agentEventBindingParentConversationId: 'parent-conversation', + _agentEventBindingParentAgentId: 'parent-agent', + _agentEventBindingTenantId: TENANT_ID, + _agentEventBindingRetention: { isTemporary: true, expiredAt }, + }; + mockGetConvo.mockResolvedValue({ + conversationId: 'parent-conversation', + agent_id: 'parent-agent', + tenantId: TENANT_ID, + createdAt: new Date('2026-08-22T00:00:00.000Z'), + }); + return expiredAt; + }; + + describe('event-bound actor resume lifecycle', () => { + it('leaves the approval pending when the previous segment still owns the lease', async () => { + configureEventActorResume(); + mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); + mockAcquireEventChildGenerationLease.mockResolvedValue(null); + + const res = await post(approveBody()); + + expect(res.status).toBe(409); + expect(res.body).toMatchObject({ code: 'EVENT_ACTOR_NOT_READY' }); + expect(mockAcquireEventChildGenerationLease).toHaveBeenCalled(); + expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled(); + expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled(); + }); + + it('classifies an expired binding as ended when no lease can be acquired', async () => { + configureEventActorResume(new Date(Date.now() - 1)); + mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); + mockAcquireEventChildGenerationLease.mockResolvedValue(null); + + const res = await post(approveBody()); + + expect(res.status).toBe(409); + expect(res.body).toMatchObject({ code: 'EVENT_BINDING_PARENT_ENDED' }); + expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled(); + }); + + it('owns the lease before consuming approval and preserves the inherited deadline', async () => { + const expiredAt = configureEventActorResume(); + mockGenerationJobManager.getJob.mockResolvedValue( + makeToolApprovalJob({ metadata: { isTemporary: true } }), + ); + + const res = await post(approveBody()); + expect(res.status).toBe(200); + await settled; + await flush(); + + expect(mockAcquireEventChildGenerationLease.mock.invocationCallOrder[0]).toBeLessThan( + mockGenerationJobManager.approvals.resolve.mock.invocationCallOrder[0], + ); + expect(mockIsSubagentOwnerAdmissible.mock.invocationCallOrder[0]).toBeLessThan( + mockGenerationJobManager.approvals.resolve.mock.invocationCallOrder[0], + ); + expect(mockAcquireEventChildGenerationLease).toHaveBeenCalledWith( + expect.objectContaining({ retentionExpiresAt: expiredAt }), + ); + expect(mockSaveMessage).toHaveBeenCalledWith( + expect.objectContaining({ isTemporary: true, expiredAt }), + expect.anything(), + expect.anything(), + ); + expect(mockReleaseEventChildLease).toHaveBeenCalledTimes(1); + }); + + it('preserves the inherited deadline when the resumed actor pauses again', async () => { + const expiredAt = configureEventActorResume(); + mockGenerationJobManager.getJob.mockResolvedValue( + makeToolApprovalJob({ metadata: { isTemporary: true } }), + ); + mockInitializeClient.mockResolvedValue({ + client: makeClient({ + pendingApproval: { actionId: NEXT_ACTION_ID }, + contentParts: [{ type: 'text', text: 'partial' }], + }), + userMCPAuthMap: {}, + }); + + const res = await post(approveBody()); + expect(res.status).toBe(200); + await settled; + await flush(); + + expect(mockSaveMessage).toHaveBeenCalledWith( + expect.objectContaining({ isTemporary: true, expiredAt }), + expect.objectContaining({ unfinished: true }), + expect.objectContaining({ + context: 'api/server/controllers/agents/resume.js - re-pause progress persist', + }), + ); + }); + + it('defers a resume when the owner admission fence is temporarily closed', async () => { + configureEventActorResume(); + mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); + mockIsSubagentOwnerAdmissible.mockResolvedValue(false); + + const res = await post(approveBody()); + + expect(res.status).toBe(409); + expect(res.body).toMatchObject({ code: 'EVENT_ACTOR_NOT_READY' }); + expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled(); + expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled(); + }); + + it('rejects a binding that expires after the route guard but before approval consumption', async () => { + configureEventActorResume(new Date(Date.now() - 1)); + mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); + + const res = await post(approveBody()); + + expect(res.body).toMatchObject({ code: 'EVENT_BINDING_PARENT_ENDED' }); + expect(res.status).toBe(409); + expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled(); + expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled(); + }); + + it('uses the guard-normalized tenant for a legacy untenanted event actor', async () => { + const expiredAt = configureEventActorResume(); + requestStateOverrides._agentEventBindingTenantId = undefined; + mockGetConvo.mockResolvedValue({ + conversationId: 'parent-conversation', + agent_id: 'parent-agent', + }); + mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); + + const res = await post(approveBody()); + expect(res.status).toBe(200); + await settled; + await flush(); + + expect(mockAcquireEventChildGenerationLease).toHaveBeenCalledWith( + expect.objectContaining({ tenantId: undefined, retentionExpiresAt: expiredAt }), + ); + }); + }); + describe('scheduled occurrence lifecycle', () => { const scheduledFor = '2026-08-17T12:00:00.000Z'; const makeScheduledJob = () => diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 7c3f59c9f1..cb37cd6a31 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -152,6 +152,19 @@ const db = require('~/models'); const loadAgent = (params) => loadAgentFn(params, { getAgent: db.getAgent, getMCPServerTools }); +function getInterruptTtlMs(checkpointerCfg, req) { + const configuredTtlMs = getApprovalTtlMs(checkpointerCfg); + const retention = req?._agentEventBindingRetention; + if (retention?.expiredAt == null) { + return configuredTtlMs; + } + const bindingDeadline = new Date(retention.expiredAt).getTime(); + if (!Number.isFinite(bindingDeadline)) { + return configuredTtlMs; + } + return Math.min(configuredTtlMs, Math.max(0, bindingDeadline - Date.now())); +} + const MEMORY_INPUT_CHARS_PER_TOKEN = 8; function getUserFacingRequestError(baseMessage, error, appConfig) { @@ -3088,7 +3101,7 @@ class AgentClient extends BaseClient { // thread_id was bound to conversationId at run config (config.configurable); // fall back to it when the SDK doesn't echo threadId on the interrupt. threadId: interrupt.threadId ?? this.conversationId, - ttlMs: getApprovalTtlMs(checkpointerCfg), + ttlMs: getInterruptTtlMs(checkpointerCfg, this.options.req), // Pin the graph-determining request fields so resume can't rebuild this paused // run on a different agent/tool set (esp. ephemeral agents, whose agent_id is // undefined so the id guard can't tell two configs apart). diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index e803f699ea..3f4db27cfc 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -516,6 +516,57 @@ describe('AgentClient - interrupt discovery persistence', () => { expect(paused?.status).toBe('requires_action'); expect(paused?.metadata.discoveredTools).toEqual(['save_issue_mcp_linear']); }); + + it('caps an event-bound pause at the inherited binding deadline', async () => { + const now = Date.now(); + const streamId = 'conversation-event-bound-pause'; + const job = await GenerationJobManager.createJob(streamId, 'user-123', streamId); + const client = new AgentClient({ + req: { + user: { id: 'user-123' }, + body: { endpoint: EModelEndpoint.agents, agent_id: 'agent-123' }, + config: { endpoints: { [EModelEndpoint.agents]: { checkpointer: { ttl: 3600 } } } }, + _agentEventBindingRetention: { + /** RetentionMode.ALL conversations are not temporary but still have a deadline. */ + isTemporary: false, + expiredAt: new Date(now + 5_000), + }, + }, + res: {}, + agent: { + id: 'agent-123', + endpoint: EModelEndpoint.openAI, + provider: EModelEndpoint.openAI, + model_parameters: { model: 'gpt-4' }, + }, + contentParts: [], + collectedUsage: [], + artifactPromises: [], + }); + client.conversationId = streamId; + client.responseMessageId = 'response-event-bound-pause'; + client.jobCreatedAt = job.createdAt; + + await client.handleRunInterrupt( + { + getInterrupt: () => ({ + interruptId: 'ask-interrupt', + threadId: streamId, + payload: { + type: 'ask_user_question', + question: { question: 'Proceed?' }, + }, + }), + getDiscoveredTools: () => [], + getRunMessages: () => [], + }, + streamId, + ); + + const paused = await GenerationJobManager.getJob(streamId); + expect(paused?.metadata.pendingAction.expiresAt).toBeGreaterThanOrEqual(now + 4_900); + expect(paused?.metadata.pendingAction.expiresAt).toBeLessThanOrEqual(now + 5_000); + }); }); jest.mock('~/server/services/Config', () => ({ diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index a28a568bd6..905723225d 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -29,6 +29,7 @@ const { deleteAgentCheckpoint, getAttachmentTitleText, createMCPRuntimeRequestBody, + isAgentEventRetentionActive, } = require('@librechat/api'); const { disposeClient } = require('~/server/cleanup'); const { @@ -37,7 +38,16 @@ const { } = require('~/server/services/MCPRequestContext'); const { logViolation } = require('~/cache'); const { recordScheduleOutcome, isScheduleLive } = require('~/server/services/Schedules'); -const { saveMessage, getMessages, getConvo, isAgentTriggerPrincipalActive } = require('~/models'); +const { + saveMessage, + getMessages, + getConvo, + isAgentTriggerPrincipalActive, + isSubagentOwnerAdmissible, +} = require('~/models'); +const { + acquireEventChildGenerationLease, +} = require('~/server/services/Endpoints/agents/eventChildLease'); const { GENERATION_PROTOCOL_HEADER, GENERATION_PROTOCOL_V2, @@ -1022,6 +1032,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit let client = null; let jobCreatedAt; let providerExecutionId; + let releaseEventChildLease; let scheduleTerminalOutcomeRecorded = false; const settleScheduledRun = async ({ status, error, clearConversationId = false }) => { if (!scheduleId) { @@ -1085,7 +1096,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit agent_id: endpointOption.agent_id ?? req.body?.agent_id, // Persist temporary-chat state so a HITL resume keeps the resumed response // non-persisted instead of trusting the resume request to re-send the flag. - isTemporary: req.body?.isTemporary, + isTemporary: req._agentEventBindingRetention?.isTemporary ?? req.body?.isTemporary, ...(isRegenerate && { isRegenerate: true }), ...(scheduleId ? { @@ -1120,6 +1131,58 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit status: 409, }); } + if (req._agentEventBindingParentConversationId != null) { + /** The generation job is the durable marker that a deletion on another replica + * can abort. Recheck only after that marker exists: either the deletion fence + * wins and this run stops here, or the deletion observes and drains this job. */ + releaseEventChildLease = await acquireEventChildGenerationLease({ + userId, + tenantId: req._agentEventBindingTenantId, + conversationId, + streamId, + jobCreatedAt, + retentionExpiresAt: req._agentEventBindingRetention?.expiredAt, + }); + if (releaseEventChildLease == null) { + const bindingActive = isAgentEventRetentionActive( + req._agentEventBindingRetention?.expiredAt, + ); + throw Object.assign( + new Error( + bindingActive + ? 'The event actor is already handling another turn' + : 'The event binding parent is no longer available', + ), + { + code: bindingActive ? 'EVENT_ACTOR_NOT_READY' : 'EVENT_BINDING_PARENT_ENDED', + status: 409, + }, + ); + } + const [eventParent, ownerAdmissible] = await Promise.all([ + getConvo(userId, req._agentEventBindingParentConversationId), + isSubagentOwnerAdmissible(userId), + ]); + if (!ownerAdmissible) { + throw Object.assign(new Error('The event actor is temporarily unavailable'), { + code: 'EVENT_ACTOR_NOT_READY', + status: 409, + }); + } + if ( + eventParent == null || + eventParent.subagentThread != null || + eventParent.agent_id !== req._agentEventBindingParentAgentId || + (eventParent.tenantId ?? undefined) !== req._agentEventBindingTenantId || + !isAgentEventRetentionActive(req._agentEventBindingRetention?.expiredAt) || + !isAgentEventRetentionActive(eventParent.expiredAt) + ) { + throw Object.assign(new Error('The event binding parent is no longer available'), { + code: 'EVENT_BINDING_PARENT_ENDED', + status: 409, + }); + } + } if ( scheduleId && !(await isScheduleLive(scheduleId, scheduleConfigRevision, { @@ -1248,7 +1311,8 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit saveMessage( { userId, - isTemporary: req?.body?.isTemporary, + isTemporary: req?._agentEventBindingRetention?.isTemporary ?? req?.body?.isTemporary, + expiredAt: req?._agentEventBindingRetention?.expiredAt, interfaceConfig: req?.config?.interfaceConfig, }, partialMessage, @@ -1676,7 +1740,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit const savedUserMessage = await saveMessage( { userId, - isTemporary: req?.body?.isTemporary, + isTemporary: + req?._agentEventBindingRetention?.isTemporary ?? req?.body?.isTemporary, + expiredAt: req?._agentEventBindingRetention?.expiredAt, interfaceConfig: req?.config?.interfaceConfig, }, userMessage, @@ -1696,7 +1762,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit const savedResponseMessage = await saveMessage( { userId, - isTemporary: req?.body?.isTemporary, + isTemporary: + req?._agentEventBindingRetention?.isTemporary ?? req?.body?.isTemporary, + expiredAt: req?._agentEventBindingRetention?.expiredAt, interfaceConfig: req?.config?.interfaceConfig, }, { @@ -1855,7 +1923,8 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit // where client refetch happens before database is updated const reqCtx = { userId: req?.user?.id, - isTemporary: req?.body?.isTemporary, + isTemporary: req?._agentEventBindingRetention?.isTemporary ?? req?.body?.isTemporary, + expiredAt: req?._agentEventBindingRetention?.expiredAt, interfaceConfig: req?.config?.interfaceConfig, }; @@ -2162,6 +2231,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit providerExecutionId, ); } + await releaseEventChildLease?.(); }) .catch((drainError) => { logger.warn( @@ -2302,6 +2372,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit ); }); } + await releaseEventChildLease?.(); } }; diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.js index 11ad6e741f..572f3d1644 100644 --- a/api/server/controllers/agents/resume.js +++ b/api/server/controllers/agents/resume.js @@ -38,6 +38,7 @@ const { toPendingSteer, createMCPRuntimeRequestBody, getSafeErrorMetadata, + isAgentEventRetentionActive, } = require('@librechat/api'); const { disposeClient } = require('~/server/cleanup'); const { decryptMetadata } = require('~/server/services/ActionService'); @@ -55,7 +56,11 @@ const { getActions, getUserMemories, getRoleByName, + isSubagentOwnerAdmissible, } = require('~/models'); +const { + acquireEventChildGenerationLease, +} = require('~/server/services/Endpoints/agents/eventChildLease'); const { recordScheduleOutcome, claimScheduleResume, @@ -240,6 +245,7 @@ async function persistRePauseProgress({ req, client, job, streamId, conversation { userId, isTemporary: meta.isTemporary ?? req.body?.isTemporary, + expiredAt: req._agentEventBindingRetention?.expiredAt, interfaceConfig: req?.config?.interfaceConfig, }, { @@ -454,7 +460,12 @@ async function finalizeResumedTurn({ let terminalPublicationStarted = false; try { const savedResponseMessage = await saveMessage( - { userId, isTemporary, interfaceConfig: req?.config?.interfaceConfig }, + { + userId, + isTemporary, + expiredAt: req._agentEventBindingRetention?.expiredAt, + interfaceConfig: req?.config?.interfaceConfig, + }, responseMessage, { context: 'api/server/controllers/agents/resume.js - resumed response end' }, ); @@ -1091,122 +1102,229 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) } }; - // Atomically claim the resume. The single winner drives the run; a racing second - // submit (double-click, two tabs) gets false and must not re-drive — that would - // re-execute tools and double-bill. - // - // The claim runs AFTER the slot increment above but BEFORE the run's own try/finally - // that releases it, so a store/Redis error here (unlike the clean `!claimed` branch) - // would leak the concurrency slot until the counter TTL expires — spuriously 429'ing - // the user when they retry the still-paused approval. Release the slot on that path too. - let claimed; + let releaseEventChildLease; + let eventLeaseTransferredToRun = false; const providerExecutionId = randomUUID(); try { - /** The CAS that reopens steering must also publish THIS owner's seal - * capability. A separate write after status=`running` leaves a window in - * which steer/arm requests read the previous replica's capability. */ - claimed = await GenerationJobManager.approvals.resolve( - streamId, - pendingAction.actionId, - { - preemptCapable: isSteerPreemptSupported(), - providerExecutionId, - providerDrained: true, - ...(resolvedAskUserQuestion && { resolvedAskUserQuestions }), - }, - job.createdAt, - ); - } catch (err) { - const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); - await rollbackUnconsumedScheduleClaim(currentJob); - await releaseScheduleFence(); - await decrementPendingRequest(userId); - logger.error('[ResumeAgentController] Failed to claim resume', getSafeErrorMetadata(err)); - return sendGenerationJson(res, 500, { error: 'Failed to resume' }, generationProtocolVersion); - } - if (!claimed) { - await decrementPendingRequest(userId); - const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); - await rollbackUnconsumedScheduleClaim(currentJob); - await releaseScheduleFence(); - if (currentJob != null && currentJob.createdAt !== job.createdAt) { - return sendGenerationJson(res, 409, { code: 'RUN_REPLACED' }, generationProtocolVersion); - } - return sendGenerationJson( - res, - 409, - { error: 'This action was already resolved or has expired' }, - generationProtocolVersion, - ); - } - - // Linearize the consumed approval against the schedule's live config. The schedule - // document fence was acquired only after all async policy reads, and this atomic - // consume checks its token/revision/enabled state immediately after the approval CAS. - // An edit/disable that won first makes this fail; one that lands afterward is ordered - // after the continuation has started. Never begin provider execution on a stale claim. - if (scheduleId) { - let scheduleClaimCurrent = false; - try { - scheduleClaimCurrent = await finalizeScheduleResumeClaim( - scheduleId, - scheduleResumeClaimToken, - scheduleResumeLeaseBy, - scheduleResumeOptions, - ); - } catch (error) { - logger.error('[ResumeAgentController] Failed to finalize scheduled resume fence', error); - await releaseScheduleFence(); - } - if (!scheduleClaimCurrent) { - await decrementPendingRequest(userId); - let stopped = false; + if (req._agentEventBindingParentConversationId != null) { try { - const abortResult = await GenerationJobManager.abortJob(streamId, { - expectedCreatedAt: job.createdAt, - awaitProviderDrain: true, + releaseEventChildLease = await acquireEventChildGenerationLease({ + userId, + tenantId: req._agentEventBindingTenantId, + conversationId, + streamId, + jobCreatedAt: job.createdAt, + retentionExpiresAt: req._agentEventBindingRetention?.expiredAt, }); - // Same authoritative gate as the inactive-schedule path above: only a landed - // abort (or an already-terminal, drained generation) may settle this occurrence. - stopped = isStopConfirmed(abortResult); } catch (error) { - logger.warn('[ResumeAgentController] Failed to stop stale scheduled resume', error); - } - if (!stopped) { + logger.warn('[ResumeAgentController] Event actor resume lease is unavailable', error); + const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); + await rollbackUnconsumedScheduleClaim(currentJob); + await releaseScheduleFence(); + await decrementPendingRequest(userId); res.set('Retry-After', '1'); return sendGenerationJson( res, 503, { - code: 'SCHEDULE_STOP_UNCONFIRMED', - error: 'The stale scheduled resume could not be confirmed stopped.', + code: 'EVENT_ACTOR_LEASE_UNAVAILABLE', + error: 'The event actor lease is temporarily unavailable', }, generationProtocolVersion, ); } - await recordScheduleOutcome({ - scheduleId, - scheduledFor, + if (releaseEventChildLease == null) { + const bindingActive = isAgentEventRetentionActive( + req._agentEventBindingRetention?.expiredAt, + ); + const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); + await rollbackUnconsumedScheduleClaim(currentJob); + await releaseScheduleFence(); + await decrementPendingRequest(userId); + if (bindingActive) { + res.set('Retry-After', '1'); + } + return sendGenerationJson( + res, + 409, + { + code: bindingActive ? 'EVENT_ACTOR_NOT_READY' : 'EVENT_BINDING_PARENT_ENDED', + error: bindingActive + ? 'The event actor is still finishing its previous segment' + : 'The event binding parent is no longer available', + }, + generationProtocolVersion, + ); + } + + /** Validate the durable parent/owner fence before consuming the HITL action. + * Once `approvals.resolve` wins its CAS, the action is irreversibly spent; a + * retryable fence rejection after that point could never replay the user's + * decision. A deletion that starts after this check observes the generation + * job plus the event-child lease and owns the corresponding abort. */ + let eventActorRejection; + try { + const [eventParent, ownerAdmissible] = await Promise.all([ + getConvo(userId, req._agentEventBindingParentConversationId), + isSubagentOwnerAdmissible(userId), + ]); + if (!ownerAdmissible) { + eventActorRejection = { + code: 'EVENT_ACTOR_NOT_READY', + error: 'The event actor owner is temporarily unavailable', + }; + } else if ( + eventParent == null || + eventParent.subagentThread != null || + eventParent.agent_id !== req._agentEventBindingParentAgentId || + (eventParent.tenantId ?? undefined) !== req._agentEventBindingTenantId || + !isAgentEventRetentionActive(req._agentEventBindingRetention?.expiredAt) || + !isAgentEventRetentionActive(eventParent.expiredAt) + ) { + eventActorRejection = { + code: 'EVENT_BINDING_PARENT_ENDED', + error: 'The event binding parent is no longer available', + }; + } + } catch (error) { + logger.warn('[ResumeAgentController] Event actor fence recheck failed', error); + eventActorRejection = { + code: 'EVENT_ACTOR_NOT_READY', + error: 'The event actor owner is temporarily unavailable', + }; + } + if (eventActorRejection != null) { + const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); + await rollbackUnconsumedScheduleClaim(currentJob); + await releaseScheduleFence(); + await decrementPendingRequest(userId); + return sendGenerationJson(res, 409, eventActorRejection, generationProtocolVersion); + } + } + + // Atomically claim the resume. The single winner drives the run; a racing second + // submit (double-click, two tabs) gets false and must not re-drive — that would + // re-execute tools and double-bill. + // + // The claim runs AFTER the slot increment above but BEFORE the run's own try/finally + // that releases it, so a store/Redis error here (unlike the clean `!claimed` branch) + // would leak the concurrency slot until the counter TTL expires — spuriously 429'ing + // the user when they retry the still-paused approval. Release the slot on that path too. + let claimed; + try { + /** The CAS that reopens steering must also publish THIS owner's seal + * capability. A separate write after status=`running` leaves a window in + * which steer/arm requests read the previous replica's capability. */ + claimed = await GenerationJobManager.approvals.resolve( streamId, - jobCreatedAt: job.createdAt, - status: 'interrupted', - conversationId, - error: 'Schedule was disabled, changed, or deleted before approval', - }); - if (checkpointNamespace !== '') { - await deleteAgentCheckpoint(conversationId, checkpointerCfg, undefined, { - checkpointNamespace, - }).catch((error) => { - logger.warn('[ResumeAgentController] Failed to prune stale schedule checkpoint', error); - }); + pendingAction.actionId, + { + preemptCapable: isSteerPreemptSupported(), + providerExecutionId, + providerDrained: true, + ...(resolvedAskUserQuestion && { resolvedAskUserQuestions }), + }, + job.createdAt, + ); + } catch (err) { + const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); + await rollbackUnconsumedScheduleClaim(currentJob); + await releaseScheduleFence(); + await decrementPendingRequest(userId); + logger.error('[ResumeAgentController] Failed to claim resume', getSafeErrorMetadata(err)); + return sendGenerationJson(res, 500, { error: 'Failed to resume' }, generationProtocolVersion); + } + if (!claimed) { + await decrementPendingRequest(userId); + const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); + await rollbackUnconsumedScheduleClaim(currentJob); + await releaseScheduleFence(); + if (currentJob != null && currentJob.createdAt !== job.createdAt) { + return sendGenerationJson(res, 409, { code: 'RUN_REPLACED' }, generationProtocolVersion); } return sendGenerationJson( res, 409, - { code: 'SCHEDULE_NO_LONGER_ACTIVE', error: 'This schedule can no longer be resumed' }, + { error: 'This action was already resolved or has expired' }, generationProtocolVersion, ); } + + // Linearize the consumed approval against the schedule's live config. The schedule + // document fence was acquired only after all async policy reads, and this atomic + // consume checks its token/revision/enabled state immediately after the approval CAS. + // An edit/disable that won first makes this fail; one that lands afterward is ordered + // after the continuation has started. Never begin provider execution on a stale claim. + if (scheduleId) { + let scheduleClaimCurrent = false; + try { + scheduleClaimCurrent = await finalizeScheduleResumeClaim( + scheduleId, + scheduleResumeClaimToken, + scheduleResumeLeaseBy, + scheduleResumeOptions, + ); + } catch (error) { + logger.error('[ResumeAgentController] Failed to finalize scheduled resume fence', error); + await releaseScheduleFence(); + } + if (!scheduleClaimCurrent) { + await decrementPendingRequest(userId); + let stopped = false; + try { + const abortResult = await GenerationJobManager.abortJob(streamId, { + expectedCreatedAt: job.createdAt, + awaitProviderDrain: true, + }); + // Same authoritative gate as the inactive-schedule path above: only a landed + // abort (or an already-terminal, drained generation) may settle this occurrence. + stopped = isStopConfirmed(abortResult); + } catch (error) { + logger.warn('[ResumeAgentController] Failed to stop stale scheduled resume', error); + } + if (!stopped) { + res.set('Retry-After', '1'); + return sendGenerationJson( + res, + 503, + { + code: 'SCHEDULE_STOP_UNCONFIRMED', + error: 'The stale scheduled resume could not be confirmed stopped.', + }, + generationProtocolVersion, + ); + } + await recordScheduleOutcome({ + scheduleId, + scheduledFor, + streamId, + jobCreatedAt: job.createdAt, + status: 'interrupted', + conversationId, + error: 'Schedule was disabled, changed, or deleted before approval', + }); + if (checkpointNamespace !== '') { + await deleteAgentCheckpoint(conversationId, checkpointerCfg, undefined, { + checkpointNamespace, + }).catch((error) => { + logger.warn('[ResumeAgentController] Failed to prune stale schedule checkpoint', error); + }); + } + return sendGenerationJson( + res, + 409, + { code: 'SCHEDULE_NO_LONGER_ACTIVE', error: 'This schedule can no longer be resumed' }, + generationProtocolVersion, + ); + } + } + + eventLeaseTransferredToRun = true; + } finally { + if (!eventLeaseTransferredToRun) { + await releaseEventChildLease?.(); + releaseEventChildLease = undefined; + } } /** @@ -1581,6 +1699,7 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) ).catch((drainError) => { logger.warn('[ResumeAgentController] Failed to record provider drain', drainError); }); + await releaseEventChildLease?.(); } } }; diff --git a/api/server/middleware/validate/subagentThreadTurn.js b/api/server/middleware/validate/subagentThreadTurn.js index 07374a325b..7a626466ce 100644 --- a/api/server/middleware/validate/subagentThreadTurn.js +++ b/api/server/middleware/validate/subagentThreadTurn.js @@ -1,8 +1,17 @@ -const { createSubagentThreadTurnGuard } = require('@librechat/api'); +const { createSubagentThreadTurnGuard, GenerationJobManager } = require('@librechat/api'); const subagentThreadTaskStore = require('~/server/services/Endpoints/agents/subagentThreadStore'); const db = require('~/models'); module.exports = createSubagentThreadTurnGuard({ getConvo: db.getConvo, + getEventBinding: db.getAgentEventBinding, + isHumanResumeAllowed: async ({ userId, tenantId, conversationId }) => { + const job = await GenerationJobManager.getJob(conversationId); + return ( + job?.status === 'requires_action' && + job.metadata?.userId === userId && + (job.metadata?.tenantId ?? undefined) === tenantId + ); + }, store: subagentThreadTaskStore, }); diff --git a/api/server/routes/__test-utils__/convos-route-mocks.js b/api/server/routes/__test-utils__/convos-route-mocks.js index e40b577eab..abd32fc437 100644 --- a/api/server/routes/__test-utils__/convos-route-mocks.js +++ b/api/server/routes/__test-utils__/convos-route-mocks.js @@ -1,8 +1,13 @@ const archiveAllHandler = jest.fn(); +const generationJobManager = { + getJob: jest.fn().mockResolvedValue(null), + abortJob: jest.fn().mockResolvedValue({ success: true }), +}; const subagentActivityHandlerInputs = []; module.exports = { archiveAllHandler, + generationJobManager, subagentActivityHandlerInputs, agents: () => ({ sleep: jest.fn() }), @@ -35,6 +40,10 @@ module.exports = { return archiveAllHandler; }), createSubagentThreadViewHandler: jest.fn(() => (_req, res) => res.status(200).json({})), + GenerationJobManager: generationJobManager, + isStopConfirmed: jest.fn( + (result) => result?.success === true || result?.failureReason === 'already_settled', + ), createSubagentActivityStreamHandler: jest.fn((deps, stream) => { subagentActivityHandlerInputs.push({ deps, stream }); return (_req, res) => res.status(200).end(); @@ -73,6 +82,7 @@ module.exports = { getConvosByCursor: jest.fn(), getConvo: jest.fn(), deleteConvos: jest.fn(), + deleteMessages: jest.fn().mockResolvedValue({ deletedCount: 0 }), saveConvo: jest.fn(), }), @@ -82,6 +92,7 @@ module.exports = { getConvosByCursor: jest.fn(), getConvo: jest.fn(), deleteConvos: jest.fn(), + deleteMessages: jest.fn().mockResolvedValue({ deletedCount: 0 }), archiveAllConvos: jest.fn(), saveConvo: jest.fn(), setConvoPinned: jest.fn(), diff --git a/api/server/routes/__tests__/convos.spec.js b/api/server/routes/__tests__/convos.spec.js index d4948ec893..635503a8ea 100644 --- a/api/server/routes/__tests__/convos.spec.js +++ b/api/server/routes/__tests__/convos.spec.js @@ -2,7 +2,7 @@ const express = require('express'); const request = require('supertest'); const MOCKS = '../__test-utils__/convos-route-mocks'; -const { archiveAllHandler, subagentActivityHandlerInputs } = require(MOCKS); +const { archiveAllHandler, generationJobManager, subagentActivityHandlerInputs } = require(MOCKS); jest.mock('@librechat/agents', () => require(MOCKS).agents()); jest.mock('@librechat/api', () => @@ -33,7 +33,13 @@ jest.mock('~/server/services/Endpoints/agents/subagentThreadStore', () => describe('Convos Routes', () => { let app; let convosRouter; - const { deleteToolCalls, deleteConvos, getConvo, saveConvo } = require('~/models'); + const { + deleteToolCalls, + deleteConvos, + deleteMessages, + getConvo, + saveConvo, + } = require('~/models'); const { deleteAgentCheckpoints, deleteAllSharedLinksWithCleanup, @@ -73,6 +79,8 @@ describe('Convos Routes', () => { beforeEach(() => { jest.clearAllMocks(); + generationJobManager.getJob.mockResolvedValue(null); + generationJobManager.abortJob.mockResolvedValue({ success: true }); }); it('binds the activity subscription adapter to the subagent task store', () => { @@ -303,6 +311,31 @@ describe('Convos Routes', () => { expect(subagentThreadStore.cancelAndDrainForOwner).not.toHaveBeenCalled(); }); + it('drains a paused event actor after owner-wide deletion removes its conversation', async () => { + const createdAt = Date.now(); + deleteConvos.mockResolvedValue({ + deletedCount: 1, + conversationIds: ['paused-event-child'], + }); + generationJobManager.getJob.mockImplementation(async (conversationId) => + conversationId === 'paused-event-child' + ? { + metadata: { userId: 'test-user-123' }, + status: 'requires_action', + createdAt, + } + : null, + ); + + const response = await request(app).delete('/api/convos/all'); + + expect(response.status).toBe(201); + expect(generationJobManager.abortJob).toHaveBeenCalledWith('paused-event-child', { + expectedCreatedAt: createdAt, + awaitProviderDrain: true, + }); + }); + it('should delete all conversations, tool calls, and shared links for a user', async () => { const mockDbResponse = { deletedCount: 5, @@ -322,7 +355,11 @@ describe('Convos Routes', () => { expect(response.body).toEqual(mockDbResponse); /** Verify deleteConvos was called with correct userId */ - expect(deleteConvos).toHaveBeenCalledWith('test-user-123', {}); + expect(deleteConvos).toHaveBeenCalledWith( + 'test-user-123', + {}, + expect.objectContaining({ beforeDelete: expect.any(Function) }), + ); expect(deleteConvos).toHaveBeenCalledTimes(1); /** Verify deleteToolCalls was called with correct userId */ @@ -501,7 +538,54 @@ describe('Convos Routes', () => { expect(subagentThreadStore.withOwnerDeletionFence).toHaveBeenCalledTimes(1); expect(subagentThreadStore.withOwnerDeletionFence.mock.calls[0][0]).toBe('test-user-123'); expect(subagentThreadStore.cancelAndDrainForOwner).not.toHaveBeenCalled(); - expect(deleteConvos).toHaveBeenCalledWith('test-user-123', {}); + expect(deleteConvos).toHaveBeenCalledWith( + 'test-user-123', + {}, + expect.objectContaining({ beforeDelete: expect.any(Function) }), + ); + }); + + it('drains a paused event actor after an empty-filter deletion removes it', async () => { + const createdAt = Date.now(); + deleteConvos.mockResolvedValue({ + deletedCount: 1, + conversationIds: ['paused-event-child'], + }); + generationJobManager.getJob.mockImplementation(async (conversationId) => + conversationId === 'paused-event-child' + ? { + metadata: { userId: 'test-user-123' }, + status: 'requires_action', + createdAt, + } + : null, + ); + + const response = await request(app) + .delete('/api/convos') + .send({ arg: { thread_id: 'thread-abc' } }); + + expect(response.status).toBe(201); + expect(generationJobManager.abortJob).toHaveBeenCalledWith('paused-event-child', { + expectedCreatedAt: createdAt, + awaitProviderDrain: true, + }); + }); + + it('fails closed before checkpoint pruning when generation lookup stays unavailable', async () => { + deleteConvos.mockResolvedValue({ + deletedCount: 1, + conversationIds: ['paused-event-child'], + }); + generationJobManager.getJob.mockRejectedValue(new Error('generation store unavailable')); + + const response = await request(app) + .delete('/api/convos') + .send({ arg: { thread_id: 'thread-abc' } }); + + expect(response.status).toBe(500); + expect(generationJobManager.getJob).toHaveBeenCalledTimes(3); + expect(deleteAgentCheckpoints).not.toHaveBeenCalled(); }); it('cancels root and descendant leases and cleans every cascaded conversation', async () => { @@ -545,6 +629,117 @@ describe('Convos Routes', () => { ]); }); + it('drains an active child generation and removes persistence that races deletion', async () => { + const createdAt = Date.now(); + deleteConvos.mockResolvedValue({ + deletedCount: 2, + conversationIds: ['parent-conversation', 'child-conversation'], + }); + generationJobManager.getJob.mockImplementation(async (conversationId) => + conversationId === 'child-conversation' + ? { metadata: { userId: 'test-user-123' }, status: 'running', createdAt } + : null, + ); + + const response = await request(app) + .delete('/api/convos') + .send({ arg: { conversationId: 'parent-conversation' } }); + + expect(response.status).toBe(201); + expect(generationJobManager.abortJob).toHaveBeenCalledWith('child-conversation', { + expectedCreatedAt: createdAt, + awaitProviderDrain: true, + }); + expect(deleteConvos).toHaveBeenNthCalledWith(2, 'test-user-123', { + conversationId: { $in: ['parent-conversation', 'child-conversation'] }, + }); + expect(deleteMessages).toHaveBeenCalledWith({ + user: 'test-user-123', + conversationId: { $in: ['parent-conversation', 'child-conversation'] }, + }); + }); + + it('does not prune generation persistence when provider stop is unconfirmed', async () => { + const createdAt = Date.now(); + let deletionCommitted = false; + deleteConvos.mockImplementation(async (_userId, _filter, options) => { + await options.beforeDelete(['child-conversation']); + deletionCommitted = true; + return { + deletedCount: 2, + conversationIds: ['parent-conversation', 'child-conversation'], + }; + }); + generationJobManager.getJob.mockImplementation(async (conversationId) => + conversationId === 'child-conversation' + ? { metadata: { userId: 'test-user-123' }, status: 'running', createdAt } + : null, + ); + generationJobManager.abortJob.mockResolvedValue({ + success: false, + failureReason: 'job_still_active', + }); + + const response = await request(app) + .delete('/api/convos') + .send({ arg: { conversationId: 'parent-conversation' } }); + + expect(response.status).toBe(500); + expect(deleteConvos).toHaveBeenCalledTimes(1); + expect(deletionCommitted).toBe(false); + expect(deleteMessages).not.toHaveBeenCalled(); + }); + + it('drains terminal persistence only for leases removed by this deletion', async () => { + const createdAt = Date.now(); + deleteConvos.mockResolvedValue({ + deletedCount: 2, + conversationIds: ['parent-conversation', 'child-conversation'], + }); + subagentThreadStore.planCancellationForConversations.mockResolvedValueOnce({ + userId: 'test-user-123', + conversationIds: ['parent-conversation'], + scopes: [], + leases: [ + { + taskId: 'related-generation', + parentConversationId: 'parent-conversation', + conversationId: 'child-conversation', + }, + { + taskId: 'unrelated-generation', + parentConversationId: 'other-parent', + conversationId: 'other-child', + }, + ], + }); + let relatedReads = 0; + generationJobManager.getJob.mockImplementation(async (conversationId) => { + if (conversationId !== 'related-generation') return null; + relatedReads += 1; + return { + status: 'complete', + createdAt, + metadata: { + userId: 'test-user-123', + terminalPersistencePending: relatedReads === 1, + }, + }; + }); + + const response = await request(app) + .delete('/api/convos') + .send({ arg: { conversationId: 'parent-conversation' } }); + + expect(response.status).toBe(201); + expect(generationJobManager.abortJob).toHaveBeenCalledWith('related-generation', { + expectedCreatedAt: createdAt, + awaitProviderDrain: true, + }); + expect(generationJobManager.getJob).not.toHaveBeenCalledWith('unrelated-generation'); + expect(deleteConvos).toHaveBeenCalledTimes(2); + }); + it('should delete a single conversation, tool calls, and associated shared links', async () => { const mockConversationId = 'conv-123'; const mockDbResponse = { @@ -571,9 +766,11 @@ describe('Convos Routes', () => { expect(response.body).toEqual(mockDbResponse); /** Verify deleteConvos was called with correct parameters */ - expect(deleteConvos).toHaveBeenCalledWith('test-user-123', { - conversationId: mockConversationId, - }); + expect(deleteConvos).toHaveBeenCalledWith( + 'test-user-123', + { conversationId: mockConversationId }, + expect.objectContaining({ beforeDelete: expect.any(Function) }), + ); /** Verify deleteToolCalls was called */ expect(deleteToolCalls).toHaveBeenCalledWith('test-user-123', mockConversationId); diff --git a/api/server/routes/agents/__tests__/events.spec.js b/api/server/routes/agents/__tests__/events.spec.js index 126703f757..034266a7c7 100644 --- a/api/server/routes/agents/__tests__/events.spec.js +++ b/api/server/routes/agents/__tests__/events.spec.js @@ -5,6 +5,8 @@ const mockEnqueueAgentTrigger = jest.fn(); const mockGetAgentTriggerDeliveryStatus = jest.fn(); const mockEnqueueEvent = jest.fn((_req, res) => res.status(202).json({ id: 'trigger-1' })); const mockGetEvent = jest.fn((_req, res) => res.status(200).json({ status: 'succeeded' })); +const mockRegisterBinding = jest.fn((_req, res) => res.status(201).json({ id: 'evtbind-1' })); +const mockResolveBinding = jest.fn((_req, _res, next) => next()); let mockIngressDependencies; const mockCreateAgentTriggerIngressHandlers = jest.fn((dependencies) => { mockIngressDependencies = dependencies; @@ -15,10 +17,21 @@ const mockCreateAgentTriggerIngressHandlers = jest.fn((dependencies) => { }); jest.mock('@librechat/api', () => ({ + createAgentEventBindingHandlers: () => ({ + register: mockRegisterBinding, + resolve: mockResolveBinding, + }), createAgentTriggerIngressHandlers: mockCreateAgentTriggerIngressHandlers, createMessageFilterPii: () => (_req, _res, next) => next(), })); +jest.mock('~/models', () => ({ + getAgent: jest.fn(), + getConvo: jest.fn(), + getAgentEventBinding: jest.fn(), + reserveSubagentThread: jest.fn(), +})); + jest.mock('~/server/controllers/agents/openai', () => ({ OpenAIChatCompletionController: jest.fn(), ListModelsController: jest.fn(), @@ -56,6 +69,17 @@ describe('Remote Agents event routes', () => { beforeEach(() => { mockEnqueueEvent.mockClear(); mockGetEvent.mockClear(); + mockRegisterBinding.mockClear(); + mockResolveBinding.mockClear(); + }); + + it('registers a source-bound child actor thread', async () => { + const response = await request(app) + .post('/api/agents/v1/events/bindings') + .send({ target: { agentId: 'agent-1' } }); + + expect(response.status).toBe(201); + expect(mockRegisterBinding).toHaveBeenCalledTimes(1); }); it('wires durable event admission to the trigger service', async () => { diff --git a/api/server/routes/agents/openai.js b/api/server/routes/agents/openai.js index 44008bb35f..92ceee96b3 100644 --- a/api/server/routes/agents/openai.js +++ b/api/server/routes/agents/openai.js @@ -19,7 +19,12 @@ * } */ const express = require('express'); -const { createAgentTriggerIngressHandlers, createMessageFilterPii } = require('@librechat/api'); +const { + createAgentEventBindingHandlers, + createAgentTriggerIngressHandlers, + createMessageFilterPii, + isEnabled, +} = require('@librechat/api'); const { OpenAIChatCompletionController, ListModelsController, @@ -37,18 +42,40 @@ const { requireRemoteAgentAuth, checkRemoteAgentsFeature, } = require('./middleware'); +const db = require('~/models'); const router = express.Router(); const eventHandlers = createAgentTriggerIngressHandlers({ enqueue: enqueueAgentTrigger, getDeliveryStatus: getAgentTriggerDeliveryStatus, }); +const eventBindingHandlers = createAgentEventBindingHandlers({ + getAgent: db.getAgent, + getConvo: db.getConvo, + getBinding: db.getAgentEventBinding, + getMessage: db.getMessage, + deleteConvos: db.deleteConvos, + reserveThread: db.reserveSubagentThread, + enabled: () => isEnabled(process.env.ENABLE_AGENT_EVENT_CHILD_TURNS), +}); router.use(preAuthTenantMiddleware); router.use(requireRemoteAgentAuth); router.use(configMiddleware); router.use(checkRemoteAgentsFeature); +/** + * @route POST /v1/events/bindings + * @desc Bind one authenticated source key to a durable child actor thread + * @access Private (API key auth required) + */ +router.post( + '/events/bindings', + agentEventUserLimiter, + checkAgentTriggerPermission, + eventBindingHandlers.register, +); + /** * @route POST /v1/events * @desc Durably deliver a source-neutral event to an agent @@ -58,6 +85,7 @@ router.post( '/events', agentEventUserLimiter, createMessageFilterPii({ getConfig: (req) => req.config?.messageFilter?.pii }), + eventBindingHandlers.resolve, checkAgentTriggerPermission, eventHandlers.enqueueEvent, ); diff --git a/api/server/routes/convos.js b/api/server/routes/convos.js index 86e3981bbc..a549f07a2b 100644 --- a/api/server/routes/convos.js +++ b/api/server/routes/convos.js @@ -16,6 +16,8 @@ const { isContentFilterError, contentFilterBlockResponse, extractConversationTitleContent, + GenerationJobManager, + isStopConfirmed, } = require('@librechat/api'); const { logger } = require('@librechat/data-schemas'); const { CacheKeys, EModelEndpoint } = require('librechat-data-provider'); @@ -152,6 +154,24 @@ router.get('/gen_title/:conversationId', async (req, res) => { const POST_DELETE_CANCEL_ATTEMPTS = 3; const POST_DELETE_CANCEL_BACKOFF_MS = 250; +const GENERATION_PERSISTENCE_DRAIN_TIMEOUT_MS = 45_000; +const GENERATION_PERSISTENCE_DRAIN_POLL_MS = 100; +const GENERATION_LOOKUP_ATTEMPTS = 3; + +async function readGenerationForDeletion(conversationId) { + let lastError; + for (let attempt = 1; attempt <= GENERATION_LOOKUP_ATTEMPTS; attempt += 1) { + try { + return await GenerationJobManager.getJob(conversationId); + } catch (error) { + lastError = error; + if (attempt < GENERATION_LOOKUP_ATTEMPTS) { + await new Promise((resolve) => setTimeout(resolve, 25 * attempt)); + } + } + } + throw lastError; +} /** Replays a cancellation plan after deletion, retrying a transiently unreachable * owner rather than losing the only pass that can stop a late-admitted child. */ @@ -170,6 +190,92 @@ async function retryPostDeleteCancellation(cancellationPlan, deletedConversation } } +/** Confirms every exact generation is stopped before its conversation wave is removed. */ +async function confirmAgentGenerationsDrained(userId, conversationIds, leaseTaskIds = []) { + let foundActiveGeneration = false; + const drainErrors = []; + const generationIds = [...new Set([...conversationIds, ...leaseTaskIds])]; + await Promise.all( + generationIds.map(async (conversationId) => { + let job; + try { + job = await readGenerationForDeletion(conversationId); + } catch (error) { + logger.warn('Deleted child generation lookup failed', error); + foundActiveGeneration = true; + drainErrors.push(error); + return; + } + if (job == null || job.metadata?.userId !== userId) { + return; + } + const needsDrain = + job.status === 'running' || + job.status === 'requires_action' || + job.metadata?.terminalPersistencePending === true; + if (!needsDrain) return; + foundActiveGeneration = true; + try { + const abortResult = await GenerationJobManager.abortJob(conversationId, { + expectedCreatedAt: job.createdAt, + awaitProviderDrain: true, + }); + if (!isStopConfirmed(abortResult)) { + throw new Error( + `Could not confirm generation stop for ${conversationId}: ${abortResult?.failureReason ?? 'unknown'}`, + ); + } + const deadline = Date.now() + GENERATION_PERSISTENCE_DRAIN_TIMEOUT_MS; + while (true) { + const current = await GenerationJobManager.getJob(conversationId); + if ( + current == null || + current.createdAt !== job.createdAt || + current.metadata?.terminalPersistencePending !== true + ) { + break; + } + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for generation persistence: ${conversationId}`); + } + await new Promise((resolve) => setTimeout(resolve, GENERATION_PERSISTENCE_DRAIN_POLL_MS)); + } + } catch (error) { + logger.warn('Deleted child generation drain failed', error); + drainErrors.push(error); + } + }), + ); + if (!foundActiveGeneration) { + return false; + } + if (drainErrors.length > 0) { + throw new Error('One or more deleted child generations could not be confirmed drained.'); + } + return true; +} + +/** Stops event-bound child generations on their owning replica and then removes + * persistence that raced the first conversation cascade. */ +async function drainDeletedAgentGenerations(userId, conversationIds, leaseTaskIds = []) { + const foundActiveGeneration = await confirmAgentGenerationsDrained( + userId, + conversationIds, + leaseTaskIds, + ); + if (!foundActiveGeneration) { + return; + } + try { + await db.deleteConvos(userId, { conversationId: { $in: conversationIds } }); + } catch { + // Expected when no generation raced the first cascade. + } + await db + .deleteMessages({ user: userId, conversationId: { $in: conversationIds } }) + .catch((error) => logger.warn('Deleted child message remnant cleanup failed', error)); +} + router.delete('/', configMiddleware, async (req, res) => { let filter = {}; const { conversationId, source, thread_id, endpoint } = req.body?.arg ?? {}; @@ -217,12 +323,18 @@ router.delete('/', configMiddleware, async (req, res) => { tenantId, ); await subagentThreadTaskStore.cancelPlan(cancellationPlan); - dbResponse = await db.deleteConvos(req.user.id, filter); + dbResponse = await db.deleteConvos(req.user.id, filter, { + beforeDelete: (conversationIds) => + confirmAgentGenerationsDrained(req.user.id, conversationIds), + }); } else { /** An empty filter deletes every conversation this owner has, so it runs behind * the same admission fence as `DELETE /all` rather than a bare drain. */ dbResponse = await subagentThreadTaskStore.withOwnerDeletionFence(req.user.id, tenantId, () => - db.deleteConvos(req.user.id, filter), + db.deleteConvos(req.user.id, filter, { + beforeDelete: (conversationIds) => + confirmAgentGenerationsDrained(req.user.id, conversationIds), + }), ); } const deletedConversationIds = @@ -235,6 +347,23 @@ router.delete('/', configMiddleware, async (req, res) => { * stop a child admitted after the first one. It cannot fail the request — the * deletion already committed — so it retries briefly before giving up. */ await retryPostDeleteCancellation(cancellationPlan, deletedConversationIds); + await drainDeletedAgentGenerations( + req.user.id, + deletedConversationIds, + cancellationPlan.leases + .filter( + (lease) => + deletedConversationIds.includes(lease.parentConversationId) || + deletedConversationIds.includes(lease.conversationId), + ) + .map((lease) => lease.taskId), + ); + } else if (deletedConversationIds.length > 0) { + /** Owner-wide deletion drains lease-backed tasks before the cascade, but a + * requires_action event actor has intentionally released its lease. Its durable + * generation is still addressable by the deleted conversation id and must be + * terminalized before its checkpoint is pruned. */ + await drainDeletedAgentGenerations(req.user.id, deletedConversationIds); } // HITL: prune the deleted conversations' durable checkpoints — a paused run's // checkpoint would otherwise persist until the Mongo TTL. Never throws. @@ -267,8 +396,17 @@ router.delete('/all', configMiddleware, async (req, res) => { const dbResponse = await subagentThreadTaskStore.withOwnerDeletionFence( req.user.id, tenantId, - () => db.deleteConvos(req.user.id, {}), + () => + db.deleteConvos( + req.user.id, + {}, + { + beforeDelete: (conversationIds) => + confirmAgentGenerationsDrained(req.user.id, conversationIds), + }, + ), ); + await drainDeletedAgentGenerations(req.user.id, dbResponse.conversationIds ?? []); // HITL: prune ALL the deleted conversations' durable checkpoints in one bulk pass. await deleteAgentCheckpoints( dbResponse.conversationIds, diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js index e662e1bb31..f7f493a864 100644 --- a/api/server/services/Agents/triggers.js +++ b/api/server/services/Agents/triggers.js @@ -1,16 +1,25 @@ const { createAgentTriggerService, + createAgentEventContinueResolver, createSubagentCompletionWakeupResolver, GenerationJobManager, + isEnabled, } = require('@librechat/api'); const methods = require('~/models'); +const completionResolver = createSubagentCompletionWakeupResolver({ + methods, + getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId), +}); + const service = createAgentTriggerService({ methods, isPrincipalActive: methods.isAgentTriggerPrincipalActive, - prepareContinue: createSubagentCompletionWakeupResolver({ + prepareContinue: createAgentEventContinueResolver({ methods, getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId), + fallback: completionResolver, + enabled: () => isEnabled(process.env.ENABLE_AGENT_EVENT_CHILD_TURNS), }), }); diff --git a/api/server/services/Endpoints/agents/eventChildLease.js b/api/server/services/Endpoints/agents/eventChildLease.js new file mode 100644 index 0000000000..c8077f7043 --- /dev/null +++ b/api/server/services/Endpoints/agents/eventChildLease.js @@ -0,0 +1,24 @@ +const librechatApi = require('@librechat/api'); +const { GenerationJobManager } = librechatApi; +const { + acquireSubagentThreadLease, + renewSubagentThreadLease, + releaseSubagentThreadLease, +} = require('~/models'); + +let acquireLease; + +function acquireEventChildGenerationLease(input) { + acquireLease ??= librechatApi.createEventChildGenerationLeaseAcquirer({ + methods: { + acquireSubagentThreadLease, + renewSubagentThreadLease, + releaseSubagentThreadLease, + }, + abortGeneration: (streamId, options) => GenerationJobManager.abortJob(streamId, options), + }); + + return acquireLease(input); +} + +module.exports = { acquireEventChildGenerationLease }; diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.js b/api/server/services/Endpoints/agents/subagentThreadStore.js index 534629a38a..a42e9cee84 100644 --- a/api/server/services/Endpoints/agents/subagentThreadStore.js +++ b/api/server/services/Endpoints/agents/subagentThreadStore.js @@ -6,6 +6,7 @@ const { duplicateIoRedisClient, createSubagentThreadTaskStore, createSubagentCompletionWakeupHandler, + GenerationJobManager, RedisSubagentTaskControlTransport, RedisEventTransport, SubagentActivityStream, @@ -17,6 +18,40 @@ const { enqueueAgentTrigger } = require('../../Agents/triggers'); * permanently reject the new `continue` envelope. Enable only after every API * replica runs a release that understands completion wakeups. */ const completionWakeupsEnabled = isEnabled(process.env.ENABLE_SUBAGENT_COMPLETION_WAKEUPS); +const GENERATION_DRAIN_TIMEOUT_MS = 45_000; +const GENERATION_DRAIN_POLL_MS = 100; + +async function cancelUnroutedGeneration({ userId, tenantId, taskId }) { + let job = await GenerationJobManager.getJob(taskId); + if ( + job == null || + job.metadata?.userId !== userId || + (job.metadata?.tenantId ?? undefined) !== tenantId + ) { + return false; + } + await GenerationJobManager.abortJob(taskId, { + expectedCreatedAt: job.createdAt, + awaitProviderDrain: true, + }); + const deadline = Date.now() + GENERATION_DRAIN_TIMEOUT_MS; + while (true) { + job = await GenerationJobManager.getJob(taskId); + if (job == null || job.metadata?.terminalPersistencePending !== true) { + return ( + job == null || + (job.metadata?.userId === userId && + (job.metadata?.tenantId ?? undefined) === tenantId && + job.status !== 'running' && + job.status !== 'requires_action') + ); + } + if (Date.now() >= deadline) { + return false; + } + await new Promise((resolve) => setTimeout(resolve, GENERATION_DRAIN_POLL_MS)); + } +} /** Durable logical threads use normal LibreChat conversations/messages. Mongo * fences continuation; optional Redis routing reaches the live owning process. */ @@ -42,6 +77,7 @@ const subagentThreadTaskStore = createSubagentThreadTaskStore( fenceOwnerAdmission: db.fenceSubagentAdmission, renewOwnerAdmission: db.renewSubagentAdmission, releaseOwnerAdmission: db.releaseSubagentAdmission, + cancelUnroutedTask: cancelUnroutedGeneration, ...(completionWakeupsEnabled && { onTaskPrepared: createSubagentCompletionWakeupHandler(enqueueAgentTrigger), }), diff --git a/api/server/services/Files/retention.js b/api/server/services/Files/retention.js index e7394e2952..a5fe2054d5 100644 --- a/api/server/services/Files/retention.js +++ b/api/server/services/Files/retention.js @@ -11,6 +11,18 @@ const getRetentionDependencies = () => ({ logger, }); +/** Event-bound actors inherit the binding's server-authenticated deadline. Never + * extend files beyond the child conversation that makes them addressable. */ +function getEventBindingRetention(req) { + const retention = req?._agentEventBindingRetention; + if (retention?.expiredAt == null) { + return null; + } + const expiredAt = + retention.expiredAt instanceof Date ? retention.expiredAt : new Date(retention.expiredAt); + return Number.isNaN(expiredAt.getTime()) ? null : { expiredAt }; +} + /** * Returns `{ expiredAt }` when the request indicates data retention applies, otherwise `{}`. * Spread into file data objects before calling createFile. @@ -18,6 +30,10 @@ const getRetentionDependencies = () => ({ * @returns {Promise<{ expiredAt?: Date | null }>} */ async function getRetentionExpiry(req) { + const inherited = getEventBindingRetention(req); + if (inherited != null) { + return inherited; + } return getRetentionExpiryWithDeps(req, getRetentionDependencies()); } @@ -31,6 +47,10 @@ async function getRetentionExpiry(req) { * @returns {Promise<{ expiredAt?: Date | null }>} */ async function getAgentFileRetentionExpiry({ tool_resource, toolResource, ...params }) { + const inherited = getEventBindingRetention(params.req); + if (inherited != null) { + return inherited; + } return getAgentFileRetentionExpiryWithDeps( { ...params, toolResource: tool_resource ?? toolResource }, getRetentionDependencies(), diff --git a/api/server/services/Files/retention.spec.js b/api/server/services/Files/retention.spec.js new file mode 100644 index 0000000000..e9a365d530 --- /dev/null +++ b/api/server/services/Files/retention.spec.js @@ -0,0 +1,33 @@ +const mockGetRetentionExpiry = jest.fn(); +const mockGetAgentFileRetentionExpiry = jest.fn(); + +jest.mock('@librechat/api', () => ({ + getRetentionExpiry: (...args) => mockGetRetentionExpiry(...args), + getAgentFileRetentionExpiry: (...args) => mockGetAgentFileRetentionExpiry(...args), +})); +jest.mock('@librechat/data-schemas', () => ({ + logger: {}, + createTempChatExpirationDate: jest.fn(), +})); +jest.mock('~/models', () => ({ getConvo: jest.fn() })); + +const { getRetentionExpiry, getAgentFileRetentionExpiry } = require('./retention'); + +describe('event-bound file retention', () => { + const expiredAt = new Date('2026-08-22T12:00:00.000Z'); + const req = { _agentEventBindingRetention: { isTemporary: false, expiredAt } }; + + beforeEach(() => jest.clearAllMocks()); + + it('uses the trusted binding deadline for generated files', async () => { + await expect(getRetentionExpiry(req)).resolves.toEqual({ expiredAt }); + expect(mockGetRetentionExpiry).not.toHaveBeenCalled(); + }); + + it('uses the same binding deadline for agent resource files', async () => { + await expect( + getAgentFileRetentionExpiry({ req, tool_resource: 'execute_code' }), + ).resolves.toEqual({ expiredAt }); + expect(mockGetAgentFileRetentionExpiry).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/agents/eventRetention.ts b/packages/api/src/agents/eventRetention.ts new file mode 100644 index 0000000000..7637de5f8c --- /dev/null +++ b/packages/api/src/agents/eventRetention.ts @@ -0,0 +1,9 @@ +/** A missing deadline is durable; an invalid or elapsed deadline is never active. */ +export function isAgentEventRetentionActive(expiredAt: unknown, now: number = Date.now()): boolean { + if (expiredAt == null) { + return true; + } + const deadline = + expiredAt instanceof Date ? expiredAt.getTime() : new Date(String(expiredAt)).getTime(); + return Number.isFinite(deadline) && deadline > now; +} diff --git a/packages/api/src/agents/guard.spec.ts b/packages/api/src/agents/guard.spec.ts index ee2a69c9b2..8ad2979ba7 100644 --- a/packages/api/src/agents/guard.spec.ts +++ b/packages/api/src/agents/guard.spec.ts @@ -20,6 +20,7 @@ function childConversation(): IConversation { parentConversationId: 'parent-conversation', parentMessageId: 'parent-message', parentToolCallId: 'parent-tool-call', + parentAgentId: 'parent-agent', subagentType: 'child-agent', subagentKind: 'agent', depth: 1, @@ -46,20 +47,43 @@ function makeStore(): SubagentThreadTaskStore { }); } -function createApp(getConvo: AllMethods['getConvo'], store: SubagentThreadTaskStore) { +function createApp( + getConvo: AllMethods['getConvo'], + store: SubagentThreadTaskStore, + getEventBinding?: AllMethods['getAgentEventBinding'], + isHumanResumeAllowed?: () => Promise, +) { const app = express(); app.use(express.json()); app.use((req, _res, next) => { req.user = { id: 'user-1', tenantId: 'tenant-1' }; + (req as typeof req & { _isAgentTrigger?: boolean })._isAgentTrigger = + req.get('x-test-trigger') === '1'; next(); }); - app.post('/chat', createSubagentThreadTurnGuard({ getConvo, store }), (req, res) => { + const guard = createSubagentThreadTurnGuard({ + getConvo, + store, + getEventBinding, + isHumanResumeAllowed, + }); + const handler = (req: express.Request, res: express.Response) => { res.json({ ok: true, resolvedConversationId: (req as typeof req & { resolvedConversation?: IConversation | null }) .resolvedConversation?.conversationId, + parentConversationId: ( + req as typeof req & { _agentEventBindingParentConversationId?: string } + )._agentEventBindingParentConversationId, + retention: ( + req as typeof req & { + _agentEventBindingRetention?: { isTemporary?: boolean; expiredAt?: Date }; + } + )._agentEventBindingRetention, }); - }); + }; + app.post('/chat', guard, handler); + app.post('/resume', guard, handler); return app; } @@ -98,6 +122,145 @@ describe('subagent child-thread write policy', () => { expect(response.body).toEqual({ error: CHILD_THREAD_READ_ONLY_ERROR }); }); + it('allows only the authenticated trigger bound to this child conversation', async () => { + const store = makeStore(); + const getEventBinding = jest.fn(async () => ({ + conversationId: 'child-conversation', + agentId: 'child-agent', + tenantId: 'tenant-1', + isTemporary: true, + expiredAt: new Date('2099-08-22T00:00:00.000Z'), + binding: { + bindingId: `evtbind_${'a'.repeat(48)}`, + sourceKeyId: 'source-key', + actorId: 'player', + }, + lineage: childConversation().subagentThread!, + })); + const app = createApp( + jest.fn(async (_user, conversationId) => + conversationId === 'parent-conversation' + ? ({ + conversationId, + agent_id: 'parent-agent', + tenantId: 'tenant-1', + } as IConversation) + : childConversation(), + ), + store, + getEventBinding as AllMethods['getAgentEventBinding'], + ); + + const response = await request(app) + .post('/chat') + .set('x-test-trigger', '1') + .set('x-lc-agent-event-binding', `evtbind_${'a'.repeat(48)}`) + .set('x-lc-agent-event-source-key', 'source-key') + .send({ conversationId: 'child-conversation' }); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ + parentConversationId: 'parent-conversation', + retention: { isTemporary: true, expiredAt: '2099-08-22T00:00:00.000Z' }, + }); + expect(getEventBinding).toHaveBeenCalledWith({ + user: 'user-1', + tenantId: 'tenant-1', + bindingId: `evtbind_${'a'.repeat(48)}`, + sourceKeyId: 'source-key', + }); + }); + + it('allows only an exact pending human resume for a bound child', async () => { + const store = makeStore(); + const reservedThreadId = createSubagentThreadId('scope', 'bound-child'); + const boundChild = { + ...childConversation(), + conversationId: reservedThreadId, + tenantId: 'tenant-1', + isTemporary: true, + agentEventBinding: { + bindingId: `evtbind_${'b'.repeat(48)}`, + sourceKeyId: 'source-key', + actorId: 'player', + }, + } as unknown as IConversation; + const isHumanResumeAllowed = jest.fn(async () => true); + const app = createApp( + jest.fn(async (_user, conversationId) => + conversationId === 'parent-conversation' + ? ({ conversationId, agent_id: 'parent-agent', tenantId: 'tenant-1' } as IConversation) + : boundChild, + ), + store, + undefined, + isHumanResumeAllowed, + ); + + const response = await request(app) + .post('/resume') + .send({ conversationId: reservedThreadId, actionId: 'action-1' }); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ + resolvedConversationId: reservedThreadId, + parentConversationId: 'parent-conversation', + retention: { isTemporary: true }, + }); + expect(isHumanResumeAllowed).toHaveBeenCalledWith({ + userId: 'user-1', + tenantId: 'tenant-1', + conversationId: reservedThreadId, + }); + }); + + it('rejects trigger continuations and human resumes after binding retention expires', async () => { + const store = makeStore(); + const expiredChild = { + ...childConversation(), + tenantId: 'tenant-1', + expiredAt: new Date(0), + agentEventBinding: { + bindingId: `evtbind_${'c'.repeat(48)}`, + sourceKeyId: 'source-key', + actorId: 'player', + }, + } as unknown as IConversation; + const getEventBinding = jest.fn(async () => ({ + conversationId: expiredChild.conversationId, + agentId: expiredChild.agent_id, + tenantId: 'tenant-1', + expiredAt: expiredChild.expiredAt, + binding: expiredChild.agentEventBinding, + lineage: expiredChild.subagentThread!, + })); + const getConvo = jest.fn(async (_user, conversationId) => + conversationId === 'parent-conversation' + ? ({ conversationId, agent_id: 'parent-agent', tenantId: 'tenant-1' } as IConversation) + : expiredChild, + ); + const isHumanResumeAllowed = jest.fn(async () => true); + const testApp = createApp( + getConvo, + store, + getEventBinding as AllMethods['getAgentEventBinding'], + isHumanResumeAllowed, + ); + + const trigger = await request(testApp) + .post('/chat') + .set('x-test-trigger', '1') + .set('x-lc-agent-event-binding', `evtbind_${'c'.repeat(48)}`) + .set('x-lc-agent-event-source-key', 'source-key') + .send({ conversationId: 'child-conversation' }); + const human = await request(testApp) + .post('/resume') + .send({ conversationId: 'child-conversation', actionId: 'action-1' }); + + expect(trigger.status).toBe(409); + expect(human.status).toBe(409); + }); + it('rejects a provisional child before its conversation becomes durable', async () => { const store = makeStore(); jest.spyOn(store, 'isThreadActiveForOwner').mockReturnValue(true); @@ -128,7 +291,7 @@ describe('subagent child-thread write policy', () => { expect(response.status).toBe(409); expect(response.body).toEqual({ error: CHILD_THREAD_READ_ONLY_ERROR }); - expect(getConvo).not.toHaveBeenCalled(); + expect(getConvo).toHaveBeenCalledWith('user-1', reservedThreadId); }); it('keeps the shared policy owner-scoped and treats child lineage as immutable', async () => { diff --git a/packages/api/src/agents/guard.ts b/packages/api/src/agents/guard.ts index c185bdaa4c..94d69f2460 100644 --- a/packages/api/src/agents/guard.ts +++ b/packages/api/src/agents/guard.ts @@ -3,6 +3,7 @@ import type { ConversationMethods, IConversation } from '@librechat/data-schemas import type { NextFunction, Request, RequestHandler, Response } from 'express'; import type { SubagentThreadTaskStore } from './subagentThreads'; import { isReservedSubagentThreadId } from './subagentThreadIds'; +import { isAgentEventRetentionActive } from './eventRetention'; export const CHILD_THREAD_READ_ONLY_ERROR = 'This subagent thread is view-only. Continue it from its parent agent or create a separate chat.'; @@ -20,6 +21,8 @@ interface SubagentTurnUser { export interface SubagentThreadWriteGuardDeps { getConvo: ConversationMethods['getConvo']; + getEventBinding?: ConversationMethods['getAgentEventBinding']; + isHumanResumeAllowed?: (target: SubagentThreadWriteTarget) => Promise; store: SubagentThreadTaskStore; } @@ -36,6 +39,75 @@ interface SubagentThreadWriteResolution { interface ResolvedConversationRequest extends Request { resolvedConversation?: IConversation | null; + _isAgentTrigger?: boolean; + _agentEventBindingRetention?: { + isTemporary?: boolean; + expiredAt?: Date; + }; + _agentEventBindingParentConversationId?: string; + _agentEventBindingParentAgentId?: string; + _agentEventBindingTenantId?: string; +} + +function applyEventBindingContext( + request: ResolvedConversationRequest, + conversation: IConversation, +): void { + request.resolvedConversation = conversation; + request._agentEventBindingRetention = { + ...(conversation.isTemporary == null ? {} : { isTemporary: conversation.isTemporary }), + ...(conversation.expiredAt == null ? {} : { expiredAt: conversation.expiredAt }), + }; + request._agentEventBindingParentConversationId = + conversation.subagentThread?.parentConversationId; + request._agentEventBindingParentAgentId = conversation.subagentThread?.parentAgentId; + request._agentEventBindingTenantId = conversation.tenantId; +} + +async function isBoundEventContinuation( + deps: SubagentThreadWriteGuardDeps, + request: ResolvedConversationRequest, + target: SubagentThreadWriteTarget, +): Promise { + if (request._isAgentTrigger !== true || deps.getEventBinding == null) { + return null; + } + const bindingId = request.get('x-lc-agent-event-binding'); + const sourceKeyId = request.get('x-lc-agent-event-source-key'); + if (bindingId == null || sourceKeyId == null) { + return null; + } + const binding = await deps.getEventBinding({ + user: target.userId, + bindingId, + sourceKeyId, + ...(target.tenantId == null ? {} : { tenantId: target.tenantId }), + }); + if ( + binding?.conversationId !== target.conversationId || + !isAgentEventRetentionActive(binding?.expiredAt) + ) { + return null; + } + const parent = await deps.getConvo(target.userId, binding.lineage.parentConversationId); + if ( + parent == null || + parent.subagentThread != null || + parent.agent_id !== binding.lineage.parentAgentId || + (parent.tenantId ?? undefined) !== target.tenantId || + !isAgentEventRetentionActive(parent.expiredAt) + ) { + return null; + } + return { + conversationId: binding.conversationId, + agent_id: binding.agentId, + ...(binding.tenantId == null ? {} : { tenantId: binding.tenantId }), + ...(binding.isTemporary == null ? {} : { isTemporary: binding.isTemporary }), + ...(binding.expiredAt == null ? {} : { expiredAt: binding.expiredAt }), + agentEventBinding: binding.binding, + subagentThread: binding.lineage, + } as unknown as IConversation; } async function resolveSubagentThreadWrite( @@ -45,7 +117,8 @@ async function resolveSubagentThreadWrite( /** New child IDs are returned synchronously by the SDK before Mongo creation can * finish. Their reserved UUID namespace closes that brief window on every replica. */ if (isReservedSubagentThreadId(conversationId)) { - return { blocked: true }; + const conversation = await getConvo(userId, conversationId); + return { blocked: true, conversation }; } if (store.isThreadActiveForOwner(userId, conversationId, tenantId)) { return { blocked: true }; @@ -98,6 +171,49 @@ export function createSubagentThreadTurnGuard(deps: SubagentThreadWriteGuardDeps next(); return; } + const resolvedRequest = request as ResolvedConversationRequest; + const resolvedConversation = resolved.conversation; + const lineage = resolvedConversation?.subagentThread; + const humanResume = deps.isHumanResumeAllowed; + if ( + request.path === '/resume' && + resolvedConversation?.agentEventBinding != null && + lineage != null && + isAgentEventRetentionActive(resolvedConversation.expiredAt) && + humanResume != null && + (await humanResume({ + userId, + conversationId: candidateConversationId, + ...(tenantId == null ? {} : { tenantId }), + })) + ) { + const parent = await deps.getConvo(userId, lineage.parentConversationId); + if ( + parent != null && + parent.subagentThread == null && + parent.agent_id === lineage.parentAgentId && + (parent.tenantId ?? undefined) === tenantId && + isAgentEventRetentionActive(parent.expiredAt) + ) { + applyEventBindingContext(resolvedRequest, resolvedConversation); + next(); + return; + } + } + const boundConversation = await isBoundEventContinuation( + deps, + request as ResolvedConversationRequest, + { + userId, + conversationId: candidateConversationId, + ...(tenantId == null ? {} : { tenantId }), + }, + ); + if (boundConversation != null) { + applyEventBindingContext(resolvedRequest, boundConversation); + next(); + return; + } res.status(409).json({ error: CHILD_THREAD_READ_ONLY_ERROR }); } catch (error) { next(error); diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts index 80cfb09bef..4e4b04eb94 100644 --- a/packages/api/src/agents/index.ts +++ b/packages/api/src/agents/index.ts @@ -11,6 +11,7 @@ export * from './conversation'; export * from './discovery'; export * from './edges'; export * from './errors'; +export * from './eventRetention'; export * from './envelope'; export * from './execution'; export * from './handlers'; diff --git a/packages/api/src/agents/subagentThreads.spec.ts b/packages/api/src/agents/subagentThreads.spec.ts index b8ca282635..47b62429bb 100644 --- a/packages/api/src/agents/subagentThreads.spec.ts +++ b/packages/api/src/agents/subagentThreads.spec.ts @@ -2925,6 +2925,62 @@ describe('SubagentThreadTaskStore', () => { ]); }); + it('delegates an owner drain for host work that is not in the task store', async () => { + const userId = 'host-generation-drain-user'; + const parentConversationId = randomUUID(); + const conversationId = randomUUID(); + const taskId = randomUUID(); + const token = randomUUID(); + await saveParent(userId, parentConversationId); + await methods.saveConvo( + { userId }, + { + conversationId, + endpoint: EModelEndpoint.agents, + title: 'Event actor', + agent_id: 'child-agent', + subagentThread: { + rootConversationId: parentConversationId, + parentConversationId, + parentMessageId: 'parent-message', + parentToolCallId: 'event-binding', + parentAgentId: 'parent-agent', + subagentType: 'child-agent', + subagentKind: 'agent', + depth: 1, + }, + }, + ); + await methods.acquireSubagentThreadLease({ + user: userId, + conversationId, + taskId, + token, + now: new Date(), + expiresAt: new Date(Date.now() + 30_000), + }); + const cancelUnroutedTask = jest.fn(async () => { + await methods.releaseSubagentThreadLease({ user: userId, conversationId, token }); + return true; + }); + const deletingStore = new SubagentThreadTaskStore(methods, { + cancelUnroutedTask, + ownerDrainPollMs: 1, + }); + + await deletingStore.cancelAndDrainForOwner(userId); + + expect(cancelUnroutedTask).toHaveBeenCalledWith({ + userId, + parentConversationId, + taskId, + tenantId: undefined, + }); + expect(await methods.countActiveSubagentThreadLeases({ user: userId, now: new Date() })).toBe( + 0, + ); + }); + it('bounds durable delegation depth to one by default', async () => { const userId = 'depth-user'; const rootConversationId = randomUUID(); diff --git a/packages/api/src/agents/subagentThreads.ts b/packages/api/src/agents/subagentThreads.ts index 08053c0c49..b90b859be4 100644 --- a/packages/api/src/agents/subagentThreads.ts +++ b/packages/api/src/agents/subagentThreads.ts @@ -192,6 +192,14 @@ export interface SubagentThreadTaskStoreOptions extends InMemorySubagentTaskStor fenceOwnerAdmission?: (userId: string, token: string, fencedUntil: Date) => Promise; renewOwnerAdmission?: (userId: string, token: string, fencedUntil: Date) => Promise; releaseOwnerAdmission?: (userId: string, token: string) => Promise; + /** Host-owned work may share the durable child lease protocol without living in + * this in-memory task store. Return true only after that work is stopped. */ + cancelUnroutedTask?: (target: { + userId: string; + parentConversationId: string; + taskId: string; + tenantId?: string; + }) => Promise; onTaskPrepared?: (registration: SubagentTaskWakeupRegistration) => Promise | void; } @@ -473,6 +481,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { ) => Promise; private readonly releaseOwnerAdmission?: (userId: string, token: string) => Promise; + private readonly cancelUnroutedTask?: SubagentThreadTaskStoreOptions['cancelUnroutedTask']; private readonly onTaskPrepared?: SubagentThreadTaskStoreOptions['onTaskPrepared']; private taskControlTransport?: SubagentTaskControlTransport; private activityStream = new SubagentActivityStream(new InMemoryEventTransport()); @@ -506,6 +515,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { this.fenceOwnerAdmission = options.fenceOwnerAdmission; this.renewOwnerAdmission = options.renewOwnerAdmission; this.releaseOwnerAdmission = options.releaseOwnerAdmission; + this.cancelUnroutedTask = options.cancelUnroutedTask; this.onTaskPrepared = options.onTaskPrepared; } @@ -1513,6 +1523,11 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { * an unconfirmed delivery, retried once the owner republishes itself. */ if (result.status === 'cancelled' || result.status === 'not_running') { answered.add(key); + } else if (result.status === 'not_found' && this.cancelUnroutedTask != null) { + const stopped = await this.cancelUnroutedTask(target); + if (stopped) { + answered.add(key); + } } } catch (error) { logger.warn('[subagentThreads] Retrying an unconfirmed child cancellation', error); diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md index 610188837d..3c15c163c6 100644 --- a/packages/api/src/agents/triggers/README.md +++ b/packages/api/src/agents/triggers/README.md @@ -14,6 +14,8 @@ envelope and calls `enqueueAgentTrigger`; the adapter does not invoke an agent r - Use `continue` only with a persisted `conversationId` and exact `parentMessageId`. The host defers that delivery while the parent generation is still running or paused, so it cannot replace the generation it is meant to follow. +- External sources never supply a child `conversationId`, `parentMessageId`, or `agentId` on a + continue delivery. Register an event binding once, then address only its opaque binding id. - Use `orderingKey` only when deliveries must remain ordered across different event sources. Without an override, ordering is scoped to the user, source, mode, agent, and conversation. @@ -105,3 +107,56 @@ Poll that location to read `pending`, `leased`, `succeeded`, or `dead` state. Su results include the conversation and generation identity needed for a later `steer` event. Status responses never expose the stored source payload, ordering key, retry history, or worker identity. Callers must sanitize `event.payload`; credentials and transport secrets must not be persisted. + +### Event-driven child actors + +Register a direct child agent once under the same Remote Agents API key that will deliver events. +The parent must be an ordinary agent conversation, and the target must be enabled in that parent +agent's direct `subagents.agent_ids` list (or be an allowed self-spawn). The reserved child +conversation is hidden from conversation lists and remains read-only to human chat routes. +`ENABLE_AGENT_EVENT_CHILD_TURNS` defaults to false; enable it only after every API replica runs a +release that understands bound child continuations, otherwise an older worker could permanently +reject a new envelope during a rolling deployment. + +```http +POST /api/agents/v1/events/bindings +Authorization: Bearer +Idempotency-Key: championship-7-player-hanae +Content-Type: application/json + +{ + "actorId": "hanae-kobayashi", + "parentConversationId": "director-conversation-id", + "parentMessageId": "director-message-id", + "target": { "agentId": "agent-hanae" } +} +``` + +The response contains an opaque `id` and the child `threadId`. Store the binding id with the +source actor. Deliver every later turn with a source-stable event id and the same API key: + +```http +POST /api/agents/v1/events +Authorization: Bearer +Idempotency-Key: game-12-ply-17-hanae +Content-Type: application/json + +{ + "mode": "continue", + "bindingId": "evtbind_…", + "event": { + "id": "game-12-ply-17", + "type": "chess.turn.ready", + "occurredAt": 1786968000000, + "source": { "id": "speed-chess", "type": "mcp" }, + "payload": { "gameId": "game-12", "expectedPly": 17 } + }, + "input": "Your clock is running. Read the position and submit one legal move." +} +``` + +LibreChat resolves the bound agent and child conversation from `(user, tenant, API key, binding)`; +caller-supplied target fields are discarded. It also resolves the latest assistant branch leaf +immediately before dispatch, so queued events do not persist stale chat topology. Each actor binding +is its default ordering lane. A short-lived internal trigger token plus a second binding lookup is +required to pass the child-thread write guard; possessing a binding id alone grants no access. diff --git a/packages/api/src/agents/triggers/bindingResolver.spec.ts b/packages/api/src/agents/triggers/bindingResolver.spec.ts new file mode 100644 index 0000000000..a98773ec4e --- /dev/null +++ b/packages/api/src/agents/triggers/bindingResolver.spec.ts @@ -0,0 +1,200 @@ +import { AIMessage } from '@langchain/core/messages'; +import type { IConversation } from '@librechat/data-schemas'; +import { createAgentTriggerEnvelope, type AgentContinueTriggerEnvelope } from './envelope'; +import { createAgentEventContinueResolver } from './bindingResolver'; + +const bindingId = `evtbind_${'a'.repeat(48)}`; +const sourceKeyId = '507f191e810c19729de860eb'; + +function envelope(): AgentContinueTriggerEnvelope { + return createAgentTriggerEnvelope({ + mode: 'continue', + requestId: 'request-1', + deliveryId: 'delivery-1', + receivedAt: 1, + principal: { id: 'user-1', tenantId: 'tenant-1' }, + event: { + id: 'event-1', + type: 'chess.turn.ready', + occurredAt: 1, + source: { id: 'chess', type: 'webhook' }, + }, + input: 'Your turn.', + target: { + agentId: 'agent-player', + conversationId: 'child-thread', + parentMessageId: 'placeholder', + bindingId, + sourceKeyId, + }, + }) as AgentContinueTriggerEnvelope; +} + +describe('agent event continuation resolver', () => { + it('defers without consuming attempts while the rollout gate is disabled', async () => { + const resolver = createAgentEventContinueResolver({ + enabled: () => false, + methods: { + getAgentEventBinding: jest.fn(), + getConvo: jest.fn(), + getMessages: jest.fn(), + } as never, + }); + + await expect( + resolver(envelope(), { idempotencyKey: 'trigger-1' } as never), + ).rejects.toMatchObject({ + code: 'EVENT_BINDING_DISABLED', + retryable: true, + deferWithoutAttempt: true, + }); + }); + + it('re-resolves the latest assistant leaf immediately before dispatch', async () => { + const getMessages = jest.fn(async () => [ + Object.assign(new AIMessage('done'), { + messageId: 'assistant-1', + isCreatedByUser: false, + createdAt: new Date(2), + }), + ]) as never; + const resolver = createAgentEventContinueResolver({ + enabled: () => true, + methods: { + getAgentEventBinding: jest.fn(async () => ({ + conversationId: 'child-thread', + agentId: 'agent-player', + tenantId: 'tenant-1', + binding: { bindingId, sourceKeyId, actorId: 'player' }, + lineage: { + parentConversationId: 'parent-thread', + parentAgentId: 'agent-director', + } as never, + })), + getConvo: jest.fn( + async () => + ({ + conversationId: 'parent-thread', + agent_id: 'agent-director', + tenantId: 'tenant-1', + }) as IConversation, + ), + getMessages, + }, + }); + + await expect(resolver(envelope(), { idempotencyKey: 'trigger-1' } as never)).resolves.toEqual({ + status: 'ready', + input: 'Your turn.', + parentMessageId: 'assistant-1', + }); + expect(getMessages).toHaveBeenCalledWith( + { user: 'user-1', conversationId: 'child-thread', isCreatedByUser: false }, + 'messageId createdAt', + { sort: { createdAt: -1, _id: -1 }, limit: 1 }, + ); + }); + + it('fails closed when the durable binding target changed', async () => { + const resolver = createAgentEventContinueResolver({ + enabled: () => true, + methods: { + getAgentEventBinding: jest.fn(async () => ({ + conversationId: 'another-thread', + agentId: 'agent-player', + binding: { bindingId, sourceKeyId, actorId: 'player' }, + lineage: {} as never, + })), + getConvo: jest.fn(), + getMessages: jest.fn(async () => []) as never, + }, + }); + + await expect( + resolver(envelope(), { idempotencyKey: 'trigger-1' } as never), + ).rejects.toMatchObject({ code: 'EVENT_BINDING_INVALID', retryable: false }); + }); + + it('defers an event while the actor has an active generation', async () => { + const resolver = createAgentEventContinueResolver({ + enabled: () => true, + getGenerationJob: jest.fn(async () => ({ status: 'running' })), + methods: { + getAgentEventBinding: jest.fn(async () => ({ + conversationId: 'child-thread', + agentId: 'agent-player', + tenantId: 'tenant-1', + binding: { bindingId, sourceKeyId, actorId: 'player' }, + lineage: { + parentConversationId: 'parent-thread', + parentAgentId: 'agent-director', + } as never, + })), + getConvo: jest.fn( + async () => + ({ + conversationId: 'parent-thread', + agent_id: 'agent-director', + tenantId: 'tenant-1', + }) as IConversation, + ), + getMessages: jest.fn(), + }, + }); + + await expect( + resolver(envelope(), { idempotencyKey: 'trigger-1' } as never), + ).rejects.toMatchObject({ + code: 'EVENT_ACTOR_NOT_READY', + retryable: true, + deferWithoutAttempt: true, + }); + }); + + it('fails closed after the binding parent is removed', async () => { + const resolver = createAgentEventContinueResolver({ + enabled: () => true, + methods: { + getAgentEventBinding: jest.fn(async () => ({ + conversationId: 'child-thread', + agentId: 'agent-player', + tenantId: 'tenant-1', + binding: { bindingId, sourceKeyId, actorId: 'player' }, + lineage: { parentConversationId: 'missing-parent' } as never, + })), + getConvo: jest.fn(async () => null), + getMessages: jest.fn(), + }, + }); + + await expect( + resolver(envelope(), { idempotencyKey: 'trigger-1' } as never), + ).rejects.toMatchObject({ code: 'EVENT_BINDING_INVALID', retryable: false }); + }); + + it('fails closed when the binding or its parent passed its retention deadline', async () => { + const getAgentEventBinding = jest.fn(async () => ({ + conversationId: 'child-thread', + agentId: 'agent-player', + tenantId: 'tenant-1', + expiredAt: new Date(0), + binding: { bindingId, sourceKeyId, actorId: 'player' }, + lineage: { + parentConversationId: 'parent-thread', + parentAgentId: 'agent-director', + } as never, + })); + const resolver = createAgentEventContinueResolver({ + enabled: () => true, + methods: { + getAgentEventBinding, + getConvo: jest.fn(), + getMessages: jest.fn(), + }, + }); + + await expect( + resolver(envelope(), { idempotencyKey: 'trigger-1' } as never), + ).rejects.toMatchObject({ code: 'EVENT_BINDING_INVALID', retryable: false }); + }); +}); diff --git a/packages/api/src/agents/triggers/bindingResolver.ts b/packages/api/src/agents/triggers/bindingResolver.ts new file mode 100644 index 0000000000..92d4274867 --- /dev/null +++ b/packages/api/src/agents/triggers/bindingResolver.ts @@ -0,0 +1,166 @@ +import { Constants } from 'librechat-data-provider'; +import type { ConversationMethods, MessageMethods } from '@librechat/data-schemas'; +import type { AgentTriggerContinuePreparation, AgentTriggerExecutionHostDeps } from './host'; +import type { AgentContinueTriggerEnvelope } from './envelope'; +import type { AgentTriggerDispatchContext } from './dispatch'; +import { isAgentEventRetentionActive } from '../eventRetention'; +import { AgentTriggerExecutionError } from './host'; + +type ContinueResolver = NonNullable; + +export interface AgentEventContinueResolverDeps { + methods: Pick & + Pick; + getGenerationJob?: (conversationId: string) => Promise< + | { + status?: string; + metadata?: { terminalPersistencePending?: boolean }; + } + | null + | undefined + >; + fallback?: ContinueResolver; + enabled?: () => boolean; +} + +function invalidBinding(message: string, retryable = false): AgentTriggerExecutionError { + return new AgentTriggerExecutionError(message, { + mode: 'continue', + certainty: 'definite', + retryable, + code: 'EVENT_BINDING_INVALID', + status: retryable ? 503 : 404, + }); +} + +/** Resolves the branch leaf at dispatch time so queued events never persist a stale parent. */ +export function createAgentEventContinueResolver({ + methods, + getGenerationJob, + fallback, + enabled, +}: AgentEventContinueResolverDeps): ContinueResolver { + return async ( + envelope: AgentContinueTriggerEnvelope, + context: AgentTriggerDispatchContext, + ): Promise => { + const { bindingId, sourceKeyId } = envelope.target; + if (bindingId == null || sourceKeyId == null) { + return fallback?.(envelope, context); + } + if (enabled?.() !== true) { + throw new AgentTriggerExecutionError( + 'Event-driven child turns are disabled on this worker.', + { + mode: 'continue', + certainty: 'definite', + retryable: true, + deferWithoutAttempt: true, + code: 'EVENT_BINDING_DISABLED', + status: 503, + }, + ); + } + + let binding; + let latestAssistant; + try { + binding = await methods.getAgentEventBinding({ + user: envelope.principal.userId, + bindingId, + sourceKeyId, + ...(envelope.principal.tenantId == null ? {} : { tenantId: envelope.principal.tenantId }), + }); + } catch (error) { + throw invalidBinding( + `Event binding state is temporarily unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + true, + ); + } + if ( + binding == null || + binding.conversationId !== envelope.target.conversationId || + binding.agentId !== envelope.target.agentId || + binding.binding.bindingId !== bindingId || + binding.binding.sourceKeyId !== sourceKeyId || + !isAgentEventRetentionActive(binding.expiredAt) + ) { + throw invalidBinding('The event binding no longer authorizes this child thread.'); + } + let parent; + try { + parent = await methods.getConvo( + envelope.principal.userId, + binding.lineage.parentConversationId, + ); + } catch (error) { + throw invalidBinding( + `Event binding parent state is temporarily unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + true, + ); + } + if ( + parent == null || + parent.subagentThread != null || + parent.agent_id !== binding.lineage.parentAgentId || + (parent.tenantId ?? undefined) !== envelope.principal.tenantId || + !isAgentEventRetentionActive(parent.expiredAt) + ) { + throw invalidBinding('The event binding parent no longer authorizes this child thread.'); + } + if (getGenerationJob != null) { + let active; + try { + active = await getGenerationJob(binding.conversationId); + } catch (error) { + throw invalidBinding( + `Event actor generation state is temporarily unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + true, + ); + } + if ( + active?.status === 'running' || + active?.status === 'requires_action' || + active?.metadata?.terminalPersistencePending === true + ) { + throw new AgentTriggerExecutionError('The event actor is still handling an earlier turn.', { + mode: 'continue', + certainty: 'definite', + retryable: true, + deferWithoutAttempt: true, + code: 'EVENT_ACTOR_NOT_READY', + status: 409, + }); + } + } + try { + [latestAssistant] = await methods.getMessages( + { + user: envelope.principal.userId, + conversationId: binding.conversationId, + isCreatedByUser: false, + }, + 'messageId createdAt', + { sort: { createdAt: -1, _id: -1 }, limit: 1 }, + ); + } catch (error) { + throw invalidBinding( + `Event actor history is temporarily unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + true, + ); + } + return { + status: 'ready', + input: envelope.input, + parentMessageId: latestAssistant?.messageId ?? Constants.NO_PARENT, + }; + }; +} diff --git a/packages/api/src/agents/triggers/bindings.spec.ts b/packages/api/src/agents/triggers/bindings.spec.ts new file mode 100644 index 0000000000..ff465ab159 --- /dev/null +++ b/packages/api/src/agents/triggers/bindings.spec.ts @@ -0,0 +1,418 @@ +import express from 'express'; +import request from 'supertest'; +import type { IConversation } from '@librechat/data-schemas'; +import { createAgentEventBindingHandlers } from './bindings'; + +const USER_ID = '507f191e810c19729de860ea'; +const SOURCE_KEY_ID = '507f191e810c19729de860eb'; +const PARENT_ID = 'parent-conversation'; +const PARENT_MESSAGE_ID = 'parent-message'; +const PARENT_AGENT_ID = 'agent_director'; +const CHILD_AGENT_ID = 'agent_player'; + +function parent(): IConversation { + return { + conversationId: PARENT_ID, + user: USER_ID, + tenantId: 'tenant-1', + agent_id: PARENT_AGENT_ID, + } as IConversation; +} + +function dependencies() { + const reserveThread = jest.fn(async (input) => ({ + created: true, + conversation: { + ...input.conversation, + user: input.user, + conversationId: input.conversationId, + }, + })); + return { + getAgent: jest.fn, [Record]>(async ({ id }) => + id === PARENT_AGENT_ID + ? { + id: PARENT_AGENT_ID, + subagents: { enabled: true, allowSelf: false, agent_ids: [CHILD_AGENT_ID] }, + } + : { id }, + ), + getConvo: jest.fn, [string, string]>(async () => parent()), + getBinding: jest.fn, [Record]>(async () => null), + getMessage: jest.fn(async () => ({ + messageId: PARENT_MESSAGE_ID, + conversationId: PARENT_ID, + user: USER_ID, + })), + deleteConvos: jest.fn(async () => ({ deletedCount: 1 })), + reserveThread, + enabled: () => true, + }; +} + +function app(deps = dependencies()) { + const handlers = createAgentEventBindingHandlers(deps as never); + const server = express(); + server.use(express.json()); + server.use((req, _res, next) => { + Object.assign(req, { + user: { id: USER_ID, tenantId: 'tenant-1' }, + apiKeyId: { toString: () => SOURCE_KEY_ID }, + }); + next(); + }); + server.post('/bindings', handlers.register); + server.post('/resolve', handlers.resolve, (req, res) => { + res.json(req.body); + }); + return { server, deps }; +} + +describe('agent event bindings', () => { + it('keeps registration off until every API replica supports child turns', async () => { + const deps = dependencies(); + deps.enabled = () => false; + const { server } = app(deps); + const response = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'disabled-binding') + .send({ + actorId: 'player-a', + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + target: { agentId: CHILD_AGENT_ID }, + }); + + expect(response.status).toBe(503); + expect(response.body.error.code).toBe('event_binding_unavailable'); + expect(deps.reserveThread).not.toHaveBeenCalled(); + }); + + it('reserves a hidden depth-one actor thread bound to the authenticated API key', async () => { + const { server, deps } = app(); + const response = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'championship-player-a') + .send({ + actorId: 'player-a', + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + target: { agentId: CHILD_AGENT_ID }, + }); + + expect(response.status).toBe(201); + expect(response.body).toMatchObject({ + id: expect.stringMatching(/^evtbind_/), + actorId: 'player-a', + agentId: CHILD_AGENT_ID, + threadId: expect.any(String), + }); + expect(deps.reserveThread).toHaveBeenCalledWith( + expect.objectContaining({ + user: USER_ID, + tenantId: 'tenant-1', + conversation: expect.objectContaining({ + agent_id: CHILD_AGENT_ID, + agentEventBinding: expect.objectContaining({ sourceKeyId: SOURCE_KEY_ID }), + subagentThread: expect.objectContaining({ + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + parentAgentId: PARENT_AGENT_ID, + subagentType: CHILD_AGENT_ID, + depth: 1, + }), + }), + }), + ); + }); + + it('rejects a target that is not a configured direct child', async () => { + const { server, deps } = app(); + deps.getAgent.mockResolvedValueOnce({ + id: PARENT_AGENT_ID, + subagents: { enabled: true, allowSelf: false, agent_ids: [] }, + } as never); + const response = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'not-configured') + .send({ + actorId: 'player-a', + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + target: { agentId: CHILD_AGENT_ID }, + }); + + expect(response.status).toBe(403); + expect(deps.reserveThread).not.toHaveBeenCalled(); + }); + + it('resolves a bound continue without accepting a caller-selected target', async () => { + const deps = dependencies(); + deps.getBinding.mockResolvedValue({ + conversationId: 'child-thread', + agentId: CHILD_AGENT_ID, + tenantId: 'tenant-1', + binding: { + bindingId: `evtbind_${'a'.repeat(48)}`, + sourceKeyId: SOURCE_KEY_ID, + actorId: 'player-a', + }, + lineage: { + rootConversationId: PARENT_ID, + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + parentToolCallId: 'event-binding', + parentAgentId: PARENT_AGENT_ID, + subagentType: CHILD_AGENT_ID, + subagentKind: 'agent', + depth: 1, + }, + }); + const { server } = app(deps); + const response = await request(server) + .post('/resolve') + .send({ + mode: 'continue', + bindingId: `evtbind_${'a'.repeat(48)}`, + orderingKey: 'attacker-selected-lane', + target: { agentId: 'agent_attacker', conversationId: 'foreign-thread' }, + }); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ + mode: 'continue', + orderingKey: `evtbind_${'a'.repeat(48)}`, + target: { + agentId: CHILD_AGENT_ID, + conversationId: 'child-thread', + bindingId: `evtbind_${'a'.repeat(48)}`, + sourceKeyId: SOURCE_KEY_ID, + }, + }); + expect(response.body.target.agentId).not.toBe('agent_attacker'); + expect(response.body.orderingKey).not.toBe('attacker-selected-lane'); + expect(deps.getBinding).toHaveBeenCalledWith({ + user: USER_ID, + tenantId: 'tenant-1', + bindingId: `evtbind_${'a'.repeat(48)}`, + sourceKeyId: SOURCE_KEY_ID, + }); + }); + + it('rejects a parent message outside the selected conversation', async () => { + const deps = dependencies(); + deps.getMessage.mockResolvedValueOnce({ + messageId: PARENT_MESSAGE_ID, + conversationId: 'another-conversation', + user: USER_ID, + }); + const { server } = app(deps); + const response = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'bad-parent-message') + .send({ + actorId: 'player-a', + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + target: { agentId: CHILD_AGENT_ID }, + }); + + expect(response.status).toBe(404); + expect(deps.reserveThread).not.toHaveBeenCalled(); + }); + + it('returns an idempotency conflict before reserving under a different parent', async () => { + const deps = dependencies(); + const { server } = app(deps); + const first = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'cross-parent-replay') + .send({ + actorId: 'player-a', + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + target: { agentId: CHILD_AGENT_ID }, + }); + + expect(first.status).toBe(201); + const reservation = await deps.reserveThread.mock.results[0].value; + deps.getConvo.mockResolvedValueOnce({ + ...parent(), + conversationId: 'other-parent', + } as unknown as IConversation); + deps.getMessage.mockResolvedValueOnce({ + messageId: PARENT_MESSAGE_ID, + conversationId: 'other-parent', + user: USER_ID, + }); + deps.getBinding.mockResolvedValueOnce({ + conversationId: reservation.conversation.conversationId, + agentId: reservation.conversation.agent_id, + tenantId: reservation.conversation.tenantId, + binding: reservation.conversation.agentEventBinding, + lineage: reservation.conversation.subagentThread, + } as never); + const response = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'cross-parent-replay') + .send({ + actorId: 'player-a', + parentConversationId: 'other-parent', + parentMessageId: PARENT_MESSAGE_ID, + target: { agentId: CHILD_AGENT_ID }, + }); + + expect(response.status).toBe(409); + expect(deps.reserveThread).toHaveBeenCalledTimes(1); + }); + + it('rolls back a new binding when its parent loses the registration race', async () => { + const deps = dependencies(); + deps.getConvo.mockResolvedValueOnce(parent()).mockResolvedValueOnce(null); + const { server } = app(deps); + const response = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'parent-delete-race') + .send({ + actorId: 'player-a', + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + target: { agentId: CHILD_AGENT_ID }, + }); + + expect(response.status).toBe(409); + expect(deps.deleteConvos).toHaveBeenCalledWith( + USER_ID, + expect.objectContaining({ conversationId: expect.any(String) }), + ); + }); + + it('rejects an idempotent replay when the parent disappears after the first read', async () => { + const deps = dependencies(); + /** Fill the deterministic binding id after the request computes it. */ + deps.getBinding.mockImplementationOnce(async (input) => ({ + conversationId: 'child-thread', + agentId: CHILD_AGENT_ID, + tenantId: 'tenant-1', + binding: { + bindingId: input.bindingId, + sourceKeyId: SOURCE_KEY_ID, + actorId: 'player-a', + }, + lineage: { + rootConversationId: PARENT_ID, + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + parentToolCallId: `event-binding:${input.bindingId}`, + parentAgentId: PARENT_AGENT_ID, + subagentType: CHILD_AGENT_ID, + subagentKind: 'agent', + depth: 1, + }, + })); + deps.getConvo.mockResolvedValueOnce(parent()).mockResolvedValueOnce(null); + const { server } = app(deps); + + const response = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'parent-replay-race') + .send({ + actorId: 'player-a', + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + target: { agentId: CHILD_AGENT_ID }, + }); + + expect(response.status).toBe(409); + expect(response.body.error.code).toBe('event_binding_parent_ended'); + expect(deps.deleteConvos).toHaveBeenCalledWith(USER_ID, { + conversationId: 'child-thread', + }); + expect(deps.reserveThread).not.toHaveBeenCalled(); + }); + + it('surfaces a failed rollback and lets a retry reconcile the orphan', async () => { + const deps = dependencies(); + deps.getConvo.mockResolvedValueOnce(parent()).mockResolvedValueOnce(null); + deps.deleteConvos + .mockRejectedValueOnce(new Error('stepdown')) + .mockRejectedValueOnce(new Error('stepdown')) + .mockRejectedValueOnce(new Error('stepdown')); + const { server } = app(deps); + const body = { + actorId: 'player-a', + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + target: { agentId: CHILD_AGENT_ID }, + }; + + const first = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'rollback-recovery') + .send(body); + + expect(first.status).toBe(503); + expect(first.body.error.code).toBe('event_binding_cleanup_failed'); + const reservation = await deps.reserveThread.mock.results[0].value; + deps.getConvo.mockResolvedValue(null); + deps.getBinding.mockResolvedValue({ + conversationId: reservation.conversation.conversationId, + agentId: reservation.conversation.agent_id, + tenantId: reservation.conversation.tenantId, + binding: reservation.conversation.agentEventBinding, + lineage: reservation.conversation.subagentThread, + } as never); + + const retry = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'rollback-recovery') + .send(body); + + expect(retry.status).toBe(409); + expect(retry.body.error.code).toBe('event_binding_parent_ended'); + expect(deps.deleteConvos).toHaveBeenCalledTimes(4); + }); + + it('rejects registration after the parent retention deadline', async () => { + const deps = dependencies(); + deps.getConvo.mockResolvedValueOnce({ ...parent(), expiredAt: new Date(0) } as IConversation); + const { server } = app(deps); + + const response = await request(server) + .post('/bindings') + .set('Idempotency-Key', 'expired-parent') + .send({ + actorId: 'player-a', + parentConversationId: PARENT_ID, + parentMessageId: PARENT_MESSAGE_ID, + target: { agentId: CHILD_AGENT_ID }, + }); + + expect(response.status).toBe(404); + expect(deps.reserveThread).not.toHaveBeenCalled(); + }); + + it('leaves fire and steer deliveries unchanged', async () => { + const { server, deps } = app(); + const response = await request(server) + .post('/resolve') + .send({ mode: 'fire', target: { agentId: CHILD_AGENT_ID } }); + expect(response.status).toBe(200); + expect(response.body).toEqual({ mode: 'fire', target: { agentId: CHILD_AGENT_ID } }); + expect(deps.getBinding).not.toHaveBeenCalled(); + }); + + it('does not reinterpret another mode merely because it contains a binding id', async () => { + const { server, deps } = app(); + const response = await request(server) + .post('/resolve') + .send({ + mode: 'fire', + bindingId: `evtbind_${'a'.repeat(48)}`, + target: { agentId: CHILD_AGENT_ID }, + }); + + expect(response.status).toBe(400); + expect(deps.getBinding).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/agents/triggers/bindings.ts b/packages/api/src/agents/triggers/bindings.ts new file mode 100644 index 0000000000..1cec1d2f30 --- /dev/null +++ b/packages/api/src/agents/triggers/bindings.ts @@ -0,0 +1,462 @@ +import { createHash } from 'node:crypto'; +import { Constants, EModelEndpoint } from 'librechat-data-provider'; +import type { + AgentMethods, + ConversationMethods, + IAgent, + IAgentEventBindingRecord, + IConversation, + MessageMethods, +} from '@librechat/data-schemas'; +import type { Request, RequestHandler, Response } from 'express'; +import { isAgentEventRetentionActive } from '../eventRetention'; +import { createSubagentThreadId } from '../subagentThreadIds'; + +const BINDING_ID_PATTERN = /^evtbind_[a-f0-9]{48}$/; +const IDEMPOTENCY_KEY_PATTERN = /^[A-Za-z0-9._~:/+=-]+$/; +const MAX_ACTOR_ID_LENGTH = 128; +const MAX_REGISTRATION_KEY_LENGTH = 256; + +interface EventBindingUser { + id?: string; + role?: string; + tenantId?: string; +} + +interface EventBindingRequest extends Request { + apiKeyId?: { toString(): string } | string; + user?: EventBindingUser; + _agentEventBindingResolved?: boolean; +} + +interface RegisterBindingBody { + actorId?: unknown; + parentConversationId?: unknown; + parentMessageId?: unknown; + target?: { agentId?: unknown }; +} + +export interface AgentEventBindingDependencies { + getAgent: AgentMethods['getAgent']; + getConvo: ConversationMethods['getConvo']; + getBinding: ConversationMethods['getAgentEventBinding']; + getMessage: MessageMethods['getMessage']; + deleteConvos: ConversationMethods['deleteConvos']; + reserveThread: ConversationMethods['reserveSubagentThread']; + enabled?: () => boolean; +} + +class AgentEventBindingError extends Error { + constructor( + message: string, + readonly status = 400, + readonly code = 'invalid_event_binding', + ) { + super(message); + this.name = 'AgentEventBindingError'; + } +} + +function requireString(value: unknown, name: string, max = 256): string { + if (typeof value !== 'string' || value.trim() === '' || value.length > max) { + throw new AgentEventBindingError(`${name} must be a non-empty string of at most ${max} bytes`); + } + return value; +} + +function requirePrincipal(req: EventBindingRequest): { + userId: string; + tenantId?: string; + sourceKeyId: string; +} { + const userId = requireString(req.user?.id, 'Authenticated user ID'); + const sourceKeyId = requireString(req.apiKeyId?.toString(), 'Authenticated API key ID'); + return { + userId, + sourceKeyId, + ...(typeof req.user?.tenantId === 'string' && req.user.tenantId !== '' + ? { tenantId: req.user.tenantId } + : {}), + }; +} + +function tenantMatches(actual: string | undefined, expected: string | undefined): boolean { + return actual == null ? expected == null : actual === expected; +} + +function configuredChild(parentAgent: IAgent, targetAgentId: string): boolean { + const parentId = typeof parentAgent.id === 'string' ? parentAgent.id : undefined; + const subagents = parentAgent.subagents as + | { enabled?: boolean; allowSelf?: boolean; agent_ids?: unknown[] } + | undefined; + if (subagents?.enabled !== true) { + return false; + } + if (targetAgentId === parentId && subagents.allowSelf !== false) { + return true; + } + return subagents.agent_ids?.includes(targetAgentId) === true; +} + +function bindingId( + userId: string, + tenantId: string | undefined, + sourceKeyId: string, + registrationKey: string, +): string { + const digest = createHash('sha256') + .update( + `librechat:agent-event-binding:v1\u0000${userId}\u0000${tenantId ?? ''}\u0000${sourceKeyId}\u0000${registrationKey}`, + ) + .digest('hex'); + return `evtbind_${digest.slice(0, 48)}`; +} + +function registrationKey(req: Request): string { + const values: string[] = []; + for (let index = 0; index < req.rawHeaders.length; index += 2) { + if (req.rawHeaders[index]?.toLowerCase() === 'idempotency-key') { + values.push(req.rawHeaders[index + 1] ?? ''); + } + } + if (values.length !== 1) { + throw new AgentEventBindingError('Exactly one Idempotency-Key header is required'); + } + const value = requireString(values[0].trim(), 'Idempotency-Key', MAX_REGISTRATION_KEY_LENGTH); + if (!IDEMPOTENCY_KEY_PATTERN.test(value)) { + throw new AgentEventBindingError('Idempotency-Key contains invalid characters'); + } + return value; +} + +function publicBinding(record: IAgentEventBindingRecord) { + return { + id: record.binding.bindingId, + actorId: record.binding.actorId, + agentId: record.agentId, + threadId: record.conversationId, + }; +} + +function assertReplay( + record: IAgentEventBindingRecord, + expected: { + bindingId: string; + sourceKeyId: string; + actorId: string; + parentConversationId: string; + parentMessageId: string; + parentAgentId: string; + targetAgentId: string; + }, +): void { + const binding = record.binding; + const lineage = record.lineage; + if ( + binding?.bindingId !== expected.bindingId || + binding.sourceKeyId !== expected.sourceKeyId || + binding.actorId !== expected.actorId || + record.agentId !== expected.targetAgentId || + lineage?.parentConversationId !== expected.parentConversationId || + lineage.parentMessageId !== expected.parentMessageId || + lineage.parentAgentId !== expected.parentAgentId || + lineage.subagentType !== expected.targetAgentId || + lineage.subagentKind !== 'agent' || + lineage.depth !== 1 + ) { + throw new AgentEventBindingError( + 'Idempotency-Key was already used for a different event binding', + 409, + 'idempotency_conflict', + ); + } +} + +function bindingRecord(conversation: IConversation): IAgentEventBindingRecord { + if ( + conversation.agentEventBinding == null || + conversation.subagentThread == null || + typeof conversation.agent_id !== 'string' + ) { + throw new AgentEventBindingError('Reserved event binding is incomplete', 500); + } + return { + conversationId: conversation.conversationId, + agentId: conversation.agent_id, + ...(conversation.tenantId == null ? {} : { tenantId: conversation.tenantId }), + ...(conversation.isTemporary == null ? {} : { isTemporary: conversation.isTemporary }), + ...(conversation.expiredAt == null ? {} : { expiredAt: conversation.expiredAt }), + binding: conversation.agentEventBinding, + lineage: conversation.subagentThread, + }; +} + +function sendError(res: Response, error: unknown): void { + if (error instanceof AgentEventBindingError) { + res.status(error.status).json({ + error: { message: error.message, type: 'invalid_request_error', code: error.code }, + }); + return; + } + throw error; +} + +function requireEnabled(deps: AgentEventBindingDependencies): void { + if (deps.enabled?.() !== true) { + throw new AgentEventBindingError( + 'Event-driven child turns are not enabled on this deployment', + 503, + 'event_binding_unavailable', + ); + } +} + +export function createAgentEventBindingHandlers(deps: AgentEventBindingDependencies): { + register: RequestHandler; + resolve: RequestHandler; +} { + const register: RequestHandler = async (baseReq, res, next) => { + const req = baseReq as EventBindingRequest; + try { + requireEnabled(deps); + const principal = requirePrincipal(req); + const body = (req.body ?? {}) as RegisterBindingBody; + const actorId = requireString(body.actorId, 'actorId', MAX_ACTOR_ID_LENGTH); + const parentConversationId = requireString(body.parentConversationId, 'parentConversationId'); + const parentMessageId = requireString(body.parentMessageId, 'parentMessageId'); + const targetAgentId = requireString(body.target?.agentId, 'target.agentId'); + const id = bindingId( + principal.userId, + principal.tenantId, + principal.sourceKeyId, + registrationKey(req), + ); + const bindingQuery = { + user: principal.userId, + bindingId: id, + sourceKeyId: principal.sourceKeyId, + ...(principal.tenantId == null ? {} : { tenantId: principal.tenantId }), + }; + const cleanupBinding = async (conversationId: string): Promise => { + let lastError: unknown; + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + await deps.deleteConvos(principal.userId, { conversationId }); + return; + } catch (error) { + lastError = error; + } + } + throw new AgentEventBindingError( + `Reserved event binding cleanup failed: ${lastError instanceof Error ? lastError.message : String(lastError)}`, + 503, + 'event_binding_cleanup_failed', + ); + }; + const expectedBinding = (parentAgentId: string) => ({ + bindingId: id, + sourceKeyId: principal.sourceKeyId, + actorId, + parentConversationId, + parentMessageId, + parentAgentId, + targetAgentId, + }); + const [parent, parentMessage] = await Promise.all([ + deps.getConvo(principal.userId, parentConversationId), + deps.getMessage({ user: principal.userId, messageId: parentMessageId }), + ]); + if ( + parent == null || + parent.subagentThread != null || + !tenantMatches(parent.tenantId, principal.tenantId) || + typeof parent.agent_id !== 'string' || + !isAgentEventRetentionActive(parent.expiredAt) + ) { + const orphan = await deps.getBinding(bindingQuery); + if (orphan != null) { + const orphanParentAgentId = orphan.lineage.parentAgentId; + if (typeof orphanParentAgentId !== 'string' || orphanParentAgentId === '') { + await cleanupBinding(orphan.conversationId); + throw new AgentEventBindingError( + 'Parent agent conversation ended during binding registration', + 409, + 'event_binding_parent_ended', + ); + } + assertReplay(orphan, expectedBinding(orphanParentAgentId)); + await cleanupBinding(orphan.conversationId); + throw new AgentEventBindingError( + 'Parent agent conversation ended during binding registration', + 409, + 'event_binding_parent_ended', + ); + } + throw new AgentEventBindingError('Parent agent conversation was not found', 404); + } + if (parentMessage?.conversationId !== parentConversationId) { + throw new AgentEventBindingError('Parent agent message was not found', 404); + } + const resolvedParentAgent = await deps.getAgent({ id: parent.agent_id }); + if (resolvedParentAgent == null || !configuredChild(resolvedParentAgent, targetAgentId)) { + throw new AgentEventBindingError( + 'Target agent is not configured as a direct child of the parent agent', + 403, + 'event_binding_forbidden', + ); + } + + const scopeId = JSON.stringify({ + userId: principal.userId, + parentConversationId, + ...(principal.tenantId == null ? {} : { tenantId: principal.tenantId }), + }); + const threadId = createSubagentThreadId(scopeId, id); + const expected = expectedBinding(parent.agent_id); + const assertCurrentParent = async (): Promise => { + const currentParent = await deps.getConvo(principal.userId, parentConversationId); + if ( + currentParent == null || + currentParent.subagentThread != null || + currentParent.agent_id !== parent.agent_id || + !tenantMatches(currentParent.tenantId, principal.tenantId) || + !isAgentEventRetentionActive(currentParent.expiredAt) + ) { + throw new AgentEventBindingError( + 'Parent agent conversation ended during binding registration', + 409, + 'event_binding_parent_ended', + ); + } + }; + const assertCurrentParentOrCleanup = async ( + record: IAgentEventBindingRecord, + ): Promise => { + try { + await assertCurrentParent(); + } catch (error) { + if ( + !(error instanceof AgentEventBindingError) || + error.code !== 'event_binding_parent_ended' + ) { + throw error; + } + await cleanupBinding(record.conversationId); + throw error; + } + }; + const replay = await deps.getBinding(bindingQuery); + if (replay != null) { + assertReplay(replay, expected); + await assertCurrentParentOrCleanup(replay); + res.status(200).json(publicBinding(replay)); + return; + } + let reserved; + try { + reserved = await deps.reserveThread({ + user: principal.userId, + conversationId: threadId, + ...(principal.tenantId == null ? {} : { tenantId: principal.tenantId }), + conversation: { + conversationId: threadId, + endpoint: EModelEndpoint.agents, + title: `Agent actor: ${actorId}`.slice(0, 120), + agent_id: targetAgentId, + ...(parent.isTemporary == null ? {} : { isTemporary: parent.isTemporary }), + ...(parent.expiredAt == null ? {} : { expiredAt: parent.expiredAt }), + ...(principal.tenantId == null ? {} : { tenantId: principal.tenantId }), + agentEventBinding: { bindingId: id, sourceKeyId: principal.sourceKeyId, actorId }, + subagentThread: { + rootConversationId: parentConversationId, + parentConversationId, + parentMessageId, + parentToolCallId: `event-binding:${id}`, + parentAgentId: parent.agent_id, + subagentType: targetAgentId, + subagentKind: 'agent', + depth: 1, + }, + }, + }); + } catch (error) { + if ((error as { code?: number }).code !== 11000) { + throw error; + } + const winner = await deps.getBinding(bindingQuery); + if (winner == null) { + throw error; + } + assertReplay(winner, expected); + await assertCurrentParentOrCleanup(winner); + res.status(200).json(publicBinding(winner)); + return; + } + const record = bindingRecord(reserved.conversation); + assertReplay(record, expected); + await assertCurrentParentOrCleanup(record); + res.status(reserved.created ? 201 : 200).json(publicBinding(record)); + } catch (error) { + try { + sendError(res, error); + } catch (unexpected) { + next(unexpected); + } + } + }; + + const resolve: RequestHandler = async (baseReq, res, next) => { + const req = baseReq as EventBindingRequest; + try { + const principal = requirePrincipal(req); + const body = (req.body ?? {}) as Record; + if (body.mode !== 'continue') { + if (body.bindingId != null) { + throw new AgentEventBindingError('bindingId is valid only for continue events'); + } + next(); + return; + } + requireEnabled(deps); + const id = requireString(body.bindingId, 'bindingId'); + if (!BINDING_ID_PATTERN.test(id)) { + throw new AgentEventBindingError('bindingId is invalid'); + } + const binding = await deps.getBinding({ + user: principal.userId, + bindingId: id, + sourceKeyId: principal.sourceKeyId, + ...(principal.tenantId == null ? {} : { tenantId: principal.tenantId }), + }); + if (binding == null) { + throw new AgentEventBindingError( + 'Event binding was not found', + 404, + 'event_binding_not_found', + ); + } + req.body = { + ...body, + orderingKey: id, + mode: 'continue', + target: { + agentId: binding.agentId, + conversationId: binding.conversationId, + parentMessageId: Constants.NO_PARENT, + bindingId: id, + sourceKeyId: principal.sourceKeyId, + }, + }; + req._agentEventBindingResolved = true; + next(); + } catch (error) { + try { + sendError(res, error); + } catch (unexpected) { + next(unexpected); + } + } + }; + + return { register, resolve }; +} diff --git a/packages/api/src/agents/triggers/envelope.spec.ts b/packages/api/src/agents/triggers/envelope.spec.ts index 4a19e37ae6..69afb73154 100644 --- a/packages/api/src/agents/triggers/envelope.spec.ts +++ b/packages/api/src/agents/triggers/envelope.spec.ts @@ -125,6 +125,34 @@ describe('createAgentTriggerEnvelope', () => { expect(parseAgentTriggerEnvelope(JSON.parse(JSON.stringify(envelope)))).toEqual(envelope); }); + it('preserves only complete authenticated binding metadata on continuations', () => { + const envelope = createAgentTriggerEnvelope({ + ...createFireInput(), + mode: 'continue', + target: { + agentId: 'agent-1', + conversationId: 'conversation-1', + parentMessageId: 'response-1', + bindingId: `evtbind_${'a'.repeat(48)}`, + sourceKeyId: 'source-key', + }, + }); + + expect(parseAgentTriggerEnvelope(JSON.parse(JSON.stringify(envelope)))).toEqual(envelope); + expect(() => + createAgentTriggerEnvelope({ + ...createFireInput(), + mode: 'continue', + target: { + agentId: 'agent-1', + conversationId: 'conversation-1', + parentMessageId: 'response-1', + bindingId: `evtbind_${'a'.repeat(48)}`, + }, + }), + ).toThrow('target.bindingId and target.sourceKeyId must be provided together'); + }); + it('builds a stable generation-compatible idempotency key per delivery target', () => { const first = createAgentTriggerEnvelope(createFireInput()); const retry = createAgentTriggerEnvelope({ diff --git a/packages/api/src/agents/triggers/envelope.ts b/packages/api/src/agents/triggers/envelope.ts index d6b4dcdbb7..ee3607b497 100644 --- a/packages/api/src/agents/triggers/envelope.ts +++ b/packages/api/src/agents/triggers/envelope.ts @@ -58,6 +58,10 @@ export interface AgentContinueTarget extends AgentTriggerTarget { conversationId: string; /** Persisted branch leaf below which the new turn is appended. */ parentMessageId: string; + /** Present only after an authenticated source binding resolved the target. */ + bindingId?: string; + /** API-key identity captured by the ingress adapter and rechecked at dispatch. */ + sourceKeyId?: string; } export interface AgentSteerTarget extends AgentTriggerTarget { @@ -261,6 +265,11 @@ export function createAgentTriggerEnvelope( } if (input.mode === 'continue') { + const bindingId = input.target?.bindingId; + const sourceKeyId = input.target?.sourceKeyId; + if ((bindingId == null) !== (sourceKeyId == null)) { + throw error('target.bindingId and target.sourceKeyId must be provided together'); + } return { ...base, mode: input.mode, @@ -268,6 +277,12 @@ export function createAgentTriggerEnvelope( agentId: requireString(input.target?.agentId, 'target.agentId'), conversationId: requireString(input.target?.conversationId, 'target.conversationId'), parentMessageId: requireString(input.target?.parentMessageId, 'target.parentMessageId'), + ...(bindingId == null + ? {} + : { + bindingId: requireString(bindingId, 'target.bindingId'), + sourceKeyId: requireString(sourceKeyId, 'target.sourceKeyId'), + }), }, }; } @@ -338,6 +353,9 @@ export function parseAgentTriggerEnvelope(input: unknown): AgentTriggerEnvelope } if (mode === 'continue') { + if ((target.bindingId == null) !== (target.sourceKeyId == null)) { + throw error('target.bindingId and target.sourceKeyId must be provided together'); + } return { ...base, mode, @@ -345,6 +363,12 @@ export function parseAgentTriggerEnvelope(input: unknown): AgentTriggerEnvelope agentId: requireString(target.agentId, 'target.agentId'), conversationId: requireString(target.conversationId, 'target.conversationId'), parentMessageId: requireString(target.parentMessageId, 'target.parentMessageId'), + ...(target.bindingId == null + ? {} + : { + bindingId: requireString(target.bindingId, 'target.bindingId'), + sourceKeyId: requireString(target.sourceKeyId, 'target.sourceKeyId'), + }), }, }; } @@ -387,6 +411,8 @@ export function getAgentTriggerIdempotencyKey(envelope: AgentTriggerEnvelope): s envelope.target.agentId, envelope.mode === 'fire' ? '' : envelope.target.conversationId, envelope.mode === 'continue' ? envelope.target.parentMessageId : '', + envelope.mode === 'continue' ? (envelope.target.bindingId ?? '') : '', + envelope.mode === 'continue' ? (envelope.target.sourceKeyId ?? '') : '', ]), ) .digest('hex'); diff --git a/packages/api/src/agents/triggers/host.spec.ts b/packages/api/src/agents/triggers/host.spec.ts index 7c820d4c44..66a5c500aa 100644 --- a/packages/api/src/agents/triggers/host.spec.ts +++ b/packages/api/src/agents/triggers/host.spec.ts @@ -575,31 +575,56 @@ describe('createAgentTriggerExecutionHost continue adapter', () => { }); }); - it('retries without consuming the logical delivery when the parent is not settled', async () => { - expect.hasAssertions(); - const host = createAgentTriggerExecutionHost( - deps( - fetchMock(async () => - response( - { code: 'PARENT_NOT_READY', error: 'The parent is still running.' }, - { status: 409 }, - ), - ), - ), + it('carries server-resolved binding identity only on bound child continuations', async () => { + const base = createContinueEnvelope(); + const envelope = { + ...base, + target: { + ...base.target, + bindingId: `evtbind_${'a'.repeat(48)}`, + sourceKeyId: 'source-key', + }, + }; + const fetcher = fetchMock(async () => + response({ + streamId: 'conversation-1', + conversationId: 'conversation-1', + status: 'started', + }), ); - await host.dispatch(createContinueEnvelope()).catch((error: unknown) => { - expectExecutionError(error, { - mode: 'continue', - certainty: 'definite', - retryable: true, - deferWithoutAttempt: true, - code: 'PARENT_NOT_READY', - status: 409, - }); - }); + await createAgentTriggerExecutionHost(deps(fetcher)).dispatch(envelope); + + const headers = fetcher.mock.calls[0][1]?.headers as Record; + expect(headers['x-lc-agent-event-binding']).toBe(`evtbind_${'a'.repeat(48)}`); + expect(headers['x-lc-agent-event-source-key']).toBe('source-key'); }); + it.each(['PARENT_NOT_READY', 'EVENT_ACTOR_NOT_READY'])( + 'retries without consuming the logical delivery for temporary admission code %s', + async (code) => { + expect.hasAssertions(); + const host = createAgentTriggerExecutionHost( + deps( + fetchMock(async () => + response({ code, error: 'The actor is still busy.' }, { status: 409 }), + ), + ), + ); + + await host.dispatch(createContinueEnvelope()).catch((error: unknown) => { + expectExecutionError(error, { + mode: 'continue', + certainty: 'definite', + retryable: true, + deferWithoutAttempt: true, + code, + status: 409, + }); + }); + }, + ); + it('releases a prepared durable result after a definite admission rejection', async () => { const releaseOnDefiniteFailure = jest.fn(async () => undefined); const host = createAgentTriggerExecutionHost( diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.ts index 55e7a317c0..342d919738 100644 --- a/packages/api/src/agents/triggers/host.ts +++ b/packages/api/src/agents/triggers/host.ts @@ -505,7 +505,11 @@ function canReleasePreparedResult(error: AgentTriggerExecutionError): boolean { if (error.code === 'START_ABORTED' || error.status == null) { return true; } - if (error.code === 'PARENT_NOT_READY' || error.code === 'PARENT_STATE_UNAVAILABLE') { + if ( + error.code === 'PARENT_NOT_READY' || + error.code === 'PARENT_STATE_UNAVAILABLE' || + error.code === 'EVENT_ACTOR_NOT_READY' + ) { return true; } return error.status >= 400 && error.status < 500 && error.status !== 408 && error.status !== 409; @@ -554,7 +558,8 @@ async function startRun( conversationId: envelope.target.conversationId, }; } - const input = preparation?.status === 'ready' ? preparation.input : envelope.input; + const readyPreparation = preparation?.status === 'ready' ? preparation : undefined; + const input = readyPreparation?.input ?? envelope.input; const parentMessageId = resolveParentMessageId(preparation, envelope); const [token, resolvedTimezone, baseUrl] = await Promise.all([ setupValue( @@ -590,6 +595,12 @@ async function startRun( 'User-Agent': TRIGGER_USER_AGENT, 'x-lc-agent-trigger': '1', 'x-request-id': context.idempotencyKey, + ...(envelope.mode === 'continue' && envelope.target.bindingId != null + ? { + 'x-lc-agent-event-binding': envelope.target.bindingId, + 'x-lc-agent-event-source-key': envelope.target.sourceKeyId!, + } + : {}), [GENERATION_PROTOCOL_HEADER]: '2', }, body: JSON.stringify({ @@ -666,18 +677,15 @@ async function startRun( if (!response.ok) { const message = errorMessage(payload) ?? (boundedBody.text.slice(0, 300) || 'request rejected'); + const deferredContinue = + mode === 'continue' && + response.status === 409 && + ['PARENT_NOT_READY', 'EVENT_ACTOR_NOT_READY'].includes(errorCode(payload) ?? ''); throw executionError(`Agent trigger ${mode} was rejected (${response.status}): ${message}`, { mode, certainty: 'definite', - retryable: - isRetryableStatus(response.status) || - (mode === 'continue' && - response.status === 409 && - errorCode(payload) === 'PARENT_NOT_READY'), - deferWithoutAttempt: - mode === 'continue' && - response.status === 409 && - errorCode(payload) === 'PARENT_NOT_READY', + retryable: isRetryableStatus(response.status) || deferredContinue, + deferWithoutAttempt: deferredContinue, code: errorCode(payload) ?? (mode === 'fire' ? 'FIRE_REJECTED' : 'CONTINUE_REJECTED'), status: response.status, ...(response.headers.get('retry-after') != null && { diff --git a/packages/api/src/agents/triggers/index.ts b/packages/api/src/agents/triggers/index.ts index e3e2f8c6a3..92ad867e8f 100644 --- a/packages/api/src/agents/triggers/index.ts +++ b/packages/api/src/agents/triggers/index.ts @@ -1,7 +1,10 @@ export * from './dispatch'; +export * from './bindings'; +export * from './bindingResolver'; export * from './delivery'; export * from './envelope'; export * from './host'; export * from './ingress'; +export * from './lease'; export * from './service'; export * from './engine'; diff --git a/packages/api/src/agents/triggers/ingress.spec.ts b/packages/api/src/agents/triggers/ingress.spec.ts index 754b646a41..ceef680280 100644 --- a/packages/api/src/agents/triggers/ingress.spec.ts +++ b/packages/api/src/agents/triggers/ingress.spec.ts @@ -62,6 +62,7 @@ function createApp( role: 'USER', tenantId: 'tenant-1', }, + bindingResolved = false, ): Application { const app = express(); const handlers = createAgentTriggerIngressHandlers(deps); @@ -71,6 +72,7 @@ function createApp( user: user ?? undefined, apiKeyId: API_KEY_ID, requestId: 'request-from-context', + _agentEventBindingResolved: bindingResolved, }); next(); }); @@ -169,6 +171,40 @@ describe('agent trigger event ingress', () => { ); }); + it('admits continue only after a source binding resolved its trusted target', async () => { + const event = { + mode: 'continue', + event: fireEvent().event, + target: { + agentId: 'agent-player', + conversationId: 'child-thread', + parentMessageId: 'placeholder', + bindingId: `evtbind_${'a'.repeat(48)}`, + sourceKeyId: 'source-key', + }, + input: 'Make the next move.', + }; + const rejected = dependencies(); + const accepted = dependencies(); + + const directResponse = await request(createApp(rejected)) + .post('/api/agents/v1/events') + .set('Idempotency-Key', 'continue-direct') + .send(event); + const boundResponse = await request(createApp(accepted, undefined, true)) + .post('/api/agents/v1/events') + .set('Idempotency-Key', 'continue-bound') + .send(event); + + expect(directResponse.status).toBe(400); + expect(rejected.enqueue).not.toHaveBeenCalled(); + expect(boundResponse.status).toBe(202); + expect(accepted.enqueue).toHaveBeenCalledWith( + expect.objectContaining({ mode: 'continue', target: event.target }), + {}, + ); + }); + it('fails closed when the idempotency header is absent or duplicated', async () => { const deps = dependencies(); const app = createApp(deps); diff --git a/packages/api/src/agents/triggers/ingress.ts b/packages/api/src/agents/triggers/ingress.ts index 1c591f4aa1..4ed1dc5649 100644 --- a/packages/api/src/agents/triggers/ingress.ts +++ b/packages/api/src/agents/triggers/ingress.ts @@ -3,6 +3,7 @@ import { logger } from '@librechat/data-schemas'; import type { Request, RequestHandler, Response } from 'express'; import type { AgentFireTarget, + AgentContinueTarget, AgentSteerTarget, AgentTriggerEvent, AgentTriggerMode, @@ -28,12 +29,13 @@ interface AgentTriggerIngressRequest extends Request { apiKeyId?: { toString(): string } | string; requestId?: string; user?: AgentTriggerIngressUser; + _agentEventBindingResolved?: boolean; } interface AgentTriggerIngressBody { mode?: AgentTriggerMode; event?: AgentTriggerEvent; - target?: AgentFireTarget | AgentSteerTarget; + target?: AgentContinueTarget | AgentFireTarget | AgentSteerTarget; input?: string; orderingKey?: string; } @@ -216,18 +218,31 @@ export function createAgentTriggerIngressHandlers(deps: AgentTriggerIngressDepen }, input: body.input as string, }; - const envelope = - body.mode === 'fire' - ? createAgentTriggerEnvelope({ - ...common, - mode: body.mode, - target: body.target as AgentFireTarget, - }) - : createAgentTriggerEnvelope({ - ...common, - mode: body.mode as 'steer', - target: body.target as AgentSteerTarget, - }); + if (body.mode === 'continue' && req._agentEventBindingResolved !== true) { + throw new AgentTriggerIngressError( + 'Continue events require an authenticated agent-event binding', + ); + } + let envelope; + if (body.mode === 'fire') { + envelope = createAgentTriggerEnvelope({ + ...common, + mode: 'fire', + target: body.target as AgentFireTarget, + }); + } else if (body.mode === 'continue') { + envelope = createAgentTriggerEnvelope({ + ...common, + mode: 'continue', + target: body.target as AgentContinueTarget, + }); + } else { + envelope = createAgentTriggerEnvelope({ + ...common, + mode: 'steer', + target: body.target as AgentSteerTarget, + }); + } const receipt = await deps.enqueue(envelope, enqueueOptions(body)); logger.info('[agent-trigger-ingress] delivery accepted', { diff --git a/packages/api/src/agents/triggers/lease.spec.ts b/packages/api/src/agents/triggers/lease.spec.ts new file mode 100644 index 0000000000..a40125a493 --- /dev/null +++ b/packages/api/src/agents/triggers/lease.spec.ts @@ -0,0 +1,230 @@ +import type { ConversationMethods } from '@librechat/data-schemas'; +import type { AbortResult } from '../../stream/interfaces/IJobStore'; +import { createEventChildGenerationLeaseAcquirer } from './lease'; + +type LeaseMethods = Pick< + ConversationMethods, + 'acquireSubagentThreadLease' | 'renewSubagentThreadLease' | 'releaseSubagentThreadLease' +>; + +const stoppedResult = (failureReason?: AbortResult['failureReason']): AbortResult => ({ + success: failureReason == null, + failureReason, + jobData: null, + content: [], + finalEvent: null, + text: '', + collectedUsage: [], +}); + +describe('event child generation lease', () => { + const acquireSubagentThreadLease = jest.fn< + ReturnType, + Parameters + >(); + const renewSubagentThreadLease = jest.fn< + ReturnType, + Parameters + >(); + const releaseSubagentThreadLease = jest.fn< + ReturnType, + Parameters + >(); + const abortGeneration = jest.fn< + Promise, + [string, { expectedCreatedAt: number; awaitProviderDrain: true }] + >(); + const acquireEventChildGenerationLease = createEventChildGenerationLeaseAcquirer({ + methods: { + acquireSubagentThreadLease, + renewSubagentThreadLease, + releaseSubagentThreadLease, + }, + abortGeneration, + }); + + beforeEach(() => { + jest.useFakeTimers({ now: new Date('2026-08-22T00:00:00.000Z') }); + jest.clearAllMocks(); + acquireSubagentThreadLease.mockResolvedValue(true); + renewSubagentThreadLease.mockResolvedValue(true); + releaseSubagentThreadLease.mockResolvedValue(true); + abortGeneration.mockResolvedValue(stoppedResult()); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('rejects and releases an initial lease that resolves after its deadline', async () => { + let resolveAcquisition: (acquired: boolean) => void = () => undefined; + acquireSubagentThreadLease.mockImplementation( + () => + new Promise((resolve) => { + resolveAcquisition = resolve; + }), + ); + const acquisition = acquireEventChildGenerationLease({ + userId: 'user-1', + tenantId: 'tenant-1', + conversationId: 'child-1', + streamId: 'child-1', + jobCreatedAt: 123, + }); + + jest.setSystemTime(new Date('2026-08-22T00:00:30.001Z')); + resolveAcquisition(true); + + await expect(acquisition).resolves.toBeNull(); + expect(releaseSubagentThreadLease).toHaveBeenCalledWith( + expect.objectContaining({ + user: 'user-1', + tenantId: 'tenant-1', + conversationId: 'child-1', + }), + ); + expect(abortGeneration).not.toHaveBeenCalled(); + expect(jest.getTimerCount()).toBe(0); + }); + + it('refreshes a near-expiry initial lease before accepting it', async () => { + let resolveAcquisition: (acquired: boolean) => void = () => undefined; + acquireSubagentThreadLease.mockImplementation( + () => + new Promise((resolve) => { + resolveAcquisition = resolve; + }), + ); + const acquisition = acquireEventChildGenerationLease({ + userId: 'user-1', + conversationId: 'child-1', + streamId: 'child-1', + jobCreatedAt: 123, + }); + + jest.setSystemTime(new Date('2026-08-22T00:00:25.000Z')); + resolveAcquisition(true); + const release = await acquisition; + + expect(renewSubagentThreadLease).toHaveBeenCalledWith( + expect.objectContaining({ + now: new Date('2026-08-22T00:00:25.000Z'), + expiresAt: new Date('2026-08-22T00:00:55.000Z'), + }), + ); + expect(release).not.toBeNull(); + await release?.(); + }); + + it('aborts when a renewal lands after continuous ownership expired', async () => { + let resolveRenewal: (renewed: boolean) => void = () => undefined; + renewSubagentThreadLease.mockImplementation( + () => + new Promise((resolve) => { + resolveRenewal = resolve; + }), + ); + const release = await acquireEventChildGenerationLease({ + userId: 'user-1', + tenantId: 'tenant-1', + conversationId: 'child-1', + streamId: 'child-1', + jobCreatedAt: 123, + }); + + jest.advanceTimersByTime(10_000); + await Promise.resolve(); + jest.setSystemTime(new Date('2026-08-22T00:00:30.001Z')); + resolveRenewal(true); + await Promise.resolve(); + await Promise.resolve(); + + expect(abortGeneration).toHaveBeenCalledWith('child-1', { + expectedCreatedAt: 123, + awaitProviderDrain: true, + }); + await release?.(); + }); + + it('aborts when renewal throws instead of silently running past expiry', async () => { + renewSubagentThreadLease.mockRejectedValue(new Error('mongo unavailable')); + const release = await acquireEventChildGenerationLease({ + userId: 'user-1', + conversationId: 'child-1', + streamId: 'child-1', + jobCreatedAt: 123, + }); + + await jest.advanceTimersByTimeAsync(10_000); + + expect(abortGeneration).toHaveBeenCalledWith('child-1', { + expectedCreatedAt: 123, + awaitProviderDrain: true, + }); + await release?.(); + }); + + it('caps ownership and aborts the exact generation at the inherited retention deadline', async () => { + const retentionExpiresAt = new Date('2026-08-22T00:00:05.000Z'); + const release = await acquireEventChildGenerationLease({ + userId: 'user-1', + tenantId: 'tenant-1', + conversationId: 'child-1', + streamId: 'child-1', + jobCreatedAt: 123, + retentionExpiresAt, + }); + + expect(acquireSubagentThreadLease).toHaveBeenCalledWith( + expect.objectContaining({ expiresAt: retentionExpiresAt }), + ); + await jest.advanceTimersByTimeAsync(4_999); + expect(abortGeneration).not.toHaveBeenCalled(); + + await jest.advanceTimersByTimeAsync(1); + expect(abortGeneration).toHaveBeenCalledWith('child-1', { + expectedCreatedAt: 123, + awaitProviderDrain: true, + }); + await release?.(); + }); + + it('retains the fence and retries an unconfirmed deadline abort', async () => { + abortGeneration + .mockResolvedValueOnce(stoppedResult('job_still_active')) + .mockResolvedValueOnce(stoppedResult()); + const release = await acquireEventChildGenerationLease({ + userId: 'user-1', + conversationId: 'child-1', + streamId: 'child-1', + jobCreatedAt: 123, + retentionExpiresAt: new Date('2026-08-22T00:00:05.000Z'), + }); + + await jest.advanceTimersByTimeAsync(5_000); + expect(abortGeneration).toHaveBeenCalledTimes(1); + expect(releaseSubagentThreadLease).not.toHaveBeenCalled(); + + await jest.advanceTimersByTimeAsync(250); + expect(abortGeneration).toHaveBeenCalledTimes(2); + await release?.(); + }); + + it('retains the fence and retries when the deadline abort throws', async () => { + abortGeneration + .mockRejectedValueOnce(new Error('abort store unavailable')) + .mockResolvedValueOnce(stoppedResult()); + const release = await acquireEventChildGenerationLease({ + userId: 'user-1', + conversationId: 'child-1', + streamId: 'child-1', + jobCreatedAt: 123, + retentionExpiresAt: new Date('2026-08-22T00:00:05.000Z'), + }); + + await jest.advanceTimersByTimeAsync(5_250); + expect(abortGeneration).toHaveBeenCalledTimes(2); + expect(releaseSubagentThreadLease).not.toHaveBeenCalled(); + await release?.(); + }); +}); diff --git a/packages/api/src/agents/triggers/lease.ts b/packages/api/src/agents/triggers/lease.ts new file mode 100644 index 0000000000..ebd5df076d --- /dev/null +++ b/packages/api/src/agents/triggers/lease.ts @@ -0,0 +1,228 @@ +import { randomUUID } from 'node:crypto'; +import { logger } from '@librechat/data-schemas'; +import type { ConversationMethods } from '@librechat/data-schemas'; +import type { AbortResult } from '../../stream/interfaces/IJobStore'; +import { isStopConfirmed } from '../../stream/interfaces/IJobStore'; + +const EVENT_CHILD_LEASE_TTL_MS = 30_000; +const EVENT_CHILD_LEASE_HEARTBEAT_MS = 10_000; +const EVENT_CHILD_ABORT_RETRY_MS = 250; +const MAX_TIMER_DELAY_MS = 2_147_483_647; + +type EventChildLeaseMethods = Pick< + ConversationMethods, + 'acquireSubagentThreadLease' | 'renewSubagentThreadLease' | 'releaseSubagentThreadLease' +>; + +interface AbortGenerationOptions { + expectedCreatedAt: number; + awaitProviderDrain: true; +} + +export interface EventChildGenerationLeaseDependencies { + methods: EventChildLeaseMethods; + abortGeneration: (streamId: string, options: AbortGenerationOptions) => Promise; +} + +export interface EventChildGenerationLeaseInput { + userId: string; + tenantId?: string; + conversationId: string; + streamId: string; + jobCreatedAt: number; + retentionExpiresAt?: Date | string | number; +} + +export type ReleaseEventChildGenerationLease = () => Promise; + +/** Makes an event-driven child generation visible to the durable deletion protocol. */ +export function createEventChildGenerationLeaseAcquirer({ + methods, + abortGeneration, +}: EventChildGenerationLeaseDependencies) { + return async function acquireEventChildGenerationLease({ + userId, + tenantId, + conversationId, + streamId, + jobCreatedAt, + retentionExpiresAt, + }: EventChildGenerationLeaseInput): Promise { + const token = randomUUID(); + const leaseIdentity = { + user: userId, + conversationId, + token, + ...(tenantId == null ? {} : { tenantId }), + }; + const initialTime = Date.now(); + const retentionDeadline = + retentionExpiresAt == null ? undefined : new Date(retentionExpiresAt).getTime(); + if ( + retentionDeadline != null && + (!Number.isFinite(retentionDeadline) || retentionDeadline <= initialTime) + ) { + return null; + } + const initialLeaseDeadline = Math.min( + initialTime + EVENT_CHILD_LEASE_TTL_MS, + retentionDeadline ?? Number.POSITIVE_INFINITY, + ); + const releaseRejectedLease = async (): Promise => { + await methods.releaseSubagentThreadLease(leaseIdentity).catch((error) => { + logger.warn('[EventChildLease] Failed to release a rejected initial lease', { error }); + }); + }; + const acquired = await methods.acquireSubagentThreadLease({ + ...leaseIdentity, + taskId: streamId, + now: new Date(initialTime), + expiresAt: new Date(initialLeaseDeadline), + }); + if (!acquired) { + return null; + } + const acquiredAt = Date.now(); + if (acquiredAt >= initialLeaseDeadline) { + await releaseRejectedLease(); + return null; + } + + let stopped = false; + let leaseLost = false; + let heldUntil = initialLeaseDeadline; + if ( + initialLeaseDeadline !== retentionDeadline && + initialLeaseDeadline - acquiredAt <= EVENT_CHILD_LEASE_HEARTBEAT_MS + ) { + const refreshedUntil = Math.min( + acquiredAt + EVENT_CHILD_LEASE_TTL_MS, + retentionDeadline ?? Number.POSITIVE_INFINITY, + ); + let refreshed: boolean; + try { + refreshed = await methods.renewSubagentThreadLease({ + ...leaseIdentity, + now: new Date(acquiredAt), + expiresAt: new Date(refreshedUntil), + }); + } catch (error) { + await releaseRejectedLease(); + throw error; + } + if (!refreshed || Date.now() >= initialLeaseDeadline) { + await releaseRejectedLease(); + return null; + } + heldUntil = refreshedUntil; + } + let renewalInFlight: Promise | undefined; + let abortInFlight: Promise | undefined; + let deadlineTimer: NodeJS.Timeout | undefined; + const abortForLostLease = (message: string, error?: unknown): Promise => { + if (stopped) { + return Promise.resolve(); + } + if (abortInFlight != null) { + return abortInFlight; + } + leaseLost = true; + logger.warn(message, error == null ? undefined : { error }); + /** Retain the durable fence until the exact generation is confirmed stopped. + * An abort reply can be ambiguous (`job_still_active`, `job_not_found`) and a + * store/provider failure can throw after the deadline has already fired. The + * owner therefore retries until abort is authoritative or its own provider + * finishes and calls `release`, which is the alternate proof of drain. */ + abortInFlight = (async () => { + while (!stopped) { + try { + const result = await abortGeneration(streamId, { + expectedCreatedAt: jobCreatedAt, + awaitProviderDrain: true, + }); + if (isStopConfirmed(result)) { + return; + } + logger.warn('[EventChildLease] Generation stop was not confirmed; retrying', { + streamId, + failureReason: result.failureReason, + }); + } catch (abortError) { + logger.warn('[EventChildLease] Failed to stop generation after lease loss; retrying', { + streamId, + error: abortError, + }); + } + await new Promise((resolve) => setTimeout(resolve, EVENT_CHILD_ABORT_RETRY_MS)); + } + })(); + return abortInFlight; + }; + const renew = (): void => { + if (stopped || leaseLost || renewalInFlight != null) { + return; + } + renewalInFlight = (async () => { + const previousDeadline = heldUntil; + const renewalTime = Date.now(); + const renewedUntil = Math.min( + renewalTime + EVENT_CHILD_LEASE_TTL_MS, + retentionDeadline ?? Number.POSITIVE_INFINITY, + ); + if (renewedUntil <= renewalTime) { + await abortForLostLease( + '[EventChildLease] Generation reached its inherited retention deadline', + ); + return; + } + const held = await methods.renewSubagentThreadLease({ + ...leaseIdentity, + now: new Date(renewalTime), + expiresAt: new Date(renewedUntil), + }); + if (!held || Date.now() >= previousDeadline) { + await abortForLostLease( + '[EventChildLease] Generation lost continuous ownership of its lease', + ); + return; + } + heldUntil = renewedUntil; + })() + .catch((error) => + abortForLostLease('[EventChildLease] Renewal failed; stopping generation', error), + ) + .finally(() => { + renewalInFlight = undefined; + }); + }; + const armRetentionDeadline = (): void => { + if (retentionDeadline == null || stopped || leaseLost) { + return; + } + const remaining = retentionDeadline - Date.now(); + if (remaining <= 0) { + void abortForLostLease( + '[EventChildLease] Generation reached its inherited retention deadline', + ); + return; + } + deadlineTimer = setTimeout(armRetentionDeadline, Math.min(remaining, MAX_TIMER_DELAY_MS)); + }; + const heartbeat = setInterval(renew, EVENT_CHILD_LEASE_HEARTBEAT_MS); + armRetentionDeadline(); + + return async () => { + if (stopped) { + return; + } + stopped = true; + clearInterval(heartbeat); + clearTimeout(deadlineTimer); + await renewalInFlight; + await abortInFlight; + await methods.releaseSubagentThreadLease(leaseIdentity).catch((error) => { + logger.warn('[EventChildLease] Release failed', { error }); + }); + }; + }; +} diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 52932db21e..b06064ceab 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -21,9 +21,12 @@ const endpointsConfig: TEndpointsConfig = { }; describe('excludedKeys', () => { - it.each(['_id', 'user', 'conversationId', '__v'])('excludes system field "%s"', (field) => { - expect(excludedKeys.has(field)).toBe(true); - }); + it.each(['_id', 'user', 'conversationId', 'agentEventBinding', '__v'])( + 'excludes system field "%s"', + (field) => { + expect(excludedKeys.has(field)).toBe(true); + }, + ); it('does not exclude tenantId (plugin-level guard owns this)', () => { expect(excludedKeys.has('tenantId')).toBe(false); diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 87363eb2c9..e512f36da4 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -69,6 +69,7 @@ export const defaultRetrievalModels = [ export const excludedKeys = new Set([ 'conversationId', + 'agentEventBinding', 'subagentThread', 'title', 'iconURL', diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index 3d1507286d..637613aa88 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -1,5 +1,5 @@ -import mongoose from 'mongoose'; import { v4 as uuidv4 } from 'uuid'; +import mongoose, { type FilterQuery } from 'mongoose'; import { MongoMemoryServer } from 'mongodb-memory-server'; import { EModelEndpoint, RetentionMode } from 'librechat-data-provider'; import type { @@ -114,6 +114,7 @@ describe('Conversation Operations', () => { let mockCtx: { userId: string; isTemporary?: boolean; + expiredAt?: Date; interfaceConfig?: { temporaryChatRetention?: number; retentionMode?: RetentionMode }; }; let mockConversationData: { @@ -984,6 +985,18 @@ describe('Conversation Operations', () => { ); }); + it('preserves an exact inherited expiration instead of recomputing retention', async () => { + const inheritedExpiration = new Date('2026-08-22T03:04:05.000Z'); + mockCtx.isTemporary = true; + mockCtx.expiredAt = inheritedExpiration; + mockCtx.interfaceConfig = { temporaryChatRetention: 48 }; + + const result = await saveConvo(mockCtx, mockConversationData); + + expect(result?.isTemporary).toBe(true); + expect(result?.expiredAt).toEqual(inheritedExpiration); + }); + it('should save a conversation without expiredAt when isTemporary is false', async () => { mockCtx.isTemporary = false; @@ -1727,11 +1740,17 @@ describe('Conversation Operations', () => { }, ]); deleteMessages.mockResolvedValue({ acknowledged: true, deletedCount: 3 }); + const beforeDelete = jest.fn(async (conversationIds: string[]) => { + expect( + await Conversation.countDocuments({ conversationId: { $in: conversationIds } }), + ).toBe(conversationIds.length); + }); - const result = await deleteConvos('user123', { conversationId: parentId }); + const result = await deleteConvos('user123', { conversationId: parentId }, { beforeDelete }); expect(result.deletedCount).toBe(3); expect(result.conversationIds).toEqual([parentId, childId, grandchildId]); + expect(beforeDelete.mock.calls).toEqual([[[parentId]], [[childId]], [[grandchildId]]]); expect(deleteMessages).toHaveBeenCalledWith({ conversationId: { $in: [parentId, childId, grandchildId] }, user: 'user123', @@ -1740,6 +1759,82 @@ describe('Conversation Operations', () => { expect(await Conversation.findOne({ conversationId: otherUsersChildId })).not.toBeNull(); }); + it('reports a partial cascade failure instead of silently succeeding', async () => { + const parentId = uuidv4(); + const childId = uuidv4(); + const project = await ChatProject.create({ + user: 'user123', + name: 'Partial Cascade', + conversationCount: 1, + lastConversationId: parentId, + }); + await ConversationTag.create({ user: 'user123', tag: 'work', count: 1, position: 1 }); + await Conversation.create([ + { + conversationId: parentId, + user: 'user123', + endpoint: EModelEndpoint.agents, + chatProjectId: project._id!.toString(), + tags: ['work'], + }, + { + conversationId: childId, + user: 'user123', + endpoint: EModelEndpoint.agents, + subagentThread: { + rootConversationId: parentId, + parentConversationId: parentId, + parentMessageId: 'message-1', + parentToolCallId: 'tool-1', + subagentType: 'agent-child', + subagentKind: 'agent', + depth: 1, + }, + }, + ]); + const realFind = Conversation.find.bind(Conversation); + const findSpy = jest.spyOn(Conversation, 'find').mockImplementation(((filter) => { + if ( + filter != null && + Object.prototype.hasOwnProperty.call(filter, 'subagentThread.parentConversationId') + ) { + return { + select: () => ({ lean: () => Promise.reject(new Error('stepdown')) }), + }; + } + return realFind(filter as FilterQuery); + }) as typeof Conversation.find); + + await expect(deleteConvos('user123', { conversationId: parentId })).rejects.toThrow( + 'stepdown', + ); + expect(findSpy).toHaveBeenCalledTimes(4); + expect(await Conversation.findOne({ conversationId: parentId })).toBeNull(); + expect(await Conversation.findOne({ conversationId: childId })).not.toBeNull(); + expect(deleteMessages).not.toHaveBeenCalled(); + expect((await ConversationTag.findOne({ user: 'user123', tag: 'work' }).lean())?.count).toBe( + 0, + ); + expect( + (await ChatProject.findById(project._id).lean())?.conversationCount, + ).toBe(0); + + findSpy.mockRestore(); + deleteMessages.mockResolvedValue({ acknowledged: true, deletedCount: 2 }); + const recovered = await deleteConvos('user123', { conversationId: parentId }); + + expect(recovered.conversationIds).toEqual([parentId, childId]); + expect(await Conversation.findOne({ conversationId: childId })).toBeNull(); + expect(deleteMessages).toHaveBeenCalledWith({ + conversationId: { $in: [parentId, childId] }, + user: 'user123', + }); + expect((await ConversationTag.findOne({ user: 'user123', tag: 'work' }).lean())?.count).toBe( + 0, + ); + findSpy.mockRestore(); + }); + it('does not delete a parent when deleting one child thread', async () => { const parentId = uuidv4(); const childId = uuidv4(); @@ -3494,6 +3589,70 @@ describe('Conversation Operations', () => { ); }); + it('resolves an event binding only through its owner, tenant, and API key', async () => { + const conversationId = uuidv4(); + const bindingId = `evtbind_${'a'.repeat(48)}`; + await Conversation.create({ + conversationId, + user: 'binding-user', + tenantId: 'tenant-a', + endpoint: EModelEndpoint.agents, + agent_id: 'agent-player', + agentEventBinding: { bindingId, sourceKeyId: 'key-a', actorId: 'player-a' }, + subagentThread: { + rootConversationId: 'parent', + parentConversationId: 'parent', + parentMessageId: 'parent-message', + parentToolCallId: 'event-binding', + parentAgentId: 'agent-director', + subagentType: 'agent-player', + subagentKind: 'agent', + depth: 1, + }, + }); + + await expect( + methods.getAgentEventBinding({ + user: 'binding-user', + tenantId: 'tenant-a', + bindingId, + sourceKeyId: 'key-a', + }), + ).resolves.toMatchObject({ + conversationId, + agentId: 'agent-player', + binding: { bindingId, sourceKeyId: 'key-a', actorId: 'player-a' }, + }); + await expect( + methods.getAgentEventBinding({ + user: 'binding-user', + tenantId: 'tenant-a', + bindingId, + sourceKeyId: 'key-b', + }), + ).resolves.toBeNull(); + await expect( + methods.getAgentEventBinding({ + user: 'binding-user', + tenantId: 'tenant-b', + bindingId, + sourceKeyId: 'key-a', + }), + ).resolves.toBeNull(); + await Conversation.updateOne({ conversationId }, { expiredAt: new Date(0) }); + await expect( + methods.getAgentEventBinding({ + user: 'binding-user', + tenantId: 'tenant-a', + bindingId, + sourceKeyId: 'key-a', + }), + ).resolves.toBeNull(); + expect(await methods.getConvo('binding-user', conversationId)).not.toHaveProperty( + 'agentEventBinding', + ); + }); + it('admits one cross-replica owner and fences renewal and release by token', async () => { const conversationId = uuidv4(); await Conversation.create({ diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index c821e89684..b09cbca203 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -2,6 +2,7 @@ import { RetentionMode } from 'librechat-data-provider'; import type { FilterQuery, Model, SortOrder, Types } from 'mongoose'; import type { DeleteResult } from 'mongoose'; import type { + IAgentEventBindingRecord, AppConfig, IChatProjectDocument, IActiveSubagentThreadLease, @@ -120,7 +121,12 @@ export interface ConversationMethods { messages: { deletedCount?: number }; }>; saveConvo( - ctx: { userId: string; isTemporary?: boolean; interfaceConfig?: AppConfig['interfaceConfig'] }, + ctx: { + userId: string; + isTemporary?: boolean; + expiredAt?: Date; + interfaceConfig?: AppConfig['interfaceConfig']; + }, data: { conversationId: string; newConversationId?: string; [key: string]: unknown }, metadata?: { context?: string; @@ -167,6 +173,12 @@ export interface ConversationMethods { conversationId: string; tenantId?: string; }): Promise; + getAgentEventBinding(input: { + user: string; + bindingId: string; + sourceKeyId: string; + tenantId?: string; + }): Promise; reserveSubagentThread(input: { user: string; conversationId: string; @@ -219,6 +231,7 @@ export interface ConversationMethods { deleteConvos( user: string, filter: FilterQuery, + options?: { beforeDelete?: (conversationIds: string[]) => Promise }, ): Promise; archiveAllConvos(user: string): Promise<{ archivedCount: number }>; } @@ -297,6 +310,43 @@ export function createConversationMethods( } } + /** Resolves an event target only when the API key, owner, and tenant all match. */ + async function getAgentEventBinding(input: { + user: string; + bindingId: string; + sourceKeyId: string; + tenantId?: string; + }): Promise { + const Conversation = mongoose.models.Conversation as Model; + const conversation = await Conversation.findOne({ + user: input.user, + 'agentEventBinding.bindingId': input.bindingId, + 'agentEventBinding.sourceKeyId': input.sourceKeyId, + ...subagentLeaseTenantFilter(input.tenantId), + ...activeExpirationFilter(), + }) + .select( + 'conversationId agent_id tenantId isTemporary expiredAt subagentThread +agentEventBinding', + ) + .lean(); + if ( + conversation?.agentEventBinding == null || + conversation.subagentThread == null || + typeof conversation.agent_id !== 'string' + ) { + return null; + } + return { + conversationId: conversation.conversationId, + agentId: conversation.agent_id, + ...(conversation.tenantId == null ? {} : { tenantId: conversation.tenantId }), + ...(conversation.isTemporary == null ? {} : { isTemporary: conversation.isTemporary }), + ...(conversation.expiredAt == null ? {} : { expiredAt: conversation.expiredAt }), + binding: conversation.agentEventBinding, + lineage: conversation.subagentThread, + }; + } + /** Creates immutable child lineage exactly once without overwriting a concurrent winner. */ async function reserveSubagentThread(input: { user: string; @@ -322,7 +372,7 @@ export function createConversationMethods( }, }, { new: true, upsert: true, includeResultMetadata: true, setDefaultsOnInsert: true }, - )) as unknown as ConversationUpdateResult; + ).select('+agentEventBinding')) as unknown as ConversationUpdateResult; if (result.value == null) { throw new Error('Unable to reserve the subagent thread.'); } @@ -334,7 +384,9 @@ export function createConversationMethods( /** Concurrent upserts can race at the unique index. The document that won is * the reservation; callers still validate its immutable lineage before use. */ if ((error as { code?: number }).code === 11000) { - const existing = await Conversation.findOne(filter).lean(); + const existing = await Conversation.findOne(filter) + .select('+agentEventBinding') + .lean(); if (existing != null) { return { conversation: existing, created: false }; } @@ -570,10 +622,12 @@ export function createConversationMethods( { userId, isTemporary, + expiredAt, interfaceConfig, }: { userId: string; isTemporary?: boolean; + expiredAt?: Date; interfaceConfig?: AppConfig['interfaceConfig']; }, { @@ -640,7 +694,12 @@ export function createConversationMethods( update.conversationId = newConversationId; } - if (interfaceConfig?.retentionMode === RetentionMode.ALL) { + if (expiredAt instanceof Date && !Number.isNaN(expiredAt.getTime())) { + if (typeof isTemporary === 'boolean') { + update.isTemporary = isTemporary; + } + update.expiredAt = expiredAt; + } else if (interfaceConfig?.retentionMode === RetentionMode.ALL) { if (typeof isTemporary === 'boolean') { update.isTemporary = isTemporary; } @@ -1347,16 +1406,56 @@ export function createConversationMethods( /** * Deletes conversations and their associated messages for a given user and filter. */ - async function deleteConvos(user: string, filter: FilterQuery) { + async function deleteConvos( + user: string, + filter: FilterQuery, + options?: { beforeDelete?: (conversationIds: string[]) => Promise }, + ) { try { const Conversation = mongoose.models.Conversation as Model; - const { deleteMessages } = getMessageMethods(); + const { deleteMessages, getMessages } = getMessageMethods(); const userFilter = { ...filter, user }; type DeletionConversation = Pick; - const conversations = await Conversation.find(userFilter) + const retryCascadeOperation = async (operation: () => PromiseLike | T): Promise => { + let lastError: unknown; + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + return await operation(); + } catch (error) { + lastError = error; + if (attempt < 3) { + await new Promise((resolve) => setTimeout(resolve, 25 * attempt)); + } + } + } + throw lastError; + }; + let conversations = await Conversation.find(userFilter) .select('conversationId chatProjectId tags') .lean(); - if (!conversations.length) { + const recoveryConversationIds: string[] = []; + if (!conversations.length && typeof filter.conversationId === 'string') { + /** A prior attempt may have deleted the root before a descendant read failed. + * Resume from immutable root lineage and retain the root id for message, + * checkpoint, and tool cleanup. The message probe distinguishes that partial + * commit from a conversation id that never existed. */ + const [descendants, rootMessages] = await Promise.all([ + retryCascadeOperation(() => + Conversation.find({ + user, + 'subagentThread.rootConversationId': filter.conversationId, + }) + .select('conversationId chatProjectId tags') + .lean(), + ), + getMessages({ user, conversationId: filter.conversationId }, '_id', { limit: 1 }), + ]); + if (descendants.length === 0 && rootMessages.length === 0) { + throw new Error('Conversation not found or already deleted.'); + } + conversations = descendants; + recoveryConversationIds.push(filter.conversationId); + } else if (!conversations.length) { throw new Error('Conversation not found or already deleted.'); } @@ -1372,72 +1471,72 @@ export function createConversationMethods( let pending = conversations; let acknowledged = true; let deletedCount = 0; + const reconcileDeletedWave = async ( + wave: DeletionConversation[], + waveDeletedCount: number, + ): Promise => { + if (waveDeletedCount === 0) { + return; + } + + /** + * Commit derived metadata while the deleted documents are still available in + * memory. Descendant discovery can fail after this point; deferring the + * reconciliation until the whole walk completes would make the root's tags and + * project impossible to recover on a later retry. + */ + const tagDecrements: string[] = []; + for (const conversation of wave) { + for (const tag of new Set(conversation.tags ?? [])) { + tagDecrements.push(tag); + } + } + await decrementTagCounts(mongoose, user, tagDecrements); + + const waveProjectIds = new Set( + wave + .map((conversation) => conversation.chatProjectId) + .filter((projectId): projectId is string => Boolean(projectId)), + ); + if (waveProjectIds.size > 0) { + try { + await refreshChatProjectStatsInBatches(mongoose, user, waveProjectIds); + } catch (error) { + logger.error('[deleteConvos] Conversations deleted but stats refresh failed', error); + } + } + }; while (pending.length > 0) { const wave = pending.filter((conversation) => !seen.has(conversation.conversationId)); if (wave.length === 0) { break; } const waveIds = wave.map((conversation) => conversation.conversationId); - try { - const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } }); - acknowledged &&= result.acknowledged; - deletedCount += result.deletedCount; - for (const conversation of wave) { - seen.add(conversation.conversationId); - deletedConversations.push(conversation); - } - pending = await Conversation.find({ + await options?.beforeDelete?.(waveIds); + const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } }); + acknowledged &&= result.acknowledged; + deletedCount += result.deletedCount; + await reconcileDeletedWave(wave, result.deletedCount); + for (const conversation of wave) { + seen.add(conversation.conversationId); + deletedConversations.push(conversation); + } + pending = await retryCascadeOperation(() => + Conversation.find({ user, 'subagentThread.parentConversationId': { $in: waveIds }, }) .select('conversationId chatProjectId tags') - .lean(); - } catch (error) { - if (deletedConversations.length === 0) { - throw error; - } - logger.error('[deleteConvos] Root deleted but child-thread cascade failed', error); - break; - } + .lean(), + ); } - const conversationIds = deletedConversations.map((c) => c.conversationId); - const projectIds = new Set( - deletedConversations - .map((conversation) => conversation.chatProjectId) - .filter((projectId): projectId is string => Boolean(projectId)), - ); - - /** - * One entry per (conversation, tag) association: each conversation's tags are - * deduped so a duplicate tag entry within a single conversation only decrements - * the bookmark count once. - */ - const tagDecrements: string[] = []; - for (const conversation of deletedConversations) { - if (!conversation.tags?.length) { - continue; - } - for (const tag of new Set(conversation.tags)) { - tagDecrements.push(tag); - } - } + const conversationIds = [ + ...recoveryConversationIds, + ...deletedConversations.map((conversation) => conversation.conversationId), + ]; const deleteConvoResult: DeleteResult = { acknowledged, deletedCount }; - const deleted = deleteConvoResult.deletedCount > 0; - - /** - * Reconcile bookmark counts from the deletion before message cleanup: if - * `deleteMessages` later throws, the conversation is already gone and a retry - * finds nothing, so the count must be reconciled here or it would stay stale. - * The decrement is best-effort and never throws, so it cannot block message - * cleanup. The `deletedCount` guard skips a losing concurrent delete whose - * pre-delete snapshot would otherwise decrement a conversation it did not - * actually remove. - */ - if (deleted) { - await decrementTagCounts(mongoose, user, tagDecrements); - } /** * Post-delete cleanup is best-effort: the conversations are already gone, so a @@ -1455,19 +1554,6 @@ export function createConversationMethods( logger.error('[deleteConvos] Conversations deleted but message cleanup failed', error); } - /** - * Refresh project stats after message cleanup so a stats-refresh error cannot - * prevent `deleteMessages` from running, which would orphan the deleted - * conversations' messages. - */ - if (deleted && projectIds.size > 0) { - try { - await refreshChatProjectStatsInBatches(mongoose, user, projectIds); - } catch (error) { - logger.error('[deleteConvos] Conversations deleted but stats refresh failed', error); - } - } - // conversationIds lets callers run sibling cleanup that lives in higher layers // (e.g. pruning the conversations' durable agent checkpoints) without re-querying // documents that no longer exist. @@ -1617,6 +1703,7 @@ export function createConversationMethods( getConvosQueried, getConvo, getSubagentThreadForParent, + getAgentEventBinding, reserveSubagentThread, acquireSubagentThreadLease, renewSubagentThreadLease, diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.ts index 8c4bfd366c..2541ef1ce7 100644 --- a/packages/data-schemas/src/methods/message.spec.ts +++ b/packages/data-schemas/src/methods/message.spec.ts @@ -74,6 +74,7 @@ describe('Message Operations', () => { let mockCtx: { userId: string; isTemporary?: boolean; + expiredAt?: Date; interfaceConfig?: { temporaryChatRetention?: number; retentionMode?: RetentionMode }; }; let mockMessageData: Partial = { @@ -1310,6 +1311,18 @@ describe('Message Operations', () => { ); }); + it('preserves an exact inherited expiration instead of recomputing retention', async () => { + const inheritedExpiration = new Date('2026-08-22T03:04:05.000Z'); + mockCtx.isTemporary = true; + mockCtx.expiredAt = inheritedExpiration; + mockCtx.interfaceConfig = { temporaryChatRetention: 48 }; + + const result = await saveMessage(mockCtx, mockMessageData); + + expect(result?.isTemporary).toBe(true); + expect(result?.expiredAt).toEqual(inheritedExpiration); + }); + it('should save a message without expiredAt when isTemporary is false', async () => { mockCtx.isTemporary = false; diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts index ded02af276..7752d86492 100644 --- a/packages/data-schemas/src/methods/message.ts +++ b/packages/data-schemas/src/methods/message.ts @@ -258,7 +258,12 @@ export type SubagentThreadViewMessageRecord = Pick< export interface MessageMethods { saveMessage( - ctx: { userId: string; isTemporary?: boolean; interfaceConfig?: AppConfig['interfaceConfig'] }, + ctx: { + userId: string; + isTemporary?: boolean; + expiredAt?: Date; + interfaceConfig?: AppConfig['interfaceConfig']; + }, params: Partial & { newMessageId?: string }, metadata?: { context?: string }, ): Promise; @@ -347,10 +352,12 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa { userId, isTemporary, + expiredAt, interfaceConfig, }: { userId: string; isTemporary?: boolean; + expiredAt?: Date; interfaceConfig?: AppConfig['interfaceConfig']; }, params: Partial & { newMessageId?: string }, @@ -376,7 +383,12 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa messageId: params.newMessageId || params.messageId, }; - if (interfaceConfig?.retentionMode === RetentionMode.ALL) { + if (expiredAt instanceof Date && !Number.isNaN(expiredAt.getTime())) { + if (typeof isTemporary === 'boolean') { + update.isTemporary = isTemporary; + } + update.expiredAt = expiredAt; + } else if (interfaceConfig?.retentionMode === RetentionMode.ALL) { if (typeof isTemporary === 'boolean') { update.isTemporary = isTemporary; } diff --git a/packages/data-schemas/src/schema/convo.ts b/packages/data-schemas/src/schema/convo.ts index 4b58f63165..12ab6be0a7 100644 --- a/packages/data-schemas/src/schema/convo.ts +++ b/packages/data-schemas/src/schema/convo.ts @@ -54,6 +54,18 @@ const convoSchema: Schema = new Schema( default: undefined, select: false, }, + /** Authenticated event sources address child actors through this opaque binding. + * The source never supplies the stored agent/thread target during delivery. */ + agentEventBinding: { + type: { + bindingId: { type: String, required: true }, + sourceKeyId: { type: String, required: true }, + actorId: { type: String, required: true }, + }, + _id: false, + default: undefined, + select: false, + }, tags: { type: [String], default: [], @@ -106,6 +118,10 @@ convoSchema.index({ user: 1, isTemporary: 1, expiredAt: 1 }); /** Owner-scoped child-thread cascade lookup used when a parent is deleted. */ convoSchema.index({ user: 1, 'subagentThread.parentConversationId': 1 }); convoSchema.index({ user: 1, 'subagentThreadLease.expiresAt': 1 }); +convoSchema.index( + { 'agentEventBinding.bindingId': 1 }, + { unique: true, sparse: true, name: 'agent_event_binding_unique' }, +); // index for MeiliSearch sync operations convoSchema.index({ _meiliIndex: 1, isTemporary: 1, expiredAt: 1 }); diff --git a/packages/data-schemas/src/types/convo.ts b/packages/data-schemas/src/types/convo.ts index 387720cf9f..02534cf74a 100644 --- a/packages/data-schemas/src/types/convo.ts +++ b/packages/data-schemas/src/types/convo.ts @@ -7,6 +7,23 @@ export interface ISubagentThreadLease { expiresAt: Date; } +/** Server-private route from one authenticated event source to a child actor thread. */ +export interface IAgentEventBinding { + bindingId: string; + sourceKeyId: string; + actorId: string; +} + +export interface IAgentEventBindingRecord { + conversationId: string; + agentId: string; + tenantId?: string; + isTemporary?: boolean; + expiredAt?: Date; + binding: IAgentEventBinding; + lineage: TSubagentThreadLineage; +} + export interface IActiveSubagentThreadLease { conversationId: string; parentConversationId: string; @@ -56,6 +73,8 @@ export interface IConversation extends Document { subagentThread?: TSubagentThreadLineage; /** Internal execution fence. Excluded from ordinary conversation reads. */ subagentThreadLease?: ISubagentThreadLease; + /** Internal event-source identity. Excluded from ordinary conversation reads. */ + agentEventBinding?: IAgentEventBinding; assistant_id?: string; instructions?: string; stop?: string[];