diff --git a/packages/api/src/agents/responses/index.ts b/packages/api/src/agents/responses/index.ts index 7f3248a4d8..7014f90d1c 100644 --- a/packages/api/src/agents/responses/index.ts +++ b/packages/api/src/agents/responses/index.ts @@ -15,6 +15,7 @@ export { } from './persistence'; export { createBackgroundResponseService } from './background'; +export { createResponseExecutionRuntime } from './runtime'; export { createResponseDecisionSubmission } from './decision'; export { createResponseResumeValidation, createResponseResumeAdmission } from './resume'; export { createBackgroundResponseAuthority } from './authority'; @@ -23,7 +24,7 @@ export type { ResponseDecisionResult, ResponseDecisionSubmission, } from './decision'; - +export { pauseResponseForReview, type ResponsePauseDependencies } from './pause'; export { createBackgroundResponseQueue, backgroundResponseDeliveryKey } from './queue'; export { createBackgroundResponseWorkerAuth, diff --git a/packages/api/src/agents/responses/pause.spec.ts b/packages/api/src/agents/responses/pause.spec.ts new file mode 100644 index 0000000000..b755bc6c05 --- /dev/null +++ b/packages/api/src/agents/responses/pause.spec.ts @@ -0,0 +1,1804 @@ +import { z } from 'zod'; +import express from 'express'; +import mongoose from 'mongoose'; +import request from 'supertest'; +import { tool } from '@langchain/core/tools'; +import { FileSources } from 'librechat-data-provider'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { Run, Providers, Constants } from '@librechat/agents'; +import { FakeChatModel } from '@langchain/core/utils/testing'; +import { AIMessageChunk, HumanMessage } from '@langchain/core/messages'; +import { + createResponseStorage, + tenantStorage, + runAsSystem, + createModels, + createMethods, +} from '@librechat/data-schemas'; +import type { AppConfig, ResponseStorage } from '@librechat/data-schemas'; +import type { RunInterruptResult, RunConfig } from '@librechat/agents'; +import type { BaseMessage } from '@langchain/core/messages'; +import type { Agent } from 'librechat-data-provider'; +import type { AgentExecutionEnrollment } from '../remote/lifecycle'; +import type { ResponseExecutionRuntime } from './runtime'; +import type { ServerRequest } from '~/types/http'; +import type { createRun } from '../run'; +import { + buildAggregatedResponse, + createAggregatorEventHandlers, + createResponseAggregator, + createResponseContext, +} from './service'; +import { + createBackgroundResponseWorkerAuth, + mintResponseResumeWorkerToken, + mintBackgroundResponseWorkerToken, +} from './worker'; +import { + buildPendingAction, + computeAgentRequestFingerprint, + computeLegacyAgentRequestFingerprint, +} from '../hitl/policy'; +import { + getAgentCheckpointer, + LIBRECHAT_CHECKPOINT_NAMESPACE_KEY, + __resetCheckpointerForTests, +} from '../checkpointer'; +import { buildEffectiveToolApprovalPolicy, markToolApprovalAllowAlways } from '../hitl/allow'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { createResponseResumeAdmission, createResponseResumeValidation } from './resume'; +import { createResponseCompletion, createResponseExecutionRelease } from './completion'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { createAskUserQuestionTool } from '../hitl/askUserQuestionTool'; +import { createResponseAccountingCollector } from './accounting'; +import { NO_CHAT_PROJECT_CONTEXT_KEY } from '~/projects/context'; +import { buildPendingBackgroundResponse } from './background'; +import { createResponseDecisionSubmission } from './decision'; +import { resolveToolApprovalResume } from '../hitl/resume'; +import { enrollAgentExecution } from '../remote/lifecycle'; +import { createResponseExecutionRuntime } from './runtime'; +import { captureRunPendingAction } from '../hitl/capture'; +import { cleanupResponseCheckpoint } from './checkpoint'; +import { createResponseRetirement } from './retirement'; +import { buildHITLRunWiring } from '../hitl/runtime'; +import { GenerationJobManagerClass } from '~/stream'; +import { createAgentRunEnvelope } from '../envelope'; +import { pauseResponseForReview } from './pause'; +import { executeAgentRun } from '../remote/host'; + +let mongo: MongoMemoryReplSet; +let storage: ResponseStorage; +let manager: GenerationJobManagerClass; +const owner = { userId: '000000000000000000000012', tenantId: 'sdk-pause-tenant' }; +const responseId = 'resp_sdk_pause'; +const conversationId = 'sdk-pause-conversation'; +const scoped = (work: () => Promise) => tenantStorage.run(owner, work); +const checkpointer = { type: 'mongo' as const, ttl: 3600 }; + +function initializedReviewAgent(id: string): Parameters[0]['agents'][number] { + return { + id, + name: id, + description: null, + created_at: 1, + avatar: null, + provider: 'openAI', + model: 'fixture', + model_parameters: { + temperature: null, + maxContextTokens: null, + max_context_tokens: null, + max_output_tokens: null, + top_p: null, + frequency_penalty: null, + presence_penalty: null, + }, + }; +} + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + storage = createResponseStorage(mongoose); +}); +beforeEach(async () => { + __resetCheckpointerForTests(); + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + await mongoose.models.ResponseScope?.deleteMany({}); + }); + manager = new GenerationJobManagerClass(); + manager.configure({ + jobStore: new InMemoryJobStore({ ttlAfterComplete: 60_000 }), + eventTransport: new InMemoryEventTransport(), + cleanupOnComplete: false, + }); + manager.initialize(); +}); +afterEach(async () => { + await manager.destroy(); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); + +async function reserveOriginal() { + return storage.reserve({ + ...owner, + responseId, + requestKey: 'request', + fingerprint: 'request', + agentId: 'agent', + sourceKeyId: 'key', + requestJSON: JSON.stringify({ + model: 'agent', + input: 'Ask which environment.', + background: true, + store: true, + }), + conversationId, + parentMessageId: 'root', + }); +} + +async function setup( + onEnrolled?: (execution: AgentExecutionEnrollment) => Promise, + processRun?: ResponseExecutionRuntime['processRun'], + toolReviews = false, + reviewedToolName = 'review_lookup', +) { + const execution = await enrollAgentExecution( + { + runId: responseId, + userId: owner.userId, + conversationId, + agentId: 'agent', + protocol: 'responses', + retainTerminalPersistence: true, + isPrincipalActive: async () => true, + }, + { manager }, + ); + await reserveOriginal(); + if (onEnrolled) await onEnrolled(execution); + else + await storage.publish(owner, responseId, 0, { + status: 'in_progress', + responseJSON: '{}', + generationCreatedAt: execution.createdAt, + }); + await execution.beginProviderExecution(); + const job = (await manager.getJob(responseId))!; + const saver = await getAgentCheckpointer(checkpointer); + if (!saver || !job.metadata.checkpointNamespace) throw new Error('Missing real durable saver'); + const config = { + configurable: { + thread_id: conversationId, + [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: job.metadata.checkpointNamespace, + }, + version: 'v2' as const, + }; + let modelCalls = 0; + let toolExecutions = 0; + const buildRun = async ( + customHandlers?: RunConfig['customHandlers'], + remembered?: readonly string[], + ) => { + const wiring = toolReviews + ? buildHITLRunWiring( + buildEffectiveToolApprovalPolicy({ enabled: true, allowAlways: true }, [], remembered), + ) + : undefined; + const run = await Run.create({ + runId: responseId, + skipCleanup: true, + tokenCounter: () => 1, + customHandlers, + humanInTheLoop: { enabled: true }, + hooks: wiring?.hooks, + interruptingToolNames: toolReviews ? [] : ['ask_user_question'], + graphConfig: { + type: 'standard', + maxContextTokens: 100_000, + llmConfig: { provider: Providers.OPENAI, streaming: false, streamUsage: false }, + compileOptions: { checkpointer: saver }, + tools: toolReviews + ? [ + tool( + async ({ value }) => { + toolExecutions++; + return value; + }, + { + name: reviewedToolName, + description: 'A reviewed lookup', + schema: z.object({ value: z.string() }), + }, + ), + ] + : [createAskUserQuestionTool()], + }, + }); + if (!run.Graph) throw new Error('Missing SDK graph'); + const model = new FakeChatModel({}); + model._generate = async (messages: BaseMessage[]) => { + modelCalls++; + const answer = messages.find((message) => message.getType() === 'tool'); + if (toolReviews) { + const results = messages.filter((message) => message.getType() === 'tool').length; + return { + generations: [ + { + text: '', + message: + results < 2 + ? new AIMessageChunk({ + content: '', + tool_calls: [ + { + id: `review-${results + 1}`, + name: reviewedToolName, + args: { value: `result-${results + 1}` }, + }, + ], + }) + : new AIMessageChunk({ content: 'Both reviewed calls completed.' }), + }, + ], + }; + } + const message = answer + ? new AIMessageChunk({ + content: `Received ${String(answer.content)}`, + usage_metadata: { input_tokens: 5, output_tokens: 5, total_tokens: 10 }, + }) + : new AIMessageChunk({ + content: '', + usage_metadata: { input_tokens: 7, output_tokens: 7, total_tokens: 14 }, + tool_calls: [ + { + id: 'ask-environment', + name: 'ask_user_question', + args: { questions: [{ id: 'environment', question: 'Which environment?' }] }, + }, + ], + }); + return { + generations: [ + { message, text: typeof message.content === 'string' ? message.content : '' }, + ], + }; + }; + run.Graph.overrideModel = model; + return run; + }; + const run = await buildRun(); + const input = { messages: [new HumanMessage('Ask which environment.')] }; + if (processRun) await processRun(run, input, config); + else await run.processStream(input, config); + const record = (await storage.read(owner, responseId))!; + const partial = buildPendingBackgroundResponse(record); + let unfinishedWrites = 0; + const deps = { + storage, + jobs: manager, + buildAction: async (interrupt: RunInterruptResult) => + buildPendingAction(interrupt.payload, { + streamId: responseId, + responseMessageId: responseId, + conversationId, + threadId: interrupt.threadId ?? conversationId, + runId: interrupt.checkpointNs ?? responseId, + interruptId: interrupt.interruptId, + ttlMs: 60_000, + projectContextKey: 'private-project-binding', + resumeAgentIds: ['agent'], + requestFingerprint: computeLegacyAgentRequestFingerprint(JSON.parse(record.requestJSON)), + requestFingerprintV2: computeAgentRequestFingerprint(JSON.parse(record.requestJSON)), + resumeContext: { + discoveredToolNames: ['retained_lookup'], + responseParentMessageId: 'original-user-message', + }, + }), + persistUnfinished: async () => { + await mongoose.connection + .db!.collection('sdk_unfinished_turns') + .insertOne({ responseId, generationCreatedAt: execution.createdAt }); + unfinishedWrites++; + }, + }; + return { + run, + buildRun, + execution, + record, + partial, + config, + deps, + modelCalls: () => modelCalls, + toolExecutions: () => toolExecutions, + unfinishedWrites: () => unfinishedWrites, + }; +} + +it.each([ + 'live', + 'write_failure', + 'owner_deleted', + 'conversation_deleted', + 'expired', + 'owner_deleted_missing_job', +] as const)('settles a real SDK pause under %s', (outcome) => + scoped(async () => { + const failPersistence = outcome === 'write_failure'; + let pausedNamespace: string | undefined; + const reserved = await reserveOriginal(); + if (!reserved.ok) throw new Error('Reservation failed'); + const secret = 'runtime-pause-test-secret'; + const user = new mongoose.models.User({ + _id: owner.userId, + tenantId: owner.tenantId, + email: 'pause@example.test', + }); + const errors: unknown[] = []; + const complete = jest.fn(async () => 'must not complete'); + const order: string[] = []; + let done!: () => void; + const finished = new Promise((resolve) => { + done = resolve; + }); + const app = express(); + app.use(express.json()); + app.use( + createBackgroundResponseWorkerAuth( + { + storage, + signingSecret: () => secret, + findUser: async () => user, + authorize: async () => true, + admitResume: async () => ({ ok: false, code: 'response_resume_unavailable' }), + onError: (error) => { + errors.push(error); + }, + }, + (_req, res) => { + res.status(401).end(); + }, + ), + ); + app.post('/', async (req, res) => { + try { + (req as ServerRequest).config = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + endpoints: { agents: { checkpointer } }, + }; + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: manager, + }); + expect( + runtime.inputChain({ + messages: [{ messageId: 'original-user-message' }], + parentMessageId: 'original-user-message', + }).messages, + ).toHaveLength(1); + expect(await runtime.graphOptions()).toMatchObject({ hitlCapable: true }); + const f = await setup( + async (execution) => { + await runtime.hooks.onEnrolled!(execution); + }, + (run, input, config) => + runtime.processRun(run, input, { + ...config, + configurable: { + thread_id: 'wrong-thread', + [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: 'wrong-owner-namespace', + }, + }), + ); + await runtime + .completeTurn( + { + run: f.run, + execution: f.execution, + agents: [], + visibleOutputTokens: 0, + buildPartial: () => f.partial, + persistence: { + saveConversation: async () => { + order.push('conversation'); + }, + saveInput: async () => { + order.push('input'); + }, + saveOutput: async (partial) => { + order.push('output'); + if (failPersistence) throw new Error('Injected unfinished output failure'); + await mongoose.connection.db!.collection('runtime_unfinished_turns').insertOne({ + responseId, + epoch: f.execution.createdAt, + responseJSON: JSON.stringify(partial), + }); + }, + }, + }, + complete, + ) + .catch(async (error) => { + if (!failPersistence) throw error; + await f.execution.settle(error); + await runtime.hooks.afterSettle!(f.execution, error); + }); + await f.execution.settle(); + pausedNamespace = (await manager.getJob(responseId))?.metadata.checkpointNamespace; + if (f.execution.isPaused && outcome !== 'live') { + const methods = createMethods(mongoose); + if (outcome.startsWith('owner_deleted')) { + await mongoose.models.User.updateOne( + { _id: owner.userId }, + { $set: { tenantId: owner.tenantId, email: 'pause-owner@example.test' } }, + { upsert: true }, + ); + await methods.deleteUserById(owner.userId); + } else if (outcome === 'conversation_deleted') { + await methods.deleteConvos(owner.userId, { conversationId }); + } else { + await methods.saveConvo( + { userId: owner.userId, expiredAt: new Date(Date.now() - 1000) }, + { conversationId, endpoint: 'agents' }, + ); + } + if (outcome === 'owner_deleted_missing_job') + await manager.getJobStore().deleteJob(responseId, f.execution.createdAt); + } + if (f.execution.isPaused) await runtime.hooks.afterPause!(f.execution); + else await runtime.hooks.afterSettle!(f.execution, undefined); + expect(f.execution.isPaused).toBe(!failPersistence); + expect(f.modelCalls()).toBe(1); + } catch (error) { + errors.push(error); + if (!res.writableEnded) res.status(500).end(); + } finally { + done(); + } + }); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(reserved.record, secret), { type: 'bearer' }) + .send({}) + .expect(202); + await finished; + expect(errors).toEqual([]); + expect(complete).not.toHaveBeenCalled(); + expect(order).toEqual(['conversation', 'input', 'output']); + const record = (await storage.read(owner, responseId))!; + if (outcome !== 'live' && !failPersistence) { + expect(record).toBeNull(); + const retainedJob = await manager.getJob(responseId); + if (outcome === 'owner_deleted_missing_job') expect(retainedJob).toBeUndefined(); + else + expect(retainedJob).toMatchObject({ + status: 'aborted', + metadata: { providerDrained: true, terminalPersistencePending: false }, + }); + expect( + await (await getAgentCheckpointer(checkpointer))!.getTuple({ + configurable: { + thread_id: conversationId, + [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: pausedNamespace, + }, + }), + ).toBeUndefined(); + return; + } + if (failPersistence) { + expect(record.status).toBe('failed'); + expect(record.review).toBeUndefined(); + expect(record.pendingTerminal).toBeUndefined(); + const job = (await manager.getJob(responseId))!; + expect( + await (await getAgentCheckpointer(checkpointer))!.getTuple({ + configurable: { + thread_id: conversationId, + [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: job.metadata.checkpointNamespace, + }, + }), + ).toBeUndefined(); + return; + } + expect(record.status).toBe('requires_action'); + expect(record.pendingTerminal).toBeUndefined(); + expect(JSON.parse(record.review!.pendingActionJSON!)).toMatchObject({ + streamId: responseId, + responseMessageId: responseId, + conversationId, + payload: { type: 'ask_user_question' }, + requestFingerprintV2: expect.any(String), + }); + const saved = await mongoose.connection + .db!.collection('runtime_unfinished_turns') + .findOne({ responseId, epoch: record.generationCreatedAt }); + expect(JSON.parse(saved!.responseJSON)).toMatchObject({ + id: responseId, + status: 'in_progress', + }); + expect(JSON.parse(record.responseJSON!).output).toEqual([]); + }), +); + +it('captures a real SDK question interrupt and cancels its actual durable checkpoint safely', () => + scoped(async () => { + const f = await setup(); + const interrupt = f.run.getInterrupt(); + expect(interrupt?.payload.type).toBe('ask_user_question'); + expect(interrupt?.checkpointId).toEqual(expect.any(String)); + expect({ + recordId: f.record.responseId, + executionId: f.execution.runId, + epoch: f.record.generationCreatedAt === f.execution.createdAt, + metadata: (await manager.getJob(responseId))!.metadata, + partialId: f.partial.id, + partialModel: f.partial.model, + thread: interrupt?.threadId, + }).toMatchObject({ + recordId: responseId, + executionId: responseId, + epoch: true, + metadata: { + userId: owner.userId, + tenantId: owner.tenantId, + agent_id: 'agent', + endpoint: 'responses', + conversationId, + checkpointNamespace: expect.stringContaining('lcg:v2:'), + }, + partialId: responseId, + partialModel: 'agent', + thread: conversationId, + }); + expect(await pauseResponseForReview({ ...f, checkpointer }, f.deps)).toBe(true); + await f.execution.settle(); + expect(f.execution.isPaused).toBe(true); + expect(f.unfinishedWrites()).toBe(1); + const saved = (await storage.read(owner, responseId))!; + expect(saved).toMatchObject({ + status: 'requires_action', + review: { + checkpointId: interrupt!.checkpointId, + checkpointNs: interrupt!.checkpointNs ?? '', + pendingActionJSON: expect.stringContaining('Which environment?'), + }, + }); + expect(saved.responseJSON).not.toContain('Which environment?'); + expect(saved.responseJSON).not.toContain('private-project-binding'); + const retire = createResponseRetirement({ + storage, + jobs: manager, + cleanupCheckpoint: cleanupResponseCheckpoint, + }); + expect(await retire(owner, responseId)).toBe('stopped'); + expect((await storage.read(owner, responseId))?.status).toBe('cancelled'); + const saver = await getAgentCheckpointer(checkpointer); + expect(await saver!.getTuple(f.config)).toBeUndefined(); + })); + +it.each(['cleanup', 'stage'] as const)( + 'recovers a real SDK paused Stop after %s failure with default job cleanup', + (failure) => + scoped(async () => { + manager.configure({ + jobStore: new InMemoryJobStore({ ttlAfterComplete: 60_000 }), + eventTransport: new InMemoryEventTransport(), + }); + const f = await setup(); + expect( + await pauseResponseForReview({ ...f, visibleOutputTokens: 0, checkpointer }, f.deps), + ).toBe(true); + await f.execution.settle(); + let failing = true; + const originalStage = storage.stageTerminal.bind(storage); + if (failure === 'stage') + jest.spyOn(storage, 'stageTerminal').mockImplementation(async (...args) => { + if (failing) throw new Error('Injected paused cancellation stage failure'); + return originalStage(...args); + }); + const cleanup = async ( + record: Parameters[0], + namespace: string, + ) => { + if (failure === 'cleanup' && failing) + throw new Error('Injected paused checkpoint cleanup failure'); + await cleanupResponseCheckpoint(record, namespace); + }; + const freshRetirement = () => { + const completion = createResponseCompletion({ + storage, + cleanupCheckpoint: cleanup, + releaseExecution: createResponseExecutionRelease(manager), + }); + return createResponseRetirement({ + storage, + jobs: manager, + cleanupCheckpoint: cleanup, + recoverTerminal: completion.finish, + }); + }; + await expect(freshRetirement()(owner, responseId)).rejects.toThrow(); + const saved = (await storage.read(owner, responseId))!; + expect(saved.status).toBe('requires_action'); + if (failure === 'stage') { + expect(saved.pendingTerminal).toBeUndefined(); + expect(await manager.getJob(responseId)).toMatchObject({ + status: 'aborted', + createdAt: f.execution.createdAt, + metadata: { providerDrained: true, terminalPersistencePending: true }, + }); + } else { + expect(saved.pendingTerminal?.status).toBe('cancelled'); + expect(await manager.getJob(responseId)).toMatchObject({ + status: 'aborted', + createdAt: f.execution.createdAt, + metadata: { + providerExecutionId: saved.pendingTerminal!.providerExecutionId, + providerDrained: true, + terminalPersistencePending: true, + }, + }); + } + expect(await (await getAgentCheckpointer(checkpointer))!.getTuple(f.config)).toBeDefined(); + failing = false; + expect(await freshRetirement()(owner, responseId)).toBe('stopped'); + const cancelled = (await storage.read(owner, responseId))!; + expect(cancelled.status).toBe('cancelled'); + expect(cancelled.pendingTerminal).toBeUndefined(); + expect(await manager.getJob(responseId)).toBeUndefined(); + expect(await (await getAgentCheckpointer(checkpointer))!.getTuple(f.config)).toBeUndefined(); + expect(f.modelCalls()).toBe(1); + }), +); + +it.each(['unchanged', 'environment', 'definition', 'workspace'] as const)( + 'checks the actual resumed native-code target before SDK tool effects (%s)', + (change) => + scoped(async () => { + const f = await setup(undefined, undefined, true, Constants.BASH_TOOL); + const agent: Parameters[0]['agents'][number] = { + id: 'agent', + name: 'Code agent', + description: null, + created_at: 1, + avatar: null, + provider: 'openAI', + model: 'fixture', + model_parameters: { + temperature: null, + maxContextTokens: null, + max_context_tokens: null, + max_output_tokens: null, + top_p: null, + frequency_penalty: null, + presence_penalty: null, + }, + codeExecutionContext: { + baseUrl: 'https://bridge.example/v1', + codeSessionKey: 'private-session', + executionProfile: 'stateful', + statefulSessions: true, + environmentId: 'environment-a', + bridgeWorkerId: 'worker-a', + codeWorkspace: { + environmentId: 'environment-a', + workspaceId: 'project-a', + workspaceInstanceId: 'a'.repeat(64), + operations: ['execute_command'], + environment: { fingerprint: 'a'.repeat(64), actions: ['typecheck'] }, + }, + }, + }; + const buildAction = f.deps.buildAction; + f.deps.buildAction = async (interrupt) => + captureRunPendingAction({ + run: f.run, + payload: interrupt.payload, + agents: [agent], + resumeAgentIds: ['agent'], + aliases: [], + context: { ...(await buildAction(interrupt)), ttlMs: 60_000 }, + }); + expect(f.toolExecutions()).toBe(0); + expect( + await pauseResponseForReview({ ...f, visibleOutputTokens: 0, checkpointer }, f.deps), + ).toBe(true); + await f.execution.settle(); + const paused = (await storage.read(owner, responseId))!; + const action = JSON.parse(paused.review!.pendingActionJSON!); + expect(action.codeExecutionBinding?.targets).toHaveLength(1); + expect(JSON.stringify(action.codeExecutionBinding)).not.toContain('private-session'); + const decisions = [{ tool_call_id: 'review-1', decision: 'approve' as const }]; + const mapped = resolveToolApprovalResume(action.payload, decisions); + if (!('resumeValue' in mapped)) throw new Error('Missing native-code decision'); + const reserved = await storage.reserveResume( + owner, + responseId, + { + generationCreatedAt: f.execution.createdAt, + intentId: 'native-code-intent', + actionId: action.actionId, + previousProviderExecutionId: paused.review!.providerExecutionId, + providerExecutionId: 'native-code-provider', + fingerprint: 'native-code-review', + decisionJSON: JSON.stringify({ actionId: action.actionId, decisions }), + resolutionJSON: JSON.stringify(mapped.resumeValue), + }, + Date.now(), + ); + if (!reserved.ok) throw new Error('Missing native-code resume intent'); + const original = agent.codeExecutionContext!; + const rebuiltAgent = { + ...agent, + codeExecutionContext: { + ...original, + environmentId: change === 'environment' ? 'environment-b' : original.environmentId, + codeWorkspace: { + ...original.codeWorkspace!, + workspaceInstanceId: + change === 'workspace' ? 'b'.repeat(64) : original.codeWorkspace!.workspaceInstanceId, + environment: { + ...original.codeWorkspace!.environment!, + fingerprint: + change === 'definition' + ? 'b'.repeat(64) + : original.codeWorkspace!.environment!.fingerprint, + }, + }, + }, + }; + const admit = createResponseResumeAdmission({ + storage, + jobs: manager, + authorize: async () => true, + validate: async () => ({ ok: true, provenance: {} }), + }); + const user = new mongoose.models.User({ _id: owner.userId, tenantId: owner.tenantId }); + const errors: string[] = []; + let finish!: () => void; + const finished = new Promise((resolve) => { + finish = resolve; + }); + const app = express(); + app.use(express.json()); + app.use( + createBackgroundResponseWorkerAuth( + { + storage, + signingSecret: () => 'native-code-test-secret', + findUser: async () => user, + authorize: async () => true, + admitResume: (record, _user, intentId) => admit(record, record.responseId, intentId), + onError: () => { + errors.push('authentication'); + }, + }, + (_req, res) => { + res.status(401).end(); + }, + ), + ); + app.post('/', async (req, res) => { + (req as ServerRequest).config = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + endpoints: { agents: { checkpointer } }, + }; + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: manager, + }); + try { + await executeAgentRun( + { + envelope: createAgentRunEnvelope({ + protocol: 'responses', + requestId: responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: req.body, + }), + runId: responseId, + conversationId, + ...runtime.hooks, + isPrincipalActive: async () => true, + execute: async (execution) => { + expect( + (await runtime.graphOptions([rebuiltAgent])).reviewedToolApprovals?.decisions, + ).toEqual(decisions); + const rebuilt = await f.buildRun(); + await runtime.processRun(rebuilt, { messages: [] }, f.config); + await runtime.completeTurn( + { + run: rebuilt, + execution, + agents: [rebuiltAgent], + buildPartial: () => f.partial, + visibleOutputTokens: 0, + persistence: { + saveConversation: async () => {}, + saveInput: async () => {}, + saveOutput: async () => {}, + }, + }, + async () => { + throw new Error('Expected the second native-code review'); + }, + ); + }, + }, + { manager }, + ); + } catch (error) { + errors.push(error instanceof Error ? error.name : 'unexpected'); + } finally { + finish(); + } + }); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintResponseResumeWorkerToken(reserved.record, 'native-code-test-secret'), { + type: 'bearer', + }) + .send({}) + .expect(202); + await finished; + expect(f.toolExecutions()).toBe(change === 'unchanged' ? 1 : 0); + expect(f.modelCalls()).toBe(change === 'unchanged' ? 2 : 1); + expect(errors).toEqual( + change === 'unchanged' ? [] : ['CodeExecutionApprovalTargetChangedError'], + ); + expect((await storage.read(owner, responseId))?.status).toBe( + change === 'unchanged' ? 'requires_action' : 'failed', + ); + }), +); + +it('executes reviewed SDK tool calls once across two same-epoch pauses', () => + scoped(async () => { + const f = await setup(undefined, undefined, true); + let run = f.run; + let record = f.record; + let execution = f.execution; + for (let round = 0; round < 2; round++) { + expect(run.getInterrupt()?.payload.type).toBe('tool_approval'); + expect(f.toolExecutions()).toBe(round); + if (round === 0) { + expect( + await pauseResponseForReview( + { run, record, execution, partial: f.partial, visibleOutputTokens: 0, checkpointer }, + f.deps, + ), + ).toBe(true); + await execution.settle(); + } + record = (await storage.read(owner, responseId))!; + const action = JSON.parse(record.review!.pendingActionJSON!); + const decisions = [{ tool_call_id: `review-${round + 1}`, decision: 'approve' as const }]; + const mapped = resolveToolApprovalResume(action.payload, decisions); + if (!('resumeValue' in mapped)) throw new Error('Missing SDK tool decision'); + const reserved = await storage.reserveResume( + owner, + responseId, + { + generationCreatedAt: execution.createdAt, + intentId: `repeat-intent-${round}`, + actionId: action.actionId, + previousProviderExecutionId: record.review!.providerExecutionId, + providerExecutionId: `repeat-provider-${round}`, + fingerprint: `repeat-${round}`, + decisionJSON: JSON.stringify({ actionId: action.actionId, decisions }), + resolutionJSON: JSON.stringify(mapped.resumeValue), + }, + Date.now(), + ); + expect(reserved.ok).toBe(true); + const admit = createResponseResumeAdmission({ + storage, + jobs: manager, + authorize: async () => true, + validate: async () => ({ ok: true, provenance: {} }), + }); + const admitted = await admit(owner, responseId, `repeat-intent-${round}`); + if (!admitted.ok) throw new Error('Missing repeated SDK admission'); + record = admitted.record; + run = await f.buildRun(); + const nextRun = run; + await executeAgentRun( + { + envelope: createAgentRunEnvelope({ + protocol: 'responses', + requestId: responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: JSON.parse(record.requestJSON), + }), + runId: responseId, + conversationId, + isPrincipalActive: async () => true, + retainTerminalPersistence: true, + resume: { + generationCreatedAt: f.execution.createdAt, + providerExecutionId: record.resume!.providerExecutionId, + }, + execute: async (currentExecution) => { + execution = currentExecution; + await nextRun.resume(mapped.resumeValue, { + ...f.config, + configurable: { + ...f.config.configurable, + checkpoint_id: record.review!.checkpointId, + checkpoint_ns: record.review!.checkpointNs, + }, + }); + if (nextRun.getInterrupt()) { + expect( + await pauseResponseForReview( + { + run: nextRun, + record, + execution: currentExecution, + partial: f.partial, + visibleOutputTokens: 0, + checkpointer, + }, + f.deps, + ), + ).toBe(true); + } + }, + }, + { manager }, + ); + if (round === 0) { + record = (await storage.read(owner, responseId))!; + expect(record.status).toBe('requires_action'); + expect(record.generationCreatedAt).toBe(f.execution.createdAt); + } + } + expect(f.toolExecutions()).toBe(2); + expect(f.modelCalls()).toBe(3); + await cleanupResponseCheckpoint( + record, + (await manager.getJob(responseId))!.metadata.checkpointNamespace!, + ); + })); + +it('rebuilds the pinned SDK graph from the captured Mongo checkpoint and consumes the answer once', () => + scoped(async () => { + const f = await setup(); + expect(await pauseResponseForReview({ ...f, checkpointer }, f.deps)).toBe(true); + await f.execution.settle(); + expect(f.modelCalls()).toBe(1); + const paused = (await storage.read(owner, responseId))!; + const reserved = await storage.reserveResume( + owner, + responseId, + { + generationCreatedAt: f.execution.createdAt, + intentId: 'sdk-resume-intent', + actionId: paused.review!.actionId, + previousProviderExecutionId: paused.review!.providerExecutionId, + providerExecutionId: 'sdk-resumed-provider', + fingerprint: 'sdk-owner-answer', + resolutionJSON: JSON.stringify({ answers: { environment: 'staging' } }), + }, + Date.now(), + ); + if (!reserved.ok) throw new Error('Failed to reserve SDK resume'); + const admit = createResponseResumeAdmission({ + storage, + jobs: manager, + authorize: async () => true, + // The host's full graph/tool-binding validator is a separate production integration. + validate: async () => ({ ok: true, provenance: {} }), + }); + const admitted = await admit(owner, responseId, 'sdk-resume-intent'); + if (!admitted.ok) throw new Error('Failed to admit SDK resume'); + const envelope = createAgentRunEnvelope({ + protocol: 'responses', + requestId: 'sdk-resume-http', + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: JSON.parse(admitted.record.requestJSON), + }); + let final: ReturnType | undefined; + const response = await executeAgentRun( + { + envelope, + runId: responseId, + conversationId, + isPrincipalActive: async () => true, + retainTerminalPersistence: true, + resume: { + generationCreatedAt: f.execution.createdAt, + providerExecutionId: 'sdk-resumed-provider', + }, + execute: async () => { + const aggregator = createResponseAggregator(); + const rebuilt = await f.buildRun(createAggregatorEventHandlers(aggregator)); + await rebuilt.resume(JSON.parse(admitted.record.resume!.resolutionJSON), f.config); + expect(rebuilt.getInterrupt()).toBeUndefined(); + final = { + ...buildAggregatedResponse( + createResponseContext(envelope.payload, responseId), + aggregator, + ), + background: true, + store: true, + }; + return final; + }, + afterSettle: async () => { + if (!final) throw new Error('Missing resumed output'); + const current = (await storage.read(owner, responseId))!; + await cleanupResponseCheckpoint( + current, + (await manager.getJob(responseId))!.metadata.checkpointNamespace!, + ); + expect( + ( + await storage.publish(owner, responseId, current.revision, { + status: final.status, + responseJSON: JSON.stringify(final), + generationCreatedAt: f.execution.createdAt, + }) + ).ok, + ).toBe(true); + }, + }, + { manager }, + ); + expect(f.modelCalls()).toBe(2); + expect(response).toMatchObject({ + id: responseId, + model: 'agent', + status: 'completed', + background: true, + store: true, + }); + expect(JSON.stringify(response.output)).toContain('staging'); + expect(await storage.read(owner, responseId)).toMatchObject({ + status: 'completed', + generationCreatedAt: f.execution.createdAt, + responseJSON: JSON.stringify(response), + }); + expect(await manager.getJob(responseId)).toMatchObject({ + status: 'complete', + createdAt: f.execution.createdAt, + metadata: { providerExecutionId: 'sdk-resumed-provider', providerDrained: true }, + }); + })); + +it('refuses a missing durable checkpoint before publishing a review action', () => + scoped(async () => { + const f = await setup(); + const real = f.run.getInterrupt()!; + await expect( + pauseResponseForReview( + { + ...f, + checkpointer, + run: { getInterrupt: () => ({ ...real, checkpointId: 'not-a-durable-checkpoint' }) }, + }, + f.deps, + ), + ).rejects.toThrow('Response interrupt checkpoint is unavailable'); + expect((await storage.read(owner, responseId))?.status).toBe('in_progress'); + expect(f.unfinishedWrites()).toBe(0); + await f.execution.settle(new Error('Checkpoint unavailable')); + })); + +it.each([ + 'completed', + 'aborted', + 'thrown', + 'added_eager', + 'added_lazy', + 'added_graph', + 'added_graph_config', +] as const)( + 'replays a real SDK checkpoint through authenticated HTTP and the actual runtime publisher (%s)', + (mode) => + scoped(async () => { + const f = await setup(); + const prefix = createResponseAggregator(); + prefix.addText('Before the checkpoint. '); + prefix.toolCalls.set('ask-environment', { + id: 'ask-environment', + name: 'ask_user_question', + arguments: JSON.stringify({ + questions: [{ id: 'environment', question: 'Which environment?' }], + }), + }); + const priorAccounting = createResponseAccountingCollector({ + quote: () => ({ amount: '0.1', basis: 'application_billing' }), + }); + priorAccounting.record({ input_tokens: 7, output_tokens: 7 }); + const partial = { + ...f.partial, + output: buildAggregatedResponse({ responseId, model: 'agent', createdAt: 1 }, prefix) + .output, + usage: { + input_tokens: 7, + output_tokens: 7, + total_tokens: 14, + input_tokens_details: { cached_tokens: 0 }, + output_tokens_details: { reasoning_tokens: 0 }, + }, + accounting: priorAccounting.snapshot(), + }; + expect( + await pauseResponseForReview( + { ...f, partial, visibleOutputTokens: 7, checkpointer }, + f.deps, + ), + ).toBe(true); + await f.execution.settle(); + const paused = (await storage.read(owner, responseId))!; + const reserved = await storage.reserveResume( + owner, + responseId, + { + generationCreatedAt: f.execution.createdAt, + intentId: 'http-sdk-intent', + actionId: paused.review!.actionId, + previousProviderExecutionId: paused.review!.providerExecutionId, + providerExecutionId: 'http-sdk-provider', + fingerprint: 'owner-answer', + decisionJSON: JSON.stringify({ + actionId: paused.review!.actionId, + answers: { environment: 'staging' }, + }), + resolutionJSON: JSON.stringify({ answers: { environment: 'staging' } }), + }, + Date.now(), + ); + if (!reserved.ok) throw new Error('Missing HTTP SDK intent'); + const admit = createResponseResumeAdmission({ + storage, + jobs: manager, + authorize: async () => true, + validate: async () => ({ ok: true, provenance: {} }), + }); + const secret = 'isolated-sdk-http-resume-secret'; + const user = new mongoose.models.User({ + _id: owner.userId, + tenantId: owner.tenantId, + email: 'sdk-resume@example.test', + }); + const errors: unknown[] = []; + let finish!: () => void; + const finished = new Promise((resolve) => { + finish = resolve; + }); + const app = express(); + app.use(express.json()); + app.use( + createBackgroundResponseWorkerAuth( + { + storage, + signingSecret: () => secret, + findUser: async () => user, + authorize: async (record) => record.sourceKeyId === 'key', + admitResume: (record, _user, intentId) => admit(record, responseId, intentId), + onError: (error) => { + errors.push(error); + }, + }, + (_req, res) => { + res.status(401).end(); + }, + ), + ); + app.post('/', async (req, res) => { + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: manager, + }); + expect( + runtime.inputChain({ + messages: [{ messageId: 'fresh-duplicate-must-not-save' }], + parentMessageId: 'fresh-duplicate-must-not-save', + }), + ).toEqual({ + messages: [], + parentMessageId: 'original-user-message', + }); + expect(await runtime.graphOptions([])).toMatchObject({ + sessionStartSource: 'resume', + discoveredToolNames: ['retained_lookup'], + reviewedToolApprovals: { decisions: [] }, + }); + const envelope = createAgentRunEnvelope({ + protocol: 'responses', + requestId: responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: req.body, + }); + try { + await executeAgentRun( + { + envelope, + runId: responseId, + conversationId, + ...runtime.hooks, + isPrincipalActive: async () => true, + execute: async (execution) => { + if (mode === 'aborted') { + await manager.abortJob(responseId, { expectedCreatedAt: execution.createdAt }); + return; + } + if (mode === 'thrown') throw new Error('Private resumed provider failure'); + if (mode.startsWith('added_')) { + const root = initializedReviewAgent('agent'); + const child = initializedReviewAgent('added-child'); + const expanded = { + ...root, + subagentAgentConfigs: mode === 'added_eager' ? [child] : undefined, + lazySubagentConfigs: + mode === 'added_lazy' + ? [{ ...child, configId: 'added-child', resolve: async () => child }] + : undefined, + subagentGraphMemberMetadata: mode === 'added_graph' ? [child] : undefined, + subagentGraphConfigs: + mode === 'added_graph_config' + ? [ + { + definition: { + type: 'added-graph', + name: 'Added graph', + description: 'Added member', + agent_ids: ['added-child'], + edges: [], + entry_agent_id: 'added-child', + result_agent_id: 'added-child', + }, + memberConfigs: [child], + }, + ] + : undefined, + }; + await runtime.graphOptions([expanded]); + throw new Error('Expanded graph reached execution'); + } + const aggregator = createResponseAggregator(); + runtime.seedAggregator(aggregator); + const rebuilt = await f.buildRun(createAggregatorEventHandlers(aggregator)); + await runtime.processRun(rebuilt, { messages: [] }, f.config); + const accounting = createResponseAccountingCollector({ + quote: () => ({ amount: '0.2', basis: 'application_billing' }), + }); + accounting.record({ + input_tokens: aggregator.usage.inputTokens, + output_tokens: aggregator.usage.outputTokens, + }); + expect(runtime.visibleOutputTokens(aggregator.usage.outputTokens)).toBe(12); + res.json( + runtime.composeResponse( + buildAggregatedResponse( + createResponseContext(envelope.payload, responseId), + aggregator, + undefined, + accounting.snapshot(), + ), + ), + ); + }, + onSettlementError: (error) => { + errors.push(error); + }, + }, + { manager }, + ); + } catch (error) { + errors.push(error); + if (!res.writableEnded) res.status(409).end(); + } finally { + finish(); + } + }); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintResponseResumeWorkerToken(reserved.record, secret), { type: 'bearer' }) + .send({ answers: { environment: 'attacker override' }, model: 'wrong-agent' }) + .expect(202); + await finished; + if (mode === 'completed' || mode === 'aborted') expect(errors).toEqual([]); + else { + expect(errors).toHaveLength(1); + expect((errors[0] as Error).message).toBe( + mode === 'thrown' + ? 'Private resumed provider failure' + : 'Response resume participants changed', + ); + } + let status = 'failed'; + if (mode === 'completed') status = 'completed'; + else if (mode === 'aborted') status = 'cancelled'; + const complete = (await storage.read(owner, responseId))!; + expect(complete).toMatchObject({ + status, + generationCreatedAt: f.execution.createdAt, + }); + const response = JSON.parse(complete.responseJSON!); + expect(response).toMatchObject({ + id: responseId, + model: 'agent', + status, + background: true, + store: true, + }); + if (mode === 'completed') { + expect(JSON.stringify(response.output)).toContain('staging'); + expect(JSON.stringify(response.output)).not.toContain('attacker override'); + expect(JSON.stringify(response.output)).toContain('Before the checkpoint.'); + expect( + response.output.filter((item: { type: string }) => item.type === 'function_call_output'), + ).toHaveLength(1); + expect(response.usage).toMatchObject({ + input_tokens: 12, + output_tokens: 12, + total_tokens: 24, + }); + expect(response.accounting).toMatchObject({ status: 'available', amount: '0.3' }); + expect(f.modelCalls()).toBe(2); + } else { + expect(response.output).toEqual(partial.output); + expect(response.usage).toEqual(partial.usage); + expect(response.accounting).toEqual(partial.accounting); + expect(response.required_action).toBeUndefined(); + expect(JSON.stringify(response)).not.toContain('Private resumed provider failure'); + expect(f.modelCalls()).toBe(1); + expect(f.toolExecutions()).toBe(0); + } + expect(await (await getAgentCheckpointer(checkpointer))!.getTuple(f.config)).toBeUndefined(); + }), +); + +it('uses production validation for a real retained question and rejects changed graph, project, access and checkpoint', () => + scoped(async () => { + const f = await setup(); + const requestBody = JSON.parse(f.record.requestJSON); + const buildAction = f.deps.buildAction; + f.deps.buildAction = async (interrupt) => ({ + ...(await buildAction(interrupt)), + projectContextKey: NO_CHAT_PROJECT_CONTEXT_KEY, + requestFingerprint: computeLegacyAgentRequestFingerprint(requestBody), + requestFingerprintV2: computeAgentRequestFingerprint(requestBody), + }); + expect(await pauseResponseForReview({ ...f, checkpointer }, f.deps)).toBe(true); + await f.execution.settle(); + await mongoose.models.User.create({ + _id: owner.userId, + tenantId: owner.tenantId, + role: 'USER', + email: 'resume-validation@example.test', + }); + const methods = createMethods(mongoose); + const leanUser = await methods.findUser({ _id: owner.userId }); + expect(leanUser?._id.toString()).toBe(owner.userId); + expect(leanUser?.id).toBeUndefined(); + const agent: Agent = { + id: 'agent', + name: 'Agent', + description: null, + created_at: 1, + avatar: null, + provider: 'openAI', + model: 'fixture', + model_parameters: { + temperature: null, + maxContextTokens: null, + max_context_tokens: null, + max_output_tokens: null, + top_p: null, + frequency_penalty: null, + presence_penalty: null, + }, + tools: ['ask_user_question'], + }; + const config: AppConfig = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + endpoints: { agents: { checkpointer } }, + }; + let allowed = true; + let projectId: string | null = null; + const validation = createResponseResumeValidation({ + jobs: manager, + findUser: (userId) => methods.findUser({ _id: userId }), + getConfig: async (currentUser) => { + expect(currentUser.id).toBe(owner.userId); + return config; + }, + projects: { + getConvo: async () => ({ + conversationId, + user: owner.userId, + tenantId: owner.tenantId, + chatProjectId: projectId, + }), + getChatProject: async () => ({ + _id: 'changed', + instructions: 'changed', + contextRevision: 1, + file_ids: [], + tenantId: owner.tenantId, + }), + getProjectFiles: async () => [], + }, + content: { + getAgentCheckpointer, + getAgent: async () => agent, + canAccessAgent: async (_agent, currentUser) => + allowed && currentUser.id === owner.userId && currentUser.tenantId === owner.tenantId, + checkAccess: async () => true, + getMessages: async () => [], + getFiles: async () => [], + getActions: async () => [], + getUserMemories: async () => [], + getRoleByName: async () => null, + decryptMetadata: async (metadata) => metadata, + }, + }); + const paused = (await storage.read(owner, responseId))!; + const action = JSON.parse(paused.review!.pendingActionJSON!); + const job = (await manager.getJob(responseId))!; + const decision = { actionId: action.actionId, answers: { environment: 'staging' } }; + const input = { + record: paused, + job, + action, + decision, + resolution: { type: 'ask_user_question' as const, value: { answers: decision.answers } }, + }; + expect(await validation.validateDecision(input)).toBe(true); + allowed = false; + expect(await validation.validateDecision(input)).toBe(false); + allowed = true; + projectId = 'changed'; + expect(await validation.validateDecision(input)).toBe(false); + projectId = null; + expect( + await validation.validateDecision({ + ...input, + action: { ...action, requestFingerprintV2: 'changed' }, + }), + ).toBe(false); + const enqueued: string[] = []; + const submit = createResponseDecisionSubmission({ + storage, + jobs: manager, + authorize: async () => true, + validate: validation.validateDecision, + enqueue: async (record) => { + enqueued.push(record.resume!.intentId); + }, + }); + const submitted = await submit(owner, responseId, decision); + expect(submitted.ok).toBe(true); + expect(enqueued).toHaveLength(1); + const admit = createResponseResumeAdmission({ + storage, + jobs: manager, + authorize: async () => true, + validate: validation.validateResume, + }); + const admitted = await admit(owner, responseId, enqueued[0]); + expect(admitted.ok).toBe(true); + const current = (await storage.read(owner, responseId))!; + const rebuilt = await f.buildRun(); + await executeAgentRun( + { + envelope: createAgentRunEnvelope({ + protocol: 'responses', + requestId: responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: requestBody, + }), + runId: responseId, + conversationId, + isPrincipalActive: async () => true, + retainTerminalPersistence: true, + resume: { + generationCreatedAt: job.createdAt, + providerExecutionId: current.resume!.providerExecutionId, + }, + execute: async () => + rebuilt.resume(JSON.parse(current.resume!.resolutionJSON), { + ...f.config, + configurable: { + ...f.config.configurable, + checkpoint_id: current.review!.checkpointId, + checkpoint_ns: current.review!.checkpointNs, + }, + }), + }, + { manager }, + ); + expect(f.modelCalls()).toBe(2); + await cleanupResponseCheckpoint(current, job.metadata.checkpointNamespace!); + expect(await validation.validateResume(current, (await manager.getJob(responseId))!)).toEqual({ + ok: false, + }); + expect(f.modelCalls()).toBe(2); + })); + +it('does not expose a resumable action when unfinished-turn persistence fails', () => + scoped(async () => { + const f = await setup(); + const failure = new Error('private unfinished-turn storage failure'); + await expect( + pauseResponseForReview( + { ...f, checkpointer }, + { + ...f.deps, + persistUnfinished: async () => { + throw failure; + }, + }, + ), + ).rejects.toBe(failure); + await f.execution.settle(failure); + expect(f.execution.isPaused).toBe(false); + const record = (await storage.read(owner, responseId))!; + expect(record.review).toBeUndefined(); + expect(record.status).toBe('in_progress'); + const job = (await manager.getJob(responseId))!; + expect(job).toMatchObject({ status: 'error', metadata: { providerDrained: true } }); + expect(JSON.stringify(job)).not.toContain(failure.message); + })); + +it('persists a session approval before compiling the resumed SDK graph and avoids a second same-tool pause', () => + scoped(async () => { + const policy = { enabled: true, allowAlways: true }; + const methods = createMethods(mongoose); + await mongoose.models.Conversation.deleteMany({ user: owner.userId, conversationId }); + await mongoose.models.Conversation.create({ + user: owner.userId, + conversationId, + endpoint: 'agents', + }); + const f = await setup(undefined, undefined, true); + const buildAction = f.deps.buildAction; + f.deps.buildAction = async (interrupt) => { + const action = await buildAction(interrupt); + if (action.payload.type !== 'tool_approval') throw new Error('Expected tool approval'); + return { ...action, payload: markToolApprovalAllowAlways(action.payload, { policy }) }; + }; + await pauseResponseForReview({ ...f, checkpointer }, f.deps); + await f.execution.settle(); + const paused = (await storage.read(owner, responseId))!; + const action = JSON.parse(paused.review!.pendingActionJSON!); + const decisions = [ + { tool_call_id: 'review-1', decision: 'approve' as const, scope: 'session' as const }, + ]; + const mapped = resolveToolApprovalResume(action.payload, decisions); + if (!('resumeValue' in mapped)) throw new Error('Missing reviewed resolution'); + const reserved = await storage.reserveResume( + owner, + responseId, + { + generationCreatedAt: f.execution.createdAt, + intentId: 'session-intent', + actionId: action.actionId, + previousProviderExecutionId: paused.review!.providerExecutionId, + providerExecutionId: 'session-provider', + fingerprint: 'session', + decisionJSON: JSON.stringify({ actionId: action.actionId, decisions }), + resolutionJSON: JSON.stringify(mapped.resumeValue), + }, + Date.now(), + ); + if (!reserved.ok) throw new Error('Missing session reservation'); + const admit = createResponseResumeAdmission({ + storage, + jobs: manager, + authorize: async () => true, + validate: async () => ({ ok: true, provenance: {} }), + }); + const secret = 'session-sdk-secret'; + const user = new mongoose.models.User({ _id: owner.userId, tenantId: owner.tenantId }); + const errors: unknown[] = []; + let finish!: () => void; + const finished = new Promise((resolve) => { + finish = resolve; + }); + const app = express(); + app.use(express.json()); + app.use( + createBackgroundResponseWorkerAuth( + { + storage, + signingSecret: () => secret, + findUser: async () => user, + authorize: async () => true, + admitResume: (record, _user, intentId) => admit(record, responseId, intentId), + onError: (error) => { + errors.push(error); + }, + }, + (_req, res) => { + res.status(401).end(); + }, + ), + ); + app.post('/', async (req, res) => { + (req as ServerRequest).config = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + endpoints: { agents: { checkpointer, toolApproval: policy } }, + }; + const writes = jest.fn(methods.addConvoToolApprovalAllows); + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: manager, + addConvoToolApprovalAllows: writes, + }); + try { + await expect(runtime.graphOptions([])).rejects.toThrow( + 'Response remembered approval ownership unavailable', + ); + expect(writes).not.toHaveBeenCalled(); + await executeAgentRun( + { + envelope: createAgentRunEnvelope({ + protocol: 'responses', + requestId: responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: req.body, + }), + runId: responseId, + conversationId, + ...runtime.hooks, + isPrincipalActive: async () => true, + execute: async () => { + await expect( + runtime.graphOptions([initializedReviewAgent('added-child')]), + ).rejects.toThrow('Response resume participants changed'); + expect(writes).not.toHaveBeenCalled(); + const options = await runtime.graphOptions([]); + expect(writes).toHaveBeenCalledTimes(1); + expect(options.toolApprovalAllows).toEqual(['review_lookup']); + // No loaded conversation snapshot is fabricated to carry the remembered policy. + expect((req as ServerRequest).resolvedConversation).toBeUndefined(); + const saved = await methods.getConvo(owner.userId, conversationId); + expect(saved?.toolApprovalAllows).toEqual(['review_lookup']); + const rebuilt = await f.buildRun(undefined, options.toolApprovalAllows); + await runtime.processRun(rebuilt, { messages: [] }, f.config); + expect(rebuilt.getInterrupt()).toBeUndefined(); + res.json({ + ...buildPendingBackgroundResponse(paused), + status: 'completed', + output: [], + completed_at: Math.floor(Date.now() / 1000), + }); + }, + }, + { manager }, + ); + } catch (error) { + errors.push(error); + if (!res.writableEnded) res.status(500).end(); + } finally { + finish(); + } + }); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintResponseResumeWorkerToken(reserved.record, secret), { type: 'bearer' }) + .send({}) + .expect(202); + await finished; + expect(errors).toEqual([]); + expect(f.toolExecutions()).toBe(2); + expect(f.modelCalls()).toBe(3); + expect((await storage.read(owner, responseId))?.status).toBe('completed'); + })); + +it.each([ + { kind: 'missing', ids: undefined }, + { kind: 'duplicate', ids: ['agent', 'agent'] }, + { kind: 'empty', ids: ['agent', ''] }, + { kind: 'foreign', ids: ['other-agent'] }, +])('rejects an incomplete private participant witness before pause persistence: $kind', ({ ids }) => + scoped(async () => { + const f = await setup(); + const buildAction = f.deps.buildAction; + f.deps.buildAction = async (interrupt) => ({ + ...(await buildAction(interrupt)), + resumeAgentIds: ids, + }); + await expect(pauseResponseForReview({ ...f, checkpointer }, f.deps)).rejects.toThrow( + 'Response review action identity changed', + ); + expect(f.unfinishedWrites()).toBe(0); + expect((await storage.read(owner, responseId))?.status).toBe('in_progress'); + await f.execution.settle(new Error('Invalid private witness')); + }), +); + +it.each(['before_commit', 'after_commit'] as const)( + 'reconciles an ambiguous SDK pause publication %s without repeating provider work', + (failure) => + scoped(async () => { + const f = await setup(); + const original = storage.publishReview.bind(storage); + const publish = jest.spyOn(storage, 'publishReview'); + publish.mockImplementationOnce(async (...args) => { + if (failure === 'after_commit') expect((await original(...args)).ok).toBe(true); + throw new Error('Injected lost database acknowledgement'); + }); + expect(await pauseResponseForReview({ ...f, checkpointer }, f.deps)).toBe(true); + await f.execution.settle(); + expect(publish).toHaveBeenCalledTimes(2); + expect(publish.mock.calls[1]).toEqual(publish.mock.calls[0]); + expect(f.unfinishedWrites()).toBe(1); + expect(f.modelCalls()).toBe(1); + expect(f.execution.isPaused).toBe(true); + const saved = (await storage.read(owner, responseId))!; + expect(saved.status).toBe('requires_action'); + expect(saved.review).toEqual(publish.mock.calls[0][4]); + expect(saved.revision).toBe(f.record.revision + 1); + }), +); + +it.each(['cancelled', 'erased', 'replaced', 'outage'] as const)( + 'does not reconcile an ambiguous SDK pause after %s', + (failure) => + scoped(async () => { + const f = await setup(); + const original = storage.publishReview.bind(storage); + const publish = jest.spyOn(storage, 'publishReview'); + publish.mockImplementationOnce(async (...args) => { + if (failure !== 'outage') expect((await original(...args)).ok).toBe(true); + if (failure === 'cancelled') + await storage.requestCancellation(owner, responseId, Date.now()); + if (failure === 'erased') await mongoose.models.StoredResponse.deleteOne({ responseId }); + if (failure === 'replaced') + await mongoose.models.StoredResponse.updateOne( + { responseId }, + { $set: { 'review.actionId': 'newer-action' } }, + ); + throw new Error('Injected ambiguous publication'); + }); + if (failure === 'outage') + publish.mockImplementation(async () => { + throw new Error('Injected persistent database outage'); + }); + await expect(pauseResponseForReview({ ...f, checkpointer }, f.deps)).rejects.toThrow(); + expect(publish).toHaveBeenCalledTimes(2); + expect(publish.mock.calls[1]).toEqual(publish.mock.calls[0]); + expect(f.unfinishedWrites()).toBe(1); + expect(f.modelCalls()).toBe(1); + const saved = await storage.read(owner, responseId); + if (failure === 'erased') expect(saved).toBeNull(); + if (failure === 'cancelled') expect(saved?.cancellationRequestedAt).toBeDefined(); + if (failure === 'replaced') expect(saved?.review?.actionId).toBe('newer-action'); + if (failure === 'outage') expect(saved?.review).toBeUndefined(); + }), +); diff --git a/packages/api/src/agents/responses/pause.ts b/packages/api/src/agents/responses/pause.ts new file mode 100644 index 0000000000..e1ca8f0b67 --- /dev/null +++ b/packages/api/src/agents/responses/pause.ts @@ -0,0 +1,139 @@ +import { MAX_SUBAGENT_GRAPH_NODES } from 'librechat-data-provider'; +import type { ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas'; +import type { Agents, TCheckpointerConfig } from 'librechat-data-provider'; +import type { RunInterruptResult } from '@librechat/agents'; +import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import type { AgentExecutionEnrollment } from '../remote/lifecycle'; +import type { GenerationJob } from '~/types/stream'; +import type { Response } from './types'; +import { hasDurableAgentInterruptCheckpoint, resolveCheckpointerConfig } from '../checkpointer'; +import { checkpointOwnerNamespacePrefix } from '~/stream/checkpoints'; +import { buildPendingBackgroundResponse } from './background'; +import { responseTenantId } from './scope'; + +export interface ResponsePauseDependencies { + storage: ResponseStorage; + jobs: Pick; + /** Capture the host's existing graph/project/code/MCP approval bindings; never caller body data. */ + buildAction: (interrupt: RunInterruptResult, job: GenerationJob) => Promise; + /** Save the ordinary unfinished assistant turn, with existing content/privacy filtering. */ + persistUnfinished: (response: Response, action: Agents.PendingAction) => Promise; +} + +/** Capture the SDK's real, durable interrupt before exposing any resumable action. */ +export async function pauseResponseForReview( + input: { + run: { getInterrupt: () => RunInterruptResult | undefined }; + execution: AgentExecutionEnrollment; + record: StoredResponseRecord; + partial: Response; + visibleOutputTokens?: number; + checkpointer?: TCheckpointerConfig; + }, + deps: ResponsePauseDependencies, +): Promise { + const interrupt = input.run.getInterrupt(); + if (!interrupt) return false; + if ( + !interrupt.payload || + !['ask_user_question', 'tool_approval'].includes(interrupt.payload.type) + ) + throw new Error('Unsupported response interrupt'); + const { record, execution } = input; + const job = await deps.jobs.getJob(record.responseId); + const namespace = job?.metadata.checkpointNamespace; + const providerExecutionId = job?.metadata.providerExecutionId; + if ( + !job || + !execution.ownsSettlement || + job.createdAt !== record.generationCreatedAt || + job.createdAt !== execution.createdAt || + job.status !== 'running' || + !providerExecutionId || + job.metadata.providerExecutionStartedId !== providerExecutionId || + job.metadata.userId !== record.userId || + responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) || + job.metadata.agent_id !== record.agentId || + job.metadata.endpoint !== 'responses' || + job.metadata.conversationId !== record.conversationId || + !namespace?.startsWith(checkpointOwnerNamespacePrefix(record.userId, record.tenantId)) || + input.partial.id !== record.responseId || + input.partial.model !== record.agentId || + !interrupt.checkpointId || + !interrupt.interruptId || + (interrupt.threadId != null && interrupt.threadId !== record.conversationId) + ) + throw new Error('Response interrupt identity changed'); + const config = resolveCheckpointerConfig(input.checkpointer); + if ( + config.type !== 'mongo' || + !(await hasDurableAgentInterruptCheckpoint(record.conversationId, input.checkpointer, { + checkpointNamespace: namespace, + checkpointId: interrupt.checkpointId, + checkpointNs: interrupt.checkpointNs, + interruptId: interrupt.interruptId, + })) + ) + throw new Error('Response interrupt checkpoint is unavailable'); + const action = await deps.buildAction(interrupt, job); + const expiresAt = action.expiresAt; + if ( + !action.resumeAgentIds?.includes(record.agentId) || + action.resumeAgentIds.length > MAX_SUBAGENT_GRAPH_NODES || + new Set(action.resumeAgentIds).size !== action.resumeAgentIds.length || + action.resumeAgentIds.some((id) => typeof id !== 'string' || !id) || + action.streamId !== record.responseId || + action.responseMessageId !== record.responseId || + action.conversationId !== record.conversationId || + action.threadId !== record.conversationId || + action.interruptId !== interrupt.interruptId || + expiresAt == null || + !Number.isSafeInteger(expiresAt) + ) + throw new Error('Response review action identity changed'); + return execution.pauseForReview(action, async () => { + await deps.persistUnfinished(input.partial, action); + const current = await deps.storage.read(record, record.responseId); + if (!current || current.generationCreatedAt !== execution.createdAt) + throw new Error('Response pause journal identity changed'); + const publicResponse = { + ...buildPendingBackgroundResponse(current), + status: 'in_progress' as const, + }; + const publication: Parameters = [ + current, + current.responseId, + { execution: current }, + execution.createdAt, + { + actionId: action.actionId, + providerExecutionId, + expiresAt, + partialResponseJSON: JSON.stringify(input.partial), + ...(input.visibleOutputTokens == null + ? {} + : { visibleOutputTokens: input.visibleOutputTokens }), + pendingActionJSON: JSON.stringify(action), + checkpointId: interrupt.checkpointId, + checkpointNs: interrupt.checkpointNs ?? '', + checkpointConfigJSON: JSON.stringify({ + type: config.type, + ttl: config.ttlSeconds, + checkpointCollectionName: config.checkpointCollectionName, + checkpointWritesCollectionName: config.checkpointWritesCollectionName, + }), + }, + JSON.stringify(publicResponse), + ]; + let published; + try { + published = await deps.storage.publishReview(...publication); + } catch { + // The write may have committed despite a lost acknowledgement. Reuse the exact + // receipt and execution guard: storage reconciles it without republishing or + // overriding cancellation, retirement, a newer review, or admitted resume. + published = await deps.storage.publishReview(...publication); + } + if (!published.ok) throw new Error('Response pause journal publication failed'); + }); +} diff --git a/packages/api/src/agents/responses/runtime.ts b/packages/api/src/agents/responses/runtime.ts new file mode 100644 index 0000000000..8620f54555 --- /dev/null +++ b/packages/api/src/agents/responses/runtime.ts @@ -0,0 +1,808 @@ +import { checkpointerSchema, responseReviewDecisionSchema } from 'librechat-data-provider'; +import type { ResponseTerminalReceipt, StoredResponseRecord } from '@librechat/data-schemas'; +import type { Request, Response as HttpResponse } from 'express'; +import type { Agents, TMessage } from 'librechat-data-provider'; +import type { Run, IState } from '@librechat/agents'; +import type { ExecuteAgentRunParams, AgentExecutionConnection } from '../remote/host'; +import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import type { MarkToolApprovalAllowAlwaysOptions } from '../hitl/allow'; +import type { AgentExecutionEnrollment } from '../remote/lifecycle'; +import type { ResponsesRunEnvelope } from '../envelope'; +import type { ResponseAggregator } from './service'; +import type { ReachableAgent } from '../traversal'; +import type { ServerRequest } from '~/types/http'; +import type { Response } from './types'; +import type { createRun } from '../run'; +import { + computeAgentRequestFingerprint, + computeLegacyAgentRequestFingerprint, + pickResumeContext, +} from '../hitl/policy'; +import { + markToolApprovalAllowAlways, + resolveRunToolApprovalAllows, + recordToolApprovalAllows, +} from '../hitl/allow'; +import { + getApprovalTtlMs, + resolveCheckpointerConfig, + LIBRECHAT_CHECKPOINT_NAMESPACE_KEY, +} from '../checkpointer'; +import { createResponseCompletion, createResponseExecutionRelease } from './completion'; +import { combineResponseSegments, seedResponseAggregator } from './segments'; +import { checkpointOwnerNamespacePrefix } from '~/stream/checkpoints'; +import { getRunDiscoveredTools, getRunMCPToolAliases } from '../run'; +import { markNativeCodeToolApprovalRequests } from '../hitl/byom'; +import { assertCodeExecutionApprovalBinding } from '../execution'; +import { getBackgroundResponseWorkerAdmission } from './worker'; +import { getChatProjectContextKey } from '~/projects/context'; +import { buildPendingBackgroundResponse } from './background'; +import { resolveResponseContinuation } from './persistence'; +import { captureRunPendingAction } from '../hitl/capture'; +import { cleanupResponseCheckpoint } from './checkpoint'; +import { createResponseRetirement } from './retirement'; +import { collectReachableAgents } from '../traversal'; +import { pauseResponseForReview } from './pause'; +import { generateResponseId } from './service'; +import { responseTenantId } from './scope'; + +type TerminalResponse = Response & { status: ResponseTerminalReceipt['status'] }; +type ReviewAgent = NonNullable[0]['agents'][number]> & + NonNullable[1][number]> & + NonNullable[number]>; +interface ReviewAgentNode + extends Omit>, + ReachableAgent {} + +/** Capture and rebuild must describe the same initialized participant identities. */ +function responseParticipantIds(primaryId: string, agents: readonly ReviewAgentNode[]): string[] { + const participants = new Set([primaryId]); + for (const agent of agents) { + if (typeof agent.id !== 'string' || !agent.id) + throw new Error('Response graph participant identity unavailable'); + participants.add(agent.id); + } + return [...participants]; +} + +function responseBody(value: unknown): value is TerminalResponse { + return ( + value != null && + typeof value === 'object' && + 'object' in value && + value.object === 'response' && + 'id' in value && + typeof value.id === 'string' && + 'model' in value && + typeof value.model === 'string' && + 'status' in value && + ['completed', 'failed', 'incomplete', 'cancelled'].includes(String(value.status)) && + 'output' in value && + Array.isArray(value.output) + ); +} + +export interface ResponseExecutionRuntime { + responseId: string; + inputChain: (chain: { messages: T[]; parentMessageId: string }) => { + messages: T[]; + parentMessageId: string; + }; + graphOptions: ( + agents?: Parameters[0]['agents'], + ) => Promise< + Pick< + Parameters[0], + | 'hitlCapable' + | 'discoveredToolNames' + | 'reviewedToolApprovals' + | 'sessionStartSource' + | 'toolApprovalStorage' + | 'toolApprovalAllows' + > + >; + seedAggregator: (aggregator: ResponseAggregator | null) => void; + composeResponse: (response: Response) => Response; + visibleOutputTokens: (current: number) => number; + completeTurn: ( + input: { + run: Pick, 'getInterrupt'> & Parameters[0]; + execution: AgentExecutionEnrollment; + agents: Parameters[0]['agents']; + buildPartial: () => Response; + visibleOutputTokens: number; + persistence: { + saveConversation: () => Promise; + saveInput: () => Promise; + saveOutput: (partial: Response) => Promise | void>; + }; + }, + complete: () => Promise, + ) => Promise; + /** A rebuilt graph restores messages from its checkpoint, not from a fresh input turn. */ + graphMessages: (messages: T[]) => T[]; + toolContext: (execution: { createdAt: number }) => { + streamId: string | null; + jobCreatedAt?: number; + }; + connection?: AgentExecutionConnection; + hooks: Pick< + ExecuteAgentRunParams, + | 'onEnrolled' + | 'afterPause' + | 'afterSettle' + | 'clientRequestId' + | 'resume' + | 'retainTerminalPersistence' + >; + processRun: ( + run: Pick, 'processStream' | 'resume'>, + ...args: Parameters['processStream']> + ) => ReturnType['processStream']>; + resolveContinuation: (envelope: ResponsesRunEnvelope) => Promise<{ + conversationId: string; + parentMessageId: string; + }>; +} + +/** Adapt only trusted background-worker requests; normal Responses keeps its synchronous lifecycle. */ +export function createResponseExecutionRuntime( + req: Request, + res: HttpResponse, + continuation: Parameters[2] & { + generateResponseId?: () => string; + jobs?: Pick< + GenerationJobManagerClass, + 'getJob' | 'abortJob' | 'finishProviderTerminalPersistence' + > & + Partial>; + pluginHookSource?: MarkToolApprovalAllowAlwaysOptions['pluginHookSource']; + toolApprovalStorage?: Parameters[0]['toolApprovalStorage']; + addConvoToolApprovalAllows?: Parameters< + typeof recordToolApprovalAllows + >[0]['addConvoToolApprovalAllows']; + }, +): ResponseExecutionRuntime { + const admission = getBackgroundResponseWorkerAdmission(req); + if ( + admission && + (!continuation.jobs?.finishProviderTerminalPersistence || !continuation.jobs.abortJob) + ) + throw new Error('Response terminal publication release capability is unavailable'); + const interactive = + admission?.interactive === true && + continuation.jobs != null && + resolveCheckpointerConfig((req as ServerRequest).config?.endpoints?.agents?.checkpointer) + .type === 'mongo'; + let checkpoint: { checkpointNamespace: string; checkpointConfigJSON: string } | undefined; + let enrolledExecution: AgentExecutionEnrollment | undefined; + let rememberedTools: string[] = []; + let responseParentMessageId: string | undefined; + let prefix: Response | undefined; + const seeded = new WeakSet(); + function previous(): Response | undefined { + if (!admission?.resume) return; + if (prefix) return prefix; + let restored: Response; + try { + restored = JSON.parse(admission.record.review?.partialResponseJSON ?? 'null'); + } catch { + throw new Error('Response resume output is unavailable'); + } + if ( + !restored || + restored.id !== admission.record.responseId || + restored.model !== admission.record.agentId || + !Array.isArray(restored.output) || + !restored.background || + !restored.store + ) + throw new Error('Response resume output identity changed'); + prefix = restored; + return prefix; + } + function visibleOutputTokens(current: number): number { + if (!admission?.resume) return current; + const prior = admission.record.review?.visibleOutputTokens; + if ( + prior == null || + !Number.isSafeInteger(prior) || + prior < 0 || + !Number.isSafeInteger(current) || + current < 0 || + !Number.isSafeInteger(prior + current) + ) + throw new Error('Response visible token counter is unavailable'); + return prior + current; + } + function composeResponse(response: Response): Response { + const prior = previous(); + return prior && admission?.resume + ? combineResponseSegments(prior, response, admission.resume.intentId) + : response; + } + let candidate: TerminalResponse | undefined; + if (admission) { + const writeJSON = res.json.bind(res); + res.json = (body: unknown) => { + if (responseBody(body)) { + if (body.id !== admission.record.responseId || body.model !== admission.record.agentId) + throw new Error('Background response output identity changed'); + candidate = { ...body, background: true, store: true }; + if (res.writableEnded) return res; + return writeJSON(candidate); + } + if (res.writableEnded) return res; + return writeJSON(body); + }; + } + const hooks: Pick< + ExecuteAgentRunParams, + | 'onEnrolled' + | 'afterPause' + | 'afterSettle' + | 'clientRequestId' + | 'resume' + | 'retainTerminalPersistence' + > = admission + ? { + retainTerminalPersistence: true, + clientRequestId: admission.record.responseId, + ...(admission.resume + ? { + resume: { + generationCreatedAt: admission.resume.generationCreatedAt, + providerExecutionId: admission.resume.providerExecutionId, + }, + } + : {}), + onEnrolled: async (execution) => { + enrolledExecution = execution; + const current = await admission.storage.read( + admission.record, + admission.record.responseId, + ); + if (admission.resume) { + if ( + !current || + current.generationCreatedAt !== execution.createdAt || + current.resume?.intentId !== admission.resume.intentId || + current.resume.state !== 'admitted' || + current.resume.providerExecutionId !== admission.resume.providerExecutionId || + current.status !== 'in_progress' + ) + throw new Error('Background response resume enrollment changed'); + admission.record = current; + if (current.cancellationRequestedAt != null) execution.abort(); + res.status(202).json({ + id: current.responseId, + object: 'response', + model: current.agentId, + status: 'in_progress', + background: true, + store: true, + }); + return; + } + if (!current || current.generationCreatedAt != null) + throw new Error('Background response execution already published'); + const publishEnrollment = ( + record: StoredResponseRecord, + status: 'queued' | 'in_progress', + ) => + admission.storage.publish(record, record.responseId, record.revision, { + status, + responseJSON: JSON.stringify({ ...buildPendingBackgroundResponse(record), status }), + generationCreatedAt: execution.createdAt, + }); + const cancelled = current.cancellationRequestedAt != null; + if (cancelled) execution.abort(); + let result = await publishEnrollment(current, cancelled ? 'queued' : 'in_progress'); + if (!result.ok && !cancelled) { + const latest = await admission.storage.read(current, current.responseId); + if ( + latest?.status === 'queued' && + latest.generationCreatedAt == null && + latest.cancellationRequestedAt != null + ) { + execution.abort(); + result = await publishEnrollment(latest, 'queued'); + } + } + if (!result.ok) throw new Error('Background response enrollment publication failed'); + admission.record = result.record; + res.status(202).json({ + id: result.record.responseId, + object: 'response', + model: result.record.agentId, + status: result.record.status, + background: true, + store: true, + }); + }, + afterPause: async (execution) => { + if (!execution.isPaused || !execution.ownsSettlement) + throw new Error('Background response pause ownership changed'); + const current = await admission.storage.read( + admission.record, + admission.record.responseId, + ); + if (current && current.generationCreatedAt !== execution.createdAt) + throw new Error('Background response pause identity changed'); + if ( + current && + current.cancellationRequestedAt == null && + !['completed', 'failed', 'cancelled', 'incomplete'].includes(current.status) + ) + return; + if (!current) { + const identity = await admission.storage.readCleanupIdentity( + admission.record, + admission.record.responseId, + { ...admission.record, generationCreatedAt: execution.createdAt, ...checkpoint }, + ); + if (!identity?.retirement || identity.generationCreatedAt !== execution.createdAt) + throw new Error('Background response pause retirement identity unavailable'); + const job = await continuation.jobs!.getJob(admission.record.responseId); + if (!job) { + // The owned settled pause proves local drain; this permits only checkpoint cleanup. + if (identity.requiresCheckpointCleanup) { + if (!identity.checkpointNamespace || !identity.checkpointConfigJSON) + throw new Error('Background response pause checkpoint identity unavailable'); + await cleanupResponseCheckpoint( + { ...identity, pendingTerminal: identity }, + identity.checkpointNamespace, + ); + } + return; + } + } + const stopped = await createResponseRetirement({ + storage: admission.storage, + jobs: continuation.jobs!, + cleanupCheckpoint: cleanupResponseCheckpoint, + })(admission.record, admission.record.responseId); + if (stopped !== 'stopped') + throw new Error('Background response pause retirement is pending'); + }, + afterSettle: async (execution, error) => { + let current = await admission.storage.read(admission.record, admission.record.responseId); + const completion = createResponseCompletion({ + storage: admission.storage, + cleanupCheckpoint: cleanupResponseCheckpoint, + releaseExecution: createResponseExecutionRelease(continuation.jobs!), + }); + if (!current || current.generationCreatedAt == null) { + const retired = await admission.storage.readCleanupIdentity( + admission.record, + admission.record.responseId, + { ...admission.record, generationCreatedAt: execution.createdAt, ...checkpoint }, + ); + if (retired?.retirement && retired.generationCreatedAt === execution.createdAt) + current = { ...admission.record, ...retired }; + else if ( + current?.status === 'cancelled' && + current.cancellationRequestedAt != null && + current.generationCreatedAt == null + ) { + // Stop won the enrollment CAS. This exact local attempt must still release its drain proof. + await completion.finish({ ...current, generationCreatedAt: execution.createdAt }); + return; + } + } + if (!current || current.generationCreatedAt !== execution.createdAt) + throw new Error('Background response settlement identity changed'); + if (['completed', 'failed', 'cancelled', 'incomplete'].includes(current.status)) { + await completion.finish(current); + return; + } + const identity = await execution.confirmTerminalIdentity(); + // Normal manager teardown aborts the local signal too; only durable outcome/intent classifies Stop. + const cancelled = + identity.outcome === 'aborted' || current.cancellationRequestedAt != null; + const result: TerminalResponse = candidate ?? { + ...(previous() ?? buildPendingBackgroundResponse(current)), + status: cancelled ? 'cancelled' : 'failed', + completed_at: Math.floor(Date.now() / 1000), + required_action: undefined, + error: cancelled + ? null + : { + type: 'server_error', + code: 'execution_failed', + message: 'The Agent could not complete this request.', + }, + }; + let final = result; + if (cancelled) + final = { + ...result, + status: 'cancelled', + completed_at: Math.floor(Date.now() / 1000), + error: null, + }; + else if ((error || identity.outcome === 'error') && result.status === 'completed') + final = { + ...result, + status: 'failed', + completed_at: Math.floor(Date.now() / 1000), + error: { + type: 'server_error', + code: 'execution_failed', + message: 'The Agent could not complete this request.', + }, + }; + + admission.record = await completion.stageAndFinish(current, { + ...identity, + status: final.status, + responseJSON: JSON.stringify(final), + ...(checkpoint ?? + (admission.resume && current.review?.checkpointConfigJSON + ? { + checkpointNamespace: admission.resume.checkpointNamespace, + checkpointConfigJSON: current.review.checkpointConfigJSON, + } + : {})), + }); + }, + } + : {}; + + return { + composeResponse, + inputChain(chain) { + if (!admission?.resume) { + responseParentMessageId = chain.parentMessageId; + return chain; + } + const action: Agents.PendingAction = JSON.parse( + admission.record.review?.pendingActionJSON ?? 'null', + ); + const parent = action?.resumeContext?.responseParentMessageId; + if (typeof parent !== 'string' || parent.length === 0) + throw new Error('Response resumed input identity unavailable'); + responseParentMessageId = parent; + return { messages: [], parentMessageId: parent }; + }, + visibleOutputTokens, + seedAggregator(aggregator) { + if (!aggregator || seeded.has(aggregator)) return; + const prior = previous(); + if (prior) seedResponseAggregator(aggregator, prior); + seeded.add(aggregator); + }, + async graphOptions(agents) { + const hitlCapable = interactive; + const request = req as ServerRequest; + const policy = () => + hitlCapable + ? { + toolApprovalStorage: continuation.toolApprovalStorage, + toolApprovalAllows: resolveRunToolApprovalAllows( + request.config?.endpoints?.agents?.toolApproval, + { + conversationId: admission!.record.conversationId, + toolApprovalAllows: [ + ...new Set([ + ...resolveRunToolApprovalAllows( + request.config?.endpoints?.agents?.toolApproval, + request.resolvedConversation, + admission!.record.conversationId, + ), + ...rememberedTools, + ]), + ], + }, + admission!.record.conversationId, + ), + } + : {}; + if (!admission?.resume) return { hitlCapable, ...policy() }; + const action: Agents.PendingAction = JSON.parse( + admission.record.review?.pendingActionJSON ?? 'null', + ); + if (!action || !agents) throw new Error('Response resume graph is unavailable'); + if ( + action.requestFingerprint !== computeLegacyAgentRequestFingerprint(request.body) || + action.requestFingerprintV2 !== computeAgentRequestFingerprint(request.body) + ) + throw new Error('Response resume graph configuration changed'); + const retained = resolveCheckpointerConfig( + checkpointerSchema.parse( + JSON.parse(admission.record.review?.checkpointConfigJSON ?? 'null'), + ), + ); + const current = resolveCheckpointerConfig( + (req as ServerRequest).config?.endpoints?.agents?.checkpointer, + ); + if ( + retained.type !== 'mongo' || + current.type !== retained.type || + current.checkpointCollectionName !== retained.checkpointCollectionName || + current.checkpointWritesCollectionName !== retained.checkpointWritesCollectionName + ) + throw new Error('Response checkpoint storage settings changed'); + const reviewedAgents = collectReachableAgents(agents); + assertCodeExecutionApprovalBinding(action.codeExecutionBinding, reviewedAgents); + const participants = new Set( + responseParticipantIds(admission.record.agentId, reviewedAgents), + ); + const retainedParticipants = action.resumeAgentIds; + if ( + !Array.isArray(retainedParticipants) || + retainedParticipants.length === 0 || + new Set(retainedParticipants).size !== retainedParticipants.length || + participants.size !== retainedParticipants.length || + retainedParticipants.some((id) => typeof id !== 'string' || !participants.has(id)) + ) + throw new Error('Response resume participants changed'); + const decision = responseReviewDecisionSchema.parse( + JSON.parse(admission.record.resume?.decisionJSON ?? 'null'), + ); + if (decision.actionId !== action.actionId) + throw new Error('Response reviewed action changed'); + const discovered = action.resumeContext?.discoveredToolNames; + if ( + discovered != null && + (!Array.isArray(discovered) || discovered.some((name) => typeof name !== 'string')) + ) + throw new Error('Response discovered tools are unavailable'); + if ( + decision.decisions?.some((item) => item.decision === 'approve' && item.scope === 'session') + ) { + const { record, resume } = admission; + const job = await continuation.jobs?.getJob(record.responseId); + if ( + !continuation.addConvoToolApprovalAllows || + !enrolledExecution?.ownsSettlement || + enrolledExecution.createdAt !== record.generationCreatedAt || + !job || + job.status !== 'running' || + job.createdAt !== record.generationCreatedAt || + job.metadata.userId !== record.userId || + responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) || + job.metadata.agent_id !== record.agentId || + job.metadata.endpoint !== 'responses' || + job.metadata.conversationId !== record.conversationId || + job.metadata.responseMessageId !== record.responseId || + job.metadata.providerExecutionId !== resume.providerExecutionId || + job.metadata.providerExecutionStartedId !== resume.providerExecutionId || + !job.metadata.providerExecutionOwnerId + ) + throw new Error('Response remembered approval ownership unavailable'); + const request = req as ServerRequest; + const persisted = await recordToolApprovalAllows({ + userId: record.userId, + conversationId: record.conversationId, + policy: request.config?.endpoints?.agents?.toolApproval, + pendingAction: action, + resolutions: decision.decisions, + agents: reviewedAgents, + hookContext: { + userId: record.userId, + tenantId: responseTenantId(record.tenantId), + conversationId: record.conversationId, + appConfig: request.config, + }, + pluginHookSource: continuation.pluginHookSource, + request, + addConvoToolApprovalAllows: continuation.addConvoToolApprovalAllows, + }); + rememberedTools = [...new Set([...rememberedTools, ...persisted])]; + } + return { + hitlCapable, + ...policy(), + sessionStartSource: 'resume', + ...(Array.isArray(discovered) ? { discoveredToolNames: discovered } : {}), + reviewedToolApprovals: { + bindings: action.toolApprovalBindings, + decisions: decision.decisions ?? [], + }, + }; + }, + async completeTurn(input, complete) { + if (!admission || !interactive || !continuation.jobs) return complete(); + if (!input.run.getInterrupt()) return complete(); + const request = req as ServerRequest; + const config = request.config?.endpoints?.agents; + const agents = collectReachableAgents(input.agents); + const aliases = getRunMCPToolAliases(input.run); + const partial = { + ...composeResponse(input.buildPartial()), + status: 'in_progress' as const, + background: true, + store: true, + }; + const paused = await pauseResponseForReview( + { + run: input.run, + execution: input.execution, + record: admission.record, + partial, + visibleOutputTokens: visibleOutputTokens(input.visibleOutputTokens), + checkpointer: config?.checkpointer, + }, + { + storage: admission.storage, + jobs: continuation.jobs, + persistUnfinished: async (response) => { + await input.persistence.saveConversation(); + await input.persistence.saveInput(); + await input.persistence.saveOutput(response); + }, + buildAction: async (interrupt) => { + const payload = + interrupt.payload.type === 'tool_approval' + ? markToolApprovalAllowAlways( + markNativeCodeToolApprovalRequests(interrupt.payload, agents), + { + policy: config?.toolApproval, + agents, + aliases, + storedTools: resolveRunToolApprovalAllows( + config?.toolApproval, + request.resolvedConversation, + admission.record.conversationId, + ), + hookContext: { + userId: admission.record.userId, + tenantId: responseTenantId(admission.record.tenantId), + conversationId: admission.record.conversationId, + appConfig: request.config, + }, + pluginHookSource: continuation.pluginHookSource, + }, + ) + : interrupt.payload; + return captureRunPendingAction({ + run: input.run, + payload, + agents, + resumeAgentIds: responseParticipantIds(admission.record.agentId, agents), + aliases, + context: { + streamId: admission.record.responseId, + responseMessageId: admission.record.responseId, + conversationId: admission.record.conversationId, + threadId: admission.record.conversationId, + runId: interrupt.checkpointNs ?? admission.record.responseId, + interruptId: interrupt.interruptId, + ttlMs: getApprovalTtlMs(config?.checkpointer), + projectContextKey: getChatProjectContextKey(request.chatProjectContext), + requestFingerprint: computeLegacyAgentRequestFingerprint(request.body), + requestFingerprintV2: computeAgentRequestFingerprint(request.body), + resumeContext: { + ...pickResumeContext(request.body, request.reasoningOverrideBase), + discoveredToolNames: getRunDiscoveredTools(input.run), + responseParentMessageId, + }, + }, + }); + }, + }, + ); + if (!paused && !input.execution.signal.aborted) + throw new Error('Response interrupt could not be retained'); + return undefined; + }, + graphMessages: (messages) => (admission?.resume ? [] : messages), + async processRun(run, input, config, options) { + if (!admission?.resume) { + if (!admission || !interactive || !continuation.jobs) + return run.processStream(input, config, options); + const record = admission.record; + const job = await continuation.jobs.getJob(record.responseId); + const namespace = job?.metadata.checkpointNamespace; + if ( + !job || + job.status !== 'running' || + job.createdAt !== record.generationCreatedAt || + job.metadata.userId !== record.userId || + responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) || + job.metadata.agent_id !== record.agentId || + job.metadata.endpoint !== 'responses' || + job.metadata.responseMessageId !== record.responseId || + job.metadata.conversationId !== record.conversationId || + !namespace?.startsWith(checkpointOwnerNamespacePrefix(record.userId, record.tenantId)) + ) + throw new Error('Response checkpoint enrollment identity unavailable'); + const settings = resolveCheckpointerConfig( + (req as ServerRequest).config?.endpoints?.agents?.checkpointer, + ); + if (settings.type !== 'mongo') + throw new Error('Response interrupt requires durable checkpoint storage'); + checkpoint = { + checkpointNamespace: namespace, + checkpointConfigJSON: JSON.stringify({ + type: settings.type, + ttl: settings.ttlSeconds, + checkpointCollectionName: settings.checkpointCollectionName, + checkpointWritesCollectionName: settings.checkpointWritesCollectionName, + }), + }; + if (!continuation.jobs?.updateMetadata) + throw new Error('Response checkpoint cleanup persistence unavailable'); + await continuation.jobs.updateMetadata( + record.responseId, + { + responseCheckpointConfigJSON: checkpoint.checkpointConfigJSON, + }, + job.createdAt, + ); + return run.processStream( + input, + { + ...config, + configurable: { + ...config.configurable, + thread_id: record.conversationId, + [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: namespace, + }, + }, + options, + ); + } + const { record, resume } = admission; + if ( + record.resume?.intentId !== resume.intentId || + !record.review?.checkpointId || + record.review.checkpointNs == null + ) + throw new Error('Response resume checkpoint identity unavailable'); + const resolution: unknown = JSON.parse(record.resume.resolutionJSON); + return run.resume( + resolution, + { + ...config, + configurable: { + ...config.configurable, + thread_id: record.conversationId, + checkpoint_id: record.review.checkpointId, + checkpoint_ns: record.review.checkpointNs, + [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: resume.checkpointNamespace, + }, + }, + options, + ); + }, + toolContext: (execution) => + admission + ? { streamId: admission.record.responseId, jobCreatedAt: execution.createdAt } + : { streamId: null }, + responseId: + admission?.record.responseId ?? (continuation.generateResponseId ?? generateResponseId)(), + connection: admission + ? undefined + : { + isClosed: () => res.destroyed === true && res.writableEnded !== true, + onClose: (listener: () => void) => { + const close = () => { + if (res.writableEnded !== true) listener(); + }; + res.once('close', close); + return () => { + res.off('close', close); + }; + }, + }, + hooks, + async resolveContinuation(envelope: ResponsesRunEnvelope) { + if (!admission) + return resolveResponseContinuation( + envelope.payload.previous_response_id, + envelope.principal.userId, + continuation, + ); + const record = admission.record; + if ( + envelope.principal.userId !== record.userId || + responseTenantId(envelope.principal.tenantId) !== responseTenantId(record.tenantId) || + envelope.payload.model !== record.agentId + ) + throw new Error('Background response principal changed'); + return { conversationId: record.conversationId, parentMessageId: record.parentMessageId }; + }, + }; +} diff --git a/packages/api/src/agents/responses/segments.spec.ts b/packages/api/src/agents/responses/segments.spec.ts new file mode 100644 index 0000000000..75047a8135 --- /dev/null +++ b/packages/api/src/agents/responses/segments.spec.ts @@ -0,0 +1,224 @@ +import type { Response, OutputItem, Usage } from './types'; +import { + buildAggregatedResponse, + createResponseAggregator, + createAggregatorEventHandlers, +} from './service'; +import { createResponseAccountingCollector, combineResponseAccounting } from './accounting'; +import { combineResponseSegments, seedResponseAggregator } from './segments'; +import { projectResponseMessage } from './persistence'; + +const usage = (count: number): Usage => ({ + input_tokens: count, + output_tokens: count, + total_tokens: count * 2, + input_tokens_details: { cached_tokens: count }, + output_tokens_details: { reasoning_tokens: count }, + primary: { input_tokens: count, output_tokens: count, total_tokens: count * 2 }, + subagent: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, +}); + +function response(output: OutputItem[], count: number, amount: string): Response { + const accounting = createResponseAccountingCollector({ + quote: () => ({ amount, basis: 'application_billing' }), + }); + accounting.record({ + input_tokens: count, + output_tokens: count, + total_tokens: count * 2, + model: 'actual-model', + provider: 'openAI', + }); + return { + ...buildAggregatedResponse( + { responseId: 'response', model: 'agent', createdAt: 1 }, + createResponseAggregator(), + usage(count), + accounting.snapshot(), + ), + output, + background: true, + store: true, + }; +} + +const message = (id: string, text: string): OutputItem => ({ + type: 'message', + id, + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text, annotations: [], logprobs: [] }], +}); +const call = (id: string, callId: string): OutputItem => ({ + type: 'function_call', + id, + call_id: callId, + name: 'lookup', + arguments: '{"key":"original"}', + status: 'completed', +}); +const result = (id: string, callId: string, output: string): OutputItem => ({ + type: 'function_call_output', + id, + call_id: callId, + output, + status: 'completed', +}); + +it('preserves the saved prefix and completes a paused tool exactly once without seeding billed usage', () => { + const previous = response( + [ + message('before', 'Before. '), + call('done-call', 'done'), + result('done-result', 'done', 'old result'), + call('waiting-call', 'waiting'), + ], + 3, + '0.1', + ); + const original = JSON.stringify(previous); + const aggregator = createResponseAggregator(); + seedResponseAggregator(aggregator, previous); + expect(aggregator.usage).toEqual({ + inputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + cachedTokens: 0, + }); + expect(aggregator.getText()).toBe(''); + const handlers = createAggregatorEventHandlers(aggregator); + handlers.on_run_step_completed.handle('on_run_step_completed', { + result: { + id: 'step', + index: 0, + type: 'tool_call', + tool_call: { + id: 'waiting', + name: 'lookup', + args: '{"key":"original"}', + output: 'new result', + progress: 1, + }, + }, + }); + aggregator.addText('After.'); + const current = { + ...response([], 5, '0.2'), + output: buildAggregatedResponse( + { responseId: 'response', model: 'agent', createdAt: 2 }, + aggregator, + ).output, + }; + const combined = combineResponseSegments(previous, current, 'segment-two'); + expect(combined.created_at).toBe(1); + expect(combined.usage).toEqual(usage(8)); + expect(combined.accounting).toMatchObject({ + status: 'available', + amount: '0.3', + coverage: 'complete', + }); + expect(combined.output.filter((item) => item.type === 'function_call')).toHaveLength(2); + expect(combined.output.filter((item) => item.type === 'function_call_output')).toEqual([ + result('done-result', 'done', 'old result'), + expect.objectContaining({ call_id: 'waiting', output: 'new result' }), + ]); + expect(combined.output[0]).toEqual(previous.output[0]); + expect(projectResponseMessage(combined).text).toBe('Before. After.'); + expect(JSON.stringify(previous)).toBe(original); +}); + +it('uses the cumulative snapshot for the next pause without duplicating earlier text, calls or usage', () => { + const first = response([message('first', 'First. ')], 1, '0.1'); + const second = combineResponseSegments( + first, + response([message('second', 'Second. ')], 2, '0.2'), + 'two', + ); + const third = combineResponseSegments( + second, + response([message('third', 'Third.')], 3, '0.3'), + 'three', + ); + expect(projectResponseMessage(third).text).toBe('First. Second. Third.'); + expect(third.usage).toEqual(usage(6)); + expect(third.accounting).toMatchObject({ amount: '0.6' }); + expect(third.accounting?.calls).toHaveLength(3); + expect(third.output.map((item) => item.id)).toEqual(['first', 'second', 'third']); +}); + +it('keeps output IDs unique even when independent segments generate the same message ID', () => { + const combined = combineResponseSegments( + response([message('same', 'Before')], 1, '0.1'), + response([message('same', 'After')], 1, '0.1'), + 'intent-two', + ); + expect(combined.output.map((item) => item.id)).toEqual(['same', 'same_intent-two']); +}); + +it('does not overwrite an already completed tool result with conflicting data', () => { + const previous = response( + [call('call', 'lookup'), result('result', 'lookup', 'original-private-result')], + 1, + '0.1', + ); + expect(() => + combineResponseSegments( + previous, + response([result('other', 'lookup', 'different-private-result')], 1, '0.1'), + 'two', + ), + ).toThrow('Response tool result changed across segments'); +}); + +it('preserves uncertainty when a previous segment has no usable usage or accounting', () => { + const previous = { ...response([], 1, '0.1'), usage: null, accounting: undefined }; + const combined = combineResponseSegments(previous, response([], 1, '0.2'), 'two'); + expect(combined.usage).toBeNull(); + expect(combined.accounting).toMatchObject({ + status: 'unavailable', + coverage: 'partial', + reason: 'partial_coverage', + }); + expect(combined.accounting).not.toHaveProperty('amount'); + expect(combined.accounting?.calls).toHaveLength(1); +}); + +it('retains past quotes rather than consulting current pricing for old calls', () => { + const prior = response([], 1, '0.1').accounting!; + const quote = jest.fn(() => ({ amount: '0.9', basis: 'application_billing' as const })); + const collector = createResponseAccountingCollector({ seed: prior, quote }); + collector.record({ input_tokens: 1, output_tokens: 1 }); + expect(quote).toHaveBeenCalledTimes(1); + expect(collector.snapshot()).toMatchObject({ amount: '1' }); + expect(prior.amount).toBe('0.1'); +}); + +it('does not erase conflicting or incomplete provenance when combining accounting', () => { + const priced = response([], 1, '0.1').accounting!; + const collector = createResponseAccountingCollector(); + collector.record({ input_tokens: 1, output_tokens: 1 }, undefined, undefined, 'same-call'); + collector.record({ input_tokens: 2, output_tokens: 1 }, undefined, undefined, 'same-call'); + expect(combineResponseAccounting(priced, collector.snapshot())).toMatchObject({ + status: 'unavailable', + reason: 'conflicting_usage', + }); + expect( + combineResponseAccounting(priced, createResponseAccountingCollector().snapshot()), + ).toMatchObject({ + status: 'unavailable', + coverage: 'partial', + reason: 'partial_coverage', + }); +}); + +it('rejects response identity changes and invalid or overflowing usage', () => { + const prior = response([], 1, '0.1'); + expect(() => combineResponseSegments(prior, { ...prior, id: 'another-response' }, 'two')).toThrow( + 'identity changed', + ); + const invalid = response([], 1, '0.1'); + invalid.usage!.output_tokens = Number.MAX_SAFE_INTEGER; + expect(() => combineResponseSegments(prior, invalid, 'two')).toThrow( + 'Response segment usage is invalid', + ); +}); diff --git a/packages/api/src/agents/responses/segments.ts b/packages/api/src/agents/responses/segments.ts new file mode 100644 index 0000000000..f4e0d8ac4f --- /dev/null +++ b/packages/api/src/agents/responses/segments.ts @@ -0,0 +1,118 @@ +import type { Response, Usage, UsageTotals, OutputItem } from './types'; +import type { ResponseAggregator } from './service'; +import { combineResponseAccounting } from './accounting'; + +/** Restore tool state only; prior text/reasoning and item IDs remain immutable in the saved prefix. */ +export function seedResponseAggregator(aggregator: ResponseAggregator, previous: Response): void { + for (const item of previous.output) { + if (item.type === 'function_call') + aggregator.toolCalls.set(item.call_id, { + id: item.call_id, + name: item.name, + arguments: item.arguments, + }); + else if (item.type === 'function_call_output') + aggregator.toolOutputs.set(item.call_id, item.output); + } +} + +function add(left: number, right: number): number { + const result = left + right; + if ( + !Number.isSafeInteger(left) || + left < 0 || + !Number.isSafeInteger(right) || + right < 0 || + !Number.isSafeInteger(result) + ) + throw new Error('Response segment usage is invalid'); + return result; +} + +function totals(left: UsageTotals, right: UsageTotals): UsageTotals { + return { + input_tokens: add(left.input_tokens, right.input_tokens), + output_tokens: add(left.output_tokens, right.output_tokens), + total_tokens: add(left.total_tokens, right.total_tokens), + }; +} + +function usage(left: Usage | null, right: Usage | null): Usage | null { + if (!left || !right) return null; + return { + ...totals(left, right), + input_tokens_details: { + cached_tokens: add( + left.input_tokens_details.cached_tokens, + right.input_tokens_details.cached_tokens, + ), + }, + output_tokens_details: { + reasoning_tokens: add( + left.output_tokens_details.reasoning_tokens, + right.output_tokens_details.reasoning_tokens, + ), + }, + ...(left.primary && right.primary ? { primary: totals(left.primary, right.primary) } : {}), + ...(left.subagent && right.subagent ? { subagent: totals(left.subagent, right.subagent) } : {}), + }; +} + +function callKey(item: OutputItem): string | undefined { + return item.type === 'function_call' || item.type === 'function_call_output' + ? `${item.type}:${item.call_id}` + : undefined; +} + +/** Compose a new provider segment once with its saved prefix; never send old usage back to billing. */ +export function combineResponseSegments( + previous: Response, + current: Response, + segmentId: string, +): Response { + if (previous.id !== current.id || previous.model !== current.model || !segmentId) + throw new Error('Response segment identity changed'); + const output = [...previous.output]; + const calls = new Map(); + const ids = new Set(); + for (let index = 0; index < output.length; index++) { + const item = output[index]; + ids.add(item.id); + const key = callKey(item); + if (key) calls.set(key, index); + } + for (const item of current.output) { + const key = callKey(item); + const index = key == null ? undefined : calls.get(key); + if (index != null) { + const prior = output[index]; + if ( + prior.type === 'function_call_output' && + item.type === 'function_call_output' && + prior.output !== item.output + ) + throw new Error('Response tool result changed across segments'); + output[index] = { ...item, id: prior.id }; + continue; + } + if ( + item.type === 'message' && + item.role === 'assistant' && + item.content.length === 0 && + output.length > 0 + ) + continue; + const id = ids.has(item.id) ? `${item.id}_${segmentId}` : item.id; + if (ids.has(id)) throw new Error('Response segment item identity collided'); + if (key) calls.set(key, output.length); + output.push({ ...item, id }); + ids.add(id); + } + return { + ...current, + created_at: previous.created_at, + output, + usage: usage(previous.usage, current.usage), + accounting: combineResponseAccounting(previous.accounting, current.accounting), + }; +} diff --git a/packages/api/src/agents/responses/worker.runtime.spec.ts b/packages/api/src/agents/responses/worker.runtime.spec.ts new file mode 100644 index 0000000000..4e3472f58f --- /dev/null +++ b/packages/api/src/agents/responses/worker.runtime.spec.ts @@ -0,0 +1,1026 @@ +import express from 'express'; +import jwt from 'jsonwebtoken'; +import request from 'supertest'; +import mongoose from 'mongoose'; +import { FileSources } from 'librechat-data-provider'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { + createModels, + createMethods, + createResponseStorage, + tenantStorage, + runAsSystem, +} from '@librechat/data-schemas'; +import type { StoredResponseRecord, ResponseStorage, IUser } from '@librechat/data-schemas'; +import type { RequestHandler, Request, Response } from 'express'; +import type { RunStreamConfig } from '@librechat/agents'; +import type { BackgroundResponseWorkerDependencies } from './worker'; +import type { ServerRequest } from '~/types/http'; +import { + createBackgroundResponseWorkerAuth, + mintBackgroundResponseWorkerToken, + getBackgroundResponseWorkerRecord, + mintResponseResumeWorkerToken, +} from './worker'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { createResponseCompletion, createResponseExecutionRelease } from './completion'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { buildPendingBackgroundResponse } from './background'; +import { createResponseExecutionRuntime } from './runtime'; +import { enrollAgentExecution } from '../remote/lifecycle'; +import { createResponseResumeAdmission } from './resume'; +import { createAgentRunEnvelope } from '../envelope'; +import { buildPendingAction } from '../hitl/policy'; +import { executeAgentRun } from '../remote/host'; +import { GenerationJobManager } from '~/stream'; + +let mongo: MongoMemoryReplSet; +let storage: ResponseStorage; +const secret = 'isolated-response-worker-test-secret'; +const owner = { userId: '000000000000000000000001', tenantId: 'worker-test' }; +const scope = (work: () => Promise) => tenantStorage.run({ tenantId: owner.tenantId }, work); +let record: StoredResponseRecord; +let user: IUser; + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + storage = createResponseStorage(mongoose); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + await mongoose.models.ResponseScope?.deleteMany({}); + }); + user = new mongoose.models.User({ + _id: owner.userId, + tenantId: owner.tenantId, + email: 'worker@example.test', + }); + await scope(async () => { + const result = await storage.reserve({ + ...owner, + responseId: 'resp_worker', + requestKey: 'request', + fingerprint: 'fingerprint', + agentId: 'agent', + sourceKeyId: 'original-key', + requestJSON: JSON.stringify({ + model: 'agent', + input: 'Saved input', + instructions: 'Saved instructions', + store: true, + background: true, + }), + conversationId: 'conversation', + parentMessageId: 'root', + }); + if (!result.ok) throw new Error('Reservation failed'); + record = result.record; + }); +}); + +function fixture( + handler?: RequestHandler, + admitResume?: BackgroundResponseWorkerDependencies['admitResume'], +) { + let authorized = true; + const errors: unknown[] = []; + const app = express(); + app.use(express.json()); + app.use( + createBackgroundResponseWorkerAuth( + { + storage, + signingSecret: () => secret, + findUser: async (id) => (id === owner.userId ? user : null), + authorize: async (stored) => authorized && stored.sourceKeyId === 'original-key', + admitResume, + onError: (error) => { + errors.push(error); + }, + }, + (_req, res) => { + res.status(418).json({ ordinaryAuth: true }); + }, + ), + ); + app.all( + /.*/, + handler ?? + ((req, res) => { + res.json({ + responseId: getBackgroundResponseWorkerRecord(req)?.responseId, + body: req.body, + userId: req.user && 'id' in req.user ? req.user.id : undefined, + }); + }), + ); + return { + app, + errors, + revoke: () => { + authorized = false; + }, + }; +} + +it('accepts only a response-bound server token and restores the exact persisted request', async () => { + const { app } = fixture(); + const response = await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(record, secret), { type: 'bearer' }) + .send({ model: 'attacker-agent', input: 'tampered', instructions: 'override' }); + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ + responseId: record.responseId, + userId: owner.userId, + body: { model: 'agent', input: 'Saved input', instructions: 'Saved instructions' }, + }); +}); + +it('preserves ordinary remote authentication when there is no worker marker', async () => { + const { app } = fixture(); + const response = await request(app).post('/').auth('ordinary-token', { type: 'bearer' }).send({}); + expect(response.status).toBe(418); + expect(response.body).toEqual({ ordinaryAuth: true }); +}); + +it('does not accept another JWT purpose or a forged worker token', async () => { + const { app } = fixture(); + for (const token of [ + jwt.sign({ id: owner.userId, scope: 'agent_trigger' }, secret), + mintBackgroundResponseWorkerToken(record, 'different-secret'), + ]) { + const response = await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + .send({}); + expect(response.status).toBe(401); + expect(response.body).not.toHaveProperty('responseId'); + } +}); + +it('rechecks original key authority rather than letting the worker token outlive revocation', async () => { + const { app, revoke } = fixture(); + const token = mintBackgroundResponseWorkerToken(record, secret); + revoke(); + const response = await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + .send({}); + expect(response.status).toBe(401); +}); + +it('cannot use a worker token to read/cancel another response through a different route', async () => { + const { app } = fixture(); + const token = mintBackgroundResponseWorkerToken(record, secret); + for (const path of ['/resp_other/cancel', '/resp_other/execute']) { + expect( + ( + await request(app) + .post(path) + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + .send({}) + ).status, + ).toBe(401); + } + expect( + (await request(app).get('/').set('x-lc-response-worker', '1').auth(token, { type: 'bearer' })) + .status, + ).toBe(401); +}); + +it('does not accept a stale token after the original key reference changes', async () => { + const { app } = fixture(); + const token = mintBackgroundResponseWorkerToken( + { ...record, sourceKeyId: 'different-key' }, + secret, + ); + expect( + ( + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + .send({}) + ).status, + ).toBe(401); +}); + +describe('background Responses execution handoff', () => { + it.each(['completed', 'cancelled', 'publication_failure'] as const)( + 'retains exact terminal proof until Responses publication with default cleanup (%s)', + async (outcome) => { + GenerationJobManager.configure({ + jobStore: new InMemoryJobStore({ ttlAfterComplete: 60_000 }), + eventTransport: new InMemoryEventTransport(), + }); + const errors: unknown[] = []; + let finish!: () => void; + const finished = new Promise((resolve) => { + finish = resolve; + }); + if (outcome === 'publication_failure') + jest + .spyOn(storage, 'finishTerminal') + .mockRejectedValueOnce(new Error('Injected Mongo publication failure')); + const { app } = fixture(async (req, res) => { + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: GenerationJobManager, + }); + try { + await executeAgentRun({ + envelope: createAgentRunEnvelope({ + protocol: 'responses', + requestId: record.responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: req.body, + }), + runId: record.responseId, + conversationId: record.conversationId, + ...runtime.hooks, + isPrincipalActive: async () => true, + execute: async (execution) => { + if (outcome === 'cancelled') { + await GenerationJobManager.abortJob(record.responseId, { + expectedCreatedAt: execution.createdAt, + }); + } + res.json({ + ...buildPendingBackgroundResponse(record), + status: 'completed', + output: [], + }); + }, + onSettlementError: (error) => { + errors.push(error); + }, + }); + } finally { + finish(); + } + }); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(record, secret), { type: 'bearer' }) + .send({}) + .expect(202); + await finished; + let saved = (await scope(() => storage.read(owner, record.responseId)))!; + if (outcome === 'publication_failure') { + expect(errors).toHaveLength(1); + expect(saved.pendingTerminal).toBeDefined(); + await GenerationJobManager.getJobStore().updateJob( + record.responseId, + { terminalPersistenceStartedAt: Date.now() - 60_000 }, + saved.generationCreatedAt, + ); + expect(await GenerationJobManager.getJob(record.responseId)).toMatchObject({ + createdAt: saved.generationCreatedAt, + status: 'complete', + metadata: { providerDrained: true, terminalPersistencePending: true }, + }); + // A fresh completion instance resumes only the persisted receipt, never the provider. + saved = await scope(() => + createResponseCompletion({ + storage, + cleanupCheckpoint: async () => {}, + releaseExecution: createResponseExecutionRelease(GenerationJobManager), + }).finish(saved), + ); + } else expect(errors).toEqual([]); + expect(saved.status).toBe(outcome === 'cancelled' ? 'cancelled' : 'completed'); + expect(saved.pendingTerminal).toBeUndefined(); + expect(await GenerationJobManager.getJob(record.responseId)).toBeUndefined(); + }, + ); + it('keeps memory-checkpointer background calls on the ordinary SDK stream with interactive host installed', async () => { + const processStream = jest.fn(async () => undefined); + const resume = jest.fn(async () => undefined); + const { app } = fixture( + async (req, res) => { + (req as ServerRequest).config = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + endpoints: { agents: { checkpointer: { type: 'memory' } } }, + }; + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: GenerationJobManager, + }); + expect(await runtime.graphOptions()).toEqual({ hitlCapable: false }); + await runtime.processRun({ processStream, resume }, { messages: [] }, { version: 'v2' }); + res.json({ ordinary: true }); + }, + async () => ({ ok: false, code: 'response_resume_unavailable' }), + ); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(record, secret), { type: 'bearer' }) + .send({}) + .expect(200, { ordinary: true }); + expect(processStream).toHaveBeenCalledTimes(1); + expect(resume).not.toHaveBeenCalled(); + }); + it('does not enable interactive execution from caller fields without an installed resume validator', async () => { + const { app } = fixture(async (req, res) => { + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: GenerationJobManager, + }); + expect(await runtime.graphOptions()).toEqual({ hitlCapable: false }); + res.json({ ok: true }); + }); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(record, secret), { type: 'bearer' }) + .send({ interactive: true, hitlCapable: true }) + .expect(200, { ok: true }); + }); + beforeEach(() => { + GenerationJobManager.configure({ + jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }), + eventTransport: new InMemoryEventTransport(), + cleanupOnComplete: false, + }); + GenerationJobManager.initialize(); + }); + afterEach(async () => { + await GenerationJobManager.destroy(); + }); + + async function prepareResume() { + await scope(async () => { + const execution = await enrollAgentExecution({ + runId: record.responseId, + userId: owner.userId, + agentId: record.agentId, + conversationId: record.conversationId, + protocol: 'responses', + retainTerminalPersistence: true, + isPrincipalActive: async () => true, + }); + await storage.publish(owner, record.responseId, 0, { + status: 'in_progress', + generationCreatedAt: execution.createdAt, + responseJSON: JSON.stringify(buildPendingBackgroundResponse(record)), + }); + await execution.beginProviderExecution(); + const job = (await GenerationJobManager.getJob(record.responseId))!; + const action = buildPendingAction( + { type: 'ask_user_question', question: { question: 'Which environment?' } }, + { + streamId: record.responseId, + responseMessageId: record.responseId, + conversationId: record.conversationId, + threadId: record.conversationId, + ttlMs: 60_000, + interruptId: 'interrupt-one', + }, + ); + await execution.pauseForReview(action, async () => { + const result = await storage.publishReview( + owner, + record.responseId, + 1, + execution.createdAt, + { + actionId: action.actionId, + providerExecutionId: job.metadata.providerExecutionId!, + expiresAt: action.expiresAt!, + partialResponseJSON: '{}', + pendingActionJSON: JSON.stringify(action), + checkpointId: 'checkpoint-exact', + checkpointNs: '', + checkpointConfigJSON: JSON.stringify({ + type: 'mongo', + checkpointCollectionName: 'worker_checkpoints', + checkpointWritesCollectionName: 'worker_checkpoint_writes', + }), + }, + JSON.stringify({ ...buildPendingBackgroundResponse(record), status: 'requires_action' }), + ); + if (!result.ok) throw new Error('Pause failed'); + }); + await execution.settle(); + const result = await storage.reserveResume( + owner, + record.responseId, + { + generationCreatedAt: execution.createdAt, + intentId: 'intent-one', + actionId: action.actionId, + previousProviderExecutionId: job.metadata.providerExecutionId!, + providerExecutionId: 'resume-provider', + fingerprint: 'answer', + resolutionJSON: '{"answer":"staging"}', + decisionJSON: JSON.stringify({ actionId: action.actionId, answer: 'staging' }), + }, + Date.now(), + ); + if (!result.ok) throw new Error('Resume reservation failed'); + record = result.record; + }); + } + + it('rejects resume-purpose tokens when no capable admission validator is installed', async () => { + await prepareResume(); + const { app } = fixture(); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintResponseResumeWorkerToken(record, secret), { type: 'bearer' }) + .send({}) + .expect(401); + expect((await scope(() => storage.read(owner, record.responseId)))?.status).toBe( + 'requires_action', + ); + }); + + it('rejects a saved resume worker token after original authority revocation without admission or provider effects', async () => { + await prepareResume(); + const handler = jest.fn((_req: Request, res: Response) => { + res.status(200).end(); + }); + const admit = jest.fn(async () => ({ + ok: false as const, + code: 'response_resume_unavailable' as const, + })); + const { app, revoke } = fixture(handler, admit); + const token = mintResponseResumeWorkerToken(record, secret); + revoke(); + + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + .send({}) + .expect(401); + + expect(admit).not.toHaveBeenCalled(); + expect(handler).not.toHaveBeenCalled(); + expect(await GenerationJobManager.getJob(record.responseId)).toMatchObject({ + status: 'requires_action', + createdAt: record.generationCreatedAt, + metadata: { providerExecutionId: record.review!.providerExecutionId }, + }); + const retained = await scope(() => storage.read(owner, record.responseId)); + expect(retained?.status).toBe('requires_action'); + expect(retained?.resume).toEqual(record.resume); + }); + + it('never lets an initial-purpose delivery replace an already-paused response', async () => { + await prepareResume(); + const handler = jest.fn((_req: Request, res: Response) => { + res.status(200).end(); + }); + const { app } = fixture(handler); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(record, secret), { type: 'bearer' }) + .send({}) + .expect(409, { code: 'RUN_ALREADY_ADMITTED' }); + expect(handler).not.toHaveBeenCalled(); + expect(await GenerationJobManager.getJob(record.responseId)).toMatchObject({ + status: 'requires_action', + createdAt: record.generationCreatedAt, + metadata: { providerExecutionId: record.review!.providerExecutionId }, + }); + }); + + it('binds resume tokens to the exact saved intent, epoch and selected provider', async () => { + await prepareResume(); + const admit = jest.fn(async () => ({ + ok: false as const, + code: 'response_resume_unavailable' as const, + })); + const { app } = fixture(undefined, admit); + for (const changed of [ + { ...record, generationCreatedAt: record.generationCreatedAt! + 1 }, + { ...record, resume: { ...record.resume!, intentId: 'other-intent' } }, + { ...record, resume: { ...record.resume!, providerExecutionId: 'other-provider' } }, + ]) { + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintResponseResumeWorkerToken(changed, secret), { type: 'bearer' }) + .send({}) + .expect(401); + } + expect(admit).not.toHaveBeenCalled(); + }); + + it('resumes through HTTP using only the saved decision and publishes after the owning segment settles', async () => { + GenerationJobManager.configure({ + jobStore: new InMemoryJobStore({ ttlAfterComplete: 60_000 }), + eventTransport: new InMemoryEventTransport(), + }); + await prepareResume(); + const originalEpoch = record.generationCreatedAt; + const admit = createResponseResumeAdmission({ + storage, + jobs: GenerationJobManager, + authorize: async () => true, + validate: async () => ({ ok: true, provenance: {} }), + }); + const processStream = jest.fn(async () => undefined); + const resume = jest.fn(async (value: unknown, config: RunStreamConfig) => { + expect(value).toEqual({ answer: 'staging' }); + expect(config.configurable).toMatchObject({ + thread_id: record.conversationId, + checkpoint_id: 'checkpoint-exact', + checkpoint_ns: '', + private_option: 'preserved', + }); + return undefined; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let done!: () => void; + const finished = new Promise((resolve) => { + done = resolve; + }); + const { app } = fixture( + async (req, res) => { + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: GenerationJobManager, + }); + const envelope = createAgentRunEnvelope({ + protocol: 'responses', + requestId: record.responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: req.body, + }); + try { + await executeAgentRun({ + envelope, + runId: runtime.responseId, + conversationId: record.conversationId, + ...runtime.hooks, + isPrincipalActive: async () => true, + execute: async () => { + await runtime.processRun( + { processStream, resume }, + { messages: [] }, + { + configurable: { + thread_id: 'tampered', + checkpoint_id: 'wrong', + private_option: 'preserved', + }, + version: 'v2', + }, + ); + await gate; + res.json({ + ...buildPendingBackgroundResponse(record), + status: 'completed', + output: [], + }); + }, + afterSettle: async (execution, error) => { + await runtime.hooks.afterSettle?.(execution, error); + done(); + }, + }); + } catch { + if (!res.writableEnded) res.status(409).json({ error: 'resume_conflict' }); + } + }, + (saved, _user, intentId) => admit(saved, saved.responseId, intentId), + ); + const token = mintResponseResumeWorkerToken(record, secret); + const send = () => + request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + .send({ resume: { answer: 'attacker answer' }, model: 'wrong' }); + await send().expect(202); + await send().expect(202); + expect(resume).toHaveBeenCalledTimes(1); + expect(processStream).not.toHaveBeenCalled(); + release(); + await finished; + expect(await scope(() => storage.read(owner, record.responseId))).toMatchObject({ + status: 'completed', + generationCreatedAt: originalEpoch, + }); + expect(await GenerationJobManager.getJob(record.responseId)).toBeUndefined(); + }); + + it('keeps final output private until provider drain and prevents a retried HTTP request starting another run', async () => { + let release!: () => void; + const trailingWrite = new Promise((resolve) => { + release = resolve; + }); + let settled!: () => void; + const finished = new Promise((resolve) => { + settled = resolve; + }); + const stage = storage.stageTerminal.bind(storage); + const stageWithConcurrentProgress = jest + .spyOn(storage, 'stageTerminal') + .mockImplementationOnce(async (...args) => { + for (let step = 0; step < 30; step++) { + const current = (await storage.read(owner, record.responseId))!; + const projected = await storage.publish(owner, current.responseId, current.revision, { + status: 'in_progress', + responseJSON: JSON.stringify({ + ...buildPendingBackgroundResponse(current), + metadata: { progress: step }, + }), + }); + expect(projected.ok).toBe(true); + } + return stage(...args); + }); + let starts = 0; + const { app } = fixture(async (req, res) => { + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: GenerationJobManager, + }); + const envelope = createAgentRunEnvelope({ + protocol: 'responses', + requestId: record.responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: req.body, + }); + try { + await executeAgentRun({ + envelope, + runId: runtime.responseId, + conversationId: record.conversationId, + connection: runtime.connection, + ...runtime.hooks, + isPrincipalActive: async () => true, + execute: async (execution) => { + starts++; + execution.track(trailingWrite); + res.json({ + ...buildPendingBackgroundResponse(record), + status: 'completed', + output: [ + { + type: 'message', + role: 'assistant', + status: 'completed', + id: 'message', + content: [ + { + type: 'output_text', + text: '# Exact final answer', + annotations: [], + logprobs: [], + }, + ], + }, + ], + }); + }, + }); + settled(); + } catch (error) { + res + .status(409) + .json({ code: error instanceof Error && 'code' in error ? error.code : 'failed' }); + } + }); + const token = mintBackgroundResponseWorkerToken(record, secret); + const first = await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + .send({}); + expect(first.status).toBe(202); + expect((await scope(async () => storage.read(owner, record.responseId)))?.status).toBe( + 'in_progress', + ); + const duplicate = await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + .send({}); + expect(duplicate.status).toBe(409); + expect(duplicate.body.code).toBe('RUN_ALREADY_ADMITTED'); + expect(starts).toBe(1); + release(); + await finished; + const saved = await scope(async () => storage.read(owner, record.responseId)); + expect(stageWithConcurrentProgress).toHaveBeenCalledTimes(1); + stageWithConcurrentProgress.mockRestore(); + expect(saved?.status).toBe('completed'); + expect(JSON.parse(saved!.responseJSON!).output[0].content[0].text).toBe('# Exact final answer'); + }); + + it('recovers runtime terminal publication failure without executing the provider again', async () => { + let done!: () => void; + const finished = new Promise((resolve) => { + done = resolve; + }); + const errors: unknown[] = []; + let starts = 0; + const publication = jest + .spyOn(storage, 'finishTerminal') + .mockRejectedValueOnce(new Error('terminal publication unavailable')); + const { app } = fixture(async (req, res) => { + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: GenerationJobManager, + }); + const envelope = createAgentRunEnvelope({ + protocol: 'responses', + requestId: record.responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: req.body, + }); + try { + await executeAgentRun({ + envelope, + runId: runtime.responseId, + conversationId: record.conversationId, + ...runtime.hooks, + isPrincipalActive: async () => true, + execute: async () => { + starts++; + res.json({ + ...buildPendingBackgroundResponse(record), + status: 'completed', + metadata: { marker: 'exact-runtime-result' }, + }); + }, + onSettlementError: (error) => { + errors.push(error); + }, + }); + } finally { + done(); + } + }); + const send = () => + request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(record, secret), { type: 'bearer' }) + .send({}); + await send().expect(202); + await finished; + expect(errors).toHaveLength(1); + const pending = (await scope(() => storage.read(owner, record.responseId)))!; + expect(pending.status).toBe('in_progress'); + expect(JSON.parse(pending.pendingTerminal!.responseJSON).metadata.marker).toBe( + 'exact-runtime-result', + ); + await send().expect(409); + const recovered = await scope(() => + createResponseCompletion({ + storage, + cleanupCheckpoint: async () => {}, + releaseExecution: createResponseExecutionRelease(GenerationJobManager), + }).finish(pending), + ); + expect(recovered.status).toBe('completed'); + expect(JSON.parse(recovered.responseJSON!).metadata.marker).toBe('exact-runtime-result'); + expect(recovered.pendingTerminal).toBeUndefined(); + expect(starts).toBe(1); + publication.mockRestore(); + }); + + it.each([ + ['owner', 'enrollment'], + ['conversation', 'enrollment'], + ['expiry', 'enrollment'], + ['owner', 'settlement'], + ['conversation', 'settlement'], + ['expiry', 'settlement'], + ] as const)( + 'releases the exact runtime after %s retirement during %s without restoring private output', + async (kind, phase) => { + const errors: unknown[] = []; + let starts = 0; + let done!: () => void; + const finished = new Promise((resolve) => { + done = resolve; + }); + const retire = async () => { + const methods = createMethods(mongoose); + if (kind === 'owner') { + await mongoose.models.User.updateOne( + { _id: owner.userId }, + { $set: { tenantId: owner.tenantId, email: 'retired-worker@example.test' } }, + { upsert: true }, + ); + await methods.deleteUserById(owner.userId); + } else if (kind === 'conversation') + await methods.deleteConvos(owner.userId, { conversationId: record.conversationId }); + else + await methods.saveConvo( + { userId: owner.userId, expiredAt: new Date(Date.now() - 1000) }, + { conversationId: record.conversationId, endpoint: 'agents' }, + ); + }; + const { app } = fixture(async (req, res) => { + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: GenerationJobManager, + }); + try { + await executeAgentRun({ + envelope: createAgentRunEnvelope({ + protocol: 'responses', + requestId: record.responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: req.body, + }), + runId: record.responseId, + conversationId: record.conversationId, + ...runtime.hooks, + onEnrolled: async (execution) => { + if (phase === 'enrollment') await retire(); + await runtime.hooks.onEnrolled!(execution); + }, + isPrincipalActive: async () => true, + execute: async () => { + starts++; + await retire(); + res.json({ + ...buildPendingBackgroundResponse(record), + status: 'completed', + output: [], + }); + }, + onSettlementError: (error) => { + errors.push(error); + }, + }); + } catch (error) { + errors.push(error); + if (!res.writableEnded) res.status(409).json({ error: 'execution_unavailable' }); + } finally { + done(); + } + }); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(record, secret), { type: 'bearer' }) + .send({}) + .expect(phase === 'enrollment' ? 409 : 202); + await finished; + expect(starts).toBe(phase === 'enrollment' ? 0 : 1); + expect(errors).toHaveLength(phase === 'enrollment' ? 1 : 0); + expect(await scope(() => storage.read(owner, record.responseId))).toBeNull(); + expect(await GenerationJobManager.getJob(record.responseId)).toMatchObject({ + status: phase === 'enrollment' ? 'error' : 'complete', + metadata: { providerDrained: true, terminalPersistencePending: false }, + }); + }, + ); + + it.each(['before enrollment', 'during publication'])( + 'honors queued cancellation %s without provider work', + async (phase) => { + if (phase === 'before enrollment') { + await scope(async () => storage.requestCancellation(owner, record.responseId, Date.now())); + } else { + const publish = storage.publish.bind(storage); + jest.spyOn(storage, 'publish').mockImplementationOnce(async (...args) => { + await storage.requestCancellation(owner, record.responseId, Date.now()); + return publish(...args); + }); + } + let settled!: () => void; + const finished = new Promise((resolve) => { + settled = resolve; + }); + let starts = 0; + const { app } = fixture(async (req, res) => { + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: GenerationJobManager, + }); + const envelope = createAgentRunEnvelope({ + protocol: 'responses', + requestId: record.responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: req.body, + }); + await executeAgentRun({ + envelope, + runId: runtime.responseId, + conversationId: record.conversationId, + connection: runtime.connection, + ...runtime.hooks, + isPrincipalActive: async () => true, + execute: async () => { + starts++; + }, + handleExecutionError: () => { + res.status(409).json({ code: 'cancelled' }); + }, + }); + settled(); + }); + const response = await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(record, secret), { type: 'bearer' }) + .send({}); + expect(response.status).toBe(202); + await finished; + expect(starts).toBe(0); + expect((await scope(async () => storage.read(owner, record.responseId)))?.status).toBe( + 'cancelled', + ); + }, + ); + + it.each(['stop', 'error'] as const)( + 'does not publish a completed candidate after %s wins settlement', + async (mode) => { + let done!: () => void; + const finished = new Promise((resolve) => { + done = resolve; + }); + const errors: unknown[] = []; + const { app } = fixture(async (req, res) => { + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: async () => null, + jobs: GenerationJobManager, + }); + const envelope = createAgentRunEnvelope({ + protocol: 'responses', + requestId: record.responseId, + receivedAt: Date.now(), + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: req.body, + }); + try { + await executeAgentRun({ + envelope, + runId: runtime.responseId, + conversationId: record.conversationId, + ...runtime.hooks, + isPrincipalActive: async () => true, + execute: async (execution) => { + if (mode === 'stop') + await GenerationJobManager.abortJob(runtime.responseId, { + expectedCreatedAt: execution.createdAt, + }); + res.json({ ...buildPendingBackgroundResponse(record), status: 'completed' }); + if (mode === 'error') throw new Error('private provider diagnostics'); + }, + onSettlementError: (error) => { + errors.push(error); + }, + }); + } catch { + /* The execution failure is represented by the canonical safe response. */ + } finally { + done(); + } + }); + await request(app) + .post('/') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(record, secret), { type: 'bearer' }) + .send({}) + .expect(202); + await finished; + expect(errors).toEqual([]); + const saved = (await scope(() => storage.read(owner, record.responseId)))!; + expect(saved.status).toBe(mode === 'stop' ? 'cancelled' : 'failed'); + expect(saved.responseJSON).not.toContain('private provider diagnostics'); + expect(saved.pendingTerminal).toBeUndefined(); + }, + ); +}); diff --git a/packages/api/src/agents/responses/worker.ts b/packages/api/src/agents/responses/worker.ts index c1460fdb85..2a6b903dbf 100644 --- a/packages/api/src/agents/responses/worker.ts +++ b/packages/api/src/agents/responses/worker.ts @@ -4,6 +4,7 @@ import { getTenantId, tenantStorage } from '@librechat/data-schemas'; import type { IUser, ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas'; import type { Request, RequestHandler } from 'express'; import type { ResumedAgentExecutionIdentity } from '../remote/lifecycle'; +import type { TurnConversationRequest } from '~/conversations/save'; import type { ResponseResumeAdmissionResult } from './types'; import type { ContextRequest } from '~/middleware/tenant'; import type { ServerRequest } from '~/types/http'; @@ -240,12 +241,19 @@ export function createBackgroundResponseWorkerAuth( checkpointNamespace: result.checkpointNamespace, }; } - const request = req as ServerRequest & Pick; + const request = req as ServerRequest & + Pick & + TurnConversationRequest; // findUser supplies a lean auth DTO; do not mutate its cached tenant representation. request.user = Object.assign({}, user, { id: record.userId, tenantId: claims.tenantId }); request.tenantId = claims.tenantId; // Caller body/headers cannot replace the persisted, already validated request. request.body = JSON.parse(record.requestJSON); + if (record.expiredAt) + request._agentEventBindingRetention = { + expiredAt: record.expiredAt, + isTemporary: record.isTemporary, + }; admitted.set(req, { record, storage: deps.storage, diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index 53af3f89bc..73cfc77af9 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -16,6 +16,7 @@ export type { ResponseOwner, ResponseStorage, ResponseTerminalGuard, + ResponseExecutionGuard, ResponseCleanupIdentity, ResponseReservation, ResponseReservationResult, diff --git a/packages/data-schemas/src/methods/response.spec.ts b/packages/data-schemas/src/methods/response.spec.ts index 8a2c222a18..104d2a0ea6 100644 --- a/packages/data-schemas/src/methods/response.spec.ts +++ b/packages/data-schemas/src/methods/response.spec.ts @@ -1615,3 +1615,183 @@ it('matches the complete reviewed and resumed execution binding rather than its (await storage.stageTerminal(owner, 'resp_one', { execution: retained }, receipt)).ok, ).toBe(true); })); + +async function runningForReview() { + await storage.reserve(reservation({ sourceKeyId: 'original-key' })); + const running = await storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + generationCreatedAt: 10, + responseJSON: '{}', + }); + if (!running.ok) throw new Error('Publication failed'); + return running.record; +} + +it('publishes one immutable pause independently of polling before and after publication', () => + scoped(async () => { + const running = await runningForReview(); + for (let i = 0; i < 30; i++) { + const current = (await storage.read(owner, 'resp_one'))!; + expect( + ( + await storage.publish(owner, 'resp_one', current.revision, { + status: 'in_progress', + responseJSON: JSON.stringify({ progress: i }), + }) + ).ok, + ).toBe(true); + } + expect( + await storage.publishReview(owner, 'resp_one', running.revision, 10, review, '{}'), + ).toEqual({ ok: false, code: 'response_conflict' }); + const attempts = await Promise.all( + Array.from({ length: 12 }, () => + storage.publishReview(owner, 'resp_one', { execution: running }, 10, review, '{}'), + ), + ); + expect(attempts.every((result) => result.ok)).toBe(true); + const paused = (await storage.read(owner, 'resp_one'))!; + expect(paused.review).toEqual(review); + expect(paused.revision).toBe(running.revision + 31); + const projected = await storage.publish(owner, 'resp_one', paused.revision, { + status: 'requires_action', + responseJSON: '{"projection":"new"}', + }); + if (!projected.ok) throw new Error('Projection failed'); + for (const execution of [running, projected.record]) { + expect( + (await storage.publishReview(owner, 'resp_one', { execution }, 10, review, '{}')).ok, + ).toBe(true); + expect( + await storage.publishReview( + owner, + 'resp_one', + { execution }, + 10, + { ...review, partialResponseJSON: '{"changed":true}' }, + '{}', + ), + ).toEqual({ ok: false, code: 'response_conflict' }); + } + const saved = (await storage.read(owner, 'resp_one'))!; + expect(saved.revision).toBe(projected.record.revision); + expect(saved.responseJSON).toBe('{"projection":"new"}'); + expect(saved.review).toEqual(review); + })); + +it.each(['agentId', 'conversationId', 'sourceKeyId', 'generationCreatedAt'] as const)( + 'rejects a pause from a mismatched %s', + (field) => + scoped(async () => { + const running = await runningForReview(); + const execution = { ...running, [field]: field === 'generationCreatedAt' ? 11 : 'other' }; + expect( + await storage.publishReview(owner, 'resp_one', { execution }, 10, review, '{}'), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect((await storage.read(owner, 'resp_one'))?.review).toBeUndefined(); + }), +); + +it('binds the next pause to the complete admitted resume identity', () => + scoped(async () => { + await pause(); + await storage.reserveResume(owner, 'resp_one', resume(), 100); + await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 101); + const retained = (await storage.read(owner, 'resp_one'))!; + const nextReview = { + ...review, + actionId: 'second-action', + providerExecutionId: 'provider-two', + }; + for (const execution of [ + { ...retained, review: undefined }, + { ...retained, review: { ...retained.review!, checkpointId: 'different' } }, + { ...retained, resume: { ...retained.resume!, fingerprint: 'different' } }, + { ...retained, resume: { ...retained.resume!, state: 'reserved' as const } }, + ]) + expect( + await storage.publishReview(owner, 'resp_one', { execution }, 10, nextReview, '{}'), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect( + ( + await storage.publishReview( + owner, + 'resp_one', + { execution: retained }, + 10, + nextReview, + '{}', + ) + ).ok, + ).toBe(true); + expect(await storage.read(owner, 'resp_one')).toMatchObject({ review: nextReview }); + expect((await storage.read(owner, 'resp_one'))?.resume).toBeUndefined(); + })); + +it.each(['cancelled', 'terminal', 'erased'] as const)( + 'does not publish a pause after %s authority wins', + (failure) => + scoped(async () => { + const running = await runningForReview(); + if (failure === 'cancelled') await storage.requestCancellation(owner, 'resp_one', Date.now()); + if (failure === 'terminal') + await storage.stageTerminal(owner, 'resp_one', { execution: running }, outcome); + if (failure === 'erased') + await mongoose.models.StoredResponse.deleteOne({ responseId: 'resp_one' }); + expect( + await storage.publishReview(owner, 'resp_one', { execution: running }, 10, review, '{}'), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect((await storage.read(owner, 'resp_one'))?.review).toBeUndefined(); + }), +); + +it.each(['live', 'owner_deleted', 'conversation_deleted', 'expired'] as const)( + 'supplements an unpublished generation only under durable retirement proof: %s', + (reason) => + scoped(async () => { + await storage.reserve(reservation()); + const retained = { + ...owner, + responseId: 'resp_one', + agentId: 'agent-one', + conversationId: 'conversation-one', + generationCreatedAt: 42, + checkpointNamespace: 'owned-checkpoint', + checkpointConfigJSON: 'pinned-settings', + }; + if (reason === 'expired') + await createResponseRetention(mongoose)( + owner, + 'conversation-one', + new Date(Date.now() - 1000), + ); + else if (reason !== 'live') + await closeResponseScopes( + mongoose, + owner, + reason === 'conversation_deleted' ? ['conversation-one'] : undefined, + ); + const identity = await storage.readCleanupIdentity(owner, 'resp_one', retained); + expect(identity).toMatchObject({ + generationCreatedAt: reason === 'live' ? undefined : 42, + retirement: reason === 'live' ? undefined : reason, + checkpointNamespace: reason === 'live' ? undefined : 'owned-checkpoint', + }); + if (reason !== 'live') { + const wrong = await storage.readCleanupIdentity(owner, 'resp_one', { + ...retained, + agentId: 'wrong-agent', + }); + expect(wrong?.generationCreatedAt).toBeUndefined(); + expect(wrong?.checkpointNamespace).toBeUndefined(); + await mongoose.models.StoredResponse.updateOne( + { responseId: 'resp_one' }, + { $set: { generationCreatedAt: 43 } }, + ); + expect(await storage.readCleanupIdentity(owner, 'resp_one', retained)).toMatchObject({ + generationCreatedAt: 43, + checkpointNamespace: undefined, + }); + } + }), +); diff --git a/packages/data-schemas/src/methods/response.ts b/packages/data-schemas/src/methods/response.ts index 670511053a..99c0676f6d 100644 --- a/packages/data-schemas/src/methods/response.ts +++ b/packages/data-schemas/src/methods/response.ts @@ -77,8 +77,8 @@ export type ResponseResumeResult = | { ok: true; record: StoredResponseRecord; replayed: boolean } | { ok: false; code: 'resume_conflict' }; -/** A drained execution's authority is independent of polling's projection revision. */ -export type ResponseTerminalGuard = +/** Canonical execution writes are independent of polling's projection revision. */ +export type ResponseExecutionGuard = | number | { execution: Pick< @@ -87,15 +87,13 @@ export type ResponseTerminalGuard = >; }; -function terminalExecutionFilter(guard: Exclude) { - const { execution } = guard; - const filter: Record = { - agentId: execution.agentId, - sourceKeyId: execution.sourceKeyId ?? { $exists: false }, - conversationId: execution.conversationId, - generationCreatedAt: execution.generationCreatedAt, - }; - const review = execution.review; +/** Compatibility name for existing terminal producers. */ +export type ResponseTerminalGuard = ResponseExecutionGuard; + +function responseReviewFilter( + review: ResponseReviewCheckpoint | undefined, +): Record { + const filter: Record = {}; if (review) { const fields: Record< keyof ResponseReviewCheckpoint, @@ -114,6 +112,18 @@ function terminalExecutionFilter(guard: Exclude) for (const [key, value] of Object.entries(fields)) filter[`review.${key}`] = value ?? { $exists: false }; } else filter.review = { $exists: false }; + return filter; +} + +function responseExecutionFilter(guard: Exclude) { + const { execution } = guard; + const filter: Record = { + agentId: execution.agentId, + sourceKeyId: execution.sourceKeyId ?? { $exists: false }, + conversationId: execution.conversationId, + generationCreatedAt: execution.generationCreatedAt, + ...responseReviewFilter(execution.review), + }; const resume = execution.resume; if (resume) { const fields: Record< @@ -153,7 +163,7 @@ export interface ResponseStorage { publishReview( owner: ResponseOwner, responseId: string, - revision: number, + guard: ResponseExecutionGuard, generationCreatedAt: number, review: ResponseReviewCheckpoint, responseJSON: string, @@ -249,7 +259,7 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp guard.execution.generationCreatedAt !== receipt.generationCreatedAt ) return { ok: false, code: 'response_conflict' }; - const execution = typeof guard === 'number' ? undefined : terminalExecutionFilter(guard); + const execution = typeof guard === 'number' ? undefined : responseExecutionFilter(guard); const pendingTerminal: ResponseTerminalReceipt = { generationCreatedAt: receipt.generationCreatedAt, providerExecutionId: receipt.providerExecutionId, @@ -329,13 +339,41 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp .lean(); return record ? { ok: true, record } : { ok: false, code: 'response_conflict' }; }, - async publishReview(owner, responseId, revision, generationCreatedAt, review, responseJSON) { + async publishReview(owner, responseId, guard, generationCreatedAt, review, responseJSON) { + if (typeof guard !== 'number' && guard.execution.generationCreatedAt !== generationCreatedAt) + return { ok: false, code: 'response_conflict' }; + const execution = typeof guard === 'number' ? undefined : responseExecutionFilter(guard); + const receiptFilter = + typeof guard === 'number' + ? undefined + : { + ...scope(owner), + responseId, + generationCreatedAt, + agentId: guard.execution.agentId, + sourceKeyId: guard.execution.sourceKeyId ?? { $exists: false }, + conversationId: guard.execution.conversationId, + ...responseReviewFilter(review), + status: 'requires_action', + resume: { $exists: false }, + pendingTerminal: { $exists: false }, + cancellationRequestedAt: { $exists: false }, + }; + if ( + typeof guard !== 'number' && + guard.execution.review?.actionId === review.actionId && + guard.execution.review.providerExecutionId === review.providerExecutionId && + guard.execution.resume == null + ) { + const existing = await model.findOne(receiptFilter).lean(); + return existing ? { ok: true, record: existing } : { ok: false, code: 'response_conflict' }; + } const record = await model .findOneAndUpdate( { ...scope(owner), responseId, - revision, + ...(execution ?? { revision: guard }), generationCreatedAt, status: { $in: ['in_progress', 'requires_action'] }, pendingTerminal: { $exists: false }, @@ -381,7 +419,10 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp { new: true, runValidators: true }, ) .lean(); - return record ? { ok: true, record } : { ok: false, code: 'response_conflict' }; + if (record) return { ok: true, record }; + if (!receiptFilter) return { ok: false, code: 'response_conflict' }; + const existing = await model.findOne(receiptFilter).lean(); + return existing ? { ok: true, record: existing } : { ok: false, code: 'response_conflict' }; }, async reserveResume(owner, responseId, input, now) { if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid resume reservation time'); @@ -599,7 +640,7 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp .lean(); // Retained data comes only from an internal job/receipt, never request input. // Validate it even when the row exists before using any fallback checkpoint fields. - const matchingRetained = + const retainedCandidate = retained && retained.responseId === responseId && retained.userId === owner.userId && @@ -607,10 +648,11 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp (!record || (retained.conversationId === record.conversationId && retained.agentId === record.agentId && - retained.generationCreatedAt === record.generationCreatedAt)) + (record.generationCreatedAt == null || + retained.generationCreatedAt === record.generationCreatedAt))) ? retained : undefined; - const source = record ?? matchingRetained; + const source = record ?? retainedCandidate; if (!source) return null; const now = new Date(); const [ownerKey, conversationKey] = responseScopeKeys(owner, source.conversationId); @@ -637,6 +679,13 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp !(conversationAnchor?.expiredAt && conversationAnchor.expiredAt <= now) ) return null; + const matchingRetained = + retainedCandidate && + (!record || + retainedCandidate.generationCreatedAt === record.generationCreatedAt || + (retirement && record.generationCreatedAt == null)) + ? retainedCandidate + : undefined; // A staged checkpoint is one binding: never fill an incomplete terminal // binding with settings from a prior review or retained provider segment. const terminalCheckpoint = record?.pendingTerminal; @@ -663,7 +712,8 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp responseId: id, agentId, conversationId, - generationCreatedAt, + generationCreatedAt: + generationCreatedAt ?? (retirement ? matchingRetained?.generationCreatedAt : undefined), expiredAt, retirement, checkpointNamespace,