const { randomUUID } = require('crypto'); const { logger } = require('@librechat/data-schemas'); const { Constants, ResourceType, EModelEndpoint, ViolationTypes, PermissionBits, } = require('librechat-data-provider'); const { checkAccess, GenerationJobManager, isPendingActionStale, mapToolApprovalResolutions, resolveAskUserQuestionResume, buildResolvedAskUserQuestion, appendResolvedAskUserQuestion, attachAskUserQuestionAnswers, findAskUserQuestionContentIndex, findUndecidedToolCalls, findDisallowedDecisions, findIncompleteDecisions, computeAgentRequestFingerprint, captureAgentCheckpointGeneration, deleteAgentCheckpoint, buildAbortedResponseMetadata, sanitizeMessageForTransmit, filterMalformedContentParts, getAgentCheckpointer, isContentFilterError, preflightResumeContent, getResumeProvenance, getUserFacingResumeError, decrementPendingRequest, checkAndIncrementPendingRequest, isSteerPreemptSupported, isStopConfirmed, toPendingSteer, createMCPRuntimeRequestBody, getSafeErrorMetadata, } = require('@librechat/api'); const { disposeClient } = require('~/server/cleanup'); const { decryptMetadata } = require('~/server/services/ActionService'); const { checkPermission } = require('~/server/services/PermissionService'); const { getMCPRequestContext, cleanupMCPRequestContextForReq, } = require('~/server/services/MCPRequestContext'); const { saveMessage, getConvo, getMessages, getFiles, getAgent, getActions, getUserMemories, getRoleByName, } = require('~/models'); const { recordScheduleOutcome, claimScheduleResume, releaseScheduleResumeClaim, finalizeScheduleResumeClaim, releaseScheduleResumeFence, isScheduleLive, } = require('~/server/services/Schedules'); const { GENERATION_PROTOCOL_HEADER, negotiateNewGenerationProtocol, negotiateExistingGenerationProtocol, } = require('./protocol'); function sendGenerationJson(res, status, body, generationProtocolVersion) { if (typeof res.set === 'function') { res.set(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion)); } else if (typeof res.setHeader === 'function') { res.setHeader(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion)); } return res.status(status).json({ ...body, generationProtocolVersion }); } /** * How long a resume waits on best-effort steering bookkeeping before answering * anyway. The approval is already consumed by that point, so a stalled Redis * must not strand the client behind a chip label and an arm. */ const STEER_RESUME_SETUP_TIMEOUT_MS = 1000; /** * New jobs are physically isolated by an immutable saver namespace, so a * terminal owner deletes the whole namespace and catches writes that landed * after an earlier read. Pre-isolation jobs share the root namespace and must * retain captured-id cleanup to avoid pruning a replacement. */ function deleteResumedGenerationCheckpoint({ conversationId, checkpointerCfg, job, checkpointGeneration, }) { const checkpointNamespace = typeof job?.metadata?.checkpointNamespace === 'string' ? job.metadata.checkpointNamespace : ''; if (checkpointNamespace !== '') { return deleteAgentCheckpoint(conversationId, checkpointerCfg, undefined, { checkpointNamespace, }); } return deleteAgentCheckpoint(conversationId, checkpointerCfg, checkpointGeneration); } /** Error-path checkpoint cleanup runs after the HTTP ACK. A storage failure * must be observable, but must not escape the controller catch and bypass the * remaining request-context/concurrency/client cleanup in `finally`. */ async function deleteFailedResumeCheckpoint(args, context) { try { await deleteResumedGenerationCheckpoint(args); } catch (error) { logger.error( `[ResumeAgentController] Failed to prune checkpoint after ${context}`, getSafeErrorMetadata(error), ); } } const GENERIC_RESUME_ERROR = 'Resume failed'; const resumeContentProtectionDependencies = { getAgentCheckpointer, checkAccess, getMessages, getFiles, getAgent, getActions, getUserMemories, getRoleByName, decryptMetadata, canAccessAgent: (agent, user) => checkPermission({ userId: user.id, role: user.role, resourceType: ResourceType.AGENT, resourceId: agent._id, requiredPermission: PermissionBits.VIEW, }), }; /** De-duplicate a merged attachment list by a stable artifact identity. */ function mergeAttachments(existing, incoming) { const seen = new Set(); const out = []; for (const attachment of [...(existing ?? []), ...(incoming ?? [])]) { if (!attachment) { continue; } const key = attachment.file_id ?? attachment.filepath ?? attachment.filename ?? JSON.stringify(attachment); if (seen.has(key)) { continue; } seen.add(key); out.push(attachment); } return out; } /** * Resolve the current segment's tool artifacts and merge them with any already * persisted on the response row. A resumed turn can span multiple pause segments; * each rebuilt client has its own `artifactPromises`, and the final finalize would * otherwise OVERWRITE the row's attachments with only the last segment's. Reading * the persisted row and merging keeps every segment's artifacts on the saved message. */ async function resolveAccumulatedAttachments({ client, conversationId, responseMessageId }) { const promises = Array.isArray(client?.artifactPromises) ? client.artifactPromises : []; const resolved = promises.length > 0 ? (await Promise.all(promises)).filter(Boolean) : []; let existing = []; if (responseMessageId) { try { const [row] = await getMessages( { conversationId, messageId: responseMessageId }, 'attachments', ); existing = Array.isArray(row?.attachments) ? row.attachments : []; } catch (err) { logger.warn( '[ResumeAgentController] Failed to read prior attachments for merge', getSafeErrorMetadata(err), ); } } return mergeAttachments(existing, resolved); } /** Resolve the segment's content for an unfinished save (mirrors finalize's source). */ async function resolveSegmentContent(client, streamId, expectedCreatedAt) { const liveContent = Array.isArray(client?.contentParts) ? client.contentParts : []; const rawContent = liveContent.length > 0 ? liveContent : ((await GenerationJobManager.getResumeState(streamId, expectedCreatedAt)) ?.aggregatedContent ?? []); return filterMalformedContentParts(rawContent); } /** * A resumed segment that streamed content / produced artifacts and then paused AGAIN * must persist that progress before returning. The next resume rebuilds a fresh client * (empty `contentParts`/`artifactPromises`), so without this an approval that later * expires or is reaped would leave only the EARLIER pause's content on the saved row — * the user loses everything streamed during this segment. Saved as a partial (`$set`, * still `unfinished`) so a subsequent successful resume overwrites it on finalize. */ async function persistRePauseProgress({ req, client, job, streamId, conversationId }) { const userId = req.user.id; const meta = job.metadata ?? {}; const responseMessageId = meta.responseMessageId ?? client.responseMessageId; if (!responseMessageId) { return; } const content = await resolveSegmentContent(client, streamId, job.createdAt); const { userSubmittedPaths, userSubmittedMessageFieldPaths } = getResumeProvenance({ content, pendingAction: meta.pendingAction, body: req.body, existingPaths: meta.userSubmittedPaths, existingMessageFieldPaths: meta.userSubmittedMessageFieldPaths, }); const attachments = await resolveAccumulatedAttachments({ client, conversationId, responseMessageId, }); if (content.length === 0 && attachments.length === 0) { return; } const savedResponseMessage = await saveMessage( { userId, isTemporary: meta.isTemporary ?? req.body?.isTemporary, interfaceConfig: req?.config?.interfaceConfig, }, { messageId: responseMessageId, conversationId, ...(content.length > 0 && { content }), ...(attachments.length > 0 && { attachments }), ...(userSubmittedPaths.length > 0 && { userSubmittedPaths }), ...(userSubmittedMessageFieldPaths.length > 0 && { userSubmittedMessageFieldPaths }), unfinished: true, user: userId, }, { context: 'api/server/controllers/agents/resume.js - re-pause progress persist' }, ); if (!savedResponseMessage) { throw new Error('Re-pause response progress could not be persisted'); } } /** Untenanted jobs (pre-multi-tenancy) remain accessible if the userId check passes. */ function hasTenantMismatch(job, user) { return job.metadata?.tenantId != null && job.metadata.tenantId !== user.tenantId; } /** * Build the SDK resume value from the wire decision payload, validating against the * pending action. Returns `{ resumeValue }` on success or `{ error }` with an HTTP * status for the route to surface. */ function resolveResumeValue(pendingAction, body) { const payload = pendingAction.payload; if (payload?.type === 'tool_approval') { const resolutions = Array.isArray(body.decisions) ? body.decisions : []; const undecided = findUndecidedToolCalls(payload, resolutions); if (undecided.length > 0) { return { status: 400, error: 'Every paused tool call must be decided', undecided }; } // Enforce the policy's per-tool allowed_decisions — a crafted POST must not // approve a tool the policy restricted to (e.g.) reject/respond. const disallowed = findDisallowedDecisions(payload, resolutions); if (disallowed.length > 0) { return { status: 403, error: 'Decision not permitted for one or more tools', disallowed }; } // `edit`/`respond` must carry their payload — otherwise toSdkDecision's defensive // defaults ({} / '') would resume with an empty input/result the user didn't approve. const incomplete = findIncompleteDecisions(resolutions); if (incomplete.length > 0) { return { status: 400, error: 'edit requires editedArguments and respond requires responseText', incomplete, }; } return { resumeValue: mapToolApprovalResolutions(resolutions) }; } if (payload?.type === 'ask_user_question') { return resolveAskUserQuestionResume(payload, body); } return { status: 400, error: 'Unsupported pending action type' }; } /** * Finalize a resumed turn that ran to completion: persist the (now complete) * response message, emit the terminal event over the existing SSE, complete the * job, and prune the checkpoint. Mirrors the abort route's save shape but for a * successful finish. Best-effort title generation for a first-turn pause. */ async function finalizeResumedTurn({ req, client, job, streamId, conversationId, addTitle, checkpointGeneration, }) { const userId = req.user.id; const checkpointerCfg = req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer; const meta = job.metadata ?? {}; const userMessage = meta.userMessage; // The response hangs off the user message; the *user* message's own parent decides // whether this is the first turn of the conversation (title eligibility). const parentMessageId = userMessage?.messageId ?? Constants.NO_PARENT; const isFirstTurn = (userMessage?.parentMessageId ?? Constants.NO_PARENT) === Constants.NO_PARENT; const responseMessageId = meta.responseMessageId ?? `${userMessage?.messageId ?? 'resumed'}_`; // Sourced from the paused job (persisted at creation), not the resume body — a // temporary chat must stay temporary on resume so its messages aren't persisted. const isTemporary = meta.isTemporary ?? req.body?.isTemporary; // Read the raw job data BEFORE completeJob deletes it — its tracked token/context // usage backs the response message's cost rollup (parity with normal completion). const jobData = await GenerationJobManager.getJobStore().getJob(streamId); // Job-replacement guard (mirrors the normal request path): jobs are keyed by streamId // (== conversationId), so a new/concurrent request reusing this conversation overwrites // the record with a fresh createdAt. If that happened while we were resuming, finalizing // now would emit `done` to / complete / delete the NEWER turn's job. Skip all terminal // side effects when the job we paused is no longer the live one; the caller's `finally` // still disposes the client + releases the slot. if (!jobData || jobData.createdAt !== job.createdAt) { logger.warn( `[ResumeAgentController] Skipping resumed finalization — job ${streamId} was replaced`, ); return; } // Prefer the resumed run's live content: it's complete (seeded with the pre-pause // content) and avoids a Redis re-read that can race appendChunk writes still in // flight. Fall back to the aggregated store content only when the live array is empty. const liveContent = Array.isArray(client?.contentParts) ? client.contentParts : []; const rawContent = liveContent.length > 0 ? liveContent : ((await GenerationJobManager.getResumeState(streamId, job.createdAt))?.aggregatedContent ?? []); // Parity with the normal agents path (AgentClient strips these before saving): // drop empty/malformed tool_call parts so a resumed turn can't persist an invalid // part that breaks reload/rendering. const content = filterMalformedContentParts(rawContent); const { userSubmittedPaths, userSubmittedMessageFieldPaths } = getResumeProvenance({ content, pendingAction: meta.pendingAction, body: req.body, existingPaths: meta.userSubmittedPaths, existingMessageFieldPaths: meta.userSubmittedMessageFieldPaths, }); /** * A resumed segment can end on an empty preempt boundary just as a fresh * one can — the boundary hook is re-registered by `buildSteerWiring` on * resume. Persisting that as complete would contradict the honest contract * the normal request path now keeps. */ const preemptStats = client?.run?.getPreemptStats?.(); const preemptIncomplete = (preemptStats?.emptyBoundaries ?? 0) > 0 || client?.run?.getHaltReason?.() === 'preempt_incomplete'; const responseMessage = { messageId: responseMessageId, parentMessageId, conversationId, content, sender: meta.sender ?? client?.sender ?? 'AI', endpoint: meta.endpoint, iconURL: meta.iconURL, model: meta.model, unfinished: preemptIncomplete, error: false, isCreatedByUser: false, user: userId, ...(userSubmittedPaths.length > 0 && { userSubmittedPaths }), ...(userSubmittedMessageFieldPaths.length > 0 && { userSubmittedMessageFieldPaths }), }; if (meta.agent_id ?? req.body?.agent_id) { responseMessage.agent_id = meta.agent_id ?? req.body.agent_id; } // Persist tool artifacts (code files, images, UI resources) the resumed continuation // produced — BaseClient.sendMessage awaits these before saving, but the lean resume // path bypasses it, so do it here or they vanish on reload / for late subscribers. // MERGE with any already on the row (earlier pause segments) rather than overwrite — // the final segment's client only holds its own segment's artifacts. const attachments = await resolveAccumulatedAttachments({ client, conversationId, responseMessageId, }); if (attachments.length > 0) { responseMessage.attachments = attachments; } // Response metadata: the resume client only sees POST-resume usage, while the job's // tracked tokenUsage is cumulative across the pause. Take the cumulative usage (+ // summary marker) from the job, and contextUsage / thoughtSignatures from the client // (which the abort-only helper drops). Cumulative usage wins so cost isn't underreported. const clientMeta = client?.buildResponseMetadata?.() ?? null; const cumulativeMeta = jobData ? buildAbortedResponseMetadata(jobData) : null; const responseMetadata = { ...(clientMeta ?? {}), ...(cumulativeMeta?.usage ? { usage: cumulativeMeta.usage } : {}), ...(cumulativeMeta?.summaryUsedTokens != null ? { summaryUsedTokens: cumulativeMeta.summaryUsedTokens } : {}), }; if (Object.keys(responseMetadata).length > 0) { responseMessage.metadata = responseMetadata; } // Carry the resumed run's context-window calibration (BaseClient.sendMessage persists // this on the response). Without it, the NEXT turn can't seed its pruner from this // run and falls back to uncalibrated token accounting. if (client?.contextMeta != null) { responseMessage.contextMeta = client.contextMeta; } // Win terminal ownership BEFORE the outcome-defining response write. Stop // and completion both write the same Mongo row; a later liveness read cannot // fence that external write, while this CAS gives exactly one side authority. // The durable pending marker keeps status/subscribers on the readiness path // until the winner has persisted and published its FINAL. const terminalClaim = await GenerationJobManager.claimTerminalJob( streamId, 'complete', undefined, job.createdAt, { persistencePending: true }, ); if (!terminalClaim) { logger.warn( `[ResumeAgentController] Skipping resumed FINAL — another terminal/pause transition won for ${streamId}`, ); return; } let terminalPublicationStarted = false; try { const savedResponseMessage = await saveMessage( { userId, isTemporary, interfaceConfig: req?.config?.interfaceConfig }, responseMessage, { context: 'api/server/controllers/agents/resume.js - resumed response end' }, ); if (!savedResponseMessage) { throw new Error('Resumed response could not be persisted before terminal publication'); } const convo = await getConvo(userId, conversationId); const conversation = { ...(convo ?? {}), conversationId }; // First-turn pause: the title was deferred when the turn paused. Generate it BEFORE // completing the stream so the `title` event still reaches the live client (emitChunk // no-ops once completeJob tears down the runtime) and the final event carries the real // title instead of "New Chat". Best-effort — a failure must not fail the resumed turn. if ( addTitle && isFirstTurn && !isTemporary && userMessage?.text && (!convo || !convo.title || convo.title === 'New Chat') ) { try { await addTitle(req, { text: userMessage.text, conversationId, client, onTitleGenerated: ({ conversationId: titleConvoId, title }) => { conversation.title = title; return GenerationJobManager.emitChunk( streamId, { event: 'title', data: { conversationId: titleConvoId, title }, }, { expectedCreatedAt: job.createdAt }, ); }, }); } catch (err) { logger.error( '[ResumeAgentController] Title generation failed after resume', getSafeErrorMetadata(err), ); } } conversation.title = conversation.title || 'New Chat'; if (meta.scheduleId) { await recordScheduleOutcome({ scheduleId: meta.scheduleId, scheduledFor: meta.scheduledFor, streamId, jobCreatedAt: job.createdAt, status: preemptIncomplete ? 'interrupted' : 'success', conversationId, ...(preemptIncomplete && { error: 'Scheduled run was interrupted before completion', }), }); } const pendingSteers = terminalClaim.drainedSteers.map(toPendingSteer); const finalEvent = { final: true, conversation, title: conversation.title, requestMessage: userMessage ? sanitizeMessageForTransmit({ ...userMessage, conversationId, isCreatedByUser: true, // job.metadata.userMessage is persisted without files; carry the restored // uploads (seeded onto req.body.files before reconstruction) so the final SSE // doesn't blank the user bubble's attachments — matching the normal path. ...(Array.isArray(req.body?.files) && req.body.files.length > 0 ? { files: req.body.files } : {}), }) : null, responseMessage: { ...responseMessage }, ...(pendingSteers.length > 0 && { pendingSteers }), }; terminalPublicationStarted = true; await GenerationJobManager.publishTerminalClaim(terminalClaim, finalEvent); } catch (error) { if (!terminalPublicationStarted) { try { await GenerationJobManager.publishTerminalClaim(terminalClaim, null); } catch (publishError) { logger.error( '[ResumeAgentController] Failed to publish terminal persistence reconciliation', getSafeErrorMetadata(publishError), ); } } throw error; } finally { try { // Cleanup must run even if persistence/publication fails. The claim // carries the exact generation/runtime identity, so this cannot tear // down a later run. await GenerationJobManager.finishTerminalJob(terminalClaim); } finally { await deleteResumedGenerationCheckpoint({ conversationId, checkpointerCfg, job, checkpointGeneration, }); } } } /** * Resume a generation that paused for human-in-the-loop review. * * The original run lives in a detached background task that exits when the run * pauses, so this REBUILDS the run from the durable checkpoint (same `thread_id`) * and continues it with the user's decision. The continuation streams over the * client's existing SSE (events flow through the same `streamId`). * * Flow: authorize → map decisions → atomically claim the resume (single-winner) → * ACK → reconstruct the client → `resumeCompletion` → finalize (or re-pause). * * Shares chat.js's middleware (auth, agent access, `buildEndpointOption`) so the * agent/endpoint are reconstructed from the request exactly like a normal turn. * * @param {express.Request} req * @param {express.Response} res * @param {express.NextFunction} next * @param {Function} initializeClient * @param {Function} addTitle */ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) => { const userId = req.user.id; let generationProtocolVersion = negotiateNewGenerationProtocol(req, GenerationJobManager); const { conversationId, actionId, generationCreatedAt } = req.body; const streamId = conversationId; if (!streamId || streamId === 'new') { return sendGenerationJson( res, 400, { error: 'conversationId is required to resume' }, generationProtocolVersion, ); } if ( generationCreatedAt != null && (!Number.isSafeInteger(generationCreatedAt) || generationCreatedAt < 0) ) { return sendGenerationJson( res, 400, { code: 'INVALID_GENERATION_IDENTITY' }, generationProtocolVersion, ); } const job = await GenerationJobManager.getJob(streamId); if (!job) { return sendGenerationJson( res, 404, { error: 'No paused generation for this conversation' }, generationProtocolVersion, ); } // Every persisted generation is owner-scoped. A missing/corrupt owner is // not a legacy wildcard: fail closed before reading or resolving its action. if (job.metadata?.userId !== userId) { return sendGenerationJson(res, 403, { error: 'Unauthorized' }, generationProtocolVersion); } if (hasTenantMismatch(job, req.user)) { return sendGenerationJson(res, 403, { error: 'Unauthorized' }, generationProtocolVersion); } generationProtocolVersion = negotiateExistingGenerationProtocol(req, job); if (generationCreatedAt != null && job.createdAt !== generationCreatedAt) { return sendGenerationJson(res, 409, { code: 'RUN_REPLACED' }, generationProtocolVersion); } // The resume must rebuild the SAME agent/endpoint that paused. Require an EXACT // agent_id match when the paused job had one — a request that omits agent_id (or // claims an ephemeral / non-agents endpoint) must not rebuild the claimed checkpoint // on a different graph. The conversation's agent is stable, so a correct client always // sends the right one. const originalAgentId = job.metadata?.agent_id; if (originalAgentId && req.body.agent_id !== originalAgentId) { return sendGenerationJson( res, 403, { error: 'Cannot resume with a different agent' }, generationProtocolVersion, ); } // Require an EXACT endpoint match (like agent_id): a request that OMITS endpoint must // not fall through — the shared chat middleware treats a missing/non-agents endpoint // as the ephemeral agent, so omitting it could rebuild the claimed checkpoint on a // different graph. A correct client always echoes the paused endpoint. const originalEndpoint = job.metadata?.endpoint; if (originalEndpoint && req.body.endpoint !== originalEndpoint) { return sendGenerationJson( res, 403, { error: 'Cannot resume on a different endpoint' }, generationProtocolVersion, ); } const scheduleId = job.metadata?.scheduleId; const scheduledFor = job.metadata?.scheduledFor; if ( scheduleId && !(await isScheduleLive(scheduleId, job.metadata?.scheduleConfigRevision, { automatic: job.metadata?.scheduleManual !== true, policy: true, // Re-validate the destination THIS occurrence recorded, not the schedule's // current one: a later fire can have redirected the schedule while this run sat // paused, and its conversation stays where it was filed. scheduledFor, })) ) { let stopped = false; try { const abortResult = await GenerationJobManager.abortJob(streamId, { expectedCreatedAt: job.createdAt, awaitProviderDrain: true, }); // `success` is the authoritative signal, exactly as the abort route gates. A // `success: false` result WITHOUT a failure reason no longer exists — an // unreached job, a replacement, or a lost CAS all report one — so the old // `failureReason == null` test settled the occurrence and pruned the // checkpoint on aborts that were never confirmed. stopped = isStopConfirmed(abortResult); } catch (error) { logger.warn('[ResumeAgentController] Failed to stop inactive scheduled run', error); } if (!stopped) { res.set('Retry-After', '1'); return sendGenerationJson( res, 503, { code: 'SCHEDULE_STOP_UNCONFIRMED', error: 'The inactive scheduled run could not be confirmed stopped. Please retry.', }, generationProtocolVersion, ); } await recordScheduleOutcome({ scheduleId, scheduledFor, streamId, jobCreatedAt: job.createdAt, status: 'interrupted', conversationId, error: 'Schedule was disabled, changed, or deleted before approval', }); const checkpointNamespace = job.metadata?.checkpointNamespace; if (typeof checkpointNamespace === 'string' && checkpointNamespace !== '') { await deleteAgentCheckpoint( conversationId, req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer, undefined, { checkpointNamespace }, ).catch((error) => { logger.warn('[ResumeAgentController] Failed to prune inactive schedule checkpoint', error); }); } return sendGenerationJson( res, 409, { code: 'SCHEDULE_NO_LONGER_ACTIVE', error: 'This schedule can no longer be resumed' }, generationProtocolVersion, ); } const pendingAction = job.metadata?.pendingAction; if (job.status !== 'requires_action') { return sendGenerationJson( res, 409, { error: 'No live pending action to resume' }, generationProtocolVersion, ); } if (isPendingActionStale({ pendingAction })) { // The action expired between the pending-action SSE and this submit. Drive the expiry // NOW (expire CAS + terminal SSE) instead of waiting for the periodic sweeper — // otherwise the job sits `requires_action` with a dead action and any attached SSE // client never gets a terminal event, so the stream appears to hang even though the // UI already reported the action as expired. try { await GenerationJobManager.expireApproval(streamId, pendingAction?.actionId, job.createdAt); } catch (err) { logger.warn( '[ResumeAgentController] Failed to expire stale action on submit', getSafeErrorMetadata(err), ); } return sendGenerationJson( res, 409, { error: 'No live pending action to resume' }, generationProtocolVersion, ); } // Require the actionId the UI sends: without it, a stale/malformed client could // resolve whatever action is currently pending (e.g. answer a different question). if (!actionId) { return sendGenerationJson( res, 400, { error: 'actionId is required to resume' }, generationProtocolVersion, ); } if (pendingAction.actionId !== actionId) { return sendGenerationJson( res, 409, { error: 'This decision targets a stale action' }, generationProtocolVersion, ); } // Pin the graph identity: the resume must rebuild the SAME agent/graph + tool set the // run paused on. The agent_id + endpoint guards above cover saved agents; the // fingerprint additionally catches an ephemeral-agent config swap (its agent_id is // undefined, so the id guard can't tell two ephemeral configs apart). Enforced only // when the paused action carries a fingerprint (in-flight pauses from before this // change won't), and recomputed from the resume body's graph-determining fields. const pinnedFingerprint = pendingAction.requestFingerprint; if (pinnedFingerprint && pinnedFingerprint !== computeAgentRequestFingerprint(req.body ?? {})) { return sendGenerationJson( res, 403, { error: 'Cannot resume with a different agent configuration' }, generationProtocolVersion, ); } const mapped = resolveResumeValue(pendingAction, req.body); if (mapped.error) { return sendGenerationJson( res, mapped.status, { error: mapped.error, ...(mapped.undecided && { undecided: mapped.undecided }), ...(mapped.disallowed && { disallowed: mapped.disallowed }), ...(mapped.incomplete && { incomplete: mapped.incomplete }), }, generationProtocolVersion, ); } let resolvedAskContentIndex; let resolvedAskContentMissing = false; if (pendingAction.payload.type === 'ask_user_question' && !pendingAction.payload.tool_call_id) { const answerSnapshot = await GenerationJobManager.getResumeState(streamId, job.createdAt); if (answerSnapshot == null) { return sendGenerationJson(res, 409, { code: 'RUN_REPLACED' }, generationProtocolVersion); } const askRequest = Array.isArray(pendingAction.payload.questions) ? { questions: pendingAction.payload.questions } : pendingAction.payload.question; const answerContent = answerSnapshot.aggregatedContent ?? []; if (answerContent.length > 0) { resolvedAskContentIndex = findAskUserQuestionContentIndex( answerContent, undefined, askRequest, ); if (resolvedAskContentIndex < 0) { resolvedAskContentIndex = undefined; resolvedAskContentMissing = true; } } else { resolvedAskContentMissing = true; } } const resolvedAskUserQuestion = buildResolvedAskUserQuestion( pendingAction, req.body, resolvedAskContentIndex, resolvedAskContentMissing, ); const resolvedAskUserQuestions = appendResolvedAskUserQuestion( job.metadata?.resolvedAskUserQuestions, resolvedAskUserQuestion, ); // A legacy job has no saver-level generation namespace, so snapshot its exact // durable ids before the atomic resume claim. New jobs can skip this indexed // read: terminal cleanup deletes their whole immutable namespace, including // writes that land while the continuation is running. // // Start the indexed read alongside the independent concurrency check so the // generation guard adds minimal time to the resume ACK path. const checkpointerCfg = req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer; const checkpointNamespace = typeof job.metadata?.checkpointNamespace === 'string' ? job.metadata.checkpointNamespace : ''; const checkpointGenerationPromise = checkpointNamespace !== '' ? Promise.resolve(undefined) : captureAgentCheckpointGeneration(conversationId, checkpointerCfg).catch((err) => { logger.warn( '[ResumeAgentController] Failed to capture checkpoint generation', getSafeErrorMetadata(err), ); return { threadId: conversationId, checkpointIds: [], }; }); // Reconstruct and inspect the exact paused input before acquiring a slot, // consuming the pending action, seeding MCP state, or acknowledging success. // A rejected answer/checkpoint therefore leaves the approval live and // retryable, with no resource initialization or externally visible side // effect beyond owner-scoped reads. req.body.parentMessageId = job.metadata.userMessage?.parentMessageId ?? Constants.NO_PARENT; // Rebuild the same persistence/retention mode as the paused turn. The resume body // is not authoritative: tools inspect this field during client initialization. req.body.isTemporary = job.metadata.isTemporary === true; const metaFiles = job.metadata.userMessage?.files; if (Array.isArray(metaFiles) && metaFiles.length > 0) { req.body.files = metaFiles; } else { let restoredFiles = false; const pausedUserMessageId = job.metadata.userMessage?.messageId; if (pausedUserMessageId) { try { const [row] = await getMessages( { conversationId, messageId: pausedUserMessageId }, 'files', ); if (Array.isArray(row?.files) && row.files.length > 0) { req.body.files = row.files; restoredFiles = true; } } catch (err) { logger.warn( '[ResumeAgentController] Failed to restore paused user message files', getSafeErrorMetadata(err), ); } } if (!restoredFiles) { req.body.files = []; } } let resumeState; let preparedContent; try { resumeState = await GenerationJobManager.getResumeState(streamId, job.createdAt); const batchedAnswer = mapped.resumeValue?.answers != null && typeof mapped.resumeValue.answers === 'object' && !Array.isArray(mapped.resumeValue.answers) ? JSON.stringify({ answers: mapped.resumeValue.answers }) : undefined; const provenanceBody = batchedAnswer === undefined ? req.body : { ...req.body, answer: batchedAnswer }; const resumeValueForInspection = batchedAnswer === undefined ? mapped.resumeValue : { ...mapped.resumeValue, answer: batchedAnswer }; const retainedAskAnswers = job.metadata.resolvedAskUserQuestions; const initialSeedContent = resumeState?.aggregatedContent ?? []; const preflightResumeState = Array.isArray(retainedAskAnswers) && retainedAskAnswers.length > 0 ? { ...(resumeState ?? {}), aggregatedContent: attachAskUserQuestionAnswers(initialSeedContent, retainedAskAnswers), } : resumeState; const preflightPendingAction = batchedAnswer !== undefined && Array.isArray(pendingAction.payload.questions) ? { ...pendingAction, payload: { ...pendingAction.payload, question: { questions: pendingAction.payload.questions }, }, } : pendingAction; preparedContent = await preflightResumeContent( { appConfig: req.config, endpointOption: req.body.endpointOption, conversationId, user: req.user, jobMetadata: job.metadata, pendingAction: preflightPendingAction, body: provenanceBody, resumeValue: resumeValueForInspection, resumeState: preflightResumeState, liveFiles: Array.isArray(req.body.files) ? req.body.files : [], isTemporary: job.metadata.isTemporary === true, checkpointNamespace, resolvedAddedAgent: req.resolvedAddedAgent, }, resumeContentProtectionDependencies, ); } catch (err) { logger.warn( '[ResumeAgentController] Resume content preflight failed', getSafeErrorMetadata(err), ); if (isContentFilterError(err)) { return sendGenerationJson(res, err.statusCode, err.body, generationProtocolVersion); } return sendGenerationJson(res, 500, { error: GENERIC_RESUME_ERROR }, generationProtocolVersion); } const { seedContent, storedMessages, userSubmittedPaths, userSubmittedMessageFieldPaths } = preparedContent; // Count the resume against the concurrency limit. The original turn released its slot // when it paused, so resuming must re-acquire one — otherwise pausing several turns // and resuming them at once would bypass LIMIT_CONCURRENT_MESSAGES. const { allowed } = await checkAndIncrementPendingRequest(userId); if (!allowed) { return sendGenerationJson( res, 429, { error: 'Too many concurrent requests' }, generationProtocolVersion, ); } // Finish the legacy checkpoint snapshot before claiming scheduled capacity. // It is independent of the approval claim, and holding a deployment-wide slot // while an indexed saver read stalls would unnecessarily block other schedules // and lengthen the Mongo-claim -> approval-CAS hand-off window below. const checkpointGeneration = await checkpointGenerationPromise; // A pause frees its scheduled-run capacity slot. Before consuming the approval, // atomically promote the run row back to `started` and claim a fresh global slot. // The database's partial unique indexes arbitrate both deployment capacity and a // concurrent active occurrence of the same schedule. let scheduleCapacitySlot; let scheduleResumeClaimToken; let scheduleResumeLeaseBy; const scheduleResumeOptions = { expectedConfigRevision: job.metadata?.scheduleConfigRevision, automatic: job.metadata?.scheduleManual !== true, }; if (scheduleId) { let scheduleClaim; try { scheduleClaim = await claimScheduleResume(scheduleId, scheduledFor, scheduleResumeOptions); } catch (err) { await decrementPendingRequest(userId); logger.error('[ResumeAgentController] Failed to claim scheduled resume capacity', err); return sendGenerationJson( res, 500, { error: 'Failed to reserve scheduled-run capacity' }, generationProtocolVersion, ); } if ('conflict' in scheduleClaim) { await decrementPendingRequest(userId); if (scheduleClaim.conflict === 'capacity' || scheduleClaim.conflict === 'overlap') { res.set('Retry-After', '1'); return sendGenerationJson( res, 429, { code: scheduleClaim.conflict === 'capacity' ? 'SCHEDULE_CAPACITY' : 'SCHEDULE_OCCURRENCE_ACTIVE', error: scheduleClaim.conflict === 'capacity' ? 'Scheduled-run capacity is currently full. Please retry.' : 'Another occurrence of this schedule is still running. Please retry.', }, generationProtocolVersion, ); } return sendGenerationJson( res, 409, { code: scheduleClaim.conflict === 'inactive' ? 'SCHEDULE_NO_LONGER_ACTIVE' : 'SCHEDULE_RUN_NOT_PAUSED', error: 'This scheduled run can no longer be resumed', }, generationProtocolVersion, ); } scheduleCapacitySlot = scheduleClaim.capacitySlot; scheduleResumeClaimToken = scheduleClaim.claimToken; scheduleResumeLeaseBy = scheduleClaim.leaseBy; } const releaseScheduleFence = async () => { if (scheduleId == null || scheduleResumeLeaseBy == null) { return; } try { await releaseScheduleResumeFence(scheduleId, scheduleResumeLeaseBy); } catch (releaseError) { logger.warn('[ResumeAgentController] Failed to release scheduled resume fence', releaseError); } }; /** Release only when the exact generation demonstrably remains paused. If the * approval CAS reply is ambiguous and the job cannot be read, retaining the slot * until reconciliation is the safe direction: releasing it could exceed the cap * while a committed continuation is already running. */ const rollbackUnconsumedScheduleClaim = async (currentJob) => { if ( scheduleId == null || scheduleCapacitySlot == null || currentJob?.createdAt !== job.createdAt || currentJob?.status !== 'requires_action' ) { return; } try { await releaseScheduleResumeClaim(scheduleId, scheduledFor, scheduleCapacitySlot); } catch (rollbackError) { logger.warn( '[ResumeAgentController] Failed to release unconsumed scheduled resume capacity', rollbackError, ); } }; // Atomically claim the resume. The single winner drives the run; a racing second // submit (double-click, two tabs) gets false and must not re-drive — that would // re-execute tools and double-bill. // // The claim runs AFTER the slot increment above but BEFORE the run's own try/finally // that releases it, so a store/Redis error here (unlike the clean `!claimed` branch) // would leak the concurrency slot until the counter TTL expires — spuriously 429'ing // the user when they retry the still-paused approval. Release the slot on that path too. let claimed; const providerExecutionId = randomUUID(); try { /** The CAS that reopens steering must also publish THIS owner's seal * capability. A separate write after status=`running` leaves a window in * which steer/arm requests read the previous replica's capability. */ claimed = await GenerationJobManager.approvals.resolve( streamId, pendingAction.actionId, { preemptCapable: isSteerPreemptSupported(), providerExecutionId, providerDrained: true, ...(resolvedAskUserQuestion && { resolvedAskUserQuestions }), }, job.createdAt, ); } catch (err) { const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); await rollbackUnconsumedScheduleClaim(currentJob); await releaseScheduleFence(); await decrementPendingRequest(userId); logger.error('[ResumeAgentController] Failed to claim resume', getSafeErrorMetadata(err)); return sendGenerationJson(res, 500, { error: 'Failed to resume' }, generationProtocolVersion); } if (!claimed) { await decrementPendingRequest(userId); const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null); await rollbackUnconsumedScheduleClaim(currentJob); await releaseScheduleFence(); if (currentJob != null && currentJob.createdAt !== job.createdAt) { return sendGenerationJson(res, 409, { code: 'RUN_REPLACED' }, generationProtocolVersion); } return sendGenerationJson( res, 409, { error: 'This action was already resolved or has expired' }, generationProtocolVersion, ); } // Linearize the consumed approval against the schedule's live config. The schedule // document fence was acquired only after all async policy reads, and this atomic // consume checks its token/revision/enabled state immediately after the approval CAS. // An edit/disable that won first makes this fail; one that lands afterward is ordered // after the continuation has started. Never begin provider execution on a stale claim. if (scheduleId) { let scheduleClaimCurrent = false; try { scheduleClaimCurrent = await finalizeScheduleResumeClaim( scheduleId, scheduleResumeClaimToken, scheduleResumeLeaseBy, scheduleResumeOptions, ); } catch (error) { logger.error('[ResumeAgentController] Failed to finalize scheduled resume fence', error); await releaseScheduleFence(); } if (!scheduleClaimCurrent) { await decrementPendingRequest(userId); let stopped = false; try { const abortResult = await GenerationJobManager.abortJob(streamId, { expectedCreatedAt: job.createdAt, awaitProviderDrain: true, }); // Same authoritative gate as the inactive-schedule path above: only a landed // abort (or an already-terminal, drained generation) may settle this occurrence. stopped = isStopConfirmed(abortResult); } catch (error) { logger.warn('[ResumeAgentController] Failed to stop stale scheduled resume', error); } if (!stopped) { res.set('Retry-After', '1'); return sendGenerationJson( res, 503, { code: 'SCHEDULE_STOP_UNCONFIRMED', error: 'The stale scheduled resume could not be confirmed stopped.', }, generationProtocolVersion, ); } await recordScheduleOutcome({ scheduleId, scheduledFor, streamId, jobCreatedAt: job.createdAt, status: 'interrupted', conversationId, error: 'Schedule was disabled, changed, or deleted before approval', }); if (checkpointNamespace !== '') { await deleteAgentCheckpoint(conversationId, checkpointerCfg, undefined, { checkpointNamespace, }).catch((error) => { logger.warn('[ResumeAgentController] Failed to prune stale schedule checkpoint', error); }); } return sendGenerationJson( res, 409, { code: 'SCHEDULE_NO_LONGER_ACTIVE', error: 'This schedule can no longer be resumed' }, generationProtocolVersion, ); } } /** * An interrupt steer enqueued just before the pause survives durably with * its `preempt` flag, but the ARM lived only in the previous owner's * runtime. Rebuild it from the queue so the resumed segment honours an * interrupt the user already had acknowledged. */ const preemptRearm = GenerationJobManager.rearmQueuedPreempts(streamId, job.createdAt).catch( (error) => { logger.error( '[ResumeAgentController] Failed to re-arm queued preempts', getSafeErrorMetadata(error), ); }, ); /** * BOUNDED, and the bound is the point. `.catch` only fires on rejection, * but ioredis queues commands while a connection is down instead of * rejecting, so either of these can simply never settle. That would block * here — after `approvals.resolve` has already consumed the action and * flipped the job to `running`, and before both `res.json` and the resume * lifecycle's own try/finally. The client times out, its retry gets a 409 * because the action is spent, and neither the continuation nor the * failed-resume cleanup ever runs. * * Re-arming is steering bookkeeping that the next tool boundary would * honour anyway, so it finishes in the background rather than holding a * resume the user is waiting on. Capability is not in this best-effort path: * it was committed atomically by the resume claim above. */ let steeringSetupTimer; await Promise.race([ preemptRearm, new Promise((resolve) => { steeringSetupTimer = setTimeout(() => { logger.warn( `[ResumeAgentController] Steering setup for ${streamId} still pending after ` + `${STEER_RESUME_SETUP_TIMEOUT_MS}ms; continuing the resume without it`, ); resolve(); }, STEER_RESUME_SETUP_TIMEOUT_MS); }), ]); clearTimeout(steeringSetupTimer); // Seed the run-scoped MCP request-context store BEFORE the ACK: once `res.json` // finishes the response, a later `getMCPRequestContext(req, res)` (from tool loading) // sees `res` as ended and returns undefined, leaving the resumed run without its MCP // connection store — approved MCP / OAuth-overlay tools would then run without their // request-scoped connections. Pre-seeding with a null `res` + `cleanupOnResponse:false` // mirrors the normal stream path (request.js); torn down in the `finally` below. req._resumableStreamId = streamId; getMCPRequestContext(req, undefined, { cleanupOnResponse: false }); // ACK immediately; the continuation streams over the client's existing SSE. sendGenerationJson( res, 200, { streamId, conversationId, status: 'resuming' }, generationProtocolVersion, ); // Restore the conversation's createdAt so temporal prompt vars ({{current_datetime}}, // {{iso_datetime}}, ...) resolve against the SAME anchor the paused graph used rather // than the resume wall-clock. initializeAgent reads `req.conversationCreatedAt`; the // normal path sets it from the convo timestamp (resolveConversationCreatedAt), so mirror // that here. (The original `timezone` is replayed onto req.body via RESUME_CONTEXT_KEYS.) try { const resumedConvo = await getConvo(userId, conversationId); const createdAt = resumedConvo?.createdAt ? new Date(resumedConvo.createdAt) : null; if (createdAt && !Number.isNaN(createdAt.getTime())) { req.conversationCreatedAt = createdAt.toISOString(); } } catch (err) { logger.warn( '[ResumeAgentController] Failed to restore conversation timestamp anchor', getSafeErrorMetadata(err), ); } let client = null; /** Re-pause progress failures use the action/epoch-scoped terminal CAS. The * generic resume catch must not subsequently call completeJob, because the * failed pause may have lost ownership to a newer action or generation. */ let pausePersistenceFailed = false; let pausePersistenceFailureFinalized = false; try { if ( !(await GenerationJobManager.beginProviderExecution( streamId, job.createdAt, providerExecutionId, )) ) { throw Object.assign(new Error('Generation stopped before provider resume'), { code: 'RUN_REPLACED', }); } if (userSubmittedPaths.length > 0) { job.metadata.userSubmittedPaths = userSubmittedPaths; } if (userSubmittedMessageFieldPaths.length > 0) { job.metadata.userSubmittedMessageFieldPaths = userSubmittedMessageFieldPaths; } if (userSubmittedPaths.length > 0 || userSubmittedMessageFieldPaths.length > 0) { await GenerationJobManager.getJobStore().updateJob( streamId, { ...(userSubmittedPaths.length > 0 && { userSubmittedPaths }), ...(userSubmittedMessageFieldPaths.length > 0 && { userSubmittedMessageFieldPaths, }), }, job.createdAt, ); } const result = await initializeClient({ req, res, endpointOption: req.body.endpointOption, signal: job.abortController.signal, jobCreatedAt: job.createdAt, checkpointNamespace, requestBody: job.metadata.mcpRequestBody ?? createMCPRuntimeRequestBody({ messageId: job.metadata.responseMessageId, conversationId: streamId, parentMessageId: job.metadata.userMessage?.messageId ?? Constants.NO_PARENT, }), }); client = result.client; // Bind the rebuilt client to the in-flight turn's identity (no new user message). client.conversationId = streamId; // The resume operates on the SAME job (it moved it running again), so its identity is // the paused job's createdAt — used by the re-pause CAS pre-check + checkpoint prune to // avoid acting on a job a newer request has since replaced. client.jobCreatedAt = job.createdAt; client.checkpointNamespace = checkpointNamespace; client.responseMessageId = job.metadata.responseMessageId; client.parentMessageId = job.metadata.userMessage?.messageId ?? Constants.NO_PARENT; if (client.contentParts) { GenerationJobManager.setContentParts(streamId, client.contentParts, job.createdAt); } await client.resumeCompletion({ resumeValue: mapped.resumeValue, seedContent, runSteps: resumeState?.runSteps ?? [], storedMessages, abortController: job.abortController, // Carry the user's MCP auth so approved MCP tools run with their credentials. userMCPAuthMap: result.userMCPAuthMap, // Replay deferred tools discovered before the pause (captured at pause). The rebuilt // graph passes `messages: []`, so without these the model would lose their schemas. discoveredToolNames: job.metadata?.discoveredTools, activityPhaseSnapshot: job.metadata?.activityPhaseSnapshot, }); // The model may pause AGAIN (another tool, or a follow-up question). The pending // action is already persisted + emitted; leave the job `requires_action`. if (client.pendingApproval) { logger.debug(`[ResumeAgentController] Re-paused for approval: ${streamId}`); const pauseActionId = client.pendingApproval.actionId; const pauseCreatedAt = client.jobCreatedAt ?? job.createdAt; const ownsPausePersistence = await GenerationJobManager.approvals.ownsPausePersistence( streamId, pauseActionId, pauseCreatedAt, ); if (ownsPausePersistence) { try { // Persist this segment's content + artifacts before the fresh client (next // resume) drops them, so an expiring re-pause doesn't lose them; finalize later // overwrites content and merges attachments onto the saved message. A failed // required write must reject into the error-finalization path rather than expose // the next action while its preceding segment is absent from durable history. await persistRePauseProgress({ req, client, job, streamId, conversationId }); } catch (pausePersistenceError) { pausePersistenceFailed = true; try { pausePersistenceFailureFinalized = (await GenerationJobManager.failPausePersistence( streamId, pauseActionId, getUserFacingResumeError(pausePersistenceError, req.config), pauseCreatedAt, )) === true; if (!pausePersistenceFailureFinalized) { logger.warn( `[ResumeAgentController] Skipping stale re-pause persistence failure — ${streamId} no longer owns its barrier`, ); } } catch (failError) { logger.error( `[ResumeAgentController] Failed to terminalize re-pause persistence error for ${streamId}`, getSafeErrorMetadata(failError), ); } throw pausePersistenceError; } const released = await GenerationJobManager.approvals.finishPausePersistence( streamId, pauseActionId, pauseCreatedAt, ); if (!released) { logger.warn( `[ResumeAgentController] Re-pause persistence barrier changed before release: ${streamId}`, ); } if (scheduleId) { await recordScheduleOutcome({ scheduleId, scheduledFor, streamId, jobCreatedAt: job.createdAt, status: 'requires_action', conversationId, }); } } else { logger.debug( `[ResumeAgentController] Skipping stale re-pause persistence — ${streamId} no longer owns its barrier`, ); } return; } // If the user aborted mid-resume, the abort route already emitted the terminal // event and finalized the job — don't double-save / double-finalize here. This // continuation is nevertheless the scheduled-run owner, so it must settle the // run row after observing its own abort; the generic Stop route deliberately // delegates a running generation's settlement to that generation owner. if (job.abortController.signal.aborted) { logger.debug( `[ResumeAgentController] Aborted during resume; abort route finalizes: ${streamId}`, ); if (scheduleId) { await recordScheduleOutcome({ scheduleId, scheduledFor, streamId, jobCreatedAt: job.createdAt, status: 'interrupted', conversationId, error: 'Scheduled run was stopped', }); } return; } await finalizeResumedTurn({ req, client, job, streamId, conversationId, addTitle, checkpointGeneration, }); } catch (err) { logger.error('[ResumeAgentController] Resume failed', getSafeErrorMetadata(err)); if (pausePersistenceFailed) { // failPausePersistence already performed the exact requires_action -> // error transition. Only its CAS winner owns this generation's checkpoint // cleanup; a stale/mismatched failure must leave the live scope intact. if (pausePersistenceFailureFinalized) { await deleteFailedResumeCheckpoint( { conversationId, checkpointerCfg, job, checkpointGeneration, }, 're-pause persistence failure', ); } if (scheduleId && pausePersistenceFailureFinalized) { await recordScheduleOutcome({ scheduleId, scheduledFor, streamId, jobCreatedAt: job.createdAt, status: 'error', conversationId, error: err?.message ?? 'Re-pause persistence failed', }); } return; } const errorMessage = getUserFacingResumeError(err, req.config); // Job-replacement guard (mirrors finalizeResumedTurn's success-path guard): if a // newer request reused this conversationId while the resume was failing, do NOT emit // the error to / complete / prune the NEWER turn's job. The finally still releases // the slot + disposes. Proceed with finalization if the replacement check itself fails. let stillLive = true; try { const liveJob = await GenerationJobManager.getJobStore().getJob(streamId); stillLive = !!liveJob && liveJob.createdAt === job.createdAt; } catch (readErr) { logger.warn( '[ResumeAgentController] Replacement check failed; finalizing anyway', getSafeErrorMetadata(readErr), ); } if (!stillLive) { logger.warn( `[ResumeAgentController] Skipping failed-resume finalization — job ${streamId} was replaced`, ); } else { // completeJob atomically claims running -> error and parks steers before // publishing. If abort or a re-pause won, it returns false; only the // terminal-CAS winner may delete this generation's checkpoint scope. let errorFinalized = false; try { errorFinalized = (await GenerationJobManager.completeJob(streamId, errorMessage, job.createdAt)) === true; } catch (completeErr) { logger.error( '[ResumeAgentController] Failed to finalize failed resume', getSafeErrorMetadata(completeErr), ); } if (errorFinalized) { await deleteFailedResumeCheckpoint( { conversationId, checkpointerCfg, job, checkpointGeneration, }, 'failed resume finalization', ); } if (scheduleId && errorFinalized) { const balanceRefusal = err?.message?.includes(ViolationTypes.TOKEN_BALANCE); await recordScheduleOutcome({ scheduleId, scheduledFor, streamId, jobCreatedAt: job.createdAt, status: balanceRefusal ? 'skipped_balance' : 'error', conversationId, ...(!balanceRefusal && { error: err?.message ?? 'Resume failed' }), }); } } } finally { try { // Tear down the MCP request-context store seeded before the ACK (parity with // request.js's finishResumableRequest). No-op if it was never seeded. await cleanupMCPRequestContextForReq(req); // Release the concurrency slot taken above — UNLESS handleRunInterrupt already // released it on a re-pause (so a fast /resume isn't 429'd). On a normal finish or // error it didn't, so release here. A re-pause re-acquires its own slot next resume. if (!client?.pendingRequestReleased) { await decrementPendingRequest(userId); } if (client) { disposeClient(client); } } finally { await GenerationJobManager.markProviderExecutionDrained?.( streamId, job.createdAt, providerExecutionId, ).catch((drainError) => { logger.warn('[ResumeAgentController] Failed to record provider drain', drainError); }); } } }; module.exports = ResumeAgentController;