🧵 feat: Execute and Resume Background Responses (#17602)

* 🧵 feat: Execute and Resume Background Responses

* WIP: guard pause publication and prepare session policy before graph creation

* wip: Align Runtime Pause and Retirement Ownership

* fix: Bind Background Review Publication to Execution Authority

* test: Express Participant Denial Cases Without Nested Conditions

* test: Read Remembered Consent Through Typed Conversation Methods

* test: Enroll Runtime Fixtures With Their Durable Journal Owner

* test: Preserve Journal Ownership in Manual SDK Resume Attempts

* feat: Own Response Segment Composition in Execution Runtime

* test: Release the Runtime During Completion Recovery

* fix: Pin Resume Participants and Preserve Retained Terminal Output

* fix: Reconcile Ambiguous Response Pause Publication
This commit is contained in:
Dustin Healy 2026-11-10 21:03:59 -08:00 • committed by GitHub
parent 4dcb391f62
commit d2126b8ac7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 4381 additions and 22 deletions

View file

@ -15,6 +15,7 @@ export {
} from './persistence';
export { createBackgroundResponseService } from './background';
export { createResponseExecutionRuntime } from './runtime';
export { createResponseDecisionSubmission } from './decision';
export { createResponseResumeValidation, createResponseResumeAdmission } from './resume';
export { createBackgroundResponseAuthority } from './authority';
@ -23,7 +24,7 @@ export type {
ResponseDecisionResult,
ResponseDecisionSubmission,
} from './decision';
export { pauseResponseForReview, type ResponsePauseDependencies } from './pause';
export { createBackgroundResponseQueue, backgroundResponseDeliveryKey } from './queue';
export {
createBackgroundResponseWorkerAuth,

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,139 @@
import { MAX_SUBAGENT_GRAPH_NODES } from 'librechat-data-provider';
import type { ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas';
import type { Agents, TCheckpointerConfig } from 'librechat-data-provider';
import type { RunInterruptResult } from '@librechat/agents';
import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import type { AgentExecutionEnrollment } from '../remote/lifecycle';
import type { GenerationJob } from '~/types/stream';
import type { Response } from './types';
import { hasDurableAgentInterruptCheckpoint, resolveCheckpointerConfig } from '../checkpointer';
import { checkpointOwnerNamespacePrefix } from '~/stream/checkpoints';
import { buildPendingBackgroundResponse } from './background';
import { responseTenantId } from './scope';
export interface ResponsePauseDependencies {
storage: ResponseStorage;
jobs: Pick<GenerationJobManagerClass, 'getJob'>;
/** Capture the host's existing graph/project/code/MCP approval bindings; never caller body data. */
buildAction: (interrupt: RunInterruptResult, job: GenerationJob) => Promise<Agents.PendingAction>;
/** Save the ordinary unfinished assistant turn, with existing content/privacy filtering. */
persistUnfinished: (response: Response, action: Agents.PendingAction) => Promise<void>;
}
/** Capture the SDK's real, durable interrupt before exposing any resumable action. */
export async function pauseResponseForReview(
input: {
run: { getInterrupt: () => RunInterruptResult | undefined };
execution: AgentExecutionEnrollment;
record: StoredResponseRecord;
partial: Response;
visibleOutputTokens?: number;
checkpointer?: TCheckpointerConfig;
},
deps: ResponsePauseDependencies,
): Promise<boolean> {
const interrupt = input.run.getInterrupt();
if (!interrupt) return false;
if (
!interrupt.payload ||
!['ask_user_question', 'tool_approval'].includes(interrupt.payload.type)
)
throw new Error('Unsupported response interrupt');
const { record, execution } = input;
const job = await deps.jobs.getJob(record.responseId);
const namespace = job?.metadata.checkpointNamespace;
const providerExecutionId = job?.metadata.providerExecutionId;
if (
!job ||
!execution.ownsSettlement ||
job.createdAt !== record.generationCreatedAt ||
job.createdAt !== execution.createdAt ||
job.status !== 'running' ||
!providerExecutionId ||
job.metadata.providerExecutionStartedId !== providerExecutionId ||
job.metadata.userId !== record.userId ||
responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) ||
job.metadata.agent_id !== record.agentId ||
job.metadata.endpoint !== 'responses' ||
job.metadata.conversationId !== record.conversationId ||
!namespace?.startsWith(checkpointOwnerNamespacePrefix(record.userId, record.tenantId)) ||
input.partial.id !== record.responseId ||
input.partial.model !== record.agentId ||
!interrupt.checkpointId ||
!interrupt.interruptId ||
(interrupt.threadId != null && interrupt.threadId !== record.conversationId)
)
throw new Error('Response interrupt identity changed');
const config = resolveCheckpointerConfig(input.checkpointer);
if (
config.type !== 'mongo' ||
!(await hasDurableAgentInterruptCheckpoint(record.conversationId, input.checkpointer, {
checkpointNamespace: namespace,
checkpointId: interrupt.checkpointId,
checkpointNs: interrupt.checkpointNs,
interruptId: interrupt.interruptId,
}))
)
throw new Error('Response interrupt checkpoint is unavailable');
const action = await deps.buildAction(interrupt, job);
const expiresAt = action.expiresAt;
if (
!action.resumeAgentIds?.includes(record.agentId) ||
action.resumeAgentIds.length > MAX_SUBAGENT_GRAPH_NODES ||
new Set(action.resumeAgentIds).size !== action.resumeAgentIds.length ||
action.resumeAgentIds.some((id) => typeof id !== 'string' || !id) ||
action.streamId !== record.responseId ||
action.responseMessageId !== record.responseId ||
action.conversationId !== record.conversationId ||
action.threadId !== record.conversationId ||
action.interruptId !== interrupt.interruptId ||
expiresAt == null ||
!Number.isSafeInteger(expiresAt)
)
throw new Error('Response review action identity changed');
return execution.pauseForReview(action, async () => {
await deps.persistUnfinished(input.partial, action);
const current = await deps.storage.read(record, record.responseId);
if (!current || current.generationCreatedAt !== execution.createdAt)
throw new Error('Response pause journal identity changed');
const publicResponse = {
...buildPendingBackgroundResponse(current),
status: 'in_progress' as const,
};
const publication: Parameters<ResponseStorage['publishReview']> = [
current,
current.responseId,
{ execution: current },
execution.createdAt,
{
actionId: action.actionId,
providerExecutionId,
expiresAt,
partialResponseJSON: JSON.stringify(input.partial),
...(input.visibleOutputTokens == null
? {}
: { visibleOutputTokens: input.visibleOutputTokens }),
pendingActionJSON: JSON.stringify(action),
checkpointId: interrupt.checkpointId,
checkpointNs: interrupt.checkpointNs ?? '',
checkpointConfigJSON: JSON.stringify({
type: config.type,
ttl: config.ttlSeconds,
checkpointCollectionName: config.checkpointCollectionName,
checkpointWritesCollectionName: config.checkpointWritesCollectionName,
}),
},
JSON.stringify(publicResponse),
];
let published;
try {
published = await deps.storage.publishReview(...publication);
} catch {
// The write may have committed despite a lost acknowledgement. Reuse the exact
// receipt and execution guard: storage reconciles it without republishing or
// overriding cancellation, retirement, a newer review, or admitted resume.
published = await deps.storage.publishReview(...publication);
}
if (!published.ok) throw new Error('Response pause journal publication failed');
});
}

View file

@ -0,0 +1,808 @@
import { checkpointerSchema, responseReviewDecisionSchema } from 'librechat-data-provider';
import type { ResponseTerminalReceipt, StoredResponseRecord } from '@librechat/data-schemas';
import type { Request, Response as HttpResponse } from 'express';
import type { Agents, TMessage } from 'librechat-data-provider';
import type { Run, IState } from '@librechat/agents';
import type { ExecuteAgentRunParams, AgentExecutionConnection } from '../remote/host';
import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import type { MarkToolApprovalAllowAlwaysOptions } from '../hitl/allow';
import type { AgentExecutionEnrollment } from '../remote/lifecycle';
import type { ResponsesRunEnvelope } from '../envelope';
import type { ResponseAggregator } from './service';
import type { ReachableAgent } from '../traversal';
import type { ServerRequest } from '~/types/http';
import type { Response } from './types';
import type { createRun } from '../run';
import {
computeAgentRequestFingerprint,
computeLegacyAgentRequestFingerprint,
pickResumeContext,
} from '../hitl/policy';
import {
markToolApprovalAllowAlways,
resolveRunToolApprovalAllows,
recordToolApprovalAllows,
} from '../hitl/allow';
import {
getApprovalTtlMs,
resolveCheckpointerConfig,
LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,
} from '../checkpointer';
import { createResponseCompletion, createResponseExecutionRelease } from './completion';
import { combineResponseSegments, seedResponseAggregator } from './segments';
import { checkpointOwnerNamespacePrefix } from '~/stream/checkpoints';
import { getRunDiscoveredTools, getRunMCPToolAliases } from '../run';
import { markNativeCodeToolApprovalRequests } from '../hitl/byom';
import { assertCodeExecutionApprovalBinding } from '../execution';
import { getBackgroundResponseWorkerAdmission } from './worker';
import { getChatProjectContextKey } from '~/projects/context';
import { buildPendingBackgroundResponse } from './background';
import { resolveResponseContinuation } from './persistence';
import { captureRunPendingAction } from '../hitl/capture';
import { cleanupResponseCheckpoint } from './checkpoint';
import { createResponseRetirement } from './retirement';
import { collectReachableAgents } from '../traversal';
import { pauseResponseForReview } from './pause';
import { generateResponseId } from './service';
import { responseTenantId } from './scope';
type TerminalResponse = Response & { status: ResponseTerminalReceipt['status'] };
type ReviewAgent = NonNullable<Parameters<typeof captureRunPendingAction>[0]['agents'][number]> &
NonNullable<Parameters<typeof markNativeCodeToolApprovalRequests>[1][number]> &
NonNullable<NonNullable<MarkToolApprovalAllowAlwaysOptions['agents']>[number]>;
interface ReviewAgentNode
extends Omit<ReviewAgent, keyof ReachableAgent<never>>,
ReachableAgent<ReviewAgentNode> {}
/** Capture and rebuild must describe the same initialized participant identities. */
function responseParticipantIds(primaryId: string, agents: readonly ReviewAgentNode[]): string[] {
const participants = new Set<string>([primaryId]);
for (const agent of agents) {
if (typeof agent.id !== 'string' || !agent.id)
throw new Error('Response graph participant identity unavailable');
participants.add(agent.id);
}
return [...participants];
}
function responseBody(value: unknown): value is TerminalResponse {
return (
value != null &&
typeof value === 'object' &&
'object' in value &&
value.object === 'response' &&
'id' in value &&
typeof value.id === 'string' &&
'model' in value &&
typeof value.model === 'string' &&
'status' in value &&
['completed', 'failed', 'incomplete', 'cancelled'].includes(String(value.status)) &&
'output' in value &&
Array.isArray(value.output)
);
}
export interface ResponseExecutionRuntime {
responseId: string;
inputChain: <T>(chain: { messages: T[]; parentMessageId: string }) => {
messages: T[];
parentMessageId: string;
};
graphOptions: (
agents?: Parameters<typeof createRun>[0]['agents'],
) => Promise<
Pick<
Parameters<typeof createRun>[0],
| 'hitlCapable'
| 'discoveredToolNames'
| 'reviewedToolApprovals'
| 'sessionStartSource'
| 'toolApprovalStorage'
| 'toolApprovalAllows'
>
>;
seedAggregator: (aggregator: ResponseAggregator | null) => void;
composeResponse: (response: Response) => Response;
visibleOutputTokens: (current: number) => number;
completeTurn: <T>(
input: {
run: Pick<Run<IState>, 'getInterrupt'> & Parameters<typeof getRunDiscoveredTools>[0];
execution: AgentExecutionEnrollment;
agents: Parameters<typeof createRun>[0]['agents'];
buildPartial: () => Response;
visibleOutputTokens: number;
persistence: {
saveConversation: () => Promise<void>;
saveInput: () => Promise<void>;
saveOutput: (partial: Response) => Promise<Partial<TMessage> | void>;
};
},
complete: () => Promise<T>,
) => Promise<T | undefined>;
/** A rebuilt graph restores messages from its checkpoint, not from a fresh input turn. */
graphMessages: <T>(messages: T[]) => T[];
toolContext: (execution: { createdAt: number }) => {
streamId: string | null;
jobCreatedAt?: number;
};
connection?: AgentExecutionConnection;
hooks: Pick<
ExecuteAgentRunParams<unknown>,
| 'onEnrolled'
| 'afterPause'
| 'afterSettle'
| 'clientRequestId'
| 'resume'
| 'retainTerminalPersistence'
>;
processRun: (
run: Pick<Run<IState>, 'processStream' | 'resume'>,
...args: Parameters<Run<IState>['processStream']>
) => ReturnType<Run<IState>['processStream']>;
resolveContinuation: (envelope: ResponsesRunEnvelope) => Promise<{
conversationId: string;
parentMessageId: string;
}>;
}
/** Adapt only trusted background-worker requests; normal Responses keeps its synchronous lifecycle. */
export function createResponseExecutionRuntime(
req: Request,
res: HttpResponse,
continuation: Parameters<typeof resolveResponseContinuation>[2] & {
generateResponseId?: () => string;
jobs?: Pick<
GenerationJobManagerClass,
'getJob' | 'abortJob' | 'finishProviderTerminalPersistence'
> &
Partial<Pick<GenerationJobManagerClass, 'updateMetadata'>>;
pluginHookSource?: MarkToolApprovalAllowAlwaysOptions['pluginHookSource'];
toolApprovalStorage?: Parameters<typeof createRun>[0]['toolApprovalStorage'];
addConvoToolApprovalAllows?: Parameters<
typeof recordToolApprovalAllows
>[0]['addConvoToolApprovalAllows'];
},
): ResponseExecutionRuntime {
const admission = getBackgroundResponseWorkerAdmission(req);
if (
admission &&
(!continuation.jobs?.finishProviderTerminalPersistence || !continuation.jobs.abortJob)
)
throw new Error('Response terminal publication release capability is unavailable');
const interactive =
admission?.interactive === true &&
continuation.jobs != null &&
resolveCheckpointerConfig((req as ServerRequest).config?.endpoints?.agents?.checkpointer)
.type === 'mongo';
let checkpoint: { checkpointNamespace: string; checkpointConfigJSON: string } | undefined;
let enrolledExecution: AgentExecutionEnrollment | undefined;
let rememberedTools: string[] = [];
let responseParentMessageId: string | undefined;
let prefix: Response | undefined;
const seeded = new WeakSet<ResponseAggregator>();
function previous(): Response | undefined {
if (!admission?.resume) return;
if (prefix) return prefix;
let restored: Response;
try {
restored = JSON.parse(admission.record.review?.partialResponseJSON ?? 'null');
} catch {
throw new Error('Response resume output is unavailable');
}
if (
!restored ||
restored.id !== admission.record.responseId ||
restored.model !== admission.record.agentId ||
!Array.isArray(restored.output) ||
!restored.background ||
!restored.store
)
throw new Error('Response resume output identity changed');
prefix = restored;
return prefix;
}
function visibleOutputTokens(current: number): number {
if (!admission?.resume) return current;
const prior = admission.record.review?.visibleOutputTokens;
if (
prior == null ||
!Number.isSafeInteger(prior) ||
prior < 0 ||
!Number.isSafeInteger(current) ||
current < 0 ||
!Number.isSafeInteger(prior + current)
)
throw new Error('Response visible token counter is unavailable');
return prior + current;
}
function composeResponse(response: Response): Response {
const prior = previous();
return prior && admission?.resume
? combineResponseSegments(prior, response, admission.resume.intentId)
: response;
}
let candidate: TerminalResponse | undefined;
if (admission) {
const writeJSON = res.json.bind(res);
res.json = (body: unknown) => {
if (responseBody(body)) {
if (body.id !== admission.record.responseId || body.model !== admission.record.agentId)
throw new Error('Background response output identity changed');
candidate = { ...body, background: true, store: true };
if (res.writableEnded) return res;
return writeJSON(candidate);
}
if (res.writableEnded) return res;
return writeJSON(body);
};
}
const hooks: Pick<
ExecuteAgentRunParams<unknown>,
| 'onEnrolled'
| 'afterPause'
| 'afterSettle'
| 'clientRequestId'
| 'resume'
| 'retainTerminalPersistence'
> = admission
? {
retainTerminalPersistence: true,
clientRequestId: admission.record.responseId,
...(admission.resume
? {
resume: {
generationCreatedAt: admission.resume.generationCreatedAt,
providerExecutionId: admission.resume.providerExecutionId,
},
}
: {}),
onEnrolled: async (execution) => {
enrolledExecution = execution;
const current = await admission.storage.read(
admission.record,
admission.record.responseId,
);
if (admission.resume) {
if (
!current ||
current.generationCreatedAt !== execution.createdAt ||
current.resume?.intentId !== admission.resume.intentId ||
current.resume.state !== 'admitted' ||
current.resume.providerExecutionId !== admission.resume.providerExecutionId ||
current.status !== 'in_progress'
)
throw new Error('Background response resume enrollment changed');
admission.record = current;
if (current.cancellationRequestedAt != null) execution.abort();
res.status(202).json({
id: current.responseId,
object: 'response',
model: current.agentId,
status: 'in_progress',
background: true,
store: true,
});
return;
}
if (!current || current.generationCreatedAt != null)
throw new Error('Background response execution already published');
const publishEnrollment = (
record: StoredResponseRecord,
status: 'queued' | 'in_progress',
) =>
admission.storage.publish(record, record.responseId, record.revision, {
status,
responseJSON: JSON.stringify({ ...buildPendingBackgroundResponse(record), status }),
generationCreatedAt: execution.createdAt,
});
const cancelled = current.cancellationRequestedAt != null;
if (cancelled) execution.abort();
let result = await publishEnrollment(current, cancelled ? 'queued' : 'in_progress');
if (!result.ok && !cancelled) {
const latest = await admission.storage.read(current, current.responseId);
if (
latest?.status === 'queued' &&
latest.generationCreatedAt == null &&
latest.cancellationRequestedAt != null
) {
execution.abort();
result = await publishEnrollment(latest, 'queued');
}
}
if (!result.ok) throw new Error('Background response enrollment publication failed');
admission.record = result.record;
res.status(202).json({
id: result.record.responseId,
object: 'response',
model: result.record.agentId,
status: result.record.status,
background: true,
store: true,
});
},
afterPause: async (execution) => {
if (!execution.isPaused || !execution.ownsSettlement)
throw new Error('Background response pause ownership changed');
const current = await admission.storage.read(
admission.record,
admission.record.responseId,
);
if (current && current.generationCreatedAt !== execution.createdAt)
throw new Error('Background response pause identity changed');
if (
current &&
current.cancellationRequestedAt == null &&
!['completed', 'failed', 'cancelled', 'incomplete'].includes(current.status)
)
return;
if (!current) {
const identity = await admission.storage.readCleanupIdentity(
admission.record,
admission.record.responseId,
{ ...admission.record, generationCreatedAt: execution.createdAt, ...checkpoint },
);
if (!identity?.retirement || identity.generationCreatedAt !== execution.createdAt)
throw new Error('Background response pause retirement identity unavailable');
const job = await continuation.jobs!.getJob(admission.record.responseId);
if (!job) {
// The owned settled pause proves local drain; this permits only checkpoint cleanup.
if (identity.requiresCheckpointCleanup) {
if (!identity.checkpointNamespace || !identity.checkpointConfigJSON)
throw new Error('Background response pause checkpoint identity unavailable');
await cleanupResponseCheckpoint(
{ ...identity, pendingTerminal: identity },
identity.checkpointNamespace,
);
}
return;
}
}
const stopped = await createResponseRetirement({
storage: admission.storage,
jobs: continuation.jobs!,
cleanupCheckpoint: cleanupResponseCheckpoint,
})(admission.record, admission.record.responseId);
if (stopped !== 'stopped')
throw new Error('Background response pause retirement is pending');
},
afterSettle: async (execution, error) => {
let current = await admission.storage.read(admission.record, admission.record.responseId);
const completion = createResponseCompletion({
storage: admission.storage,
cleanupCheckpoint: cleanupResponseCheckpoint,
releaseExecution: createResponseExecutionRelease(continuation.jobs!),
});
if (!current || current.generationCreatedAt == null) {
const retired = await admission.storage.readCleanupIdentity(
admission.record,
admission.record.responseId,
{ ...admission.record, generationCreatedAt: execution.createdAt, ...checkpoint },
);
if (retired?.retirement && retired.generationCreatedAt === execution.createdAt)
current = { ...admission.record, ...retired };
else if (
current?.status === 'cancelled' &&
current.cancellationRequestedAt != null &&
current.generationCreatedAt == null
) {
// Stop won the enrollment CAS. This exact local attempt must still release its drain proof.
await completion.finish({ ...current, generationCreatedAt: execution.createdAt });
return;
}
}
if (!current || current.generationCreatedAt !== execution.createdAt)
throw new Error('Background response settlement identity changed');
if (['completed', 'failed', 'cancelled', 'incomplete'].includes(current.status)) {
await completion.finish(current);
return;
}
const identity = await execution.confirmTerminalIdentity();
// Normal manager teardown aborts the local signal too; only durable outcome/intent classifies Stop.
const cancelled =
identity.outcome === 'aborted' || current.cancellationRequestedAt != null;
const result: TerminalResponse = candidate ?? {
...(previous() ?? buildPendingBackgroundResponse(current)),
status: cancelled ? 'cancelled' : 'failed',
completed_at: Math.floor(Date.now() / 1000),
required_action: undefined,
error: cancelled
? null
: {
type: 'server_error',
code: 'execution_failed',
message: 'The Agent could not complete this request.',
},
};
let final = result;
if (cancelled)
final = {
...result,
status: 'cancelled',
completed_at: Math.floor(Date.now() / 1000),
error: null,
};
else if ((error || identity.outcome === 'error') && result.status === 'completed')
final = {
...result,
status: 'failed',
completed_at: Math.floor(Date.now() / 1000),
error: {
type: 'server_error',
code: 'execution_failed',
message: 'The Agent could not complete this request.',
},
};
admission.record = await completion.stageAndFinish(current, {
...identity,
status: final.status,
responseJSON: JSON.stringify(final),
...(checkpoint ??
(admission.resume && current.review?.checkpointConfigJSON
? {
checkpointNamespace: admission.resume.checkpointNamespace,
checkpointConfigJSON: current.review.checkpointConfigJSON,
}
: {})),
});
},
}
: {};
return {
composeResponse,
inputChain(chain) {
if (!admission?.resume) {
responseParentMessageId = chain.parentMessageId;
return chain;
}
const action: Agents.PendingAction = JSON.parse(
admission.record.review?.pendingActionJSON ?? 'null',
);
const parent = action?.resumeContext?.responseParentMessageId;
if (typeof parent !== 'string' || parent.length === 0)
throw new Error('Response resumed input identity unavailable');
responseParentMessageId = parent;
return { messages: [], parentMessageId: parent };
},
visibleOutputTokens,
seedAggregator(aggregator) {
if (!aggregator || seeded.has(aggregator)) return;
const prior = previous();
if (prior) seedResponseAggregator(aggregator, prior);
seeded.add(aggregator);
},
async graphOptions(agents) {
const hitlCapable = interactive;
const request = req as ServerRequest;
const policy = () =>
hitlCapable
? {
toolApprovalStorage: continuation.toolApprovalStorage,
toolApprovalAllows: resolveRunToolApprovalAllows(
request.config?.endpoints?.agents?.toolApproval,
{
conversationId: admission!.record.conversationId,
toolApprovalAllows: [
...new Set([
...resolveRunToolApprovalAllows(
request.config?.endpoints?.agents?.toolApproval,
request.resolvedConversation,
admission!.record.conversationId,
),
...rememberedTools,
]),
],
},
admission!.record.conversationId,
),
}
: {};
if (!admission?.resume) return { hitlCapable, ...policy() };
const action: Agents.PendingAction = JSON.parse(
admission.record.review?.pendingActionJSON ?? 'null',
);
if (!action || !agents) throw new Error('Response resume graph is unavailable');
if (
action.requestFingerprint !== computeLegacyAgentRequestFingerprint(request.body) ||
action.requestFingerprintV2 !== computeAgentRequestFingerprint(request.body)
)
throw new Error('Response resume graph configuration changed');
const retained = resolveCheckpointerConfig(
checkpointerSchema.parse(
JSON.parse(admission.record.review?.checkpointConfigJSON ?? 'null'),
),
);
const current = resolveCheckpointerConfig(
(req as ServerRequest).config?.endpoints?.agents?.checkpointer,
);
if (
retained.type !== 'mongo' ||
current.type !== retained.type ||
current.checkpointCollectionName !== retained.checkpointCollectionName ||
current.checkpointWritesCollectionName !== retained.checkpointWritesCollectionName
)
throw new Error('Response checkpoint storage settings changed');
const reviewedAgents = collectReachableAgents<ReviewAgentNode>(agents);
assertCodeExecutionApprovalBinding(action.codeExecutionBinding, reviewedAgents);
const participants = new Set(
responseParticipantIds(admission.record.agentId, reviewedAgents),
);
const retainedParticipants = action.resumeAgentIds;
if (
!Array.isArray(retainedParticipants) ||
retainedParticipants.length === 0 ||
new Set(retainedParticipants).size !== retainedParticipants.length ||
participants.size !== retainedParticipants.length ||
retainedParticipants.some((id) => typeof id !== 'string' || !participants.has(id))
)
throw new Error('Response resume participants changed');
const decision = responseReviewDecisionSchema.parse(
JSON.parse(admission.record.resume?.decisionJSON ?? 'null'),
);
if (decision.actionId !== action.actionId)
throw new Error('Response reviewed action changed');
const discovered = action.resumeContext?.discoveredToolNames;
if (
discovered != null &&
(!Array.isArray(discovered) || discovered.some((name) => typeof name !== 'string'))
)
throw new Error('Response discovered tools are unavailable');
if (
decision.decisions?.some((item) => item.decision === 'approve' && item.scope === 'session')
) {
const { record, resume } = admission;
const job = await continuation.jobs?.getJob(record.responseId);
if (
!continuation.addConvoToolApprovalAllows ||
!enrolledExecution?.ownsSettlement ||
enrolledExecution.createdAt !== record.generationCreatedAt ||
!job ||
job.status !== 'running' ||
job.createdAt !== record.generationCreatedAt ||
job.metadata.userId !== record.userId ||
responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) ||
job.metadata.agent_id !== record.agentId ||
job.metadata.endpoint !== 'responses' ||
job.metadata.conversationId !== record.conversationId ||
job.metadata.responseMessageId !== record.responseId ||
job.metadata.providerExecutionId !== resume.providerExecutionId ||
job.metadata.providerExecutionStartedId !== resume.providerExecutionId ||
!job.metadata.providerExecutionOwnerId
)
throw new Error('Response remembered approval ownership unavailable');
const request = req as ServerRequest;
const persisted = await recordToolApprovalAllows({
userId: record.userId,
conversationId: record.conversationId,
policy: request.config?.endpoints?.agents?.toolApproval,
pendingAction: action,
resolutions: decision.decisions,
agents: reviewedAgents,
hookContext: {
userId: record.userId,
tenantId: responseTenantId(record.tenantId),
conversationId: record.conversationId,
appConfig: request.config,
},
pluginHookSource: continuation.pluginHookSource,
request,
addConvoToolApprovalAllows: continuation.addConvoToolApprovalAllows,
});
rememberedTools = [...new Set([...rememberedTools, ...persisted])];
}
return {
hitlCapable,
...policy(),
sessionStartSource: 'resume',
...(Array.isArray(discovered) ? { discoveredToolNames: discovered } : {}),
reviewedToolApprovals: {
bindings: action.toolApprovalBindings,
decisions: decision.decisions ?? [],
},
};
},
async completeTurn(input, complete) {
if (!admission || !interactive || !continuation.jobs) return complete();
if (!input.run.getInterrupt()) return complete();
const request = req as ServerRequest;
const config = request.config?.endpoints?.agents;
const agents = collectReachableAgents<ReviewAgentNode>(input.agents);
const aliases = getRunMCPToolAliases(input.run);
const partial = {
...composeResponse(input.buildPartial()),
status: 'in_progress' as const,
background: true,
store: true,
};
const paused = await pauseResponseForReview(
{
run: input.run,
execution: input.execution,
record: admission.record,
partial,
visibleOutputTokens: visibleOutputTokens(input.visibleOutputTokens),
checkpointer: config?.checkpointer,
},
{
storage: admission.storage,
jobs: continuation.jobs,
persistUnfinished: async (response) => {
await input.persistence.saveConversation();
await input.persistence.saveInput();
await input.persistence.saveOutput(response);
},
buildAction: async (interrupt) => {
const payload =
interrupt.payload.type === 'tool_approval'
? markToolApprovalAllowAlways(
markNativeCodeToolApprovalRequests(interrupt.payload, agents),
{
policy: config?.toolApproval,
agents,
aliases,
storedTools: resolveRunToolApprovalAllows(
config?.toolApproval,
request.resolvedConversation,
admission.record.conversationId,
),
hookContext: {
userId: admission.record.userId,
tenantId: responseTenantId(admission.record.tenantId),
conversationId: admission.record.conversationId,
appConfig: request.config,
},
pluginHookSource: continuation.pluginHookSource,
},
)
: interrupt.payload;
return captureRunPendingAction({
run: input.run,
payload,
agents,
resumeAgentIds: responseParticipantIds(admission.record.agentId, agents),
aliases,
context: {
streamId: admission.record.responseId,
responseMessageId: admission.record.responseId,
conversationId: admission.record.conversationId,
threadId: admission.record.conversationId,
runId: interrupt.checkpointNs ?? admission.record.responseId,
interruptId: interrupt.interruptId,
ttlMs: getApprovalTtlMs(config?.checkpointer),
projectContextKey: getChatProjectContextKey(request.chatProjectContext),
requestFingerprint: computeLegacyAgentRequestFingerprint(request.body),
requestFingerprintV2: computeAgentRequestFingerprint(request.body),
resumeContext: {
...pickResumeContext(request.body, request.reasoningOverrideBase),
discoveredToolNames: getRunDiscoveredTools(input.run),
responseParentMessageId,
},
},
});
},
},
);
if (!paused && !input.execution.signal.aborted)
throw new Error('Response interrupt could not be retained');
return undefined;
},
graphMessages: (messages) => (admission?.resume ? [] : messages),
async processRun(run, input, config, options) {
if (!admission?.resume) {
if (!admission || !interactive || !continuation.jobs)
return run.processStream(input, config, options);
const record = admission.record;
const job = await continuation.jobs.getJob(record.responseId);
const namespace = job?.metadata.checkpointNamespace;
if (
!job ||
job.status !== 'running' ||
job.createdAt !== record.generationCreatedAt ||
job.metadata.userId !== record.userId ||
responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) ||
job.metadata.agent_id !== record.agentId ||
job.metadata.endpoint !== 'responses' ||
job.metadata.responseMessageId !== record.responseId ||
job.metadata.conversationId !== record.conversationId ||
!namespace?.startsWith(checkpointOwnerNamespacePrefix(record.userId, record.tenantId))
)
throw new Error('Response checkpoint enrollment identity unavailable');
const settings = resolveCheckpointerConfig(
(req as ServerRequest).config?.endpoints?.agents?.checkpointer,
);
if (settings.type !== 'mongo')
throw new Error('Response interrupt requires durable checkpoint storage');
checkpoint = {
checkpointNamespace: namespace,
checkpointConfigJSON: JSON.stringify({
type: settings.type,
ttl: settings.ttlSeconds,
checkpointCollectionName: settings.checkpointCollectionName,
checkpointWritesCollectionName: settings.checkpointWritesCollectionName,
}),
};
if (!continuation.jobs?.updateMetadata)
throw new Error('Response checkpoint cleanup persistence unavailable');
await continuation.jobs.updateMetadata(
record.responseId,
{
responseCheckpointConfigJSON: checkpoint.checkpointConfigJSON,
},
job.createdAt,
);
return run.processStream(
input,
{
...config,
configurable: {
...config.configurable,
thread_id: record.conversationId,
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: namespace,
},
},
options,
);
}
const { record, resume } = admission;
if (
record.resume?.intentId !== resume.intentId ||
!record.review?.checkpointId ||
record.review.checkpointNs == null
)
throw new Error('Response resume checkpoint identity unavailable');
const resolution: unknown = JSON.parse(record.resume.resolutionJSON);
return run.resume(
resolution,
{
...config,
configurable: {
...config.configurable,
thread_id: record.conversationId,
checkpoint_id: record.review.checkpointId,
checkpoint_ns: record.review.checkpointNs,
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: resume.checkpointNamespace,
},
},
options,
);
},
toolContext: (execution) =>
admission
? { streamId: admission.record.responseId, jobCreatedAt: execution.createdAt }
: { streamId: null },
responseId:
admission?.record.responseId ?? (continuation.generateResponseId ?? generateResponseId)(),
connection: admission
? undefined
: {
isClosed: () => res.destroyed === true && res.writableEnded !== true,
onClose: (listener: () => void) => {
const close = () => {
if (res.writableEnded !== true) listener();
};
res.once('close', close);
return () => {
res.off('close', close);
};
},
},
hooks,
async resolveContinuation(envelope: ResponsesRunEnvelope) {
if (!admission)
return resolveResponseContinuation(
envelope.payload.previous_response_id,
envelope.principal.userId,
continuation,
);
const record = admission.record;
if (
envelope.principal.userId !== record.userId ||
responseTenantId(envelope.principal.tenantId) !== responseTenantId(record.tenantId) ||
envelope.payload.model !== record.agentId
)
throw new Error('Background response principal changed');
return { conversationId: record.conversationId, parentMessageId: record.parentMessageId };
},
};
}

View file

@ -0,0 +1,224 @@
import type { Response, OutputItem, Usage } from './types';
import {
buildAggregatedResponse,
createResponseAggregator,
createAggregatorEventHandlers,
} from './service';
import { createResponseAccountingCollector, combineResponseAccounting } from './accounting';
import { combineResponseSegments, seedResponseAggregator } from './segments';
import { projectResponseMessage } from './persistence';
const usage = (count: number): Usage => ({
input_tokens: count,
output_tokens: count,
total_tokens: count * 2,
input_tokens_details: { cached_tokens: count },
output_tokens_details: { reasoning_tokens: count },
primary: { input_tokens: count, output_tokens: count, total_tokens: count * 2 },
subagent: { input_tokens: 0, output_tokens: 0, total_tokens: 0 },
});
function response(output: OutputItem[], count: number, amount: string): Response {
const accounting = createResponseAccountingCollector({
quote: () => ({ amount, basis: 'application_billing' }),
});
accounting.record({
input_tokens: count,
output_tokens: count,
total_tokens: count * 2,
model: 'actual-model',
provider: 'openAI',
});
return {
...buildAggregatedResponse(
{ responseId: 'response', model: 'agent', createdAt: 1 },
createResponseAggregator(),
usage(count),
accounting.snapshot(),
),
output,
background: true,
store: true,
};
}
const message = (id: string, text: string): OutputItem => ({
type: 'message',
id,
role: 'assistant',
status: 'completed',
content: [{ type: 'output_text', text, annotations: [], logprobs: [] }],
});
const call = (id: string, callId: string): OutputItem => ({
type: 'function_call',
id,
call_id: callId,
name: 'lookup',
arguments: '{"key":"original"}',
status: 'completed',
});
const result = (id: string, callId: string, output: string): OutputItem => ({
type: 'function_call_output',
id,
call_id: callId,
output,
status: 'completed',
});
it('preserves the saved prefix and completes a paused tool exactly once without seeding billed usage', () => {
const previous = response(
[
message('before', 'Before. '),
call('done-call', 'done'),
result('done-result', 'done', 'old result'),
call('waiting-call', 'waiting'),
],
3,
'0.1',
);
const original = JSON.stringify(previous);
const aggregator = createResponseAggregator();
seedResponseAggregator(aggregator, previous);
expect(aggregator.usage).toEqual({
inputTokens: 0,
outputTokens: 0,
reasoningTokens: 0,
cachedTokens: 0,
});
expect(aggregator.getText()).toBe('');
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_run_step_completed.handle('on_run_step_completed', {
result: {
id: 'step',
index: 0,
type: 'tool_call',
tool_call: {
id: 'waiting',
name: 'lookup',
args: '{"key":"original"}',
output: 'new result',
progress: 1,
},
},
});
aggregator.addText('After.');
const current = {
...response([], 5, '0.2'),
output: buildAggregatedResponse(
{ responseId: 'response', model: 'agent', createdAt: 2 },
aggregator,
).output,
};
const combined = combineResponseSegments(previous, current, 'segment-two');
expect(combined.created_at).toBe(1);
expect(combined.usage).toEqual(usage(8));
expect(combined.accounting).toMatchObject({
status: 'available',
amount: '0.3',
coverage: 'complete',
});
expect(combined.output.filter((item) => item.type === 'function_call')).toHaveLength(2);
expect(combined.output.filter((item) => item.type === 'function_call_output')).toEqual([
result('done-result', 'done', 'old result'),
expect.objectContaining({ call_id: 'waiting', output: 'new result' }),
]);
expect(combined.output[0]).toEqual(previous.output[0]);
expect(projectResponseMessage(combined).text).toBe('Before. After.');
expect(JSON.stringify(previous)).toBe(original);
});
it('uses the cumulative snapshot for the next pause without duplicating earlier text, calls or usage', () => {
const first = response([message('first', 'First. ')], 1, '0.1');
const second = combineResponseSegments(
first,
response([message('second', 'Second. ')], 2, '0.2'),
'two',
);
const third = combineResponseSegments(
second,
response([message('third', 'Third.')], 3, '0.3'),
'three',
);
expect(projectResponseMessage(third).text).toBe('First. Second. Third.');
expect(third.usage).toEqual(usage(6));
expect(third.accounting).toMatchObject({ amount: '0.6' });
expect(third.accounting?.calls).toHaveLength(3);
expect(third.output.map((item) => item.id)).toEqual(['first', 'second', 'third']);
});
it('keeps output IDs unique even when independent segments generate the same message ID', () => {
const combined = combineResponseSegments(
response([message('same', 'Before')], 1, '0.1'),
response([message('same', 'After')], 1, '0.1'),
'intent-two',
);
expect(combined.output.map((item) => item.id)).toEqual(['same', 'same_intent-two']);
});
it('does not overwrite an already completed tool result with conflicting data', () => {
const previous = response(
[call('call', 'lookup'), result('result', 'lookup', 'original-private-result')],
1,
'0.1',
);
expect(() =>
combineResponseSegments(
previous,
response([result('other', 'lookup', 'different-private-result')], 1, '0.1'),
'two',
),
).toThrow('Response tool result changed across segments');
});
it('preserves uncertainty when a previous segment has no usable usage or accounting', () => {
const previous = { ...response([], 1, '0.1'), usage: null, accounting: undefined };
const combined = combineResponseSegments(previous, response([], 1, '0.2'), 'two');
expect(combined.usage).toBeNull();
expect(combined.accounting).toMatchObject({
status: 'unavailable',
coverage: 'partial',
reason: 'partial_coverage',
});
expect(combined.accounting).not.toHaveProperty('amount');
expect(combined.accounting?.calls).toHaveLength(1);
});
it('retains past quotes rather than consulting current pricing for old calls', () => {
const prior = response([], 1, '0.1').accounting!;
const quote = jest.fn(() => ({ amount: '0.9', basis: 'application_billing' as const }));
const collector = createResponseAccountingCollector({ seed: prior, quote });
collector.record({ input_tokens: 1, output_tokens: 1 });
expect(quote).toHaveBeenCalledTimes(1);
expect(collector.snapshot()).toMatchObject({ amount: '1' });
expect(prior.amount).toBe('0.1');
});
it('does not erase conflicting or incomplete provenance when combining accounting', () => {
const priced = response([], 1, '0.1').accounting!;
const collector = createResponseAccountingCollector();
collector.record({ input_tokens: 1, output_tokens: 1 }, undefined, undefined, 'same-call');
collector.record({ input_tokens: 2, output_tokens: 1 }, undefined, undefined, 'same-call');
expect(combineResponseAccounting(priced, collector.snapshot())).toMatchObject({
status: 'unavailable',
reason: 'conflicting_usage',
});
expect(
combineResponseAccounting(priced, createResponseAccountingCollector().snapshot()),
).toMatchObject({
status: 'unavailable',
coverage: 'partial',
reason: 'partial_coverage',
});
});
it('rejects response identity changes and invalid or overflowing usage', () => {
const prior = response([], 1, '0.1');
expect(() => combineResponseSegments(prior, { ...prior, id: 'another-response' }, 'two')).toThrow(
'identity changed',
);
const invalid = response([], 1, '0.1');
invalid.usage!.output_tokens = Number.MAX_SAFE_INTEGER;
expect(() => combineResponseSegments(prior, invalid, 'two')).toThrow(
'Response segment usage is invalid',
);
});

View file

@ -0,0 +1,118 @@
import type { Response, Usage, UsageTotals, OutputItem } from './types';
import type { ResponseAggregator } from './service';
import { combineResponseAccounting } from './accounting';
/** Restore tool state only; prior text/reasoning and item IDs remain immutable in the saved prefix. */
export function seedResponseAggregator(aggregator: ResponseAggregator, previous: Response): void {
for (const item of previous.output) {
if (item.type === 'function_call')
aggregator.toolCalls.set(item.call_id, {
id: item.call_id,
name: item.name,
arguments: item.arguments,
});
else if (item.type === 'function_call_output')
aggregator.toolOutputs.set(item.call_id, item.output);
}
}
function add(left: number, right: number): number {
const result = left + right;
if (
!Number.isSafeInteger(left) ||
left < 0 ||
!Number.isSafeInteger(right) ||
right < 0 ||
!Number.isSafeInteger(result)
)
throw new Error('Response segment usage is invalid');
return result;
}
function totals(left: UsageTotals, right: UsageTotals): UsageTotals {
return {
input_tokens: add(left.input_tokens, right.input_tokens),
output_tokens: add(left.output_tokens, right.output_tokens),
total_tokens: add(left.total_tokens, right.total_tokens),
};
}
function usage(left: Usage | null, right: Usage | null): Usage | null {
if (!left || !right) return null;
return {
...totals(left, right),
input_tokens_details: {
cached_tokens: add(
left.input_tokens_details.cached_tokens,
right.input_tokens_details.cached_tokens,
),
},
output_tokens_details: {
reasoning_tokens: add(
left.output_tokens_details.reasoning_tokens,
right.output_tokens_details.reasoning_tokens,
),
},
...(left.primary && right.primary ? { primary: totals(left.primary, right.primary) } : {}),
...(left.subagent && right.subagent ? { subagent: totals(left.subagent, right.subagent) } : {}),
};
}
function callKey(item: OutputItem): string | undefined {
return item.type === 'function_call' || item.type === 'function_call_output'
? `${item.type}:${item.call_id}`
: undefined;
}
/** Compose a new provider segment once with its saved prefix; never send old usage back to billing. */
export function combineResponseSegments(
previous: Response,
current: Response,
segmentId: string,
): Response {
if (previous.id !== current.id || previous.model !== current.model || !segmentId)
throw new Error('Response segment identity changed');
const output = [...previous.output];
const calls = new Map<string, number>();
const ids = new Set<string>();
for (let index = 0; index < output.length; index++) {
const item = output[index];
ids.add(item.id);
const key = callKey(item);
if (key) calls.set(key, index);
}
for (const item of current.output) {
const key = callKey(item);
const index = key == null ? undefined : calls.get(key);
if (index != null) {
const prior = output[index];
if (
prior.type === 'function_call_output' &&
item.type === 'function_call_output' &&
prior.output !== item.output
)
throw new Error('Response tool result changed across segments');
output[index] = { ...item, id: prior.id };
continue;
}
if (
item.type === 'message' &&
item.role === 'assistant' &&
item.content.length === 0 &&
output.length > 0
)
continue;
const id = ids.has(item.id) ? `${item.id}_${segmentId}` : item.id;
if (ids.has(id)) throw new Error('Response segment item identity collided');
if (key) calls.set(key, output.length);
output.push({ ...item, id });
ids.add(id);
}
return {
...current,
created_at: previous.created_at,
output,
usage: usage(previous.usage, current.usage),
accounting: combineResponseAccounting(previous.accounting, current.accounting),
};
}

File diff suppressed because it is too large Load diff

View file

@ -4,6 +4,7 @@ import { getTenantId, tenantStorage } from '@librechat/data-schemas';
import type { IUser, ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas';
import type { Request, RequestHandler } from 'express';
import type { ResumedAgentExecutionIdentity } from '../remote/lifecycle';
import type { TurnConversationRequest } from '~/conversations/save';
import type { ResponseResumeAdmissionResult } from './types';
import type { ContextRequest } from '~/middleware/tenant';
import type { ServerRequest } from '~/types/http';
@ -240,12 +241,19 @@ export function createBackgroundResponseWorkerAuth(
checkpointNamespace: result.checkpointNamespace,
};
}
const request = req as ServerRequest & Pick<ContextRequest, 'tenantId'>;
const request = req as ServerRequest &
Pick<ContextRequest, 'tenantId'> &
TurnConversationRequest;
// findUser supplies a lean auth DTO; do not mutate its cached tenant representation.
request.user = Object.assign({}, user, { id: record.userId, tenantId: claims.tenantId });
request.tenantId = claims.tenantId;
// Caller body/headers cannot replace the persisted, already validated request.
request.body = JSON.parse(record.requestJSON);
if (record.expiredAt)
request._agentEventBindingRetention = {
expiredAt: record.expiredAt,
isTemporary: record.isTemporary,
};
admitted.set(req, {
record,
storage: deps.storage,

View file

@ -16,6 +16,7 @@ export type {
ResponseOwner,
ResponseStorage,
ResponseTerminalGuard,
ResponseExecutionGuard,
ResponseCleanupIdentity,
ResponseReservation,
ResponseReservationResult,

View file

@ -1615,3 +1615,183 @@ it('matches the complete reviewed and resumed execution binding rather than its
(await storage.stageTerminal(owner, 'resp_one', { execution: retained }, receipt)).ok,
).toBe(true);
}));
async function runningForReview() {
await storage.reserve(reservation({ sourceKeyId: 'original-key' }));
const running = await storage.publish(owner, 'resp_one', 0, {
status: 'in_progress',
generationCreatedAt: 10,
responseJSON: '{}',
});
if (!running.ok) throw new Error('Publication failed');
return running.record;
}
it('publishes one immutable pause independently of polling before and after publication', () =>
scoped(async () => {
const running = await runningForReview();
for (let i = 0; i < 30; i++) {
const current = (await storage.read(owner, 'resp_one'))!;
expect(
(
await storage.publish(owner, 'resp_one', current.revision, {
status: 'in_progress',
responseJSON: JSON.stringify({ progress: i }),
})
).ok,
).toBe(true);
}
expect(
await storage.publishReview(owner, 'resp_one', running.revision, 10, review, '{}'),
).toEqual({ ok: false, code: 'response_conflict' });
const attempts = await Promise.all(
Array.from({ length: 12 }, () =>
storage.publishReview(owner, 'resp_one', { execution: running }, 10, review, '{}'),
),
);
expect(attempts.every((result) => result.ok)).toBe(true);
const paused = (await storage.read(owner, 'resp_one'))!;
expect(paused.review).toEqual(review);
expect(paused.revision).toBe(running.revision + 31);
const projected = await storage.publish(owner, 'resp_one', paused.revision, {
status: 'requires_action',
responseJSON: '{"projection":"new"}',
});
if (!projected.ok) throw new Error('Projection failed');
for (const execution of [running, projected.record]) {
expect(
(await storage.publishReview(owner, 'resp_one', { execution }, 10, review, '{}')).ok,
).toBe(true);
expect(
await storage.publishReview(
owner,
'resp_one',
{ execution },
10,
{ ...review, partialResponseJSON: '{"changed":true}' },
'{}',
),
).toEqual({ ok: false, code: 'response_conflict' });
}
const saved = (await storage.read(owner, 'resp_one'))!;
expect(saved.revision).toBe(projected.record.revision);
expect(saved.responseJSON).toBe('{"projection":"new"}');
expect(saved.review).toEqual(review);
}));
it.each(['agentId', 'conversationId', 'sourceKeyId', 'generationCreatedAt'] as const)(
'rejects a pause from a mismatched %s',
(field) =>
scoped(async () => {
const running = await runningForReview();
const execution = { ...running, [field]: field === 'generationCreatedAt' ? 11 : 'other' };
expect(
await storage.publishReview(owner, 'resp_one', { execution }, 10, review, '{}'),
).toEqual({ ok: false, code: 'response_conflict' });
expect((await storage.read(owner, 'resp_one'))?.review).toBeUndefined();
}),
);
it('binds the next pause to the complete admitted resume identity', () =>
scoped(async () => {
await pause();
await storage.reserveResume(owner, 'resp_one', resume(), 100);
await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 101);
const retained = (await storage.read(owner, 'resp_one'))!;
const nextReview = {
...review,
actionId: 'second-action',
providerExecutionId: 'provider-two',
};
for (const execution of [
{ ...retained, review: undefined },
{ ...retained, review: { ...retained.review!, checkpointId: 'different' } },
{ ...retained, resume: { ...retained.resume!, fingerprint: 'different' } },
{ ...retained, resume: { ...retained.resume!, state: 'reserved' as const } },
])
expect(
await storage.publishReview(owner, 'resp_one', { execution }, 10, nextReview, '{}'),
).toEqual({ ok: false, code: 'response_conflict' });
expect(
(
await storage.publishReview(
owner,
'resp_one',
{ execution: retained },
10,
nextReview,
'{}',
)
).ok,
).toBe(true);
expect(await storage.read(owner, 'resp_one')).toMatchObject({ review: nextReview });
expect((await storage.read(owner, 'resp_one'))?.resume).toBeUndefined();
}));
it.each(['cancelled', 'terminal', 'erased'] as const)(
'does not publish a pause after %s authority wins',
(failure) =>
scoped(async () => {
const running = await runningForReview();
if (failure === 'cancelled') await storage.requestCancellation(owner, 'resp_one', Date.now());
if (failure === 'terminal')
await storage.stageTerminal(owner, 'resp_one', { execution: running }, outcome);
if (failure === 'erased')
await mongoose.models.StoredResponse.deleteOne({ responseId: 'resp_one' });
expect(
await storage.publishReview(owner, 'resp_one', { execution: running }, 10, review, '{}'),
).toEqual({ ok: false, code: 'response_conflict' });
expect((await storage.read(owner, 'resp_one'))?.review).toBeUndefined();
}),
);
it.each(['live', 'owner_deleted', 'conversation_deleted', 'expired'] as const)(
'supplements an unpublished generation only under durable retirement proof: %s',
(reason) =>
scoped(async () => {
await storage.reserve(reservation());
const retained = {
...owner,
responseId: 'resp_one',
agentId: 'agent-one',
conversationId: 'conversation-one',
generationCreatedAt: 42,
checkpointNamespace: 'owned-checkpoint',
checkpointConfigJSON: 'pinned-settings',
};
if (reason === 'expired')
await createResponseRetention(mongoose)(
owner,
'conversation-one',
new Date(Date.now() - 1000),
);
else if (reason !== 'live')
await closeResponseScopes(
mongoose,
owner,
reason === 'conversation_deleted' ? ['conversation-one'] : undefined,
);
const identity = await storage.readCleanupIdentity(owner, 'resp_one', retained);
expect(identity).toMatchObject({
generationCreatedAt: reason === 'live' ? undefined : 42,
retirement: reason === 'live' ? undefined : reason,
checkpointNamespace: reason === 'live' ? undefined : 'owned-checkpoint',
});
if (reason !== 'live') {
const wrong = await storage.readCleanupIdentity(owner, 'resp_one', {
...retained,
agentId: 'wrong-agent',
});
expect(wrong?.generationCreatedAt).toBeUndefined();
expect(wrong?.checkpointNamespace).toBeUndefined();
await mongoose.models.StoredResponse.updateOne(
{ responseId: 'resp_one' },
{ $set: { generationCreatedAt: 43 } },
);
expect(await storage.readCleanupIdentity(owner, 'resp_one', retained)).toMatchObject({
generationCreatedAt: 43,
checkpointNamespace: undefined,
});
}
}),
);

View file

@ -77,8 +77,8 @@ export type ResponseResumeResult =
| { ok: true; record: StoredResponseRecord; replayed: boolean }
| { ok: false; code: 'resume_conflict' };
/** A drained execution's authority is independent of polling's projection revision. */
export type ResponseTerminalGuard =
/** Canonical execution writes are independent of polling's projection revision. */
export type ResponseExecutionGuard =
| number
| {
execution: Pick<
@ -87,15 +87,13 @@ export type ResponseTerminalGuard =
>;
};
function terminalExecutionFilter(guard: Exclude<ResponseTerminalGuard, number>) {
const { execution } = guard;
const filter: Record<string, unknown> = {
agentId: execution.agentId,
sourceKeyId: execution.sourceKeyId ?? { $exists: false },
conversationId: execution.conversationId,
generationCreatedAt: execution.generationCreatedAt,
};
const review = execution.review;
/** Compatibility name for existing terminal producers. */
export type ResponseTerminalGuard = ResponseExecutionGuard;
function responseReviewFilter(
review: ResponseReviewCheckpoint | undefined,
): Record<string, unknown> {
const filter: Record<string, unknown> = {};
if (review) {
const fields: Record<
keyof ResponseReviewCheckpoint,
@ -114,6 +112,18 @@ function terminalExecutionFilter(guard: Exclude<ResponseTerminalGuard, number>)
for (const [key, value] of Object.entries(fields))
filter[`review.${key}`] = value ?? { $exists: false };
} else filter.review = { $exists: false };
return filter;
}
function responseExecutionFilter(guard: Exclude<ResponseExecutionGuard, number>) {
const { execution } = guard;
const filter: Record<string, unknown> = {
agentId: execution.agentId,
sourceKeyId: execution.sourceKeyId ?? { $exists: false },
conversationId: execution.conversationId,
generationCreatedAt: execution.generationCreatedAt,
...responseReviewFilter(execution.review),
};
const resume = execution.resume;
if (resume) {
const fields: Record<
@ -153,7 +163,7 @@ export interface ResponseStorage {
publishReview(
owner: ResponseOwner,
responseId: string,
revision: number,
guard: ResponseExecutionGuard,
generationCreatedAt: number,
review: ResponseReviewCheckpoint,
responseJSON: string,
@ -249,7 +259,7 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
guard.execution.generationCreatedAt !== receipt.generationCreatedAt
)
return { ok: false, code: 'response_conflict' };
const execution = typeof guard === 'number' ? undefined : terminalExecutionFilter(guard);
const execution = typeof guard === 'number' ? undefined : responseExecutionFilter(guard);
const pendingTerminal: ResponseTerminalReceipt = {
generationCreatedAt: receipt.generationCreatedAt,
providerExecutionId: receipt.providerExecutionId,
@ -329,13 +339,41 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
.lean();
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
},
async publishReview(owner, responseId, revision, generationCreatedAt, review, responseJSON) {
async publishReview(owner, responseId, guard, generationCreatedAt, review, responseJSON) {
if (typeof guard !== 'number' && guard.execution.generationCreatedAt !== generationCreatedAt)
return { ok: false, code: 'response_conflict' };
const execution = typeof guard === 'number' ? undefined : responseExecutionFilter(guard);
const receiptFilter =
typeof guard === 'number'
? undefined
: {
...scope(owner),
responseId,
generationCreatedAt,
agentId: guard.execution.agentId,
sourceKeyId: guard.execution.sourceKeyId ?? { $exists: false },
conversationId: guard.execution.conversationId,
...responseReviewFilter(review),
status: 'requires_action',
resume: { $exists: false },
pendingTerminal: { $exists: false },
cancellationRequestedAt: { $exists: false },
};
if (
typeof guard !== 'number' &&
guard.execution.review?.actionId === review.actionId &&
guard.execution.review.providerExecutionId === review.providerExecutionId &&
guard.execution.resume == null
) {
const existing = await model.findOne(receiptFilter).lean();
return existing ? { ok: true, record: existing } : { ok: false, code: 'response_conflict' };
}
const record = await model
.findOneAndUpdate(
{
...scope(owner),
responseId,
revision,
...(execution ?? { revision: guard }),
generationCreatedAt,
status: { $in: ['in_progress', 'requires_action'] },
pendingTerminal: { $exists: false },
@ -381,7 +419,10 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
{ new: true, runValidators: true },
)
.lean();
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
if (record) return { ok: true, record };
if (!receiptFilter) return { ok: false, code: 'response_conflict' };
const existing = await model.findOne(receiptFilter).lean();
return existing ? { ok: true, record: existing } : { ok: false, code: 'response_conflict' };
},
async reserveResume(owner, responseId, input, now) {
if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid resume reservation time');
@ -599,7 +640,7 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
.lean();
// Retained data comes only from an internal job/receipt, never request input.
// Validate it even when the row exists before using any fallback checkpoint fields.
const matchingRetained =
const retainedCandidate =
retained &&
retained.responseId === responseId &&
retained.userId === owner.userId &&
@ -607,10 +648,11 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
(!record ||
(retained.conversationId === record.conversationId &&
retained.agentId === record.agentId &&
retained.generationCreatedAt === record.generationCreatedAt))
(record.generationCreatedAt == null ||
retained.generationCreatedAt === record.generationCreatedAt)))
? retained
: undefined;
const source = record ?? matchingRetained;
const source = record ?? retainedCandidate;
if (!source) return null;
const now = new Date();
const [ownerKey, conversationKey] = responseScopeKeys(owner, source.conversationId);
@ -637,6 +679,13 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
!(conversationAnchor?.expiredAt && conversationAnchor.expiredAt <= now)
)
return null;
const matchingRetained =
retainedCandidate &&
(!record ||
retainedCandidate.generationCreatedAt === record.generationCreatedAt ||
(retirement && record.generationCreatedAt == null))
? retainedCandidate
: undefined;
// A staged checkpoint is one binding: never fill an incomplete terminal
// binding with settings from a prior review or retained provider segment.
const terminalCheckpoint = record?.pendingTerminal;
@ -663,7 +712,8 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
responseId: id,
agentId,
conversationId,
generationCreatedAt,
generationCreatedAt:
generationCreatedAt ?? (retirement ? matchingRetained?.generationCreatedAt : undefined),
expiredAt,
retirement,
checkpointNamespace,