mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 15:02:02 +00:00
🧵 feat: Execute and Resume Background Responses (#17602)
* 🧵 feat: Execute and Resume Background Responses
* WIP: guard pause publication and prepare session policy before graph creation
* wip: Align Runtime Pause and Retirement Ownership
* fix: Bind Background Review Publication to Execution Authority
* test: Express Participant Denial Cases Without Nested Conditions
* test: Read Remembered Consent Through Typed Conversation Methods
* test: Enroll Runtime Fixtures With Their Durable Journal Owner
* test: Preserve Journal Ownership in Manual SDK Resume Attempts
* feat: Own Response Segment Composition in Execution Runtime
* test: Release the Runtime During Completion Recovery
* fix: Pin Resume Participants and Preserve Retained Terminal Output
* fix: Reconcile Ambiguous Response Pause Publication
This commit is contained in:
parent
4dcb391f62
commit
d2126b8ac7
11 changed files with 4381 additions and 22 deletions
|
|
@ -15,6 +15,7 @@ export {
|
|||
} from './persistence';
|
||||
export { createBackgroundResponseService } from './background';
|
||||
|
||||
export { createResponseExecutionRuntime } from './runtime';
|
||||
export { createResponseDecisionSubmission } from './decision';
|
||||
export { createResponseResumeValidation, createResponseResumeAdmission } from './resume';
|
||||
export { createBackgroundResponseAuthority } from './authority';
|
||||
|
|
@ -23,7 +24,7 @@ export type {
|
|||
ResponseDecisionResult,
|
||||
ResponseDecisionSubmission,
|
||||
} from './decision';
|
||||
|
||||
export { pauseResponseForReview, type ResponsePauseDependencies } from './pause';
|
||||
export { createBackgroundResponseQueue, backgroundResponseDeliveryKey } from './queue';
|
||||
export {
|
||||
createBackgroundResponseWorkerAuth,
|
||||
|
|
|
|||
1804
packages/api/src/agents/responses/pause.spec.ts
Normal file
1804
packages/api/src/agents/responses/pause.spec.ts
Normal file
File diff suppressed because it is too large
Load diff
139
packages/api/src/agents/responses/pause.ts
Normal file
139
packages/api/src/agents/responses/pause.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import { MAX_SUBAGENT_GRAPH_NODES } from 'librechat-data-provider';
|
||||
import type { ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas';
|
||||
import type { Agents, TCheckpointerConfig } from 'librechat-data-provider';
|
||||
import type { RunInterruptResult } from '@librechat/agents';
|
||||
import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
|
||||
import type { AgentExecutionEnrollment } from '../remote/lifecycle';
|
||||
import type { GenerationJob } from '~/types/stream';
|
||||
import type { Response } from './types';
|
||||
import { hasDurableAgentInterruptCheckpoint, resolveCheckpointerConfig } from '../checkpointer';
|
||||
import { checkpointOwnerNamespacePrefix } from '~/stream/checkpoints';
|
||||
import { buildPendingBackgroundResponse } from './background';
|
||||
import { responseTenantId } from './scope';
|
||||
|
||||
export interface ResponsePauseDependencies {
|
||||
storage: ResponseStorage;
|
||||
jobs: Pick<GenerationJobManagerClass, 'getJob'>;
|
||||
/** Capture the host's existing graph/project/code/MCP approval bindings; never caller body data. */
|
||||
buildAction: (interrupt: RunInterruptResult, job: GenerationJob) => Promise<Agents.PendingAction>;
|
||||
/** Save the ordinary unfinished assistant turn, with existing content/privacy filtering. */
|
||||
persistUnfinished: (response: Response, action: Agents.PendingAction) => Promise<void>;
|
||||
}
|
||||
|
||||
/** Capture the SDK's real, durable interrupt before exposing any resumable action. */
|
||||
export async function pauseResponseForReview(
|
||||
input: {
|
||||
run: { getInterrupt: () => RunInterruptResult | undefined };
|
||||
execution: AgentExecutionEnrollment;
|
||||
record: StoredResponseRecord;
|
||||
partial: Response;
|
||||
visibleOutputTokens?: number;
|
||||
checkpointer?: TCheckpointerConfig;
|
||||
},
|
||||
deps: ResponsePauseDependencies,
|
||||
): Promise<boolean> {
|
||||
const interrupt = input.run.getInterrupt();
|
||||
if (!interrupt) return false;
|
||||
if (
|
||||
!interrupt.payload ||
|
||||
!['ask_user_question', 'tool_approval'].includes(interrupt.payload.type)
|
||||
)
|
||||
throw new Error('Unsupported response interrupt');
|
||||
const { record, execution } = input;
|
||||
const job = await deps.jobs.getJob(record.responseId);
|
||||
const namespace = job?.metadata.checkpointNamespace;
|
||||
const providerExecutionId = job?.metadata.providerExecutionId;
|
||||
if (
|
||||
!job ||
|
||||
!execution.ownsSettlement ||
|
||||
job.createdAt !== record.generationCreatedAt ||
|
||||
job.createdAt !== execution.createdAt ||
|
||||
job.status !== 'running' ||
|
||||
!providerExecutionId ||
|
||||
job.metadata.providerExecutionStartedId !== providerExecutionId ||
|
||||
job.metadata.userId !== record.userId ||
|
||||
responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) ||
|
||||
job.metadata.agent_id !== record.agentId ||
|
||||
job.metadata.endpoint !== 'responses' ||
|
||||
job.metadata.conversationId !== record.conversationId ||
|
||||
!namespace?.startsWith(checkpointOwnerNamespacePrefix(record.userId, record.tenantId)) ||
|
||||
input.partial.id !== record.responseId ||
|
||||
input.partial.model !== record.agentId ||
|
||||
!interrupt.checkpointId ||
|
||||
!interrupt.interruptId ||
|
||||
(interrupt.threadId != null && interrupt.threadId !== record.conversationId)
|
||||
)
|
||||
throw new Error('Response interrupt identity changed');
|
||||
const config = resolveCheckpointerConfig(input.checkpointer);
|
||||
if (
|
||||
config.type !== 'mongo' ||
|
||||
!(await hasDurableAgentInterruptCheckpoint(record.conversationId, input.checkpointer, {
|
||||
checkpointNamespace: namespace,
|
||||
checkpointId: interrupt.checkpointId,
|
||||
checkpointNs: interrupt.checkpointNs,
|
||||
interruptId: interrupt.interruptId,
|
||||
}))
|
||||
)
|
||||
throw new Error('Response interrupt checkpoint is unavailable');
|
||||
const action = await deps.buildAction(interrupt, job);
|
||||
const expiresAt = action.expiresAt;
|
||||
if (
|
||||
!action.resumeAgentIds?.includes(record.agentId) ||
|
||||
action.resumeAgentIds.length > MAX_SUBAGENT_GRAPH_NODES ||
|
||||
new Set(action.resumeAgentIds).size !== action.resumeAgentIds.length ||
|
||||
action.resumeAgentIds.some((id) => typeof id !== 'string' || !id) ||
|
||||
action.streamId !== record.responseId ||
|
||||
action.responseMessageId !== record.responseId ||
|
||||
action.conversationId !== record.conversationId ||
|
||||
action.threadId !== record.conversationId ||
|
||||
action.interruptId !== interrupt.interruptId ||
|
||||
expiresAt == null ||
|
||||
!Number.isSafeInteger(expiresAt)
|
||||
)
|
||||
throw new Error('Response review action identity changed');
|
||||
return execution.pauseForReview(action, async () => {
|
||||
await deps.persistUnfinished(input.partial, action);
|
||||
const current = await deps.storage.read(record, record.responseId);
|
||||
if (!current || current.generationCreatedAt !== execution.createdAt)
|
||||
throw new Error('Response pause journal identity changed');
|
||||
const publicResponse = {
|
||||
...buildPendingBackgroundResponse(current),
|
||||
status: 'in_progress' as const,
|
||||
};
|
||||
const publication: Parameters<ResponseStorage['publishReview']> = [
|
||||
current,
|
||||
current.responseId,
|
||||
{ execution: current },
|
||||
execution.createdAt,
|
||||
{
|
||||
actionId: action.actionId,
|
||||
providerExecutionId,
|
||||
expiresAt,
|
||||
partialResponseJSON: JSON.stringify(input.partial),
|
||||
...(input.visibleOutputTokens == null
|
||||
? {}
|
||||
: { visibleOutputTokens: input.visibleOutputTokens }),
|
||||
pendingActionJSON: JSON.stringify(action),
|
||||
checkpointId: interrupt.checkpointId,
|
||||
checkpointNs: interrupt.checkpointNs ?? '',
|
||||
checkpointConfigJSON: JSON.stringify({
|
||||
type: config.type,
|
||||
ttl: config.ttlSeconds,
|
||||
checkpointCollectionName: config.checkpointCollectionName,
|
||||
checkpointWritesCollectionName: config.checkpointWritesCollectionName,
|
||||
}),
|
||||
},
|
||||
JSON.stringify(publicResponse),
|
||||
];
|
||||
let published;
|
||||
try {
|
||||
published = await deps.storage.publishReview(...publication);
|
||||
} catch {
|
||||
// The write may have committed despite a lost acknowledgement. Reuse the exact
|
||||
// receipt and execution guard: storage reconciles it without republishing or
|
||||
// overriding cancellation, retirement, a newer review, or admitted resume.
|
||||
published = await deps.storage.publishReview(...publication);
|
||||
}
|
||||
if (!published.ok) throw new Error('Response pause journal publication failed');
|
||||
});
|
||||
}
|
||||
808
packages/api/src/agents/responses/runtime.ts
Normal file
808
packages/api/src/agents/responses/runtime.ts
Normal file
|
|
@ -0,0 +1,808 @@
|
|||
import { checkpointerSchema, responseReviewDecisionSchema } from 'librechat-data-provider';
|
||||
import type { ResponseTerminalReceipt, StoredResponseRecord } from '@librechat/data-schemas';
|
||||
import type { Request, Response as HttpResponse } from 'express';
|
||||
import type { Agents, TMessage } from 'librechat-data-provider';
|
||||
import type { Run, IState } from '@librechat/agents';
|
||||
import type { ExecuteAgentRunParams, AgentExecutionConnection } from '../remote/host';
|
||||
import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
|
||||
import type { MarkToolApprovalAllowAlwaysOptions } from '../hitl/allow';
|
||||
import type { AgentExecutionEnrollment } from '../remote/lifecycle';
|
||||
import type { ResponsesRunEnvelope } from '../envelope';
|
||||
import type { ResponseAggregator } from './service';
|
||||
import type { ReachableAgent } from '../traversal';
|
||||
import type { ServerRequest } from '~/types/http';
|
||||
import type { Response } from './types';
|
||||
import type { createRun } from '../run';
|
||||
import {
|
||||
computeAgentRequestFingerprint,
|
||||
computeLegacyAgentRequestFingerprint,
|
||||
pickResumeContext,
|
||||
} from '../hitl/policy';
|
||||
import {
|
||||
markToolApprovalAllowAlways,
|
||||
resolveRunToolApprovalAllows,
|
||||
recordToolApprovalAllows,
|
||||
} from '../hitl/allow';
|
||||
import {
|
||||
getApprovalTtlMs,
|
||||
resolveCheckpointerConfig,
|
||||
LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,
|
||||
} from '../checkpointer';
|
||||
import { createResponseCompletion, createResponseExecutionRelease } from './completion';
|
||||
import { combineResponseSegments, seedResponseAggregator } from './segments';
|
||||
import { checkpointOwnerNamespacePrefix } from '~/stream/checkpoints';
|
||||
import { getRunDiscoveredTools, getRunMCPToolAliases } from '../run';
|
||||
import { markNativeCodeToolApprovalRequests } from '../hitl/byom';
|
||||
import { assertCodeExecutionApprovalBinding } from '../execution';
|
||||
import { getBackgroundResponseWorkerAdmission } from './worker';
|
||||
import { getChatProjectContextKey } from '~/projects/context';
|
||||
import { buildPendingBackgroundResponse } from './background';
|
||||
import { resolveResponseContinuation } from './persistence';
|
||||
import { captureRunPendingAction } from '../hitl/capture';
|
||||
import { cleanupResponseCheckpoint } from './checkpoint';
|
||||
import { createResponseRetirement } from './retirement';
|
||||
import { collectReachableAgents } from '../traversal';
|
||||
import { pauseResponseForReview } from './pause';
|
||||
import { generateResponseId } from './service';
|
||||
import { responseTenantId } from './scope';
|
||||
|
||||
type TerminalResponse = Response & { status: ResponseTerminalReceipt['status'] };
|
||||
type ReviewAgent = NonNullable<Parameters<typeof captureRunPendingAction>[0]['agents'][number]> &
|
||||
NonNullable<Parameters<typeof markNativeCodeToolApprovalRequests>[1][number]> &
|
||||
NonNullable<NonNullable<MarkToolApprovalAllowAlwaysOptions['agents']>[number]>;
|
||||
interface ReviewAgentNode
|
||||
extends Omit<ReviewAgent, keyof ReachableAgent<never>>,
|
||||
ReachableAgent<ReviewAgentNode> {}
|
||||
|
||||
/** Capture and rebuild must describe the same initialized participant identities. */
|
||||
function responseParticipantIds(primaryId: string, agents: readonly ReviewAgentNode[]): string[] {
|
||||
const participants = new Set<string>([primaryId]);
|
||||
for (const agent of agents) {
|
||||
if (typeof agent.id !== 'string' || !agent.id)
|
||||
throw new Error('Response graph participant identity unavailable');
|
||||
participants.add(agent.id);
|
||||
}
|
||||
return [...participants];
|
||||
}
|
||||
|
||||
function responseBody(value: unknown): value is TerminalResponse {
|
||||
return (
|
||||
value != null &&
|
||||
typeof value === 'object' &&
|
||||
'object' in value &&
|
||||
value.object === 'response' &&
|
||||
'id' in value &&
|
||||
typeof value.id === 'string' &&
|
||||
'model' in value &&
|
||||
typeof value.model === 'string' &&
|
||||
'status' in value &&
|
||||
['completed', 'failed', 'incomplete', 'cancelled'].includes(String(value.status)) &&
|
||||
'output' in value &&
|
||||
Array.isArray(value.output)
|
||||
);
|
||||
}
|
||||
|
||||
export interface ResponseExecutionRuntime {
|
||||
responseId: string;
|
||||
inputChain: <T>(chain: { messages: T[]; parentMessageId: string }) => {
|
||||
messages: T[];
|
||||
parentMessageId: string;
|
||||
};
|
||||
graphOptions: (
|
||||
agents?: Parameters<typeof createRun>[0]['agents'],
|
||||
) => Promise<
|
||||
Pick<
|
||||
Parameters<typeof createRun>[0],
|
||||
| 'hitlCapable'
|
||||
| 'discoveredToolNames'
|
||||
| 'reviewedToolApprovals'
|
||||
| 'sessionStartSource'
|
||||
| 'toolApprovalStorage'
|
||||
| 'toolApprovalAllows'
|
||||
>
|
||||
>;
|
||||
seedAggregator: (aggregator: ResponseAggregator | null) => void;
|
||||
composeResponse: (response: Response) => Response;
|
||||
visibleOutputTokens: (current: number) => number;
|
||||
completeTurn: <T>(
|
||||
input: {
|
||||
run: Pick<Run<IState>, 'getInterrupt'> & Parameters<typeof getRunDiscoveredTools>[0];
|
||||
execution: AgentExecutionEnrollment;
|
||||
agents: Parameters<typeof createRun>[0]['agents'];
|
||||
buildPartial: () => Response;
|
||||
visibleOutputTokens: number;
|
||||
persistence: {
|
||||
saveConversation: () => Promise<void>;
|
||||
saveInput: () => Promise<void>;
|
||||
saveOutput: (partial: Response) => Promise<Partial<TMessage> | void>;
|
||||
};
|
||||
},
|
||||
complete: () => Promise<T>,
|
||||
) => Promise<T | undefined>;
|
||||
/** A rebuilt graph restores messages from its checkpoint, not from a fresh input turn. */
|
||||
graphMessages: <T>(messages: T[]) => T[];
|
||||
toolContext: (execution: { createdAt: number }) => {
|
||||
streamId: string | null;
|
||||
jobCreatedAt?: number;
|
||||
};
|
||||
connection?: AgentExecutionConnection;
|
||||
hooks: Pick<
|
||||
ExecuteAgentRunParams<unknown>,
|
||||
| 'onEnrolled'
|
||||
| 'afterPause'
|
||||
| 'afterSettle'
|
||||
| 'clientRequestId'
|
||||
| 'resume'
|
||||
| 'retainTerminalPersistence'
|
||||
>;
|
||||
processRun: (
|
||||
run: Pick<Run<IState>, 'processStream' | 'resume'>,
|
||||
...args: Parameters<Run<IState>['processStream']>
|
||||
) => ReturnType<Run<IState>['processStream']>;
|
||||
resolveContinuation: (envelope: ResponsesRunEnvelope) => Promise<{
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
/** Adapt only trusted background-worker requests; normal Responses keeps its synchronous lifecycle. */
|
||||
export function createResponseExecutionRuntime(
|
||||
req: Request,
|
||||
res: HttpResponse,
|
||||
continuation: Parameters<typeof resolveResponseContinuation>[2] & {
|
||||
generateResponseId?: () => string;
|
||||
jobs?: Pick<
|
||||
GenerationJobManagerClass,
|
||||
'getJob' | 'abortJob' | 'finishProviderTerminalPersistence'
|
||||
> &
|
||||
Partial<Pick<GenerationJobManagerClass, 'updateMetadata'>>;
|
||||
pluginHookSource?: MarkToolApprovalAllowAlwaysOptions['pluginHookSource'];
|
||||
toolApprovalStorage?: Parameters<typeof createRun>[0]['toolApprovalStorage'];
|
||||
addConvoToolApprovalAllows?: Parameters<
|
||||
typeof recordToolApprovalAllows
|
||||
>[0]['addConvoToolApprovalAllows'];
|
||||
},
|
||||
): ResponseExecutionRuntime {
|
||||
const admission = getBackgroundResponseWorkerAdmission(req);
|
||||
if (
|
||||
admission &&
|
||||
(!continuation.jobs?.finishProviderTerminalPersistence || !continuation.jobs.abortJob)
|
||||
)
|
||||
throw new Error('Response terminal publication release capability is unavailable');
|
||||
const interactive =
|
||||
admission?.interactive === true &&
|
||||
continuation.jobs != null &&
|
||||
resolveCheckpointerConfig((req as ServerRequest).config?.endpoints?.agents?.checkpointer)
|
||||
.type === 'mongo';
|
||||
let checkpoint: { checkpointNamespace: string; checkpointConfigJSON: string } | undefined;
|
||||
let enrolledExecution: AgentExecutionEnrollment | undefined;
|
||||
let rememberedTools: string[] = [];
|
||||
let responseParentMessageId: string | undefined;
|
||||
let prefix: Response | undefined;
|
||||
const seeded = new WeakSet<ResponseAggregator>();
|
||||
function previous(): Response | undefined {
|
||||
if (!admission?.resume) return;
|
||||
if (prefix) return prefix;
|
||||
let restored: Response;
|
||||
try {
|
||||
restored = JSON.parse(admission.record.review?.partialResponseJSON ?? 'null');
|
||||
} catch {
|
||||
throw new Error('Response resume output is unavailable');
|
||||
}
|
||||
if (
|
||||
!restored ||
|
||||
restored.id !== admission.record.responseId ||
|
||||
restored.model !== admission.record.agentId ||
|
||||
!Array.isArray(restored.output) ||
|
||||
!restored.background ||
|
||||
!restored.store
|
||||
)
|
||||
throw new Error('Response resume output identity changed');
|
||||
prefix = restored;
|
||||
return prefix;
|
||||
}
|
||||
function visibleOutputTokens(current: number): number {
|
||||
if (!admission?.resume) return current;
|
||||
const prior = admission.record.review?.visibleOutputTokens;
|
||||
if (
|
||||
prior == null ||
|
||||
!Number.isSafeInteger(prior) ||
|
||||
prior < 0 ||
|
||||
!Number.isSafeInteger(current) ||
|
||||
current < 0 ||
|
||||
!Number.isSafeInteger(prior + current)
|
||||
)
|
||||
throw new Error('Response visible token counter is unavailable');
|
||||
return prior + current;
|
||||
}
|
||||
function composeResponse(response: Response): Response {
|
||||
const prior = previous();
|
||||
return prior && admission?.resume
|
||||
? combineResponseSegments(prior, response, admission.resume.intentId)
|
||||
: response;
|
||||
}
|
||||
let candidate: TerminalResponse | undefined;
|
||||
if (admission) {
|
||||
const writeJSON = res.json.bind(res);
|
||||
res.json = (body: unknown) => {
|
||||
if (responseBody(body)) {
|
||||
if (body.id !== admission.record.responseId || body.model !== admission.record.agentId)
|
||||
throw new Error('Background response output identity changed');
|
||||
candidate = { ...body, background: true, store: true };
|
||||
if (res.writableEnded) return res;
|
||||
return writeJSON(candidate);
|
||||
}
|
||||
if (res.writableEnded) return res;
|
||||
return writeJSON(body);
|
||||
};
|
||||
}
|
||||
const hooks: Pick<
|
||||
ExecuteAgentRunParams<unknown>,
|
||||
| 'onEnrolled'
|
||||
| 'afterPause'
|
||||
| 'afterSettle'
|
||||
| 'clientRequestId'
|
||||
| 'resume'
|
||||
| 'retainTerminalPersistence'
|
||||
> = admission
|
||||
? {
|
||||
retainTerminalPersistence: true,
|
||||
clientRequestId: admission.record.responseId,
|
||||
...(admission.resume
|
||||
? {
|
||||
resume: {
|
||||
generationCreatedAt: admission.resume.generationCreatedAt,
|
||||
providerExecutionId: admission.resume.providerExecutionId,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
onEnrolled: async (execution) => {
|
||||
enrolledExecution = execution;
|
||||
const current = await admission.storage.read(
|
||||
admission.record,
|
||||
admission.record.responseId,
|
||||
);
|
||||
if (admission.resume) {
|
||||
if (
|
||||
!current ||
|
||||
current.generationCreatedAt !== execution.createdAt ||
|
||||
current.resume?.intentId !== admission.resume.intentId ||
|
||||
current.resume.state !== 'admitted' ||
|
||||
current.resume.providerExecutionId !== admission.resume.providerExecutionId ||
|
||||
current.status !== 'in_progress'
|
||||
)
|
||||
throw new Error('Background response resume enrollment changed');
|
||||
admission.record = current;
|
||||
if (current.cancellationRequestedAt != null) execution.abort();
|
||||
res.status(202).json({
|
||||
id: current.responseId,
|
||||
object: 'response',
|
||||
model: current.agentId,
|
||||
status: 'in_progress',
|
||||
background: true,
|
||||
store: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!current || current.generationCreatedAt != null)
|
||||
throw new Error('Background response execution already published');
|
||||
const publishEnrollment = (
|
||||
record: StoredResponseRecord,
|
||||
status: 'queued' | 'in_progress',
|
||||
) =>
|
||||
admission.storage.publish(record, record.responseId, record.revision, {
|
||||
status,
|
||||
responseJSON: JSON.stringify({ ...buildPendingBackgroundResponse(record), status }),
|
||||
generationCreatedAt: execution.createdAt,
|
||||
});
|
||||
const cancelled = current.cancellationRequestedAt != null;
|
||||
if (cancelled) execution.abort();
|
||||
let result = await publishEnrollment(current, cancelled ? 'queued' : 'in_progress');
|
||||
if (!result.ok && !cancelled) {
|
||||
const latest = await admission.storage.read(current, current.responseId);
|
||||
if (
|
||||
latest?.status === 'queued' &&
|
||||
latest.generationCreatedAt == null &&
|
||||
latest.cancellationRequestedAt != null
|
||||
) {
|
||||
execution.abort();
|
||||
result = await publishEnrollment(latest, 'queued');
|
||||
}
|
||||
}
|
||||
if (!result.ok) throw new Error('Background response enrollment publication failed');
|
||||
admission.record = result.record;
|
||||
res.status(202).json({
|
||||
id: result.record.responseId,
|
||||
object: 'response',
|
||||
model: result.record.agentId,
|
||||
status: result.record.status,
|
||||
background: true,
|
||||
store: true,
|
||||
});
|
||||
},
|
||||
afterPause: async (execution) => {
|
||||
if (!execution.isPaused || !execution.ownsSettlement)
|
||||
throw new Error('Background response pause ownership changed');
|
||||
const current = await admission.storage.read(
|
||||
admission.record,
|
||||
admission.record.responseId,
|
||||
);
|
||||
if (current && current.generationCreatedAt !== execution.createdAt)
|
||||
throw new Error('Background response pause identity changed');
|
||||
if (
|
||||
current &&
|
||||
current.cancellationRequestedAt == null &&
|
||||
!['completed', 'failed', 'cancelled', 'incomplete'].includes(current.status)
|
||||
)
|
||||
return;
|
||||
if (!current) {
|
||||
const identity = await admission.storage.readCleanupIdentity(
|
||||
admission.record,
|
||||
admission.record.responseId,
|
||||
{ ...admission.record, generationCreatedAt: execution.createdAt, ...checkpoint },
|
||||
);
|
||||
if (!identity?.retirement || identity.generationCreatedAt !== execution.createdAt)
|
||||
throw new Error('Background response pause retirement identity unavailable');
|
||||
const job = await continuation.jobs!.getJob(admission.record.responseId);
|
||||
if (!job) {
|
||||
// The owned settled pause proves local drain; this permits only checkpoint cleanup.
|
||||
if (identity.requiresCheckpointCleanup) {
|
||||
if (!identity.checkpointNamespace || !identity.checkpointConfigJSON)
|
||||
throw new Error('Background response pause checkpoint identity unavailable');
|
||||
await cleanupResponseCheckpoint(
|
||||
{ ...identity, pendingTerminal: identity },
|
||||
identity.checkpointNamespace,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
const stopped = await createResponseRetirement({
|
||||
storage: admission.storage,
|
||||
jobs: continuation.jobs!,
|
||||
cleanupCheckpoint: cleanupResponseCheckpoint,
|
||||
})(admission.record, admission.record.responseId);
|
||||
if (stopped !== 'stopped')
|
||||
throw new Error('Background response pause retirement is pending');
|
||||
},
|
||||
afterSettle: async (execution, error) => {
|
||||
let current = await admission.storage.read(admission.record, admission.record.responseId);
|
||||
const completion = createResponseCompletion({
|
||||
storage: admission.storage,
|
||||
cleanupCheckpoint: cleanupResponseCheckpoint,
|
||||
releaseExecution: createResponseExecutionRelease(continuation.jobs!),
|
||||
});
|
||||
if (!current || current.generationCreatedAt == null) {
|
||||
const retired = await admission.storage.readCleanupIdentity(
|
||||
admission.record,
|
||||
admission.record.responseId,
|
||||
{ ...admission.record, generationCreatedAt: execution.createdAt, ...checkpoint },
|
||||
);
|
||||
if (retired?.retirement && retired.generationCreatedAt === execution.createdAt)
|
||||
current = { ...admission.record, ...retired };
|
||||
else if (
|
||||
current?.status === 'cancelled' &&
|
||||
current.cancellationRequestedAt != null &&
|
||||
current.generationCreatedAt == null
|
||||
) {
|
||||
// Stop won the enrollment CAS. This exact local attempt must still release its drain proof.
|
||||
await completion.finish({ ...current, generationCreatedAt: execution.createdAt });
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!current || current.generationCreatedAt !== execution.createdAt)
|
||||
throw new Error('Background response settlement identity changed');
|
||||
if (['completed', 'failed', 'cancelled', 'incomplete'].includes(current.status)) {
|
||||
await completion.finish(current);
|
||||
return;
|
||||
}
|
||||
const identity = await execution.confirmTerminalIdentity();
|
||||
// Normal manager teardown aborts the local signal too; only durable outcome/intent classifies Stop.
|
||||
const cancelled =
|
||||
identity.outcome === 'aborted' || current.cancellationRequestedAt != null;
|
||||
const result: TerminalResponse = candidate ?? {
|
||||
...(previous() ?? buildPendingBackgroundResponse(current)),
|
||||
status: cancelled ? 'cancelled' : 'failed',
|
||||
completed_at: Math.floor(Date.now() / 1000),
|
||||
required_action: undefined,
|
||||
error: cancelled
|
||||
? null
|
||||
: {
|
||||
type: 'server_error',
|
||||
code: 'execution_failed',
|
||||
message: 'The Agent could not complete this request.',
|
||||
},
|
||||
};
|
||||
let final = result;
|
||||
if (cancelled)
|
||||
final = {
|
||||
...result,
|
||||
status: 'cancelled',
|
||||
completed_at: Math.floor(Date.now() / 1000),
|
||||
error: null,
|
||||
};
|
||||
else if ((error || identity.outcome === 'error') && result.status === 'completed')
|
||||
final = {
|
||||
...result,
|
||||
status: 'failed',
|
||||
completed_at: Math.floor(Date.now() / 1000),
|
||||
error: {
|
||||
type: 'server_error',
|
||||
code: 'execution_failed',
|
||||
message: 'The Agent could not complete this request.',
|
||||
},
|
||||
};
|
||||
|
||||
admission.record = await completion.stageAndFinish(current, {
|
||||
...identity,
|
||||
status: final.status,
|
||||
responseJSON: JSON.stringify(final),
|
||||
...(checkpoint ??
|
||||
(admission.resume && current.review?.checkpointConfigJSON
|
||||
? {
|
||||
checkpointNamespace: admission.resume.checkpointNamespace,
|
||||
checkpointConfigJSON: current.review.checkpointConfigJSON,
|
||||
}
|
||||
: {})),
|
||||
});
|
||||
},
|
||||
}
|
||||
: {};
|
||||
|
||||
return {
|
||||
composeResponse,
|
||||
inputChain(chain) {
|
||||
if (!admission?.resume) {
|
||||
responseParentMessageId = chain.parentMessageId;
|
||||
return chain;
|
||||
}
|
||||
const action: Agents.PendingAction = JSON.parse(
|
||||
admission.record.review?.pendingActionJSON ?? 'null',
|
||||
);
|
||||
const parent = action?.resumeContext?.responseParentMessageId;
|
||||
if (typeof parent !== 'string' || parent.length === 0)
|
||||
throw new Error('Response resumed input identity unavailable');
|
||||
responseParentMessageId = parent;
|
||||
return { messages: [], parentMessageId: parent };
|
||||
},
|
||||
visibleOutputTokens,
|
||||
seedAggregator(aggregator) {
|
||||
if (!aggregator || seeded.has(aggregator)) return;
|
||||
const prior = previous();
|
||||
if (prior) seedResponseAggregator(aggregator, prior);
|
||||
seeded.add(aggregator);
|
||||
},
|
||||
async graphOptions(agents) {
|
||||
const hitlCapable = interactive;
|
||||
const request = req as ServerRequest;
|
||||
const policy = () =>
|
||||
hitlCapable
|
||||
? {
|
||||
toolApprovalStorage: continuation.toolApprovalStorage,
|
||||
toolApprovalAllows: resolveRunToolApprovalAllows(
|
||||
request.config?.endpoints?.agents?.toolApproval,
|
||||
{
|
||||
conversationId: admission!.record.conversationId,
|
||||
toolApprovalAllows: [
|
||||
...new Set([
|
||||
...resolveRunToolApprovalAllows(
|
||||
request.config?.endpoints?.agents?.toolApproval,
|
||||
request.resolvedConversation,
|
||||
admission!.record.conversationId,
|
||||
),
|
||||
...rememberedTools,
|
||||
]),
|
||||
],
|
||||
},
|
||||
admission!.record.conversationId,
|
||||
),
|
||||
}
|
||||
: {};
|
||||
if (!admission?.resume) return { hitlCapable, ...policy() };
|
||||
const action: Agents.PendingAction = JSON.parse(
|
||||
admission.record.review?.pendingActionJSON ?? 'null',
|
||||
);
|
||||
if (!action || !agents) throw new Error('Response resume graph is unavailable');
|
||||
if (
|
||||
action.requestFingerprint !== computeLegacyAgentRequestFingerprint(request.body) ||
|
||||
action.requestFingerprintV2 !== computeAgentRequestFingerprint(request.body)
|
||||
)
|
||||
throw new Error('Response resume graph configuration changed');
|
||||
const retained = resolveCheckpointerConfig(
|
||||
checkpointerSchema.parse(
|
||||
JSON.parse(admission.record.review?.checkpointConfigJSON ?? 'null'),
|
||||
),
|
||||
);
|
||||
const current = resolveCheckpointerConfig(
|
||||
(req as ServerRequest).config?.endpoints?.agents?.checkpointer,
|
||||
);
|
||||
if (
|
||||
retained.type !== 'mongo' ||
|
||||
current.type !== retained.type ||
|
||||
current.checkpointCollectionName !== retained.checkpointCollectionName ||
|
||||
current.checkpointWritesCollectionName !== retained.checkpointWritesCollectionName
|
||||
)
|
||||
throw new Error('Response checkpoint storage settings changed');
|
||||
const reviewedAgents = collectReachableAgents<ReviewAgentNode>(agents);
|
||||
assertCodeExecutionApprovalBinding(action.codeExecutionBinding, reviewedAgents);
|
||||
const participants = new Set(
|
||||
responseParticipantIds(admission.record.agentId, reviewedAgents),
|
||||
);
|
||||
const retainedParticipants = action.resumeAgentIds;
|
||||
if (
|
||||
!Array.isArray(retainedParticipants) ||
|
||||
retainedParticipants.length === 0 ||
|
||||
new Set(retainedParticipants).size !== retainedParticipants.length ||
|
||||
participants.size !== retainedParticipants.length ||
|
||||
retainedParticipants.some((id) => typeof id !== 'string' || !participants.has(id))
|
||||
)
|
||||
throw new Error('Response resume participants changed');
|
||||
const decision = responseReviewDecisionSchema.parse(
|
||||
JSON.parse(admission.record.resume?.decisionJSON ?? 'null'),
|
||||
);
|
||||
if (decision.actionId !== action.actionId)
|
||||
throw new Error('Response reviewed action changed');
|
||||
const discovered = action.resumeContext?.discoveredToolNames;
|
||||
if (
|
||||
discovered != null &&
|
||||
(!Array.isArray(discovered) || discovered.some((name) => typeof name !== 'string'))
|
||||
)
|
||||
throw new Error('Response discovered tools are unavailable');
|
||||
if (
|
||||
decision.decisions?.some((item) => item.decision === 'approve' && item.scope === 'session')
|
||||
) {
|
||||
const { record, resume } = admission;
|
||||
const job = await continuation.jobs?.getJob(record.responseId);
|
||||
if (
|
||||
!continuation.addConvoToolApprovalAllows ||
|
||||
!enrolledExecution?.ownsSettlement ||
|
||||
enrolledExecution.createdAt !== record.generationCreatedAt ||
|
||||
!job ||
|
||||
job.status !== 'running' ||
|
||||
job.createdAt !== record.generationCreatedAt ||
|
||||
job.metadata.userId !== record.userId ||
|
||||
responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) ||
|
||||
job.metadata.agent_id !== record.agentId ||
|
||||
job.metadata.endpoint !== 'responses' ||
|
||||
job.metadata.conversationId !== record.conversationId ||
|
||||
job.metadata.responseMessageId !== record.responseId ||
|
||||
job.metadata.providerExecutionId !== resume.providerExecutionId ||
|
||||
job.metadata.providerExecutionStartedId !== resume.providerExecutionId ||
|
||||
!job.metadata.providerExecutionOwnerId
|
||||
)
|
||||
throw new Error('Response remembered approval ownership unavailable');
|
||||
const request = req as ServerRequest;
|
||||
const persisted = await recordToolApprovalAllows({
|
||||
userId: record.userId,
|
||||
conversationId: record.conversationId,
|
||||
policy: request.config?.endpoints?.agents?.toolApproval,
|
||||
pendingAction: action,
|
||||
resolutions: decision.decisions,
|
||||
agents: reviewedAgents,
|
||||
hookContext: {
|
||||
userId: record.userId,
|
||||
tenantId: responseTenantId(record.tenantId),
|
||||
conversationId: record.conversationId,
|
||||
appConfig: request.config,
|
||||
},
|
||||
pluginHookSource: continuation.pluginHookSource,
|
||||
request,
|
||||
addConvoToolApprovalAllows: continuation.addConvoToolApprovalAllows,
|
||||
});
|
||||
rememberedTools = [...new Set([...rememberedTools, ...persisted])];
|
||||
}
|
||||
return {
|
||||
hitlCapable,
|
||||
...policy(),
|
||||
sessionStartSource: 'resume',
|
||||
...(Array.isArray(discovered) ? { discoveredToolNames: discovered } : {}),
|
||||
reviewedToolApprovals: {
|
||||
bindings: action.toolApprovalBindings,
|
||||
decisions: decision.decisions ?? [],
|
||||
},
|
||||
};
|
||||
},
|
||||
async completeTurn(input, complete) {
|
||||
if (!admission || !interactive || !continuation.jobs) return complete();
|
||||
if (!input.run.getInterrupt()) return complete();
|
||||
const request = req as ServerRequest;
|
||||
const config = request.config?.endpoints?.agents;
|
||||
const agents = collectReachableAgents<ReviewAgentNode>(input.agents);
|
||||
const aliases = getRunMCPToolAliases(input.run);
|
||||
const partial = {
|
||||
...composeResponse(input.buildPartial()),
|
||||
status: 'in_progress' as const,
|
||||
background: true,
|
||||
store: true,
|
||||
};
|
||||
const paused = await pauseResponseForReview(
|
||||
{
|
||||
run: input.run,
|
||||
execution: input.execution,
|
||||
record: admission.record,
|
||||
partial,
|
||||
visibleOutputTokens: visibleOutputTokens(input.visibleOutputTokens),
|
||||
checkpointer: config?.checkpointer,
|
||||
},
|
||||
{
|
||||
storage: admission.storage,
|
||||
jobs: continuation.jobs,
|
||||
persistUnfinished: async (response) => {
|
||||
await input.persistence.saveConversation();
|
||||
await input.persistence.saveInput();
|
||||
await input.persistence.saveOutput(response);
|
||||
},
|
||||
buildAction: async (interrupt) => {
|
||||
const payload =
|
||||
interrupt.payload.type === 'tool_approval'
|
||||
? markToolApprovalAllowAlways(
|
||||
markNativeCodeToolApprovalRequests(interrupt.payload, agents),
|
||||
{
|
||||
policy: config?.toolApproval,
|
||||
agents,
|
||||
aliases,
|
||||
storedTools: resolveRunToolApprovalAllows(
|
||||
config?.toolApproval,
|
||||
request.resolvedConversation,
|
||||
admission.record.conversationId,
|
||||
),
|
||||
hookContext: {
|
||||
userId: admission.record.userId,
|
||||
tenantId: responseTenantId(admission.record.tenantId),
|
||||
conversationId: admission.record.conversationId,
|
||||
appConfig: request.config,
|
||||
},
|
||||
pluginHookSource: continuation.pluginHookSource,
|
||||
},
|
||||
)
|
||||
: interrupt.payload;
|
||||
return captureRunPendingAction({
|
||||
run: input.run,
|
||||
payload,
|
||||
agents,
|
||||
resumeAgentIds: responseParticipantIds(admission.record.agentId, agents),
|
||||
aliases,
|
||||
context: {
|
||||
streamId: admission.record.responseId,
|
||||
responseMessageId: admission.record.responseId,
|
||||
conversationId: admission.record.conversationId,
|
||||
threadId: admission.record.conversationId,
|
||||
runId: interrupt.checkpointNs ?? admission.record.responseId,
|
||||
interruptId: interrupt.interruptId,
|
||||
ttlMs: getApprovalTtlMs(config?.checkpointer),
|
||||
projectContextKey: getChatProjectContextKey(request.chatProjectContext),
|
||||
requestFingerprint: computeLegacyAgentRequestFingerprint(request.body),
|
||||
requestFingerprintV2: computeAgentRequestFingerprint(request.body),
|
||||
resumeContext: {
|
||||
...pickResumeContext(request.body, request.reasoningOverrideBase),
|
||||
discoveredToolNames: getRunDiscoveredTools(input.run),
|
||||
responseParentMessageId,
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
if (!paused && !input.execution.signal.aborted)
|
||||
throw new Error('Response interrupt could not be retained');
|
||||
return undefined;
|
||||
},
|
||||
graphMessages: (messages) => (admission?.resume ? [] : messages),
|
||||
async processRun(run, input, config, options) {
|
||||
if (!admission?.resume) {
|
||||
if (!admission || !interactive || !continuation.jobs)
|
||||
return run.processStream(input, config, options);
|
||||
const record = admission.record;
|
||||
const job = await continuation.jobs.getJob(record.responseId);
|
||||
const namespace = job?.metadata.checkpointNamespace;
|
||||
if (
|
||||
!job ||
|
||||
job.status !== 'running' ||
|
||||
job.createdAt !== record.generationCreatedAt ||
|
||||
job.metadata.userId !== record.userId ||
|
||||
responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) ||
|
||||
job.metadata.agent_id !== record.agentId ||
|
||||
job.metadata.endpoint !== 'responses' ||
|
||||
job.metadata.responseMessageId !== record.responseId ||
|
||||
job.metadata.conversationId !== record.conversationId ||
|
||||
!namespace?.startsWith(checkpointOwnerNamespacePrefix(record.userId, record.tenantId))
|
||||
)
|
||||
throw new Error('Response checkpoint enrollment identity unavailable');
|
||||
const settings = resolveCheckpointerConfig(
|
||||
(req as ServerRequest).config?.endpoints?.agents?.checkpointer,
|
||||
);
|
||||
if (settings.type !== 'mongo')
|
||||
throw new Error('Response interrupt requires durable checkpoint storage');
|
||||
checkpoint = {
|
||||
checkpointNamespace: namespace,
|
||||
checkpointConfigJSON: JSON.stringify({
|
||||
type: settings.type,
|
||||
ttl: settings.ttlSeconds,
|
||||
checkpointCollectionName: settings.checkpointCollectionName,
|
||||
checkpointWritesCollectionName: settings.checkpointWritesCollectionName,
|
||||
}),
|
||||
};
|
||||
if (!continuation.jobs?.updateMetadata)
|
||||
throw new Error('Response checkpoint cleanup persistence unavailable');
|
||||
await continuation.jobs.updateMetadata(
|
||||
record.responseId,
|
||||
{
|
||||
responseCheckpointConfigJSON: checkpoint.checkpointConfigJSON,
|
||||
},
|
||||
job.createdAt,
|
||||
);
|
||||
return run.processStream(
|
||||
input,
|
||||
{
|
||||
...config,
|
||||
configurable: {
|
||||
...config.configurable,
|
||||
thread_id: record.conversationId,
|
||||
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: namespace,
|
||||
},
|
||||
},
|
||||
options,
|
||||
);
|
||||
}
|
||||
const { record, resume } = admission;
|
||||
if (
|
||||
record.resume?.intentId !== resume.intentId ||
|
||||
!record.review?.checkpointId ||
|
||||
record.review.checkpointNs == null
|
||||
)
|
||||
throw new Error('Response resume checkpoint identity unavailable');
|
||||
const resolution: unknown = JSON.parse(record.resume.resolutionJSON);
|
||||
return run.resume(
|
||||
resolution,
|
||||
{
|
||||
...config,
|
||||
configurable: {
|
||||
...config.configurable,
|
||||
thread_id: record.conversationId,
|
||||
checkpoint_id: record.review.checkpointId,
|
||||
checkpoint_ns: record.review.checkpointNs,
|
||||
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: resume.checkpointNamespace,
|
||||
},
|
||||
},
|
||||
options,
|
||||
);
|
||||
},
|
||||
toolContext: (execution) =>
|
||||
admission
|
||||
? { streamId: admission.record.responseId, jobCreatedAt: execution.createdAt }
|
||||
: { streamId: null },
|
||||
responseId:
|
||||
admission?.record.responseId ?? (continuation.generateResponseId ?? generateResponseId)(),
|
||||
connection: admission
|
||||
? undefined
|
||||
: {
|
||||
isClosed: () => res.destroyed === true && res.writableEnded !== true,
|
||||
onClose: (listener: () => void) => {
|
||||
const close = () => {
|
||||
if (res.writableEnded !== true) listener();
|
||||
};
|
||||
res.once('close', close);
|
||||
return () => {
|
||||
res.off('close', close);
|
||||
};
|
||||
},
|
||||
},
|
||||
hooks,
|
||||
async resolveContinuation(envelope: ResponsesRunEnvelope) {
|
||||
if (!admission)
|
||||
return resolveResponseContinuation(
|
||||
envelope.payload.previous_response_id,
|
||||
envelope.principal.userId,
|
||||
continuation,
|
||||
);
|
||||
const record = admission.record;
|
||||
if (
|
||||
envelope.principal.userId !== record.userId ||
|
||||
responseTenantId(envelope.principal.tenantId) !== responseTenantId(record.tenantId) ||
|
||||
envelope.payload.model !== record.agentId
|
||||
)
|
||||
throw new Error('Background response principal changed');
|
||||
return { conversationId: record.conversationId, parentMessageId: record.parentMessageId };
|
||||
},
|
||||
};
|
||||
}
|
||||
224
packages/api/src/agents/responses/segments.spec.ts
Normal file
224
packages/api/src/agents/responses/segments.spec.ts
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
import type { Response, OutputItem, Usage } from './types';
|
||||
import {
|
||||
buildAggregatedResponse,
|
||||
createResponseAggregator,
|
||||
createAggregatorEventHandlers,
|
||||
} from './service';
|
||||
import { createResponseAccountingCollector, combineResponseAccounting } from './accounting';
|
||||
import { combineResponseSegments, seedResponseAggregator } from './segments';
|
||||
import { projectResponseMessage } from './persistence';
|
||||
|
||||
const usage = (count: number): Usage => ({
|
||||
input_tokens: count,
|
||||
output_tokens: count,
|
||||
total_tokens: count * 2,
|
||||
input_tokens_details: { cached_tokens: count },
|
||||
output_tokens_details: { reasoning_tokens: count },
|
||||
primary: { input_tokens: count, output_tokens: count, total_tokens: count * 2 },
|
||||
subagent: { input_tokens: 0, output_tokens: 0, total_tokens: 0 },
|
||||
});
|
||||
|
||||
function response(output: OutputItem[], count: number, amount: string): Response {
|
||||
const accounting = createResponseAccountingCollector({
|
||||
quote: () => ({ amount, basis: 'application_billing' }),
|
||||
});
|
||||
accounting.record({
|
||||
input_tokens: count,
|
||||
output_tokens: count,
|
||||
total_tokens: count * 2,
|
||||
model: 'actual-model',
|
||||
provider: 'openAI',
|
||||
});
|
||||
return {
|
||||
...buildAggregatedResponse(
|
||||
{ responseId: 'response', model: 'agent', createdAt: 1 },
|
||||
createResponseAggregator(),
|
||||
usage(count),
|
||||
accounting.snapshot(),
|
||||
),
|
||||
output,
|
||||
background: true,
|
||||
store: true,
|
||||
};
|
||||
}
|
||||
|
||||
const message = (id: string, text: string): OutputItem => ({
|
||||
type: 'message',
|
||||
id,
|
||||
role: 'assistant',
|
||||
status: 'completed',
|
||||
content: [{ type: 'output_text', text, annotations: [], logprobs: [] }],
|
||||
});
|
||||
const call = (id: string, callId: string): OutputItem => ({
|
||||
type: 'function_call',
|
||||
id,
|
||||
call_id: callId,
|
||||
name: 'lookup',
|
||||
arguments: '{"key":"original"}',
|
||||
status: 'completed',
|
||||
});
|
||||
const result = (id: string, callId: string, output: string): OutputItem => ({
|
||||
type: 'function_call_output',
|
||||
id,
|
||||
call_id: callId,
|
||||
output,
|
||||
status: 'completed',
|
||||
});
|
||||
|
||||
it('preserves the saved prefix and completes a paused tool exactly once without seeding billed usage', () => {
|
||||
const previous = response(
|
||||
[
|
||||
message('before', 'Before. '),
|
||||
call('done-call', 'done'),
|
||||
result('done-result', 'done', 'old result'),
|
||||
call('waiting-call', 'waiting'),
|
||||
],
|
||||
3,
|
||||
'0.1',
|
||||
);
|
||||
const original = JSON.stringify(previous);
|
||||
const aggregator = createResponseAggregator();
|
||||
seedResponseAggregator(aggregator, previous);
|
||||
expect(aggregator.usage).toEqual({
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
cachedTokens: 0,
|
||||
});
|
||||
expect(aggregator.getText()).toBe('');
|
||||
const handlers = createAggregatorEventHandlers(aggregator);
|
||||
handlers.on_run_step_completed.handle('on_run_step_completed', {
|
||||
result: {
|
||||
id: 'step',
|
||||
index: 0,
|
||||
type: 'tool_call',
|
||||
tool_call: {
|
||||
id: 'waiting',
|
||||
name: 'lookup',
|
||||
args: '{"key":"original"}',
|
||||
output: 'new result',
|
||||
progress: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
aggregator.addText('After.');
|
||||
const current = {
|
||||
...response([], 5, '0.2'),
|
||||
output: buildAggregatedResponse(
|
||||
{ responseId: 'response', model: 'agent', createdAt: 2 },
|
||||
aggregator,
|
||||
).output,
|
||||
};
|
||||
const combined = combineResponseSegments(previous, current, 'segment-two');
|
||||
expect(combined.created_at).toBe(1);
|
||||
expect(combined.usage).toEqual(usage(8));
|
||||
expect(combined.accounting).toMatchObject({
|
||||
status: 'available',
|
||||
amount: '0.3',
|
||||
coverage: 'complete',
|
||||
});
|
||||
expect(combined.output.filter((item) => item.type === 'function_call')).toHaveLength(2);
|
||||
expect(combined.output.filter((item) => item.type === 'function_call_output')).toEqual([
|
||||
result('done-result', 'done', 'old result'),
|
||||
expect.objectContaining({ call_id: 'waiting', output: 'new result' }),
|
||||
]);
|
||||
expect(combined.output[0]).toEqual(previous.output[0]);
|
||||
expect(projectResponseMessage(combined).text).toBe('Before. After.');
|
||||
expect(JSON.stringify(previous)).toBe(original);
|
||||
});
|
||||
|
||||
it('uses the cumulative snapshot for the next pause without duplicating earlier text, calls or usage', () => {
|
||||
const first = response([message('first', 'First. ')], 1, '0.1');
|
||||
const second = combineResponseSegments(
|
||||
first,
|
||||
response([message('second', 'Second. ')], 2, '0.2'),
|
||||
'two',
|
||||
);
|
||||
const third = combineResponseSegments(
|
||||
second,
|
||||
response([message('third', 'Third.')], 3, '0.3'),
|
||||
'three',
|
||||
);
|
||||
expect(projectResponseMessage(third).text).toBe('First. Second. Third.');
|
||||
expect(third.usage).toEqual(usage(6));
|
||||
expect(third.accounting).toMatchObject({ amount: '0.6' });
|
||||
expect(third.accounting?.calls).toHaveLength(3);
|
||||
expect(third.output.map((item) => item.id)).toEqual(['first', 'second', 'third']);
|
||||
});
|
||||
|
||||
it('keeps output IDs unique even when independent segments generate the same message ID', () => {
|
||||
const combined = combineResponseSegments(
|
||||
response([message('same', 'Before')], 1, '0.1'),
|
||||
response([message('same', 'After')], 1, '0.1'),
|
||||
'intent-two',
|
||||
);
|
||||
expect(combined.output.map((item) => item.id)).toEqual(['same', 'same_intent-two']);
|
||||
});
|
||||
|
||||
it('does not overwrite an already completed tool result with conflicting data', () => {
|
||||
const previous = response(
|
||||
[call('call', 'lookup'), result('result', 'lookup', 'original-private-result')],
|
||||
1,
|
||||
'0.1',
|
||||
);
|
||||
expect(() =>
|
||||
combineResponseSegments(
|
||||
previous,
|
||||
response([result('other', 'lookup', 'different-private-result')], 1, '0.1'),
|
||||
'two',
|
||||
),
|
||||
).toThrow('Response tool result changed across segments');
|
||||
});
|
||||
|
||||
it('preserves uncertainty when a previous segment has no usable usage or accounting', () => {
|
||||
const previous = { ...response([], 1, '0.1'), usage: null, accounting: undefined };
|
||||
const combined = combineResponseSegments(previous, response([], 1, '0.2'), 'two');
|
||||
expect(combined.usage).toBeNull();
|
||||
expect(combined.accounting).toMatchObject({
|
||||
status: 'unavailable',
|
||||
coverage: 'partial',
|
||||
reason: 'partial_coverage',
|
||||
});
|
||||
expect(combined.accounting).not.toHaveProperty('amount');
|
||||
expect(combined.accounting?.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('retains past quotes rather than consulting current pricing for old calls', () => {
|
||||
const prior = response([], 1, '0.1').accounting!;
|
||||
const quote = jest.fn(() => ({ amount: '0.9', basis: 'application_billing' as const }));
|
||||
const collector = createResponseAccountingCollector({ seed: prior, quote });
|
||||
collector.record({ input_tokens: 1, output_tokens: 1 });
|
||||
expect(quote).toHaveBeenCalledTimes(1);
|
||||
expect(collector.snapshot()).toMatchObject({ amount: '1' });
|
||||
expect(prior.amount).toBe('0.1');
|
||||
});
|
||||
|
||||
it('does not erase conflicting or incomplete provenance when combining accounting', () => {
|
||||
const priced = response([], 1, '0.1').accounting!;
|
||||
const collector = createResponseAccountingCollector();
|
||||
collector.record({ input_tokens: 1, output_tokens: 1 }, undefined, undefined, 'same-call');
|
||||
collector.record({ input_tokens: 2, output_tokens: 1 }, undefined, undefined, 'same-call');
|
||||
expect(combineResponseAccounting(priced, collector.snapshot())).toMatchObject({
|
||||
status: 'unavailable',
|
||||
reason: 'conflicting_usage',
|
||||
});
|
||||
expect(
|
||||
combineResponseAccounting(priced, createResponseAccountingCollector().snapshot()),
|
||||
).toMatchObject({
|
||||
status: 'unavailable',
|
||||
coverage: 'partial',
|
||||
reason: 'partial_coverage',
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects response identity changes and invalid or overflowing usage', () => {
|
||||
const prior = response([], 1, '0.1');
|
||||
expect(() => combineResponseSegments(prior, { ...prior, id: 'another-response' }, 'two')).toThrow(
|
||||
'identity changed',
|
||||
);
|
||||
const invalid = response([], 1, '0.1');
|
||||
invalid.usage!.output_tokens = Number.MAX_SAFE_INTEGER;
|
||||
expect(() => combineResponseSegments(prior, invalid, 'two')).toThrow(
|
||||
'Response segment usage is invalid',
|
||||
);
|
||||
});
|
||||
118
packages/api/src/agents/responses/segments.ts
Normal file
118
packages/api/src/agents/responses/segments.ts
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
import type { Response, Usage, UsageTotals, OutputItem } from './types';
|
||||
import type { ResponseAggregator } from './service';
|
||||
import { combineResponseAccounting } from './accounting';
|
||||
|
||||
/** Restore tool state only; prior text/reasoning and item IDs remain immutable in the saved prefix. */
|
||||
export function seedResponseAggregator(aggregator: ResponseAggregator, previous: Response): void {
|
||||
for (const item of previous.output) {
|
||||
if (item.type === 'function_call')
|
||||
aggregator.toolCalls.set(item.call_id, {
|
||||
id: item.call_id,
|
||||
name: item.name,
|
||||
arguments: item.arguments,
|
||||
});
|
||||
else if (item.type === 'function_call_output')
|
||||
aggregator.toolOutputs.set(item.call_id, item.output);
|
||||
}
|
||||
}
|
||||
|
||||
function add(left: number, right: number): number {
|
||||
const result = left + right;
|
||||
if (
|
||||
!Number.isSafeInteger(left) ||
|
||||
left < 0 ||
|
||||
!Number.isSafeInteger(right) ||
|
||||
right < 0 ||
|
||||
!Number.isSafeInteger(result)
|
||||
)
|
||||
throw new Error('Response segment usage is invalid');
|
||||
return result;
|
||||
}
|
||||
|
||||
function totals(left: UsageTotals, right: UsageTotals): UsageTotals {
|
||||
return {
|
||||
input_tokens: add(left.input_tokens, right.input_tokens),
|
||||
output_tokens: add(left.output_tokens, right.output_tokens),
|
||||
total_tokens: add(left.total_tokens, right.total_tokens),
|
||||
};
|
||||
}
|
||||
|
||||
function usage(left: Usage | null, right: Usage | null): Usage | null {
|
||||
if (!left || !right) return null;
|
||||
return {
|
||||
...totals(left, right),
|
||||
input_tokens_details: {
|
||||
cached_tokens: add(
|
||||
left.input_tokens_details.cached_tokens,
|
||||
right.input_tokens_details.cached_tokens,
|
||||
),
|
||||
},
|
||||
output_tokens_details: {
|
||||
reasoning_tokens: add(
|
||||
left.output_tokens_details.reasoning_tokens,
|
||||
right.output_tokens_details.reasoning_tokens,
|
||||
),
|
||||
},
|
||||
...(left.primary && right.primary ? { primary: totals(left.primary, right.primary) } : {}),
|
||||
...(left.subagent && right.subagent ? { subagent: totals(left.subagent, right.subagent) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function callKey(item: OutputItem): string | undefined {
|
||||
return item.type === 'function_call' || item.type === 'function_call_output'
|
||||
? `${item.type}:${item.call_id}`
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Compose a new provider segment once with its saved prefix; never send old usage back to billing. */
|
||||
export function combineResponseSegments(
|
||||
previous: Response,
|
||||
current: Response,
|
||||
segmentId: string,
|
||||
): Response {
|
||||
if (previous.id !== current.id || previous.model !== current.model || !segmentId)
|
||||
throw new Error('Response segment identity changed');
|
||||
const output = [...previous.output];
|
||||
const calls = new Map<string, number>();
|
||||
const ids = new Set<string>();
|
||||
for (let index = 0; index < output.length; index++) {
|
||||
const item = output[index];
|
||||
ids.add(item.id);
|
||||
const key = callKey(item);
|
||||
if (key) calls.set(key, index);
|
||||
}
|
||||
for (const item of current.output) {
|
||||
const key = callKey(item);
|
||||
const index = key == null ? undefined : calls.get(key);
|
||||
if (index != null) {
|
||||
const prior = output[index];
|
||||
if (
|
||||
prior.type === 'function_call_output' &&
|
||||
item.type === 'function_call_output' &&
|
||||
prior.output !== item.output
|
||||
)
|
||||
throw new Error('Response tool result changed across segments');
|
||||
output[index] = { ...item, id: prior.id };
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
item.type === 'message' &&
|
||||
item.role === 'assistant' &&
|
||||
item.content.length === 0 &&
|
||||
output.length > 0
|
||||
)
|
||||
continue;
|
||||
const id = ids.has(item.id) ? `${item.id}_${segmentId}` : item.id;
|
||||
if (ids.has(id)) throw new Error('Response segment item identity collided');
|
||||
if (key) calls.set(key, output.length);
|
||||
output.push({ ...item, id });
|
||||
ids.add(id);
|
||||
}
|
||||
return {
|
||||
...current,
|
||||
created_at: previous.created_at,
|
||||
output,
|
||||
usage: usage(previous.usage, current.usage),
|
||||
accounting: combineResponseAccounting(previous.accounting, current.accounting),
|
||||
};
|
||||
}
|
||||
1026
packages/api/src/agents/responses/worker.runtime.spec.ts
Normal file
1026
packages/api/src/agents/responses/worker.runtime.spec.ts
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -4,6 +4,7 @@ import { getTenantId, tenantStorage } from '@librechat/data-schemas';
|
|||
import type { IUser, ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas';
|
||||
import type { Request, RequestHandler } from 'express';
|
||||
import type { ResumedAgentExecutionIdentity } from '../remote/lifecycle';
|
||||
import type { TurnConversationRequest } from '~/conversations/save';
|
||||
import type { ResponseResumeAdmissionResult } from './types';
|
||||
import type { ContextRequest } from '~/middleware/tenant';
|
||||
import type { ServerRequest } from '~/types/http';
|
||||
|
|
@ -240,12 +241,19 @@ export function createBackgroundResponseWorkerAuth(
|
|||
checkpointNamespace: result.checkpointNamespace,
|
||||
};
|
||||
}
|
||||
const request = req as ServerRequest & Pick<ContextRequest, 'tenantId'>;
|
||||
const request = req as ServerRequest &
|
||||
Pick<ContextRequest, 'tenantId'> &
|
||||
TurnConversationRequest;
|
||||
// findUser supplies a lean auth DTO; do not mutate its cached tenant representation.
|
||||
request.user = Object.assign({}, user, { id: record.userId, tenantId: claims.tenantId });
|
||||
request.tenantId = claims.tenantId;
|
||||
// Caller body/headers cannot replace the persisted, already validated request.
|
||||
request.body = JSON.parse(record.requestJSON);
|
||||
if (record.expiredAt)
|
||||
request._agentEventBindingRetention = {
|
||||
expiredAt: record.expiredAt,
|
||||
isTemporary: record.isTemporary,
|
||||
};
|
||||
admitted.set(req, {
|
||||
record,
|
||||
storage: deps.storage,
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ export type {
|
|||
ResponseOwner,
|
||||
ResponseStorage,
|
||||
ResponseTerminalGuard,
|
||||
ResponseExecutionGuard,
|
||||
ResponseCleanupIdentity,
|
||||
ResponseReservation,
|
||||
ResponseReservationResult,
|
||||
|
|
|
|||
|
|
@ -1615,3 +1615,183 @@ it('matches the complete reviewed and resumed execution binding rather than its
|
|||
(await storage.stageTerminal(owner, 'resp_one', { execution: retained }, receipt)).ok,
|
||||
).toBe(true);
|
||||
}));
|
||||
|
||||
async function runningForReview() {
|
||||
await storage.reserve(reservation({ sourceKeyId: 'original-key' }));
|
||||
const running = await storage.publish(owner, 'resp_one', 0, {
|
||||
status: 'in_progress',
|
||||
generationCreatedAt: 10,
|
||||
responseJSON: '{}',
|
||||
});
|
||||
if (!running.ok) throw new Error('Publication failed');
|
||||
return running.record;
|
||||
}
|
||||
|
||||
it('publishes one immutable pause independently of polling before and after publication', () =>
|
||||
scoped(async () => {
|
||||
const running = await runningForReview();
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const current = (await storage.read(owner, 'resp_one'))!;
|
||||
expect(
|
||||
(
|
||||
await storage.publish(owner, 'resp_one', current.revision, {
|
||||
status: 'in_progress',
|
||||
responseJSON: JSON.stringify({ progress: i }),
|
||||
})
|
||||
).ok,
|
||||
).toBe(true);
|
||||
}
|
||||
expect(
|
||||
await storage.publishReview(owner, 'resp_one', running.revision, 10, review, '{}'),
|
||||
).toEqual({ ok: false, code: 'response_conflict' });
|
||||
const attempts = await Promise.all(
|
||||
Array.from({ length: 12 }, () =>
|
||||
storage.publishReview(owner, 'resp_one', { execution: running }, 10, review, '{}'),
|
||||
),
|
||||
);
|
||||
expect(attempts.every((result) => result.ok)).toBe(true);
|
||||
const paused = (await storage.read(owner, 'resp_one'))!;
|
||||
expect(paused.review).toEqual(review);
|
||||
expect(paused.revision).toBe(running.revision + 31);
|
||||
const projected = await storage.publish(owner, 'resp_one', paused.revision, {
|
||||
status: 'requires_action',
|
||||
responseJSON: '{"projection":"new"}',
|
||||
});
|
||||
if (!projected.ok) throw new Error('Projection failed');
|
||||
for (const execution of [running, projected.record]) {
|
||||
expect(
|
||||
(await storage.publishReview(owner, 'resp_one', { execution }, 10, review, '{}')).ok,
|
||||
).toBe(true);
|
||||
expect(
|
||||
await storage.publishReview(
|
||||
owner,
|
||||
'resp_one',
|
||||
{ execution },
|
||||
10,
|
||||
{ ...review, partialResponseJSON: '{"changed":true}' },
|
||||
'{}',
|
||||
),
|
||||
).toEqual({ ok: false, code: 'response_conflict' });
|
||||
}
|
||||
const saved = (await storage.read(owner, 'resp_one'))!;
|
||||
expect(saved.revision).toBe(projected.record.revision);
|
||||
expect(saved.responseJSON).toBe('{"projection":"new"}');
|
||||
expect(saved.review).toEqual(review);
|
||||
}));
|
||||
|
||||
it.each(['agentId', 'conversationId', 'sourceKeyId', 'generationCreatedAt'] as const)(
|
||||
'rejects a pause from a mismatched %s',
|
||||
(field) =>
|
||||
scoped(async () => {
|
||||
const running = await runningForReview();
|
||||
const execution = { ...running, [field]: field === 'generationCreatedAt' ? 11 : 'other' };
|
||||
expect(
|
||||
await storage.publishReview(owner, 'resp_one', { execution }, 10, review, '{}'),
|
||||
).toEqual({ ok: false, code: 'response_conflict' });
|
||||
expect((await storage.read(owner, 'resp_one'))?.review).toBeUndefined();
|
||||
}),
|
||||
);
|
||||
|
||||
it('binds the next pause to the complete admitted resume identity', () =>
|
||||
scoped(async () => {
|
||||
await pause();
|
||||
await storage.reserveResume(owner, 'resp_one', resume(), 100);
|
||||
await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 101);
|
||||
const retained = (await storage.read(owner, 'resp_one'))!;
|
||||
const nextReview = {
|
||||
...review,
|
||||
actionId: 'second-action',
|
||||
providerExecutionId: 'provider-two',
|
||||
};
|
||||
for (const execution of [
|
||||
{ ...retained, review: undefined },
|
||||
{ ...retained, review: { ...retained.review!, checkpointId: 'different' } },
|
||||
{ ...retained, resume: { ...retained.resume!, fingerprint: 'different' } },
|
||||
{ ...retained, resume: { ...retained.resume!, state: 'reserved' as const } },
|
||||
])
|
||||
expect(
|
||||
await storage.publishReview(owner, 'resp_one', { execution }, 10, nextReview, '{}'),
|
||||
).toEqual({ ok: false, code: 'response_conflict' });
|
||||
expect(
|
||||
(
|
||||
await storage.publishReview(
|
||||
owner,
|
||||
'resp_one',
|
||||
{ execution: retained },
|
||||
10,
|
||||
nextReview,
|
||||
'{}',
|
||||
)
|
||||
).ok,
|
||||
).toBe(true);
|
||||
expect(await storage.read(owner, 'resp_one')).toMatchObject({ review: nextReview });
|
||||
expect((await storage.read(owner, 'resp_one'))?.resume).toBeUndefined();
|
||||
}));
|
||||
|
||||
it.each(['cancelled', 'terminal', 'erased'] as const)(
|
||||
'does not publish a pause after %s authority wins',
|
||||
(failure) =>
|
||||
scoped(async () => {
|
||||
const running = await runningForReview();
|
||||
if (failure === 'cancelled') await storage.requestCancellation(owner, 'resp_one', Date.now());
|
||||
if (failure === 'terminal')
|
||||
await storage.stageTerminal(owner, 'resp_one', { execution: running }, outcome);
|
||||
if (failure === 'erased')
|
||||
await mongoose.models.StoredResponse.deleteOne({ responseId: 'resp_one' });
|
||||
expect(
|
||||
await storage.publishReview(owner, 'resp_one', { execution: running }, 10, review, '{}'),
|
||||
).toEqual({ ok: false, code: 'response_conflict' });
|
||||
expect((await storage.read(owner, 'resp_one'))?.review).toBeUndefined();
|
||||
}),
|
||||
);
|
||||
|
||||
it.each(['live', 'owner_deleted', 'conversation_deleted', 'expired'] as const)(
|
||||
'supplements an unpublished generation only under durable retirement proof: %s',
|
||||
(reason) =>
|
||||
scoped(async () => {
|
||||
await storage.reserve(reservation());
|
||||
const retained = {
|
||||
...owner,
|
||||
responseId: 'resp_one',
|
||||
agentId: 'agent-one',
|
||||
conversationId: 'conversation-one',
|
||||
generationCreatedAt: 42,
|
||||
checkpointNamespace: 'owned-checkpoint',
|
||||
checkpointConfigJSON: 'pinned-settings',
|
||||
};
|
||||
if (reason === 'expired')
|
||||
await createResponseRetention(mongoose)(
|
||||
owner,
|
||||
'conversation-one',
|
||||
new Date(Date.now() - 1000),
|
||||
);
|
||||
else if (reason !== 'live')
|
||||
await closeResponseScopes(
|
||||
mongoose,
|
||||
owner,
|
||||
reason === 'conversation_deleted' ? ['conversation-one'] : undefined,
|
||||
);
|
||||
const identity = await storage.readCleanupIdentity(owner, 'resp_one', retained);
|
||||
expect(identity).toMatchObject({
|
||||
generationCreatedAt: reason === 'live' ? undefined : 42,
|
||||
retirement: reason === 'live' ? undefined : reason,
|
||||
checkpointNamespace: reason === 'live' ? undefined : 'owned-checkpoint',
|
||||
});
|
||||
if (reason !== 'live') {
|
||||
const wrong = await storage.readCleanupIdentity(owner, 'resp_one', {
|
||||
...retained,
|
||||
agentId: 'wrong-agent',
|
||||
});
|
||||
expect(wrong?.generationCreatedAt).toBeUndefined();
|
||||
expect(wrong?.checkpointNamespace).toBeUndefined();
|
||||
await mongoose.models.StoredResponse.updateOne(
|
||||
{ responseId: 'resp_one' },
|
||||
{ $set: { generationCreatedAt: 43 } },
|
||||
);
|
||||
expect(await storage.readCleanupIdentity(owner, 'resp_one', retained)).toMatchObject({
|
||||
generationCreatedAt: 43,
|
||||
checkpointNamespace: undefined,
|
||||
});
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -77,8 +77,8 @@ export type ResponseResumeResult =
|
|||
| { ok: true; record: StoredResponseRecord; replayed: boolean }
|
||||
| { ok: false; code: 'resume_conflict' };
|
||||
|
||||
/** A drained execution's authority is independent of polling's projection revision. */
|
||||
export type ResponseTerminalGuard =
|
||||
/** Canonical execution writes are independent of polling's projection revision. */
|
||||
export type ResponseExecutionGuard =
|
||||
| number
|
||||
| {
|
||||
execution: Pick<
|
||||
|
|
@ -87,15 +87,13 @@ export type ResponseTerminalGuard =
|
|||
>;
|
||||
};
|
||||
|
||||
function terminalExecutionFilter(guard: Exclude<ResponseTerminalGuard, number>) {
|
||||
const { execution } = guard;
|
||||
const filter: Record<string, unknown> = {
|
||||
agentId: execution.agentId,
|
||||
sourceKeyId: execution.sourceKeyId ?? { $exists: false },
|
||||
conversationId: execution.conversationId,
|
||||
generationCreatedAt: execution.generationCreatedAt,
|
||||
};
|
||||
const review = execution.review;
|
||||
/** Compatibility name for existing terminal producers. */
|
||||
export type ResponseTerminalGuard = ResponseExecutionGuard;
|
||||
|
||||
function responseReviewFilter(
|
||||
review: ResponseReviewCheckpoint | undefined,
|
||||
): Record<string, unknown> {
|
||||
const filter: Record<string, unknown> = {};
|
||||
if (review) {
|
||||
const fields: Record<
|
||||
keyof ResponseReviewCheckpoint,
|
||||
|
|
@ -114,6 +112,18 @@ function terminalExecutionFilter(guard: Exclude<ResponseTerminalGuard, number>)
|
|||
for (const [key, value] of Object.entries(fields))
|
||||
filter[`review.${key}`] = value ?? { $exists: false };
|
||||
} else filter.review = { $exists: false };
|
||||
return filter;
|
||||
}
|
||||
|
||||
function responseExecutionFilter(guard: Exclude<ResponseExecutionGuard, number>) {
|
||||
const { execution } = guard;
|
||||
const filter: Record<string, unknown> = {
|
||||
agentId: execution.agentId,
|
||||
sourceKeyId: execution.sourceKeyId ?? { $exists: false },
|
||||
conversationId: execution.conversationId,
|
||||
generationCreatedAt: execution.generationCreatedAt,
|
||||
...responseReviewFilter(execution.review),
|
||||
};
|
||||
const resume = execution.resume;
|
||||
if (resume) {
|
||||
const fields: Record<
|
||||
|
|
@ -153,7 +163,7 @@ export interface ResponseStorage {
|
|||
publishReview(
|
||||
owner: ResponseOwner,
|
||||
responseId: string,
|
||||
revision: number,
|
||||
guard: ResponseExecutionGuard,
|
||||
generationCreatedAt: number,
|
||||
review: ResponseReviewCheckpoint,
|
||||
responseJSON: string,
|
||||
|
|
@ -249,7 +259,7 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
|
|||
guard.execution.generationCreatedAt !== receipt.generationCreatedAt
|
||||
)
|
||||
return { ok: false, code: 'response_conflict' };
|
||||
const execution = typeof guard === 'number' ? undefined : terminalExecutionFilter(guard);
|
||||
const execution = typeof guard === 'number' ? undefined : responseExecutionFilter(guard);
|
||||
const pendingTerminal: ResponseTerminalReceipt = {
|
||||
generationCreatedAt: receipt.generationCreatedAt,
|
||||
providerExecutionId: receipt.providerExecutionId,
|
||||
|
|
@ -329,13 +339,41 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
|
|||
.lean();
|
||||
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
|
||||
},
|
||||
async publishReview(owner, responseId, revision, generationCreatedAt, review, responseJSON) {
|
||||
async publishReview(owner, responseId, guard, generationCreatedAt, review, responseJSON) {
|
||||
if (typeof guard !== 'number' && guard.execution.generationCreatedAt !== generationCreatedAt)
|
||||
return { ok: false, code: 'response_conflict' };
|
||||
const execution = typeof guard === 'number' ? undefined : responseExecutionFilter(guard);
|
||||
const receiptFilter =
|
||||
typeof guard === 'number'
|
||||
? undefined
|
||||
: {
|
||||
...scope(owner),
|
||||
responseId,
|
||||
generationCreatedAt,
|
||||
agentId: guard.execution.agentId,
|
||||
sourceKeyId: guard.execution.sourceKeyId ?? { $exists: false },
|
||||
conversationId: guard.execution.conversationId,
|
||||
...responseReviewFilter(review),
|
||||
status: 'requires_action',
|
||||
resume: { $exists: false },
|
||||
pendingTerminal: { $exists: false },
|
||||
cancellationRequestedAt: { $exists: false },
|
||||
};
|
||||
if (
|
||||
typeof guard !== 'number' &&
|
||||
guard.execution.review?.actionId === review.actionId &&
|
||||
guard.execution.review.providerExecutionId === review.providerExecutionId &&
|
||||
guard.execution.resume == null
|
||||
) {
|
||||
const existing = await model.findOne(receiptFilter).lean();
|
||||
return existing ? { ok: true, record: existing } : { ok: false, code: 'response_conflict' };
|
||||
}
|
||||
const record = await model
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
...scope(owner),
|
||||
responseId,
|
||||
revision,
|
||||
...(execution ?? { revision: guard }),
|
||||
generationCreatedAt,
|
||||
status: { $in: ['in_progress', 'requires_action'] },
|
||||
pendingTerminal: { $exists: false },
|
||||
|
|
@ -381,7 +419,10 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
|
|||
{ new: true, runValidators: true },
|
||||
)
|
||||
.lean();
|
||||
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
|
||||
if (record) return { ok: true, record };
|
||||
if (!receiptFilter) return { ok: false, code: 'response_conflict' };
|
||||
const existing = await model.findOne(receiptFilter).lean();
|
||||
return existing ? { ok: true, record: existing } : { ok: false, code: 'response_conflict' };
|
||||
},
|
||||
async reserveResume(owner, responseId, input, now) {
|
||||
if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid resume reservation time');
|
||||
|
|
@ -599,7 +640,7 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
|
|||
.lean();
|
||||
// Retained data comes only from an internal job/receipt, never request input.
|
||||
// Validate it even when the row exists before using any fallback checkpoint fields.
|
||||
const matchingRetained =
|
||||
const retainedCandidate =
|
||||
retained &&
|
||||
retained.responseId === responseId &&
|
||||
retained.userId === owner.userId &&
|
||||
|
|
@ -607,10 +648,11 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
|
|||
(!record ||
|
||||
(retained.conversationId === record.conversationId &&
|
||||
retained.agentId === record.agentId &&
|
||||
retained.generationCreatedAt === record.generationCreatedAt))
|
||||
(record.generationCreatedAt == null ||
|
||||
retained.generationCreatedAt === record.generationCreatedAt)))
|
||||
? retained
|
||||
: undefined;
|
||||
const source = record ?? matchingRetained;
|
||||
const source = record ?? retainedCandidate;
|
||||
if (!source) return null;
|
||||
const now = new Date();
|
||||
const [ownerKey, conversationKey] = responseScopeKeys(owner, source.conversationId);
|
||||
|
|
@ -637,6 +679,13 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
|
|||
!(conversationAnchor?.expiredAt && conversationAnchor.expiredAt <= now)
|
||||
)
|
||||
return null;
|
||||
const matchingRetained =
|
||||
retainedCandidate &&
|
||||
(!record ||
|
||||
retainedCandidate.generationCreatedAt === record.generationCreatedAt ||
|
||||
(retirement && record.generationCreatedAt == null))
|
||||
? retainedCandidate
|
||||
: undefined;
|
||||
// A staged checkpoint is one binding: never fill an incomplete terminal
|
||||
// binding with settings from a prior review or retained provider segment.
|
||||
const terminalCheckpoint = record?.pendingTerminal;
|
||||
|
|
@ -663,7 +712,8 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
|
|||
responseId: id,
|
||||
agentId,
|
||||
conversationId,
|
||||
generationCreatedAt,
|
||||
generationCreatedAt:
|
||||
generationCreatedAt ?? (retirement ? matchingRetained?.generationCreatedAt : undefined),
|
||||
expiredAt,
|
||||
retirement,
|
||||
checkpointNamespace,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue