mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 21:08:17 +00:00
* 🐢 fix: Back Off Waiting Completion Wake-ups and Deliver Them When Ready * fix: Signal every readiness path and make the completion wait cap configurable * fix: Mark held completion deliveries instead of pipeline expedite, and announce store-won approval expiry * fix: Scope settle expedites to the resumed conversation and close the remaining signal gaps * fix: Announce a won approval expiry once and release subagent wake-up registrations * 🧪 ci: Count System Instructions in Mock Model Usage (#16350) The e2e fake model reported input tokens for the chat messages alone, because the SDK hands a test override model the pruned messages without the systemRunnable pipe. Once #16301 gave every memory-enabled chat a system prompt, the calibrated context snapshot put used tokens below the instructions and the gauge dropped its Messages row. Count input over the complete prompt, as a real provider bills it. --------- Co-authored-by: Lia <lia@librechat.ai>
3540 lines
133 KiB
TypeScript
3540 lines
133 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { InMemorySubagentTaskStore } from '@librechat/agents';
|
|
import { logger, tenantStorage } from '@librechat/data-schemas';
|
|
import { EModelEndpoint, Constants } from 'librechat-data-provider';
|
|
import {
|
|
mapChatMessagesToStoredMessages,
|
|
mapStoredMessagesToChatMessages,
|
|
} from '@librechat/agents/langchain/messages';
|
|
import type {
|
|
InMemorySubagentTaskStoreOptions,
|
|
SubagentTaskClaim,
|
|
SubagentTaskControlCommand,
|
|
SubagentTaskControlResult,
|
|
SubagentTaskRuntime,
|
|
SubagentTaskSnapshot,
|
|
SubagentTaskStartRequest,
|
|
SubagentTaskStartResult,
|
|
SubagentTaskStore,
|
|
SubagentUpdateEvent,
|
|
} from '@librechat/agents';
|
|
import type {
|
|
AllMethods,
|
|
IActiveSubagentThreadLease,
|
|
IConversation,
|
|
ISubagentTaskControlReceipt,
|
|
IMessage,
|
|
MessageMethods,
|
|
ConversationMethods,
|
|
SubagentTaskResultClaim,
|
|
} from '@librechat/data-schemas';
|
|
import type { BaseMessage, StoredMessage } from '@librechat/agents/langchain/messages';
|
|
import type {
|
|
SubagentActivityUpdateEvent,
|
|
SubagentActivitySubscriber,
|
|
SubagentActivitySubscription,
|
|
SubagentActivityTerminalStatus,
|
|
} from './subagentActivity';
|
|
import type { SubagentTaskControlTransport } from './subagentTaskRouting';
|
|
import type { UsageMetadata } from '~/stream/interfaces/IJobStore';
|
|
import type { HostSubagentTaskConfig } from './subagentDelivery';
|
|
import {
|
|
boundedClaim,
|
|
boundedTaskList,
|
|
controlFingerprint,
|
|
SubagentTaskOwnerUnavailableError,
|
|
} from './subagentTaskRouting';
|
|
import { boundSubagentActivityUpdate, SubagentActivityStream } from './subagentActivity';
|
|
import { createSubagentAttemptKey, createSubagentThreadId } from './subagentThreadIds';
|
|
import { runWithDetachedSubagentUsage } from './subagentTaskContext';
|
|
import { SUBAGENT_COMPLETION_DELIVERY } from './subagentDelivery';
|
|
import { createConcurrencyLimiter } from '~/utils/promise';
|
|
import { projectSubagentActivity } from './activity';
|
|
import { InMemoryEventTransport } from '~/stream';
|
|
import { aggregateEmittedUsage } from './usage';
|
|
|
|
const SCOPE_VERSION = 1;
|
|
const DEFAULT_MAX_THREAD_DEPTH = 1;
|
|
const DEFAULT_LEASE_TTL_MS = 30_000;
|
|
const DEFAULT_LEASE_HEARTBEAT_MS = 10_000;
|
|
const DEFAULT_OWNER_DRAIN_TIMEOUT_MS = 45_000;
|
|
/** Keeps the admission fence alive across the deletion that follows the drain. */
|
|
const OWNER_FENCE_GRACE_MS = 5 * 60_000;
|
|
const DEFAULT_OWNER_DRAIN_POLL_MS = 100;
|
|
/** Matches the deletion drain batch so cancellation cannot burst Redis. */
|
|
const DELETION_CANCEL_CONCURRENCY = 32;
|
|
/** Bounds retained control invocations; one entry per applied command. */
|
|
const MAX_CONTROL_INVOCATIONS = 4_096;
|
|
/** Terminal controls are side-effect free, but retaining one bounded window
|
|
* prevents duplicate storage writers while preserving recent retry replay. */
|
|
const MAX_TERMINAL_CONTROL_INVOCATIONS = 64;
|
|
/** Keep same-task durable reservations below the storage CAS retry bound. Receipt
|
|
* finalization is serialized per task separately, leaving ample collision headroom. */
|
|
const CONTROL_RESERVATION_CONCURRENCY = 32;
|
|
const MAX_DURABLE_CONTROL_MESSAGE_CHARS = 4 * 1024;
|
|
const DEFAULT_CONTROL_RECEIPT_RETRY_MS = 5_000;
|
|
const SHUTDOWN_CONTROL_RECEIPT_FLUSH_ATTEMPTS = 4;
|
|
const DEFAULT_SHUTDOWN_CONTROL_RECEIPT_BACKOFF_MS = 1_000;
|
|
/** Bounds retained live-only updates while an event transport is unavailable. */
|
|
const MAX_PENDING_ACTIVITY_EVENTS = 32;
|
|
/** Live activity must never delay terminal notification indefinitely. */
|
|
const ACTIVITY_PUBLICATION_TIMEOUT_MS = 1_000;
|
|
|
|
/** A cancellation target set resolved before the conversations are removed. */
|
|
export interface SubagentCancellationPlan {
|
|
userId: string;
|
|
tenantId?: string;
|
|
conversationIds: string[];
|
|
scopes: Array<{ scopeId: string; threadIds: string[] | null }>;
|
|
leases: IActiveSubagentThreadLease[];
|
|
}
|
|
/** Three missed 10-second transport heartbeats retire a crashed owner. */
|
|
const DEFAULT_TASK_ROUTING_TTL_MS = 30_000;
|
|
const SLOW_PREPARATION_WARN_MS = 5_000;
|
|
const MAX_TRANSCRIPT_BYTES = 12 * 1024 * 1024;
|
|
const TRANSCRIPT_SELECT =
|
|
'messageId parentMessageId text createdAt +subagentTranscript +subagentTask';
|
|
const DURABLE_RESULT_SELECT =
|
|
'messageId conversationId sender text createdAt updatedAt +subagentTask';
|
|
|
|
class SubagentThreadPublicError extends Error {}
|
|
class SubagentThreadDeletedError extends SubagentThreadPublicError {}
|
|
class SubagentControlReceiptConflictError extends Error {}
|
|
|
|
type SubagentThreadMethods = Pick<
|
|
AllMethods,
|
|
| 'acquireSubagentThreadLease'
|
|
| 'claimSubagentTaskResult'
|
|
| 'countActiveSubagentThreadLeases'
|
|
| 'deleteConvos'
|
|
| 'deleteMessages'
|
|
| 'getConvo'
|
|
| 'getSubagentTaskControlReplay'
|
|
| 'getMessages'
|
|
| 'listActiveSubagentThreadLeases'
|
|
| 'reserveSubagentThread'
|
|
| 'releaseSubagentThreadLease'
|
|
| 'recordSubagentTaskControlReceipt'
|
|
| 'renewSubagentThreadLease'
|
|
| 'saveConvo'
|
|
| 'saveMessage'
|
|
>;
|
|
|
|
interface SubagentThreadScope {
|
|
version: typeof SCOPE_VERSION;
|
|
userId: string;
|
|
parentConversationId: string;
|
|
tenantId?: string;
|
|
}
|
|
|
|
interface PreparedThread {
|
|
conversation: IConversation;
|
|
initialMessages: BaseMessage[];
|
|
initialStoredMessages: StoredMessage[];
|
|
attemptKey: string;
|
|
/** Stable source-occurrence time shared by first delivery and every replay. */
|
|
taskCreatedAt: number;
|
|
userMessageId?: string;
|
|
replay?: {
|
|
status: 'completed' | 'error' | 'cancelled';
|
|
content: string;
|
|
taskId: string;
|
|
parentRunId: string;
|
|
/** Original running task closed by a retry under a new terminal message id. */
|
|
previousTaskId?: string;
|
|
};
|
|
}
|
|
|
|
interface HostSubagentTaskStartRequest extends SubagentTaskStartRequest {
|
|
completionDelivery?: typeof SUBAGENT_COMPLETION_DELIVERY;
|
|
}
|
|
|
|
type ThreadMessage = Pick<
|
|
IMessage,
|
|
'messageId' | 'parentMessageId' | 'text' | 'createdAt' | 'subagentTranscript' | 'subagentTask'
|
|
>;
|
|
|
|
type SdkControlReceipt = {
|
|
controlId: string;
|
|
action: 'steer' | 'queue' | 'interrupt';
|
|
status: 'accepted' | 'applied' | 'rejected' | 'failed';
|
|
createdAt: number;
|
|
updatedAt: number;
|
|
boundary?: 'preempt' | 'tool' | 'turn';
|
|
reason?: 'withdrawn' | 'task_completed' | 'task_cancelled' | 'task_failed';
|
|
};
|
|
|
|
type SnapshotWithControlReceipts = SubagentTaskSnapshot & {
|
|
controlReceipts?: SdkControlReceipt[];
|
|
};
|
|
|
|
type ControlInvocationRecord = {
|
|
scopeId: string;
|
|
taskId: string;
|
|
invocationId: string;
|
|
fingerprint: string;
|
|
command: SubagentTaskControlCommand;
|
|
commandMessageTruncated: boolean;
|
|
result: SubagentTaskControlResult;
|
|
createdAt: number;
|
|
/** Last authoritative SDK transition, retained for idempotent retries even
|
|
* after the bounded SDK snapshot evicts older receipt history. */
|
|
receipt?: ISubagentTaskControlReceipt;
|
|
/** True only after this invocation's current receipt is durable and therefore
|
|
* safe to evict from the bounded process-local replay window. */
|
|
receiptPersisted?: boolean;
|
|
/** The current durable write, shared by same-invocation retries so a caller
|
|
* cannot observe success before the authoritative receipt is committed. */
|
|
receiptPersistence?: Promise<void>;
|
|
};
|
|
|
|
const hasDurableControlReceipt = (invocation: ControlInvocationRecord): boolean =>
|
|
invocation.receiptPersisted === true;
|
|
|
|
interface TaskThreadLease {
|
|
scopeId: string;
|
|
idempotencyKey: string;
|
|
taskId: string;
|
|
running: boolean;
|
|
settling: boolean;
|
|
/** Resolves only after child persistence and lease cleanup finish. */
|
|
execution?: Promise<void>;
|
|
/** Ordered observational tail; canonical child settlement never awaits it. */
|
|
activityTail?: Promise<void>;
|
|
activityPending?: number;
|
|
/** Terminal settlement stops new admission but must not discard admitted events. */
|
|
activityAdmissionClosed?: boolean;
|
|
/** A failed observational publication suppresses the remainder of this task's queue. */
|
|
activityCircuitOpen?: boolean;
|
|
shared?: {
|
|
token: string;
|
|
lost: boolean;
|
|
/** Epoch ms this lease is durable until, advanced only by a confirmed renewal. */
|
|
expiresAt: number;
|
|
heartbeat?: ReturnType<typeof setInterval>;
|
|
heartbeatInFlight?: Promise<void>;
|
|
};
|
|
}
|
|
|
|
class SubagentActivityPublicationTimeoutError extends Error {
|
|
constructor() {
|
|
super('Subagent activity publication timed out.');
|
|
this.name = 'SubagentActivityPublicationTimeoutError';
|
|
}
|
|
}
|
|
|
|
async function settleActivityWithin(operation: Promise<void>): Promise<void> {
|
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
try {
|
|
await Promise.race([
|
|
operation,
|
|
new Promise<void>((_, reject) => {
|
|
timeout = setTimeout(
|
|
() => reject(new SubagentActivityPublicationTimeoutError()),
|
|
ACTIVITY_PUBLICATION_TIMEOUT_MS,
|
|
);
|
|
timeout.unref?.();
|
|
}),
|
|
]);
|
|
} finally {
|
|
if (timeout != null) clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
export interface SubagentThreadTaskStoreOptions extends InMemorySubagentTaskStoreOptions {
|
|
maxThreadDepth?: number;
|
|
leaseTtlMs?: number;
|
|
leaseHeartbeatMs?: number;
|
|
ownerDrainTimeoutMs?: number;
|
|
ownerDrainPollMs?: number;
|
|
taskRoutingTtlMs?: number;
|
|
isOwnerActive?: (userId: string) => Promise<boolean>;
|
|
maxControlInvocations?: number;
|
|
controlReceiptRetryMs?: number;
|
|
shutdownControlReceiptBackoffMs?: number;
|
|
ownerFenceGraceMs?: number;
|
|
fenceOwnerAdmission?: (userId: string, token: string, fencedUntil: Date) => Promise<void>;
|
|
renewOwnerAdmission?: (userId: string, token: string, fencedUntil: Date) => Promise<boolean>;
|
|
releaseOwnerAdmission?: (userId: string, token: string) => Promise<void>;
|
|
/** Host-owned work may share the durable child lease protocol without living in
|
|
* this in-memory task store. Return true only after that work is stopped. */
|
|
cancelUnroutedTask?: (target: {
|
|
userId: string;
|
|
parentConversationId: string;
|
|
taskId: string;
|
|
tenantId?: string;
|
|
}) => Promise<boolean>;
|
|
onTaskPrepared?: (
|
|
registration: SubagentTaskWakeupRegistration,
|
|
) => Promise<boolean | void> | boolean | void;
|
|
/** Called once a child's terminal message is durable, with the parent conversation its
|
|
* completion wake-up resumes. The child generation settles before that write, so the
|
|
* wake-up is ready only from here. Task ids include the original abandoned
|
|
* attempt when a retry persisted its result under a new id. */
|
|
onTaskSettled?: (userId: string, parentConversationId: string, taskIds: string[]) => void;
|
|
}
|
|
|
|
export interface SubagentTaskWakeupRegistration {
|
|
userId: string;
|
|
parentConversationId: string;
|
|
parentMessageId: string;
|
|
parentAgentId?: string;
|
|
tenantId?: string;
|
|
taskId: string;
|
|
threadId: string;
|
|
subagentType: string;
|
|
createdAt: number;
|
|
}
|
|
|
|
function positiveInteger(value: number | undefined, fallback: number): number {
|
|
return Number.isSafeInteger(value) && value != null && value > 0 ? value : fallback;
|
|
}
|
|
|
|
function isNonEmptyString(value: unknown): value is string {
|
|
return typeof value === 'string' && value.trim() !== '';
|
|
}
|
|
|
|
function normalizedRequestFingerprint(request: SubagentTaskStartRequest): string | undefined {
|
|
const fingerprint = request.requestFingerprint?.trim();
|
|
return fingerprint == null || fingerprint === '' ? undefined : fingerprint;
|
|
}
|
|
|
|
function parseScope(scopeId: string): SubagentThreadScope {
|
|
let parsed: unknown;
|
|
try {
|
|
parsed = JSON.parse(scopeId);
|
|
} catch {
|
|
throw new Error('Invalid subagent thread scope.');
|
|
}
|
|
if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
throw new Error('Invalid subagent thread scope.');
|
|
}
|
|
const candidate = parsed as Partial<SubagentThreadScope>;
|
|
if (
|
|
candidate.version !== SCOPE_VERSION ||
|
|
!isNonEmptyString(candidate.userId) ||
|
|
!isNonEmptyString(candidate.parentConversationId) ||
|
|
(candidate.tenantId != null && !isNonEmptyString(candidate.tenantId))
|
|
) {
|
|
throw new Error('Invalid subagent thread scope.');
|
|
}
|
|
return {
|
|
version: SCOPE_VERSION,
|
|
userId: candidate.userId,
|
|
parentConversationId: candidate.parentConversationId,
|
|
...(candidate.tenantId == null ? {} : { tenantId: candidate.tenantId }),
|
|
};
|
|
}
|
|
|
|
function serializeScope(scope: Omit<SubagentThreadScope, 'version'>): string {
|
|
return JSON.stringify({ version: SCOPE_VERSION, ...scope });
|
|
}
|
|
|
|
/** Builds the trusted live-owner routing scope after parent authorization. */
|
|
export function createSubagentThreadScopeId(scope: Omit<SubagentThreadScope, 'version'>): string {
|
|
return serializeScope(scope);
|
|
}
|
|
|
|
function matchesTenant(actual: string | undefined, expected: string | undefined): boolean {
|
|
return actual === expected;
|
|
}
|
|
|
|
function durableMessageTime(message: Pick<IMessage, 'createdAt'>, missingMessage: string): number {
|
|
const value = message.createdAt?.getTime();
|
|
if (!Number.isSafeInteger(value) || value == null || value < 0) {
|
|
throw new Error(missingMessage);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function assertParentPersistence(
|
|
value: unknown,
|
|
scope: SubagentThreadScope,
|
|
): asserts value is { message: { messageId: string; conversationId: string } } {
|
|
const message =
|
|
value != null && typeof value === 'object'
|
|
? (value as { message?: unknown }).message
|
|
: undefined;
|
|
if (
|
|
message == null ||
|
|
typeof message !== 'object' ||
|
|
!isNonEmptyString((message as { messageId?: unknown }).messageId) ||
|
|
(message as { conversationId?: unknown }).conversationId !== scope.parentConversationId
|
|
) {
|
|
throw new Error('The parent message was not persisted.');
|
|
}
|
|
}
|
|
|
|
function selectLatestBranch(messages: ThreadMessage[]): ThreadMessage[] {
|
|
const byId = new Map(messages.map((message) => [message.messageId, message]));
|
|
const branch: ThreadMessage[] = [];
|
|
const seen = new Set<string>();
|
|
let current: ThreadMessage | undefined = messages[messages.length - 1];
|
|
while (current != null && !seen.has(current.messageId)) {
|
|
branch.push(current);
|
|
seen.add(current.messageId);
|
|
const parentId: string | undefined = current.parentMessageId ?? undefined;
|
|
current =
|
|
parentId == null || parentId === '' || parentId === Constants.NO_PARENT
|
|
? undefined
|
|
: byId.get(parentId);
|
|
}
|
|
return branch.reverse();
|
|
}
|
|
|
|
function parseStoredMessages(value: string): StoredMessage[] {
|
|
const parsed = JSON.parse(value) as unknown;
|
|
if (!Array.isArray(parsed)) {
|
|
throw new SubagentThreadPublicError('Invalid persisted subagent transcript.');
|
|
}
|
|
for (const message of parsed) {
|
|
if (
|
|
message == null ||
|
|
typeof message !== 'object' ||
|
|
Array.isArray(message) ||
|
|
!isNonEmptyString((message as { type?: unknown }).type) ||
|
|
(message as { data?: unknown }).data == null ||
|
|
typeof (message as { data?: unknown }).data !== 'object' ||
|
|
Array.isArray((message as { data?: unknown }).data)
|
|
) {
|
|
throw new SubagentThreadPublicError('Invalid persisted subagent transcript.');
|
|
}
|
|
}
|
|
return parsed as StoredMessage[];
|
|
}
|
|
|
|
function restoreThreadMessages(branch: ThreadMessage[]): BaseMessage[] {
|
|
let storedMessages: StoredMessage[] = [];
|
|
for (const message of branch) {
|
|
const transcript = message.subagentTranscript;
|
|
if (transcript == null) {
|
|
continue;
|
|
}
|
|
const segment = parseStoredMessages(transcript.messagesJson);
|
|
if (transcript.mode === 'replace') {
|
|
storedMessages = segment;
|
|
} else {
|
|
storedMessages.push(...segment);
|
|
}
|
|
}
|
|
return mapStoredMessagesToChatMessages(storedMessages);
|
|
}
|
|
|
|
function isStoredPrefix(prefix: StoredMessage[], messages: StoredMessage[]): boolean {
|
|
if (prefix.length > messages.length) {
|
|
return false;
|
|
}
|
|
for (let index = 0; index < prefix.length; index += 1) {
|
|
if (JSON.stringify(prefix[index]) !== JSON.stringify(messages[index])) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
function serializeTranscript(
|
|
taskId: string,
|
|
initialMessages: StoredMessage[],
|
|
resultMessages: BaseMessage[] | undefined,
|
|
): IMessage['subagentTranscript'] {
|
|
if (resultMessages == null) {
|
|
return undefined;
|
|
}
|
|
const storedResult = mapChatMessagesToStoredMessages(resultMessages);
|
|
const storedResultJson = JSON.stringify(storedResult);
|
|
if (Buffer.byteLength(storedResultJson, 'utf8') > MAX_TRANSCRIPT_BYTES) {
|
|
throw new SubagentThreadPublicError(
|
|
'Subagent thread transcript is too large to persist safely.',
|
|
);
|
|
}
|
|
const append = isStoredPrefix(initialMessages, storedResult);
|
|
const messages = append ? storedResult.slice(initialMessages.length) : storedResult;
|
|
if (messages.length === 0) {
|
|
return undefined;
|
|
}
|
|
const messagesJson = append ? JSON.stringify(messages) : storedResultJson;
|
|
return {
|
|
taskId,
|
|
mode: append ? 'append' : 'replace',
|
|
messagesJson,
|
|
};
|
|
}
|
|
|
|
function retentionContext(conversation: IConversation): {
|
|
isTemporary?: boolean;
|
|
expiredAt?: Date;
|
|
} {
|
|
return {
|
|
...(conversation.isTemporary == null ? {} : { isTemporary: conversation.isTemporary }),
|
|
...(conversation.expiredAt == null ? {} : { expiredAt: conversation.expiredAt }),
|
|
};
|
|
}
|
|
|
|
function retentionFields(conversation: IConversation): {
|
|
isTemporary?: boolean;
|
|
expiredAt?: Date;
|
|
tenantId?: string;
|
|
} {
|
|
return {
|
|
...retentionContext(conversation),
|
|
...(conversation.tenantId == null ? {} : { tenantId: conversation.tenantId }),
|
|
};
|
|
}
|
|
|
|
function childAgentId(request: SubagentTaskStartRequest): string | undefined {
|
|
if (request.subagentKind === 'graph') {
|
|
return undefined;
|
|
}
|
|
return request.subagentType === 'self' ? request.parentAgentId : request.subagentType;
|
|
}
|
|
|
|
function publicFailureDetail(error: unknown): string {
|
|
return error instanceof SubagentThreadPublicError
|
|
? error.message.slice(0, 2_000)
|
|
: 'The child run could not be completed.';
|
|
}
|
|
|
|
/** Rebuilds the terminal claim a recovered durable result stands for. */
|
|
function recoveredClaim(
|
|
message: IMessage,
|
|
claim: Extract<SubagentTaskClaim, { status: 'claimed' }>,
|
|
): SubagentTaskClaim | undefined {
|
|
const status = message.subagentTask?.status;
|
|
const content = message.text ?? '';
|
|
/** A durable child message keeps the untruncated output, so recovering one applies
|
|
* the same bounds a routed response would have. */
|
|
if (status === 'completed') {
|
|
return boundedClaim({ status: 'completed', task: claim.task, result: content });
|
|
}
|
|
if (status === 'error' || status === 'cancelled') {
|
|
return boundedClaim({ status, task: claim.task, error: content });
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function drainKey(parentConversationId: string, taskId: string): string {
|
|
return `${parentConversationId}\u0000${taskId}`;
|
|
}
|
|
|
|
function controlTaskKey(scopeId: string, taskId: string): string {
|
|
return `${scopeId}\u0000${taskId}`;
|
|
}
|
|
|
|
function parseControlTaskKey(key: string): { scopeId: string; taskId: string } | undefined {
|
|
const separator = key.lastIndexOf('\u0000');
|
|
if (separator < 0 || separator === key.length - 1) return undefined;
|
|
return { scopeId: key.slice(0, separator), taskId: key.slice(separator + 1) };
|
|
}
|
|
|
|
function controlReceiptKey(scopeId: string, taskId: string, controlId: string): string {
|
|
return `${scopeId}\u0000${taskId}\u0000${controlId}`;
|
|
}
|
|
|
|
function boundedControlMessage(
|
|
command: SubagentTaskControlCommand,
|
|
alreadyTruncated = false,
|
|
): {
|
|
message?: string;
|
|
messageTruncated?: boolean;
|
|
} {
|
|
if (!('message' in command)) return {};
|
|
if (command.message.length <= MAX_DURABLE_CONTROL_MESSAGE_CHARS) {
|
|
return {
|
|
message: command.message,
|
|
...(alreadyTruncated ? { messageTruncated: true } : {}),
|
|
};
|
|
}
|
|
return {
|
|
message: command.message.slice(0, MAX_DURABLE_CONTROL_MESSAGE_CHARS),
|
|
messageTruncated: true,
|
|
};
|
|
}
|
|
|
|
function boundedControlCommand(command: SubagentTaskControlCommand): SubagentTaskControlCommand {
|
|
if (!('message' in command)) return command;
|
|
return {
|
|
action: command.action,
|
|
message: command.message.slice(0, MAX_DURABLE_CONTROL_MESSAGE_CHARS),
|
|
};
|
|
}
|
|
|
|
function safeErrorMessage(error: unknown): string {
|
|
return `Subagent task failed: ${publicFailureDetail(error).slice(0, 2_000)}`;
|
|
}
|
|
|
|
async function observeSlowPreparation<T>(
|
|
operation: Promise<T>,
|
|
context: { stage: string; taskId: string; threadId: string },
|
|
): Promise<T> {
|
|
const warning = setTimeout(() => {
|
|
logger.warn('[subagentThreads] Child-thread preparation is still waiting', context);
|
|
}, SLOW_PREPARATION_WARN_MS);
|
|
try {
|
|
return await operation;
|
|
} finally {
|
|
clearTimeout(warning);
|
|
}
|
|
}
|
|
|
|
/** Persists view-only logical child threads with owner-routed controls and a shared execution fence. */
|
|
export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
|
|
readonly supportsThreadContinuation = true;
|
|
private readonly activeThreads = new Map<string, TaskThreadLease>();
|
|
private readonly controlInvocations = new Map<string, ControlInvocationRecord>();
|
|
private readonly terminalControlInvocations = new Map<string, ControlInvocationRecord>();
|
|
|
|
private readonly controlInvocationByReceipt = new Map<string, ControlInvocationRecord>();
|
|
private readonly pendingControlReceipts = new Map<
|
|
string,
|
|
Map<string, { threadId: string; receipt: ISubagentTaskControlReceipt }>
|
|
>();
|
|
|
|
private readonly controlPersistenceTails = new Map<string, Promise<void>>();
|
|
private readonly controlPersistenceRetryTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
private readonly controlReservationSlot = createConcurrencyLimiter(
|
|
CONTROL_RESERVATION_CONCURRENCY,
|
|
);
|
|
|
|
private controlPersistenceStopping = false;
|
|
private controlCommandAdmissionClosed = false;
|
|
|
|
private readonly parentPersistence = new Map<string, Promise<unknown>>();
|
|
private readonly maxThreadDepth: number;
|
|
private readonly leaseTtlMs: number;
|
|
private readonly leaseHeartbeatMs: number;
|
|
private readonly ownerDrainTimeoutMs: number;
|
|
private readonly ownerDrainPollMs: number;
|
|
private readonly taskRoutingTtlMs: number;
|
|
private readonly maxControlInvocations: number;
|
|
private readonly controlReceiptRetryMs: number;
|
|
private readonly shutdownControlReceiptBackoffMs: number;
|
|
private readonly ownerFenceGraceMs: number;
|
|
private readonly isOwnerActive: (userId: string) => Promise<boolean>;
|
|
private readonly fenceOwnerAdmission?: (
|
|
userId: string,
|
|
token: string,
|
|
fencedUntil: Date,
|
|
) => Promise<void>;
|
|
|
|
private readonly renewOwnerAdmission?: (
|
|
userId: string,
|
|
token: string,
|
|
fencedUntil: Date,
|
|
) => Promise<boolean>;
|
|
|
|
private readonly releaseOwnerAdmission?: (userId: string, token: string) => Promise<void>;
|
|
private readonly cancelUnroutedTask?: SubagentThreadTaskStoreOptions['cancelUnroutedTask'];
|
|
private readonly onTaskPrepared?: SubagentThreadTaskStoreOptions['onTaskPrepared'];
|
|
private readonly onTaskSettled?: SubagentThreadTaskStoreOptions['onTaskSettled'];
|
|
/** Tasks whose completion wake-up was registered; only they announce settlement. */
|
|
private readonly wakeupTaskIds = new Set<string>();
|
|
private taskControlTransport?: SubagentTaskControlTransport;
|
|
private activityStream = new SubagentActivityStream(new InMemoryEventTransport());
|
|
|
|
constructor(
|
|
private readonly methods: SubagentThreadMethods,
|
|
options: SubagentThreadTaskStoreOptions = {},
|
|
) {
|
|
super(options);
|
|
this.maxThreadDepth =
|
|
Number.isSafeInteger(options.maxThreadDepth) && (options.maxThreadDepth ?? 0) > 0
|
|
? (options.maxThreadDepth as number)
|
|
: DEFAULT_MAX_THREAD_DEPTH;
|
|
this.leaseTtlMs = positiveInteger(options.leaseTtlMs, DEFAULT_LEASE_TTL_MS);
|
|
this.leaseHeartbeatMs = Math.min(
|
|
positiveInteger(options.leaseHeartbeatMs, DEFAULT_LEASE_HEARTBEAT_MS),
|
|
Math.max(1, Math.floor(this.leaseTtlMs / 2)),
|
|
);
|
|
this.ownerDrainTimeoutMs = positiveInteger(
|
|
options.ownerDrainTimeoutMs,
|
|
DEFAULT_OWNER_DRAIN_TIMEOUT_MS,
|
|
);
|
|
this.ownerDrainPollMs = positiveInteger(options.ownerDrainPollMs, DEFAULT_OWNER_DRAIN_POLL_MS);
|
|
this.taskRoutingTtlMs = positiveInteger(options.taskRoutingTtlMs, DEFAULT_TASK_ROUTING_TTL_MS);
|
|
this.maxControlInvocations = positiveInteger(
|
|
options.maxControlInvocations,
|
|
MAX_CONTROL_INVOCATIONS,
|
|
);
|
|
this.controlReceiptRetryMs = positiveInteger(
|
|
options.controlReceiptRetryMs,
|
|
DEFAULT_CONTROL_RECEIPT_RETRY_MS,
|
|
);
|
|
this.shutdownControlReceiptBackoffMs = positiveInteger(
|
|
options.shutdownControlReceiptBackoffMs,
|
|
DEFAULT_SHUTDOWN_CONTROL_RECEIPT_BACKOFF_MS,
|
|
);
|
|
this.ownerFenceGraceMs = positiveInteger(options.ownerFenceGraceMs, OWNER_FENCE_GRACE_MS);
|
|
this.isOwnerActive = options.isOwnerActive ?? (async () => true);
|
|
this.fenceOwnerAdmission = options.fenceOwnerAdmission;
|
|
this.renewOwnerAdmission = options.renewOwnerAdmission;
|
|
this.releaseOwnerAdmission = options.releaseOwnerAdmission;
|
|
this.cancelUnroutedTask = options.cancelUnroutedTask;
|
|
this.onTaskPrepared = options.onTaskPrepared;
|
|
this.onTaskSettled = options.onTaskSettled;
|
|
}
|
|
|
|
/** Receives payload-free authoritative transitions from the SDK task store. */
|
|
protected onControlReceipt(scopeId: string, taskId: string, receipt: SdkControlReceipt): void {
|
|
const persistence = this.queueAuthoritativeControlReceipt(scopeId, taskId, receipt);
|
|
void persistence?.catch((error) => {
|
|
logger.warn('[subagentThreads] Failed to persist a child control transition', error);
|
|
});
|
|
}
|
|
|
|
private queueAuthoritativeControlReceipt(
|
|
scopeId: string,
|
|
taskId: string,
|
|
receipt: SdkControlReceipt,
|
|
): Promise<void> | undefined {
|
|
const invocation = this.controlInvocationByReceipt.get(
|
|
controlReceiptKey(scopeId, taskId, receipt.controlId),
|
|
);
|
|
const threadId = this.get(scopeId, taskId)?.threadId;
|
|
if (invocation == null || threadId == null) return undefined;
|
|
const durable = this.durableReceipt(invocation, receipt);
|
|
invocation.receipt = durable;
|
|
invocation.result = this.controlResultFromReceipt(
|
|
invocation,
|
|
durable,
|
|
this.get(scopeId, taskId)?.pendingControls,
|
|
);
|
|
if (receipt.status !== 'accepted') {
|
|
this.controlInvocationByReceipt.delete(controlReceiptKey(scopeId, taskId, receipt.controlId));
|
|
}
|
|
invocation.receiptPersisted = false;
|
|
const persistence = this.queueControlReceipt(scopeId, taskId, threadId, durable).then(() => {
|
|
if (invocation.receipt === durable) invocation.receiptPersisted = true;
|
|
});
|
|
const tracked = persistence.finally(() => {
|
|
if (invocation.receiptPersistence === tracked) invocation.receiptPersistence = undefined;
|
|
});
|
|
invocation.receiptPersistence = tracked;
|
|
return tracked;
|
|
}
|
|
|
|
/** Keeps same-process retries aligned with the durable receipt ledger. The SDK
|
|
* can replace an accepted control with a terminal transition after the child
|
|
* settles, so the originally returned result is no longer authoritative. */
|
|
private controlResultFromReceipt(
|
|
invocation: ControlInvocationRecord,
|
|
receipt: ISubagentTaskControlReceipt,
|
|
pendingControls?: number,
|
|
): SubagentTaskControlResult {
|
|
const current = invocation.result;
|
|
if (!('task' in current)) return current;
|
|
let terminalStatus = current.task.status;
|
|
if (
|
|
(receipt.action === 'cancel' && receipt.status === 'applied') ||
|
|
receipt.reason === 'task_cancelled'
|
|
) {
|
|
terminalStatus = 'cancelled';
|
|
} else if (receipt.reason === 'task_completed') {
|
|
terminalStatus = 'completed';
|
|
} else if (receipt.reason === 'task_failed') {
|
|
terminalStatus = 'error';
|
|
}
|
|
const task: SubagentTaskSnapshot = {
|
|
...current.task,
|
|
status: terminalStatus,
|
|
updatedAt: receipt.updatedAt.getTime(),
|
|
/** A receipt can make cancellation authoritative before the assistant row
|
|
* exists. Preserve actual result materialization rather than inferring it. */
|
|
resultAvailable: current.task.resultAvailable,
|
|
pendingControls: pendingControls ?? current.task.pendingControls,
|
|
};
|
|
if (receipt.status === 'accepted') {
|
|
return {
|
|
status: 'accepted',
|
|
task,
|
|
...(receipt.controlId == null ? {} : { controlId: receipt.controlId }),
|
|
};
|
|
}
|
|
if (receipt.status === 'applied') {
|
|
return receipt.action === 'cancel'
|
|
? { status: 'cancelled', task }
|
|
: {
|
|
status: 'accepted',
|
|
task,
|
|
...(receipt.controlId == null ? {} : { controlId: receipt.controlId }),
|
|
};
|
|
}
|
|
if (
|
|
receipt.reason === 'task_not_running' ||
|
|
receipt.reason === 'task_completed' ||
|
|
receipt.reason === 'task_cancelled' ||
|
|
receipt.reason === 'task_failed'
|
|
) {
|
|
return { status: 'not_running', task };
|
|
}
|
|
if (receipt.reason === 'control_not_found' || receipt.reason === 'withdrawn') {
|
|
return { status: 'control_not_found', task };
|
|
}
|
|
return {
|
|
status: 'invalid',
|
|
message:
|
|
receipt.status === 'failed'
|
|
? 'The prior control invocation failed.'
|
|
: 'The prior control invocation was rejected.',
|
|
};
|
|
}
|
|
|
|
private durableReceipt(
|
|
invocation: ControlInvocationRecord,
|
|
receipt: SdkControlReceipt,
|
|
): ISubagentTaskControlReceipt {
|
|
return {
|
|
invocationId: invocation.invocationId,
|
|
fingerprint: invocation.fingerprint,
|
|
controlId: receipt.controlId,
|
|
action: receipt.action,
|
|
status: receipt.status,
|
|
createdAt: new Date(receipt.createdAt),
|
|
updatedAt: new Date(receipt.updatedAt),
|
|
...(receipt.boundary == null ? {} : { boundary: receipt.boundary }),
|
|
...(receipt.reason == null ? {} : { reason: receipt.reason }),
|
|
...boundedControlMessage(invocation.command, invocation.commandMessageTruncated),
|
|
};
|
|
}
|
|
|
|
private controlResultReceipt(
|
|
invocation: ControlInvocationRecord,
|
|
): ISubagentTaskControlReceipt | undefined {
|
|
const { command, result } = invocation;
|
|
if (result.status === 'not_found' || result.status === 'invalid') return undefined;
|
|
if (invocation.receipt != null) return invocation.receipt;
|
|
const snapshot = result.task as SnapshotWithControlReceipts;
|
|
if (
|
|
result.status === 'accepted' &&
|
|
result.controlId != null &&
|
|
(command.action === 'steer' || command.action === 'queue' || command.action === 'interrupt')
|
|
) {
|
|
const sdkReceipt = snapshot.controlReceipts?.find(
|
|
(receipt) => receipt.controlId === result.controlId,
|
|
);
|
|
if (sdkReceipt != null) return this.durableReceipt(invocation, sdkReceipt);
|
|
return {
|
|
invocationId: invocation.invocationId,
|
|
fingerprint: invocation.fingerprint,
|
|
controlId: result.controlId,
|
|
action: command.action,
|
|
status: 'accepted',
|
|
createdAt: new Date(invocation.createdAt),
|
|
updatedAt: new Date(invocation.createdAt),
|
|
...boundedControlMessage(command, invocation.commandMessageTruncated),
|
|
};
|
|
}
|
|
const now = new Date();
|
|
let reason: string | undefined;
|
|
if (result.status === 'not_running') {
|
|
reason = 'task_not_running';
|
|
} else if (result.status === 'control_not_found') {
|
|
reason = 'control_not_found';
|
|
}
|
|
let targetControlId: string | undefined;
|
|
if (command.action === 'cancel_message') {
|
|
targetControlId = command.controlId;
|
|
} else if (result.status === 'accepted') {
|
|
targetControlId = result.controlId;
|
|
}
|
|
return {
|
|
invocationId: invocation.invocationId,
|
|
fingerprint: invocation.fingerprint,
|
|
...(targetControlId == null ? {} : { controlId: targetControlId }),
|
|
action: command.action,
|
|
status:
|
|
result.status === 'accepted' || result.status === 'cancelled' ? 'applied' : 'rejected',
|
|
createdAt: new Date(invocation.createdAt),
|
|
updatedAt: now,
|
|
...(reason == null ? {} : { reason }),
|
|
...boundedControlMessage(command, invocation.commandMessageTruncated),
|
|
};
|
|
}
|
|
|
|
private async replayDurableControl(
|
|
scopeId: string,
|
|
taskId: string,
|
|
command: SubagentTaskControlCommand,
|
|
invocationId: string,
|
|
): Promise<SubagentTaskControlResult | undefined> {
|
|
const scope = parseScope(scopeId);
|
|
const replay = await this.runWithOwnerContext(scope, () =>
|
|
this.methods.getSubagentTaskControlReplay({
|
|
userId: scope.userId,
|
|
parentConversationId: scope.parentConversationId,
|
|
taskId,
|
|
invocationId,
|
|
...(scope.tenantId == null ? {} : { tenantId: scope.tenantId }),
|
|
}),
|
|
);
|
|
if (replay == null) return undefined;
|
|
if (replay.receipt.fingerprint !== controlFingerprint(command)) {
|
|
return {
|
|
status: 'invalid',
|
|
message: 'This control invocation id was already used for a different command.',
|
|
};
|
|
}
|
|
const { receipt, task: durableTask } = replay;
|
|
if (receipt.status === 'reserved') {
|
|
/** The prior owner fenced this invocation but did not durably prove the
|
|
* side effect. Reapplying could duplicate it; reporting acceptance would lie. */
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
const task: SubagentTaskSnapshot = {
|
|
taskId,
|
|
threadId: durableTask.threadId,
|
|
subagentType: durableTask.subagentType,
|
|
status: durableTask.status,
|
|
createdAt: durableTask.createdAt.getTime(),
|
|
updatedAt: durableTask.updatedAt.getTime(),
|
|
resultAvailable: durableTask.resultAvailable,
|
|
resultClaimed: durableTask.resultClaimed,
|
|
pendingControls: durableTask.pendingControls,
|
|
...(receipt.controlId != null &&
|
|
(receipt.action === 'steer' || receipt.action === 'queue' || receipt.action === 'interrupt')
|
|
? {
|
|
controlReceipts: [
|
|
{
|
|
controlId: receipt.controlId,
|
|
action: receipt.action,
|
|
status: receipt.status,
|
|
createdAt: receipt.createdAt.getTime(),
|
|
updatedAt: receipt.updatedAt.getTime(),
|
|
...(receipt.boundary == null ? {} : { boundary: receipt.boundary }),
|
|
...(receipt.reason === 'withdrawn' ||
|
|
receipt.reason === 'task_completed' ||
|
|
receipt.reason === 'task_cancelled' ||
|
|
receipt.reason === 'task_failed'
|
|
? { reason: receipt.reason }
|
|
: {}),
|
|
},
|
|
],
|
|
}
|
|
: {}),
|
|
};
|
|
if (receipt.status === 'accepted') {
|
|
return {
|
|
status: 'accepted',
|
|
task,
|
|
...(receipt.controlId == null ? {} : { controlId: receipt.controlId }),
|
|
};
|
|
}
|
|
if (receipt.status === 'applied') {
|
|
return command.action === 'cancel'
|
|
? { status: 'cancelled', task }
|
|
: {
|
|
status: 'accepted',
|
|
task,
|
|
...(receipt.controlId == null ? {} : { controlId: receipt.controlId }),
|
|
};
|
|
}
|
|
if (
|
|
receipt.reason === 'task_not_running' ||
|
|
receipt.reason === 'task_completed' ||
|
|
receipt.reason === 'task_cancelled' ||
|
|
receipt.reason === 'task_failed'
|
|
) {
|
|
return { status: 'not_running', task };
|
|
}
|
|
if (receipt.reason === 'control_not_found' || receipt.reason === 'withdrawn') {
|
|
return { status: 'control_not_found', task };
|
|
}
|
|
return {
|
|
status: 'invalid',
|
|
message:
|
|
receipt.status === 'failed'
|
|
? 'The prior control invocation failed.'
|
|
: 'The prior control invocation was rejected.',
|
|
};
|
|
}
|
|
|
|
private queueControlReceipt(
|
|
scopeId: string,
|
|
taskId: string,
|
|
threadId: string,
|
|
receipt: ISubagentTaskControlReceipt,
|
|
): Promise<void> {
|
|
const key = controlTaskKey(scopeId, taskId);
|
|
const pending = this.pendingControlReceipts.get(key) ?? new Map();
|
|
pending.set(receipt.invocationId, { threadId, receipt });
|
|
this.pendingControlReceipts.set(key, pending);
|
|
return this.flushControlReceipts(scopeId, taskId);
|
|
}
|
|
|
|
private flushControlReceipts(scopeId: string, taskId: string): Promise<void> {
|
|
const key = controlTaskKey(scopeId, taskId);
|
|
const prior = this.controlPersistenceTails.get(key) ?? Promise.resolve();
|
|
const operation = prior
|
|
.catch(() => undefined)
|
|
.then(async () => {
|
|
const pending = this.pendingControlReceipts.get(key);
|
|
if (pending == null) return;
|
|
const scope = parseScope(scopeId);
|
|
for (const [invocationId, candidate] of [...pending]) {
|
|
const current = pending.get(invocationId);
|
|
if (current !== candidate) continue;
|
|
const persisted = await this.runWithOwnerContext(scope, () =>
|
|
this.methods.recordSubagentTaskControlReceipt({
|
|
userId: scope.userId,
|
|
conversationId: candidate.threadId,
|
|
taskId,
|
|
...(scope.tenantId == null ? {} : { tenantId: scope.tenantId }),
|
|
receipt: candidate.receipt,
|
|
}),
|
|
);
|
|
if (persisted === 'conflict') {
|
|
if (pending.get(invocationId) === candidate) pending.delete(invocationId);
|
|
if (pending.size === 0) this.pendingControlReceipts.delete(key);
|
|
throw new SubagentControlReceiptConflictError();
|
|
}
|
|
if (!persisted) {
|
|
throw new Error('The child control receipt target is not ready.');
|
|
}
|
|
const invocation = this.retainedControlInvocation(scopeId, taskId, invocationId);
|
|
if (invocation?.receipt === candidate.receipt) {
|
|
invocation.receiptPersisted = true;
|
|
}
|
|
if (pending.get(invocationId) === candidate) {
|
|
pending.delete(invocationId);
|
|
}
|
|
}
|
|
if (pending.size === 0) {
|
|
this.pendingControlReceipts.delete(key);
|
|
const retry = this.controlPersistenceRetryTimers.get(key);
|
|
if (retry != null) clearTimeout(retry);
|
|
this.controlPersistenceRetryTimers.delete(key);
|
|
}
|
|
});
|
|
this.controlPersistenceTails.set(key, operation);
|
|
void operation.then(
|
|
() => {
|
|
if (this.controlPersistenceTails.get(key) === operation) {
|
|
this.controlPersistenceTails.delete(key);
|
|
}
|
|
this.scheduleControlReceiptRetry(scopeId, taskId);
|
|
},
|
|
() => {
|
|
if (this.controlPersistenceTails.get(key) === operation) {
|
|
this.controlPersistenceTails.delete(key);
|
|
}
|
|
this.scheduleControlReceiptRetry(scopeId, taskId);
|
|
},
|
|
);
|
|
return operation;
|
|
}
|
|
|
|
/** A terminal child may have no later caller to retrigger persistence. Keep a
|
|
* single bounded retry timer per task so transient storage failures converge
|
|
* even after result collection expires its in-memory task; restart durability
|
|
* remains AI-1737. */
|
|
private scheduleControlReceiptRetry(scopeId: string, taskId: string): void {
|
|
const key = controlTaskKey(scopeId, taskId);
|
|
if (
|
|
this.controlPersistenceStopping ||
|
|
this.controlPersistenceRetryTimers.has(key) ||
|
|
!this.pendingControlReceipts.has(key)
|
|
) {
|
|
return;
|
|
}
|
|
const timer = setTimeout(() => {
|
|
this.controlPersistenceRetryTimers.delete(key);
|
|
void this.flushControlReceipts(scopeId, taskId).catch((error) => {
|
|
logger.warn('[subagentThreads] Failed to retry child control receipts', error);
|
|
});
|
|
}, this.controlReceiptRetryMs);
|
|
this.controlPersistenceRetryTimers.set(key, timer);
|
|
}
|
|
|
|
private async flushControlReceiptsForSettlement(scopeId: string, taskId: string): Promise<void> {
|
|
try {
|
|
await this.flushControlReceipts(scopeId, taskId);
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Failed to flush child control receipts', error);
|
|
}
|
|
}
|
|
|
|
/** Enables optional cross-replica lookup after the host's Redis service is ready. */
|
|
async configureTaskControlTransport(transport: SubagentTaskControlTransport): Promise<void> {
|
|
if (this.taskControlTransport != null) {
|
|
throw new Error('Subagent task control transport is already configured.');
|
|
}
|
|
await transport.bind({
|
|
claim: (scopeId, taskId) => super.claim(scopeId, taskId),
|
|
control: (scopeId, taskId, command, invocationId) =>
|
|
this.controlInvocationAndPersist(scopeId, taskId, command, invocationId),
|
|
list: (scopeId) => super.list(scopeId),
|
|
cancelScope: (scopeId, threadIds, removedConversationIds = []) => {
|
|
const cancelled = this.cancelForScope(scopeId, threadIds);
|
|
if (removedConversationIds.length > 0) {
|
|
const scope = parseScope(scopeId);
|
|
this.dropDeletedControlReceiptWork(
|
|
scope.userId,
|
|
new Set(removedConversationIds),
|
|
scope.tenantId,
|
|
);
|
|
}
|
|
return cancelled;
|
|
},
|
|
retainsTaskOwnership: (scopeId, taskId) =>
|
|
this.pendingControlReceipts.has(controlTaskKey(scopeId, taskId)),
|
|
});
|
|
this.taskControlTransport = transport;
|
|
}
|
|
|
|
async destroyTaskControlTransport(): Promise<void> {
|
|
/** Close command admission and synchronously cancel every locally-owned child
|
|
* before the first await. The SDK emits all pending-control transitions while
|
|
* cancelling, so no receipt producer can race the final persistence snapshot. */
|
|
this.controlCommandAdmissionClosed = true;
|
|
const cancellationFlushes: Promise<void>[] = [];
|
|
for (const lease of this.activeThreads.values()) {
|
|
if (lease.taskId !== '' && this.get(lease.scopeId, lease.taskId)?.status === 'running') {
|
|
const cancellation = super.control(lease.scopeId, lease.taskId, { action: 'cancel' });
|
|
if (cancellation.status === 'cancelled') {
|
|
/** The SDK hook above is synchronous, but retain direct promises for the
|
|
* authoritative terminal snapshot as well. This makes shutdown await the
|
|
* transition even when its first storage attempt fails under load. */
|
|
const snapshot = cancellation.task as SnapshotWithControlReceipts;
|
|
for (const receipt of snapshot.controlReceipts ?? []) {
|
|
const persistence = this.queueAuthoritativeControlReceipt(
|
|
lease.scopeId,
|
|
lease.taskId,
|
|
receipt,
|
|
);
|
|
if (persistence != null) cancellationFlushes.push(persistence);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
const childSettlements = [...this.activeThreads.values()]
|
|
.map((lease) => lease.execution)
|
|
.filter((execution): execution is Promise<void> => execution != null);
|
|
let childSettlementTimedOut = false;
|
|
if (childSettlements.length > 0) {
|
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
try {
|
|
await Promise.race([
|
|
Promise.allSettled(childSettlements),
|
|
new Promise<void>((_, reject) => {
|
|
timeout = setTimeout(
|
|
() => reject(new SubagentTaskOwnerUnavailableError()),
|
|
this.ownerDrainTimeoutMs,
|
|
);
|
|
}),
|
|
]);
|
|
} catch {
|
|
childSettlementTimedOut = true;
|
|
} finally {
|
|
if (timeout != null) clearTimeout(timeout);
|
|
}
|
|
}
|
|
this.controlPersistenceStopping = true;
|
|
for (const timer of this.controlPersistenceRetryTimers.values()) clearTimeout(timer);
|
|
this.controlPersistenceRetryTimers.clear();
|
|
await Promise.allSettled(cancellationFlushes);
|
|
/** Cancellation can enqueue its terminal transition behind an already-failing
|
|
* acceptance write. Re-snapshot both maps after each round so work admitted
|
|
* synchronously before shutdown cannot appear just after the final snapshot. */
|
|
for (let attempt = 0; attempt < SHUTDOWN_CONTROL_RECEIPT_FLUSH_ATTEMPTS; attempt += 1) {
|
|
const pendingTasks = [...this.pendingControlReceipts.keys()]
|
|
.map(parseControlTaskKey)
|
|
.filter((task): task is { scopeId: string; taskId: string } => task != null);
|
|
await Promise.allSettled(
|
|
pendingTasks.map(({ scopeId, taskId }) => this.flushControlReceipts(scopeId, taskId)),
|
|
);
|
|
await Promise.allSettled([...this.controlPersistenceTails.values()]);
|
|
if (this.pendingControlReceipts.size === 0 && this.controlPersistenceTails.size === 0) {
|
|
break;
|
|
}
|
|
if (attempt + 1 < SHUTDOWN_CONTROL_RECEIPT_FLUSH_ATTEMPTS) {
|
|
await new Promise<void>((resolve) => {
|
|
setTimeout(resolve, this.shutdownControlReceiptBackoffMs * 2 ** attempt);
|
|
});
|
|
}
|
|
}
|
|
const transport = this.taskControlTransport;
|
|
this.taskControlTransport = undefined;
|
|
await transport?.destroy();
|
|
if (
|
|
childSettlementTimedOut ||
|
|
this.pendingControlReceipts.size > 0 ||
|
|
this.controlPersistenceTails.size > 0
|
|
) {
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
}
|
|
|
|
/** Replaces the process-local activity bus after the host's Redis service is ready. */
|
|
configureActivityStream(stream: SubagentActivityStream): void {
|
|
const previous = this.activityStream;
|
|
this.activityStream = stream;
|
|
previous.destroy();
|
|
}
|
|
|
|
destroyActivityStream(): void {
|
|
this.activityStream.destroy();
|
|
}
|
|
|
|
prepareActivityForShutdown(): void {
|
|
this.activityStream.prepareForShutdown();
|
|
}
|
|
|
|
subscribeActivity(
|
|
threadId: string,
|
|
taskId: string,
|
|
subscriber: SubagentActivitySubscriber,
|
|
): SubagentActivitySubscription {
|
|
return this.activityStream.subscribe(threadId, taskId, subscriber);
|
|
}
|
|
|
|
/** Publishes activity produced by a host-owned event child. Event children
|
|
* use the same bounded, demand-aware transport as detached tool children,
|
|
* but their generation lease is owned by the trigger controller instead of
|
|
* this task store. */
|
|
publishTaskActivity(threadId: string, taskId: string, event: SubagentUpdateEvent): Promise<void> {
|
|
return this.activityStream.publish(threadId, taskId, boundSubagentActivityUpdate(event));
|
|
}
|
|
|
|
private publishActivity(
|
|
lease: TaskThreadLease,
|
|
threadId: string,
|
|
taskId: string,
|
|
event: SubagentUpdateEvent,
|
|
): void {
|
|
if (
|
|
lease.activityAdmissionClosed === true ||
|
|
lease.activityCircuitOpen === true ||
|
|
(lease.activityPending ?? 0) >= MAX_PENDING_ACTIVITY_EVENTS
|
|
) {
|
|
return;
|
|
}
|
|
lease.activityPending = (lease.activityPending ?? 0) + 1;
|
|
const boundedEvent = boundSubagentActivityUpdate(event);
|
|
const publication = (lease.activityTail ?? Promise.resolve())
|
|
.then(() => {
|
|
if (lease.activityCircuitOpen === true) return;
|
|
return settleActivityWithin(this.activityStream.publish(threadId, taskId, boundedEvent));
|
|
})
|
|
.catch((error) => {
|
|
/** Any failed observational command opens the per-task circuit. Retrying every
|
|
* token during an outage only creates command/log pressure; durable state remains. */
|
|
lease.activityCircuitOpen = true;
|
|
logger.warn('[subagentThreads] Failed to publish child activity', error);
|
|
})
|
|
.finally(() => {
|
|
lease.activityPending = Math.max(0, (lease.activityPending ?? 1) - 1);
|
|
});
|
|
lease.activityTail = publication;
|
|
}
|
|
|
|
private completeActivity(
|
|
lease: TaskThreadLease,
|
|
threadId: string,
|
|
taskId: string,
|
|
status: SubagentActivityTerminalStatus,
|
|
): void {
|
|
lease.activityAdmissionClosed = true;
|
|
const terminal = (lease.activityTail ?? Promise.resolve())
|
|
.then(() => settleActivityWithin(this.activityStream.complete(threadId, taskId, status)))
|
|
.catch((error) => {
|
|
logger.warn('[subagentThreads] Failed to close child activity stream', error);
|
|
});
|
|
lease.activityTail = terminal;
|
|
void terminal.finally(() => {
|
|
if (lease.activityTail === terminal) {
|
|
lease.activityTail = undefined;
|
|
}
|
|
});
|
|
}
|
|
|
|
/** Gates child creation on the ordinary parent write without retaining request state. */
|
|
registerParentPersistence(scopeId: string, persistence: Promise<unknown>): void {
|
|
const scope = parseScope(scopeId);
|
|
const gate = Promise.resolve(persistence).then((result) => {
|
|
assertParentPersistence(result, scope);
|
|
return result;
|
|
});
|
|
this.parentPersistence.set(scopeId, gate);
|
|
void gate
|
|
.then(() => {
|
|
if (this.parentPersistence.get(scopeId) === gate) {
|
|
this.parentPersistence.delete(scopeId);
|
|
}
|
|
})
|
|
.catch(() => undefined);
|
|
}
|
|
|
|
override start(request: SubagentTaskStartRequest): SubagentTaskStartResult {
|
|
if (request.subagentKind !== 'agent' && request.subagentKind !== 'graph') {
|
|
throw new Error('Subagent task kind must be agent or graph.');
|
|
}
|
|
const scope = parseScope(request.scopeId);
|
|
const parentReady = this.parentPersistence.get(request.scopeId);
|
|
const requestedThreadId = request.threadId?.trim();
|
|
const isContinuation = requestedThreadId != null && requestedThreadId !== '';
|
|
const idempotencyKey = request.idempotencyKey.trim();
|
|
const threadId = isContinuation
|
|
? requestedThreadId
|
|
: createSubagentThreadId(request.scopeId, idempotencyKey);
|
|
const lockKey = `${request.scopeId}\u0000${threadId}`;
|
|
const active = this.activeThreads.get(lockKey);
|
|
if (active != null && active.idempotencyKey !== idempotencyKey) {
|
|
return { accepted: false, reason: 'capacity' };
|
|
}
|
|
|
|
const lease: TaskThreadLease = active ?? {
|
|
scopeId: request.scopeId,
|
|
idempotencyKey,
|
|
taskId: '',
|
|
running: false,
|
|
settling: false,
|
|
};
|
|
const ownsLease = active == null;
|
|
if (ownsLease) {
|
|
this.activeThreads.set(lockKey, lease);
|
|
}
|
|
|
|
let started: SubagentTaskStartResult;
|
|
try {
|
|
started = super.start({
|
|
...request,
|
|
threadId,
|
|
run: (runtime: SubagentTaskRuntime) => {
|
|
const execution = this.runWithOwnerContext(scope, async () => {
|
|
lease.taskId = runtime.taskId;
|
|
lease.running = true;
|
|
const detachedUsage: UsageMetadata[] = [];
|
|
let prepared: PreparedThread | undefined;
|
|
let activityTerminal: SubagentActivityTerminalStatus = 'failed';
|
|
try {
|
|
if (runtime.signal.aborted) {
|
|
throw runtime.signal.reason ?? new Error('Subagent task was cancelled.');
|
|
}
|
|
/** Publish the owner address before any provider work: a child running
|
|
* while unaddressable cannot be polled, controlled, or cancelled, and its
|
|
* side effects would already have happened by the time a heartbeat
|
|
* republished it. A failed registration fails the task closed instead. */
|
|
await this.taskControlTransport?.registerTask(
|
|
request.scopeId,
|
|
runtime.taskId,
|
|
this.taskRoutingTtlMs,
|
|
);
|
|
await parentReady;
|
|
prepared = await this.prepareThread(
|
|
request.scopeId,
|
|
scope,
|
|
threadId,
|
|
isContinuation,
|
|
request,
|
|
runtime.taskId,
|
|
lease,
|
|
);
|
|
await this.registerTaskWakeup(scope, prepared.conversation.conversationId, request, {
|
|
taskId: prepared.replay?.taskId ?? runtime.taskId,
|
|
parentRunId: prepared.replay?.parentRunId ?? request.parentRunId,
|
|
createdAt: prepared.taskCreatedAt,
|
|
});
|
|
if (prepared.replay != null) {
|
|
this.notifyTaskSettled(
|
|
scope,
|
|
prepared.replay.taskId,
|
|
prepared.replay.previousTaskId,
|
|
);
|
|
}
|
|
if (runtime.signal.aborted) {
|
|
throw runtime.signal.reason ?? new Error('Subagent task was cancelled.');
|
|
}
|
|
if (prepared.replay != null) {
|
|
if (prepared.replay.status === 'completed') {
|
|
return { content: prepared.replay.content };
|
|
}
|
|
throw new SubagentThreadPublicError(prepared.replay.content);
|
|
}
|
|
if (!(await this.renewSharedLease(scope, threadId, lease))) {
|
|
throw new SubagentThreadPublicError(
|
|
'This child thread is already being continued by another run.',
|
|
);
|
|
}
|
|
const preparedThread = prepared;
|
|
let activitySequence = 0;
|
|
const activityRuntime: SubagentTaskRuntime = {
|
|
...runtime,
|
|
reportProgress: (event) => {
|
|
const sequence = activitySequence++;
|
|
const activityEvent: SubagentActivityUpdateEvent = {
|
|
...event,
|
|
activityEventId: `${runtime.taskId}:${sequence}`,
|
|
activitySequence: sequence,
|
|
};
|
|
runtime.reportProgress(activityEvent);
|
|
this.publishActivity(
|
|
lease,
|
|
preparedThread.conversation.conversationId,
|
|
runtime.taskId,
|
|
activityEvent,
|
|
);
|
|
},
|
|
};
|
|
const result = await runWithDetachedSubagentUsage(detachedUsage, () =>
|
|
request.run(activityRuntime, preparedThread.initialMessages),
|
|
);
|
|
if (runtime.signal.aborted) {
|
|
throw runtime.signal.reason ?? new Error('Subagent task was cancelled.');
|
|
}
|
|
if (!(await this.renewSharedLease(scope, threadId, lease))) {
|
|
throw new SubagentThreadPublicError(
|
|
'This child thread is already being continued by another run.',
|
|
);
|
|
}
|
|
lease.settling = true;
|
|
await this.persistResult(
|
|
scope,
|
|
request,
|
|
runtime.taskId,
|
|
prepared,
|
|
result,
|
|
detachedUsage,
|
|
);
|
|
activityTerminal = 'completed';
|
|
return result;
|
|
} catch (error) {
|
|
/** A replay is already terminal in Mongo. A temporary wakeup-queue
|
|
* outage must not overwrite that canonical result with a new error. */
|
|
if (prepared?.replay != null) {
|
|
throw error;
|
|
}
|
|
const mayPersist =
|
|
lease.shared == null || (await this.renewSharedLease(scope, threadId, lease));
|
|
const terminalTask = this.get(request.scopeId, runtime.taskId);
|
|
if (runtime.signal.aborted && terminalTask?.status === 'cancelled') {
|
|
activityTerminal = 'cancelled';
|
|
if (mayPersist) {
|
|
await this.persistCancellation(
|
|
scope,
|
|
threadId,
|
|
request,
|
|
runtime.taskId,
|
|
detachedUsage,
|
|
).catch((persistError) => {
|
|
logger.error(
|
|
'[subagentThreads] Failed to persist child-thread cancellation',
|
|
persistError,
|
|
);
|
|
});
|
|
}
|
|
throw error;
|
|
}
|
|
logger.error('[subagentThreads] Child-thread execution failed', {
|
|
detail: publicFailureDetail(error),
|
|
errorName: error instanceof Error ? error.name : typeof error,
|
|
...(error instanceof Error && error.stack != null ? { stack: error.stack } : {}),
|
|
});
|
|
if (mayPersist) {
|
|
await this.persistFailure(
|
|
scope,
|
|
threadId,
|
|
request,
|
|
runtime.taskId,
|
|
error,
|
|
detachedUsage,
|
|
).catch((persistError) => {
|
|
logger.error(
|
|
'[subagentThreads] Failed to persist child-thread failure',
|
|
persistError,
|
|
);
|
|
});
|
|
}
|
|
throw new Error(publicFailureDetail(error));
|
|
} finally {
|
|
if (prepared != null) {
|
|
this.wakeupTaskIds.delete(prepared.replay?.taskId ?? runtime.taskId);
|
|
}
|
|
if (prepared != null && prepared.replay == null) {
|
|
this.completeActivity(
|
|
lease,
|
|
prepared.conversation.conversationId,
|
|
runtime.taskId,
|
|
activityTerminal,
|
|
);
|
|
}
|
|
await this.stopAndReleaseSharedLease(scope, threadId, lease);
|
|
if (this.activeThreads.get(lockKey) === lease) {
|
|
this.activeThreads.delete(lockKey);
|
|
}
|
|
}
|
|
});
|
|
const settlement = execution.then(
|
|
() => undefined,
|
|
() => undefined,
|
|
);
|
|
lease.execution = settlement;
|
|
return execution;
|
|
},
|
|
});
|
|
} catch (error) {
|
|
if (ownsLease && this.activeThreads.get(lockKey) === lease) {
|
|
this.activeThreads.delete(lockKey);
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
if (started.accepted && started.isNew) {
|
|
lease.taskId = started.task.taskId;
|
|
} else if (ownsLease && this.activeThreads.get(lockKey) === lease) {
|
|
this.activeThreads.delete(lockKey);
|
|
}
|
|
return started;
|
|
}
|
|
|
|
/** Detached tasks intentionally outlive the HTTP request that admitted them.
|
|
* Reconstruct only the trusted owner identity carried by the opaque task scope
|
|
* so tenant-isolated database reads and lazy child initialization do not depend
|
|
* on request AsyncLocalStorage remaining alive after the parent turn returns. */
|
|
private runWithOwnerContext<T>(scope: SubagentThreadScope, run: () => Promise<T>): Promise<T> {
|
|
return tenantStorage.run(
|
|
{
|
|
userId: scope.userId,
|
|
...(scope.tenantId == null ? {} : { tenantId: scope.tenantId }),
|
|
},
|
|
run,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Claims locally when possible, otherwise asks the registered owning replica.
|
|
*
|
|
* A child's terminal result is durable in its own thread, so collection is recorded
|
|
* there against the polling invocation rather than kept alive in the owner's memory.
|
|
* The invocation that lost a response re-acquires its own result on the next poll;
|
|
* a different invocation is told the result was already collected. Owner-side
|
|
* retention stays a fast path, free to expire, instead of the only copy.
|
|
*/
|
|
async claimTask(
|
|
scopeId: string,
|
|
taskId: string,
|
|
invocationId?: string,
|
|
): Promise<SubagentTaskClaim> {
|
|
const local = super.claim(scopeId, taskId);
|
|
const claim =
|
|
local.status !== 'not_found'
|
|
? local
|
|
: ((await this.taskControlTransport?.claim(scopeId, taskId)) ?? local);
|
|
if (invocationId == null || claim.status === 'running') {
|
|
return claim;
|
|
}
|
|
if (claim.status === 'not_found') {
|
|
return this.claimDurableTaskResult(scopeId, taskId, invocationId);
|
|
}
|
|
const threadId = claim.task.threadId;
|
|
if (threadId == null || threadId === '') {
|
|
return claim;
|
|
}
|
|
/** The durable record decides who holds this one-shot result. The invocation that
|
|
* already consumed it re-acquires and is handed it again, a second invocation is
|
|
* told it was collected instead of being given a duplicate, and a task with no
|
|
* durable record to arbitrate keeps whatever the owner just answered. */
|
|
const collected = await this.assignResultClaim(
|
|
parseScope(scopeId).userId,
|
|
threadId,
|
|
claim.task.taskId,
|
|
invocationId,
|
|
);
|
|
if (collected.status === 'claimed') {
|
|
return { status: 'claimed', task: claim.task };
|
|
}
|
|
if (collected.status === 'not_found') {
|
|
return claim;
|
|
}
|
|
return claim.status === 'claimed' ? (recoveredClaim(collected.message, claim) ?? claim) : claim;
|
|
}
|
|
|
|
/**
|
|
* Recovers a terminal task after its owning process and Redis registration are gone.
|
|
* The task id locates only a candidate; durable child lineage re-establishes the
|
|
* trusted parent scope before the one-shot result is claimed.
|
|
*/
|
|
private async claimDurableTaskResult(
|
|
scopeId: string,
|
|
taskId: string,
|
|
invocationId: string,
|
|
): Promise<SubagentTaskClaim> {
|
|
const scope = parseScope(scopeId);
|
|
let message: IMessage | undefined;
|
|
try {
|
|
[message] = await this.methods.getMessages(
|
|
{
|
|
user: scope.userId,
|
|
messageId: `${taskId}:assistant`,
|
|
'subagentTask.status': { $in: ['completed', 'error', 'cancelled'] },
|
|
},
|
|
DURABLE_RESULT_SELECT,
|
|
{ limit: 1, sort: false },
|
|
);
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Failed to locate a durable child result', error);
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
const threadId = message?.conversationId;
|
|
const status = message?.subagentTask?.status;
|
|
if (
|
|
message == null ||
|
|
!isNonEmptyString(threadId) ||
|
|
!isNonEmptyString(message.sender) ||
|
|
(status !== 'completed' && status !== 'error' && status !== 'cancelled')
|
|
) {
|
|
return { status: 'not_found' };
|
|
}
|
|
|
|
let parent: IConversation | null;
|
|
let conversation: IConversation | null;
|
|
try {
|
|
[parent, conversation] = await Promise.all([
|
|
this.methods.getConvo(scope.userId, scope.parentConversationId),
|
|
this.methods.getConvo(scope.userId, threadId),
|
|
]);
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Failed to verify durable child lineage', error);
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
const lineage = conversation?.subagentThread;
|
|
if (
|
|
parent == null ||
|
|
conversation == null ||
|
|
lineage == null ||
|
|
conversation.endpoint !== EModelEndpoint.agents ||
|
|
lineage.parentConversationId !== scope.parentConversationId ||
|
|
lineage.subagentType !== message.sender ||
|
|
lineage.depth > this.maxThreadDepth ||
|
|
!matchesTenant(parent.tenantId, scope.tenantId) ||
|
|
!matchesTenant(conversation.tenantId, scope.tenantId)
|
|
) {
|
|
return { status: 'not_found' };
|
|
}
|
|
|
|
const createdAt = message.createdAt?.getTime();
|
|
const updatedAt = message.updatedAt?.getTime() ?? createdAt;
|
|
if (createdAt == null || updatedAt == null) {
|
|
return { status: 'not_found' };
|
|
}
|
|
const task: SubagentTaskSnapshot = {
|
|
taskId,
|
|
threadId,
|
|
subagentType: lineage.subagentType,
|
|
status,
|
|
createdAt,
|
|
updatedAt,
|
|
resultAvailable: true,
|
|
resultClaimed: true,
|
|
pendingControls: 0,
|
|
...(status === 'completed' ? {} : { error: message.text ?? '' }),
|
|
};
|
|
const collected = await this.assignResultClaim(scope.userId, threadId, taskId, invocationId);
|
|
if (collected.status === 'not_found') {
|
|
return { status: 'not_found' };
|
|
}
|
|
if (collected.status === 'claimed') {
|
|
return { status: 'claimed', task };
|
|
}
|
|
return (
|
|
recoveredClaim(collected.message, { status: 'claimed', task }) ?? {
|
|
status: 'not_found',
|
|
}
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Assigns one durable terminal result to the invocation collecting it. A failed
|
|
* write is not an absent record: handing the result over without recording its
|
|
* claimant would let another invocation acquire the same one-shot output once the
|
|
* database recovers, so this reports the retryable path and leaves the result
|
|
* unclaimed for a later poll.
|
|
*/
|
|
private async assignResultClaim(
|
|
userId: string,
|
|
threadId: string,
|
|
taskId: string,
|
|
invocationId: string,
|
|
): Promise<SubagentTaskResultClaim> {
|
|
try {
|
|
return await this.methods.claimSubagentTaskResult({
|
|
userId,
|
|
conversationId: threadId,
|
|
taskId,
|
|
kind: 'manual',
|
|
claimId: invocationId,
|
|
});
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Failed to record a collected child result', error);
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Controls locally when possible, otherwise asks the registered owning replica.
|
|
* `invocationId` identifies one caller invocation: a routed retransmission of that
|
|
* invocation replays the owner's result, while a fresh invocation applies again even
|
|
* when its action and message are identical.
|
|
*/
|
|
async controlTask(
|
|
scopeId: string,
|
|
taskId: string,
|
|
command: SubagentTaskControlCommand,
|
|
invocationId: string = randomUUID(),
|
|
): Promise<SubagentTaskControlResult> {
|
|
/** A requester that has neither the task nor a retained invocation cannot be
|
|
* authoritative. Route first so a remote control pays only the owner's durable
|
|
* preflight instead of repeating the same Mongo read on both replicas. */
|
|
const hasLocalAuthority =
|
|
this.get(scopeId, taskId) != null ||
|
|
this.retainedControlInvocation(scopeId, taskId, invocationId) != null;
|
|
const local = hasLocalAuthority
|
|
? await this.controlInvocationAndPersist(scopeId, taskId, command, invocationId)
|
|
: ({ status: 'not_found' } as const);
|
|
if (local.status !== 'not_found') {
|
|
return local;
|
|
}
|
|
let routed: SubagentTaskControlResult | undefined;
|
|
try {
|
|
routed = await this.taskControlTransport?.control(scopeId, taskId, command, invocationId);
|
|
} catch (error) {
|
|
if (error instanceof SubagentTaskOwnerUnavailableError) {
|
|
const replay = await this.replayDurableControlAtBoundary(
|
|
scopeId,
|
|
taskId,
|
|
command,
|
|
invocationId,
|
|
);
|
|
if (replay != null) return replay;
|
|
}
|
|
throw error;
|
|
}
|
|
if (routed != null && routed.status !== 'not_found') return routed;
|
|
return (
|
|
(await this.replayDurableControlAtBoundary(scopeId, taskId, command, invocationId)) ??
|
|
routed ??
|
|
local
|
|
);
|
|
}
|
|
|
|
/** Durable receipt reads are part of the owner boundary. Storage ambiguity must
|
|
* remain retryable instead of escaping as an unrelated tool execution failure. */
|
|
private async replayDurableControlAtBoundary(
|
|
scopeId: string,
|
|
taskId: string,
|
|
command: SubagentTaskControlCommand,
|
|
invocationId: string,
|
|
): Promise<SubagentTaskControlResult | undefined> {
|
|
try {
|
|
return await this.replayDurableControl(scopeId, taskId, command, invocationId);
|
|
} catch (error) {
|
|
if (error instanceof SubagentTaskOwnerUnavailableError) throw error;
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
}
|
|
|
|
private retainedControlInvocation(
|
|
scopeId: string,
|
|
taskId: string,
|
|
invocationId: string,
|
|
): ControlInvocationRecord | undefined {
|
|
const key = `${scopeId}\u0000${taskId}\u0000${invocationId}`;
|
|
return this.controlInvocations.get(key) ?? this.terminalControlInvocations.get(key);
|
|
}
|
|
|
|
private async replayRetainedControl(
|
|
scopeId: string,
|
|
taskId: string,
|
|
command: SubagentTaskControlCommand,
|
|
invocationId: string,
|
|
): Promise<SubagentTaskControlResult | undefined> {
|
|
const retained = this.retainedControlInvocation(scopeId, taskId, invocationId);
|
|
if (retained == null) return undefined;
|
|
if (retained.fingerprint !== controlFingerprint(command)) {
|
|
return {
|
|
status: 'invalid',
|
|
message: 'This control invocation id was already used for a different command.',
|
|
};
|
|
}
|
|
if (hasDurableControlReceipt(retained)) {
|
|
/** Terminal materialization and one-shot collection can change after the
|
|
* receipt becomes durable. Refresh those flags from the exact durable row so
|
|
* same-owner replay agrees with replay after owner loss. */
|
|
if ('task' in retained.result && retained.result.task.status !== 'running') {
|
|
const current = this.get(scopeId, taskId);
|
|
if (current != null) {
|
|
retained.result = {
|
|
...retained.result,
|
|
task: {
|
|
...retained.result.task,
|
|
status: current.status,
|
|
updatedAt: current.updatedAt,
|
|
resultAvailable: current.resultAvailable,
|
|
resultClaimed: current.resultClaimed,
|
|
},
|
|
};
|
|
} else {
|
|
const durable = await this.replayDurableControlAtBoundary(
|
|
scopeId,
|
|
taskId,
|
|
command,
|
|
invocationId,
|
|
);
|
|
if (durable != null) retained.result = durable;
|
|
}
|
|
}
|
|
return retained.result;
|
|
}
|
|
let persistenceFailed = false;
|
|
try {
|
|
await (retained.receiptPersistence ?? this.flushControlReceipts(scopeId, taskId));
|
|
} catch {
|
|
persistenceFailed = true;
|
|
// The durable replay below distinguishes a committed result or conflict
|
|
// from a genuinely retryable storage failure.
|
|
}
|
|
if (hasDurableControlReceipt(retained)) return retained.result;
|
|
if (persistenceFailed) {
|
|
try {
|
|
const retry = this.flushControlReceipts(scopeId, taskId);
|
|
retained.receiptPersistence = retry;
|
|
await retry;
|
|
} catch {
|
|
// Durable replay below remains the authoritative discriminator.
|
|
}
|
|
}
|
|
try {
|
|
const durable = await this.replayDurableControl(scopeId, taskId, command, invocationId);
|
|
if (durable != null) {
|
|
retained.result = durable;
|
|
retained.receiptPersisted = true;
|
|
retained.receiptPersistence = undefined;
|
|
return durable;
|
|
}
|
|
} catch {
|
|
// Normalize storage outages at the owner boundary.
|
|
}
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
|
|
private retainedControlResult(
|
|
scopeId: string,
|
|
taskId: string,
|
|
command: SubagentTaskControlCommand,
|
|
invocationId: string,
|
|
): SubagentTaskControlResult | undefined {
|
|
const retained = this.retainedControlInvocation(scopeId, taskId, invocationId);
|
|
if (retained == null) return undefined;
|
|
return retained.fingerprint === controlFingerprint(command)
|
|
? retained.result
|
|
: {
|
|
status: 'invalid',
|
|
message: 'This control invocation id was already used for a different command.',
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Applies one logical control exactly once for its owning task. Idempotency lives
|
|
* here rather than in the transport so a local and a routed caller of the same
|
|
* invocation agree, and it is keyed by task as well as invocation because provider
|
|
* tool-call ids repeat across runs and agents.
|
|
*/
|
|
controlInvocation(
|
|
scopeId: string,
|
|
taskId: string,
|
|
command: SubagentTaskControlCommand,
|
|
invocationId: string,
|
|
): SubagentTaskControlResult {
|
|
if (this.controlCommandAdmissionClosed) {
|
|
return { status: 'invalid', message: 'Subagent task controls are shutting down.' };
|
|
}
|
|
const key = `${scopeId}\u0000${taskId}\u0000${invocationId}`;
|
|
const fingerprint = controlFingerprint(command);
|
|
const applied = this.retainedControlResult(scopeId, taskId, command, invocationId);
|
|
if (applied != null) return applied;
|
|
const localTask = this.get(scopeId, taskId);
|
|
if (localTask == null) {
|
|
/** Not this replica's task. Refusing here would keep the command from ever
|
|
* reaching its owner, so local load cannot veto a remote cancellation: the
|
|
* owner applies its own window to the routed request. */
|
|
return this.control(scopeId, taskId, command);
|
|
}
|
|
if (localTask.status !== 'running') {
|
|
const result = this.control(scopeId, taskId, command);
|
|
if (result.status === 'not_found' || result.status === 'invalid') return result;
|
|
if (this.terminalControlInvocations.size >= MAX_TERMINAL_CONTROL_INVOCATIONS) {
|
|
const oldestPersisted = [...this.terminalControlInvocations].find(
|
|
([, invocation]) => invocation.receiptPersisted === true,
|
|
)?.[0];
|
|
if (oldestPersisted == null) {
|
|
return {
|
|
status: 'invalid',
|
|
message:
|
|
'Too many terminal control invocations are awaiting persistence; retry shortly.',
|
|
};
|
|
}
|
|
this.terminalControlInvocations.delete(oldestPersisted);
|
|
}
|
|
this.terminalControlInvocations.set(key, {
|
|
scopeId,
|
|
taskId,
|
|
invocationId,
|
|
fingerprint,
|
|
command: boundedControlCommand(command),
|
|
commandMessageTruncated:
|
|
'message' in command && command.message.length > MAX_DURABLE_CONTROL_MESSAGE_CHARS,
|
|
result,
|
|
createdAt: Date.now(),
|
|
});
|
|
return result;
|
|
}
|
|
if (!this.makeRoomForInvocation()) {
|
|
/** Every tracked invocation belongs to a task this store still holds. Applying
|
|
* this command without room to record it would let a caller retry apply it a
|
|
* second time, so it is refused before the child is touched at all. */
|
|
logger.warn('[subagentThreads] Refused a control; live invocation records are full');
|
|
return {
|
|
status: 'invalid',
|
|
message: 'Too many control invocations are in flight for this process; retry shortly.',
|
|
};
|
|
}
|
|
const result = this.control(scopeId, taskId, command);
|
|
if (result.status === 'not_found' || result.status === 'invalid') {
|
|
return result;
|
|
}
|
|
if (result.status !== 'accepted' && result.status !== 'cancelled') {
|
|
return result;
|
|
}
|
|
const invocation: ControlInvocationRecord = {
|
|
scopeId,
|
|
taskId,
|
|
invocationId,
|
|
fingerprint,
|
|
command: boundedControlCommand(command),
|
|
commandMessageTruncated:
|
|
'message' in command && command.message.length > MAX_DURABLE_CONTROL_MESSAGE_CHARS,
|
|
result,
|
|
createdAt: Date.now(),
|
|
};
|
|
this.controlInvocations.set(key, invocation);
|
|
if (
|
|
result.status === 'accepted' &&
|
|
result.controlId != null &&
|
|
(command.action === 'steer' || command.action === 'queue' || command.action === 'interrupt')
|
|
) {
|
|
this.controlInvocationByReceipt.set(
|
|
controlReceiptKey(scopeId, taskId, result.controlId),
|
|
invocation,
|
|
);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private async controlInvocationAndPersist(
|
|
scopeId: string,
|
|
taskId: string,
|
|
command: SubagentTaskControlCommand,
|
|
invocationId: string,
|
|
): Promise<SubagentTaskControlResult> {
|
|
const retained = await this.replayRetainedControl(scopeId, taskId, command, invocationId);
|
|
if (retained != null) return retained;
|
|
try {
|
|
const durable = await this.replayDurableControl(scopeId, taskId, command, invocationId);
|
|
if (durable != null) return durable;
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Failed to preflight a child control receipt', error);
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
const localTask = this.get(scopeId, taskId);
|
|
if (localTask?.status === 'running') {
|
|
if (localTask.threadId == null || localTask.threadId === '') {
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
const scope = parseScope(scopeId);
|
|
const now = new Date();
|
|
const reservation: ISubagentTaskControlReceipt = {
|
|
invocationId,
|
|
fingerprint: controlFingerprint(command),
|
|
action: command.action,
|
|
status: 'reserved',
|
|
createdAt: now,
|
|
updatedAt: now,
|
|
...boundedControlMessage(
|
|
boundedControlCommand(command),
|
|
'message' in command && command.message.length > MAX_DURABLE_CONTROL_MESSAGE_CHARS,
|
|
),
|
|
};
|
|
let reserved: boolean | 'unchanged' | 'conflict';
|
|
try {
|
|
reserved = (await this.controlReservationSlot(() =>
|
|
this.runWithOwnerContext(scope, () =>
|
|
this.methods.recordSubagentTaskControlReceipt({
|
|
userId: scope.userId,
|
|
conversationId: localTask.threadId as string,
|
|
taskId,
|
|
...(scope.tenantId == null ? {} : { tenantId: scope.tenantId }),
|
|
receipt: reservation,
|
|
}),
|
|
),
|
|
)) as boolean | 'unchanged' | 'conflict';
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Failed to reserve a child control invocation', error);
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
if (reserved === 'conflict') {
|
|
return {
|
|
status: 'invalid',
|
|
message: 'This control invocation id was already used for a different command.',
|
|
};
|
|
}
|
|
if (reserved === 'unchanged') {
|
|
try {
|
|
const replay = await this.replayDurableControl(scopeId, taskId, command, invocationId);
|
|
if (replay != null) return replay;
|
|
} catch {
|
|
// Normalize storage ambiguity at the owner boundary below.
|
|
}
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
if (!reserved) throw new SubagentTaskOwnerUnavailableError();
|
|
}
|
|
const invocationKey = `${scopeId}\u0000${taskId}\u0000${invocationId}`;
|
|
const result = this.controlInvocation(scopeId, taskId, command, invocationId);
|
|
const retainedInvocation =
|
|
this.controlInvocations.get(invocationKey) ??
|
|
this.terminalControlInvocations.get(invocationKey);
|
|
const invocation: ControlInvocationRecord | undefined =
|
|
retainedInvocation ??
|
|
(result.status === 'not_found' || result.status === 'invalid'
|
|
? undefined
|
|
: {
|
|
scopeId,
|
|
taskId,
|
|
invocationId,
|
|
fingerprint: controlFingerprint(command),
|
|
command: boundedControlCommand(command),
|
|
commandMessageTruncated:
|
|
'message' in command && command.message.length > MAX_DURABLE_CONTROL_MESSAGE_CHARS,
|
|
result,
|
|
createdAt: Date.now(),
|
|
});
|
|
const threadId = 'task' in result ? result.task.threadId : undefined;
|
|
if (invocation == null || threadId == null) return result;
|
|
const receipt = this.controlResultReceipt(invocation);
|
|
let persistedReceipt: ISubagentTaskControlReceipt | undefined;
|
|
let persistence: Promise<void> | undefined;
|
|
try {
|
|
if (receipt != null) {
|
|
persistedReceipt = receipt;
|
|
invocation.receipt = persistedReceipt;
|
|
invocation.receiptPersisted = false;
|
|
persistence = this.queueControlReceipt(scopeId, taskId, threadId, persistedReceipt);
|
|
invocation.receiptPersistence = persistence;
|
|
await persistence;
|
|
/** A terminal SDK transition can replace the accepted projection while its
|
|
* older write is awaiting Mongo. Mark only the exact generation awaited. */
|
|
if (invocation.receipt === persistedReceipt) invocation.receiptPersisted = true;
|
|
/** Do not acknowledge an older generation while a newer authoritative SDK
|
|
* transition is still queued. There is no async gap after this loop, so the
|
|
* generation proven durable is the one returned to the caller. */
|
|
await this.awaitCurrentControlReceipt(scopeId, taskId, invocation);
|
|
}
|
|
} catch (error) {
|
|
if (error instanceof SubagentControlReceiptConflictError) {
|
|
const invalid: SubagentTaskControlResult = {
|
|
status: 'invalid',
|
|
message: 'This control invocation id was already used for a different command.',
|
|
};
|
|
invocation.result = invalid;
|
|
invocation.receiptPersisted = true;
|
|
if (result.status === 'accepted' && result.controlId != null) {
|
|
this.controlInvocationByReceipt.delete(
|
|
controlReceiptKey(scopeId, taskId, result.controlId),
|
|
);
|
|
super.control(scopeId, taskId, {
|
|
action: 'cancel_message',
|
|
controlId: result.controlId,
|
|
});
|
|
}
|
|
return invalid;
|
|
}
|
|
logger.warn('[subagentThreads] Failed to durably accept a child control', error);
|
|
throw new SubagentTaskOwnerUnavailableError();
|
|
} finally {
|
|
if (
|
|
persistence != null &&
|
|
persistedReceipt != null &&
|
|
invocation.receiptPersistence === persistence &&
|
|
invocation.receipt === persistedReceipt &&
|
|
invocation.receiptPersisted === true
|
|
) {
|
|
invocation.receiptPersistence = undefined;
|
|
}
|
|
}
|
|
return invocation.result;
|
|
}
|
|
|
|
private async awaitCurrentControlReceipt(
|
|
scopeId: string,
|
|
taskId: string,
|
|
invocation: ControlInvocationRecord,
|
|
): Promise<void> {
|
|
while (invocation.receipt != null && invocation.receiptPersisted !== true) {
|
|
const receipt = invocation.receipt;
|
|
const persistence =
|
|
invocation.receiptPersistence ?? this.flushControlReceipts(scopeId, taskId);
|
|
invocation.receiptPersistence = persistence;
|
|
await persistence;
|
|
if (invocation.receipt === receipt && hasDurableControlReceipt(invocation)) return;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Frees invocation slots by dropping records whose task the store no longer holds:
|
|
* a settled task cannot be controlled again, so its record is worthless, while a
|
|
* live one is exactly what a caller retry needs to replay instead of applying its
|
|
* command twice. The sweep runs only when the window is full and clears every dead
|
|
* record at once, so it is amortized rather than repeated per control.
|
|
*/
|
|
private makeRoomForInvocation(): boolean {
|
|
if (this.controlInvocations.size < this.maxControlInvocations) {
|
|
return true;
|
|
}
|
|
for (const [key, invocation] of this.controlInvocations) {
|
|
if (this.get(invocation.scopeId, invocation.taskId) == null) {
|
|
this.controlInvocations.delete(key);
|
|
const result = invocation.result;
|
|
if (result.status === 'accepted' && result.controlId != null) {
|
|
this.controlInvocationByReceipt.delete(
|
|
controlReceiptKey(invocation.scopeId, invocation.taskId, result.controlId),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
return this.controlInvocations.size < this.maxControlInvocations;
|
|
}
|
|
|
|
/** Returns this process's tasks plus tasks reported by registered remote owners. */
|
|
async listTasks(scopeId: string): Promise<SubagentTaskSnapshot[]> {
|
|
const local = super.list(scopeId);
|
|
const remote = (await this.taskControlTransport?.list(scopeId)) ?? [];
|
|
const byId = new Map(local.map((task) => [task.taskId, task]));
|
|
for (const task of remote) {
|
|
byId.set(task.taskId, task);
|
|
}
|
|
/** The remote aggregation and each owner's reply carry their own bound, but this
|
|
* merge is what the poll tool reads: without a cap here the list the model sees is
|
|
* that bound plus however many children this replica happens to own. */
|
|
return boundedTaskList([...byId.values()]);
|
|
}
|
|
|
|
/** Fast capability probe used while deciding whether a later turn needs the poll tool. */
|
|
async hasTasks(scopeId: string): Promise<boolean> {
|
|
if (super.list(scopeId).length > 0) {
|
|
return true;
|
|
}
|
|
return (await this.taskControlTransport?.hasTasks(scopeId)) ?? false;
|
|
}
|
|
|
|
override control(
|
|
scopeId: string,
|
|
taskId: string,
|
|
command: SubagentTaskControlCommand,
|
|
): SubagentTaskControlResult {
|
|
const snapshot = this.get(scopeId, taskId);
|
|
const lockKey = snapshot?.threadId == null ? undefined : `${scopeId}\u0000${snapshot.threadId}`;
|
|
const lease = lockKey == null ? undefined : this.activeThreads.get(lockKey);
|
|
if (snapshot != null && lease?.taskId === taskId && lease.settling) {
|
|
return { status: 'not_running', task: snapshot };
|
|
}
|
|
const result = super.control(scopeId, taskId, command);
|
|
if (
|
|
command.action === 'cancel' &&
|
|
result.status === 'cancelled' &&
|
|
lockKey != null &&
|
|
lease?.taskId === taskId &&
|
|
!lease.running
|
|
) {
|
|
this.activeThreads.delete(lockKey);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/** Finds a provisional child lease before its conversation is durable. */
|
|
isThreadActiveForOwner(userId: string, threadId: string, tenantId?: string): boolean {
|
|
const suffix = `\u0000${threadId}`;
|
|
for (const lockKey of this.activeThreads.keys()) {
|
|
if (!lockKey.endsWith(suffix)) {
|
|
continue;
|
|
}
|
|
const scope = parseScope(lockKey.slice(0, -suffix.length));
|
|
if (scope.userId === userId && matchesTenant(scope.tenantId, tenantId)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/** Cancels active descendants before their owning conversations are removed. */
|
|
cancelForConversations(
|
|
userId: string,
|
|
conversationIds: Iterable<string>,
|
|
tenantId?: string,
|
|
): number {
|
|
const targets = new Set(conversationIds);
|
|
return this.cancelMatchingThreads(
|
|
(scope, threadId) =>
|
|
scope.userId === userId &&
|
|
matchesTenant(scope.tenantId, tenantId) &&
|
|
(targets.has(scope.parentConversationId) || targets.has(threadId)),
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Resolves every cancellation target while the conversations still exist. The plan is
|
|
* replayed after deletion, when those rows can no longer be read back, so the second
|
|
* pass only has to reach registered owners through Redis.
|
|
*/
|
|
async planCancellationForConversations(
|
|
userId: string,
|
|
conversationIds: Iterable<string>,
|
|
tenantId?: string,
|
|
): Promise<SubagentCancellationPlan> {
|
|
const targetIds = [...new Set(conversationIds)];
|
|
const plan: SubagentCancellationPlan = {
|
|
userId,
|
|
...(tenantId == null ? {} : { tenantId }),
|
|
conversationIds: targetIds,
|
|
scopes: [],
|
|
leases: [],
|
|
};
|
|
if (targetIds.length === 0 || this.taskControlTransport == null) {
|
|
return plan;
|
|
}
|
|
const targets = new Set(targetIds);
|
|
const scopeIdFor = (parentConversationId: string): string =>
|
|
serializeScope({
|
|
userId,
|
|
parentConversationId,
|
|
...(tenantId ? { tenantId } : {}),
|
|
});
|
|
/** Deleting a conversation takes its whole scope; a deleted child only cancels its
|
|
* own thread inside a parent scope that survives. */
|
|
const conversations = await Promise.all(
|
|
targetIds.map((conversationId) => this.methods.getConvo(userId, conversationId)),
|
|
);
|
|
const threadTargetsByParent = new Map<string, Set<string>>();
|
|
for (const [index, conversation] of conversations.entries()) {
|
|
const parentConversationId = conversation?.subagentThread?.parentConversationId;
|
|
if (
|
|
parentConversationId == null ||
|
|
targets.has(parentConversationId) ||
|
|
!matchesTenant(conversation?.tenantId, tenantId)
|
|
) {
|
|
continue;
|
|
}
|
|
const threadIds = threadTargetsByParent.get(parentConversationId) ?? new Set<string>();
|
|
threadIds.add(targetIds[index]);
|
|
threadTargetsByParent.set(parentConversationId, threadIds);
|
|
}
|
|
plan.scopes = [
|
|
...targetIds.map((parentConversationId) => ({
|
|
scopeId: scopeIdFor(parentConversationId),
|
|
threadIds: null,
|
|
})),
|
|
...[...threadTargetsByParent].map(([parentConversationId, threadIds]) => ({
|
|
scopeId: scopeIdFor(parentConversationId),
|
|
threadIds: [...threadIds],
|
|
})),
|
|
];
|
|
/** Captured now so descendants removed by the cascade stay reachable afterwards. */
|
|
plan.leases = await this.methods.listActiveSubagentThreadLeases({
|
|
user: userId,
|
|
now: new Date(),
|
|
...(tenantId == null ? {} : { tenantId }),
|
|
});
|
|
return plan;
|
|
}
|
|
|
|
/**
|
|
* Cancels local children and replays a plan against registered remote owners.
|
|
* `removedConversationIds` extends it with the cascade a deletion reported, matched
|
|
* against leases captured before those rows were removed.
|
|
*/
|
|
async cancelPlan(
|
|
plan: SubagentCancellationPlan,
|
|
removedConversationIds: Iterable<string> = [],
|
|
): Promise<number> {
|
|
const { userId, tenantId } = plan;
|
|
const planned = new Set(plan.conversationIds);
|
|
const removed = new Set(removedConversationIds);
|
|
/** A cascade can remove descendants the plan never named — a grandchild lives in
|
|
* its own parent's scope, not the deleted root's — so every removed conversation
|
|
* is cancelled as a scope of its own. */
|
|
const targets = [...new Set([...planned, ...removed])];
|
|
let cancelled = this.cancelForConversations(userId, targets, tenantId);
|
|
const transport = this.taskControlTransport;
|
|
if (transport == null) {
|
|
if (removed.size > 0) {
|
|
this.dropDeletedControlReceiptWork(userId, removed, tenantId);
|
|
}
|
|
return cancelled;
|
|
}
|
|
const cancelSlot = createConcurrencyLimiter(DELETION_CANCEL_CONCURRENCY);
|
|
const cascadeScopes = [...removed]
|
|
.filter((conversationId) => !planned.has(conversationId))
|
|
.map((parentConversationId) => ({
|
|
scopeId: serializeScope({
|
|
userId,
|
|
parentConversationId,
|
|
...(tenantId ? { tenantId } : {}),
|
|
}),
|
|
threadIds: null,
|
|
}));
|
|
const scopeCancellations = [...plan.scopes, ...cascadeScopes].map((scope) => {
|
|
const parsed = parseScope(scope.scopeId);
|
|
const removedForScope = [
|
|
...(removed.has(parsed.parentConversationId) ? [parsed.parentConversationId] : []),
|
|
...(scope.threadIds ?? []).filter((threadId) => removed.has(threadId)),
|
|
];
|
|
return cancelSlot(() =>
|
|
transport.cancelScope(scope.scopeId, scope.threadIds, removedForScope),
|
|
);
|
|
});
|
|
const leaseCancellations = plan.leases
|
|
.filter(
|
|
(lease) => removed.has(lease.parentConversationId) || removed.has(lease.conversationId),
|
|
)
|
|
.map((lease) =>
|
|
cancelSlot(async () => {
|
|
const scopeId = serializeScope({
|
|
userId,
|
|
parentConversationId: lease.parentConversationId,
|
|
...(tenantId ? { tenantId } : {}),
|
|
});
|
|
const removedForLease = [lease.parentConversationId, lease.conversationId].filter((id) =>
|
|
removed.has(id),
|
|
);
|
|
const stopped = await transport.cancelScope(
|
|
scopeId,
|
|
[lease.conversationId],
|
|
removedForLease,
|
|
);
|
|
if (stopped > 0 || this.cancelUnroutedTask == null) return stopped;
|
|
return (await this.cancelUnroutedTask({
|
|
userId,
|
|
parentConversationId: lease.parentConversationId,
|
|
taskId: lease.taskId,
|
|
...(tenantId ? { tenantId } : {}),
|
|
}))
|
|
? 1
|
|
: 0;
|
|
}),
|
|
);
|
|
try {
|
|
for (const count of await Promise.all(scopeCancellations)) {
|
|
cancelled += count;
|
|
}
|
|
for (const count of await Promise.all(leaseCancellations)) {
|
|
cancelled += count;
|
|
}
|
|
return cancelled;
|
|
} finally {
|
|
/** Delivery may fail after the deletion committed. Receipt persistence for
|
|
* removed rows is still terminal and must not poison graceful shutdown. */
|
|
if (removed.size > 0) {
|
|
this.dropDeletedControlReceiptWork(userId, removed, tenantId);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** A successful deletion makes false receipt writes permanent, not retryable.
|
|
* Remove only work whose authorized parent or child was actually deleted. */
|
|
private dropDeletedControlReceiptWork(
|
|
userId: string,
|
|
removedConversationIds: ReadonlySet<string>,
|
|
tenantId?: string,
|
|
): void {
|
|
const matchesDeletedScope = (scopeId: string): boolean => {
|
|
const scope = parseScope(scopeId);
|
|
return (
|
|
scope.userId === userId &&
|
|
matchesTenant(scope.tenantId, tenantId) &&
|
|
removedConversationIds.has(scope.parentConversationId)
|
|
);
|
|
};
|
|
for (const [key, pending] of this.pendingControlReceipts) {
|
|
const task = parseControlTaskKey(key);
|
|
if (task == null) continue;
|
|
const deleteWholeTask = matchesDeletedScope(task.scopeId);
|
|
for (const [invocationId, candidate] of pending) {
|
|
if (deleteWholeTask || removedConversationIds.has(candidate.threadId)) {
|
|
pending.delete(invocationId);
|
|
}
|
|
}
|
|
if (pending.size === 0) {
|
|
this.pendingControlReceipts.delete(key);
|
|
const retry = this.controlPersistenceRetryTimers.get(key);
|
|
if (retry != null) clearTimeout(retry);
|
|
this.controlPersistenceRetryTimers.delete(key);
|
|
}
|
|
}
|
|
const dropInvocation = (key: string, invocation: ControlInvocationRecord): void => {
|
|
const resultThreadId =
|
|
'task' in invocation.result ? invocation.result.task.threadId : undefined;
|
|
if (
|
|
!matchesDeletedScope(invocation.scopeId) &&
|
|
(resultThreadId == null || !removedConversationIds.has(resultThreadId))
|
|
) {
|
|
return;
|
|
}
|
|
this.controlInvocations.delete(key);
|
|
this.terminalControlInvocations.delete(key);
|
|
if (invocation.result.status === 'accepted' && invocation.result.controlId != null) {
|
|
this.controlInvocationByReceipt.delete(
|
|
controlReceiptKey(invocation.scopeId, invocation.taskId, invocation.result.controlId),
|
|
);
|
|
}
|
|
};
|
|
for (const [key, invocation] of this.controlInvocations) dropInvocation(key, invocation);
|
|
for (const [key, invocation] of this.terminalControlInvocations)
|
|
dropInvocation(key, invocation);
|
|
}
|
|
|
|
/** Cancels this process's live children for one scope, optionally narrowed to threads. */
|
|
private cancelForScope(scopeId: string, threadIds: string[] | null): number {
|
|
const scope = parseScope(scopeId);
|
|
const targets = threadIds == null ? null : new Set(threadIds);
|
|
return this.cancelMatchingThreads(
|
|
(candidate, threadId) =>
|
|
candidate.userId === scope.userId &&
|
|
candidate.parentConversationId === scope.parentConversationId &&
|
|
matchesTenant(candidate.tenantId, scope.tenantId) &&
|
|
(targets == null || targets.has(threadId)),
|
|
);
|
|
}
|
|
|
|
/** Cancels every active child owned by a user before a delete-all operation. */
|
|
cancelForOwner(userId: string, tenantId?: string): number {
|
|
return this.cancelMatchingThreads(
|
|
(scope) => scope.userId === userId && matchesTenant(scope.tenantId, tenantId),
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Deletes an owner's conversations behind a durable admission fence. Draining alone
|
|
* cannot close the race: a child admitted on another replica after the drain read
|
|
* its leases would begin provider work against a parent that is about to disappear.
|
|
* Fencing first inverts that — the fence is written before any lease is read, and a
|
|
* child validates the fence after its own lease is written, so one of the two always
|
|
* observes the other. The fence expires by itself, so a process lost mid-deletion
|
|
* cannot leave the account unable to run subagents.
|
|
*/
|
|
async withOwnerDeletionFence<T>(
|
|
userId: string,
|
|
tenantId: string | undefined,
|
|
deletion: () => Promise<T>,
|
|
recoverAdditionalOwnerWork?: () => Promise<void>,
|
|
): Promise<T> {
|
|
const fenceWindowMs = this.ownerDrainTimeoutMs + this.ownerFenceGraceMs;
|
|
const token = randomUUID();
|
|
/** Only a confirmed write moves this, so a run of failed renewals leaves it in the
|
|
* past and the deletion can tell that its fence is no longer guaranteed. */
|
|
let fencedUntil = Date.now() + fenceWindowMs;
|
|
let fenceLapsed = false;
|
|
await this.fenceOwnerAdmission?.(userId, token, new Date(fencedUntil));
|
|
/** A very large account, or a stalled database, can outlast one fence window, and
|
|
* a fence that expires mid-deletion lets another replica admit a child against
|
|
* conversations being deleted. It is renewed for as long as the work runs. */
|
|
let releasing = false;
|
|
let inFlight: Promise<void> | undefined;
|
|
const renewal = setInterval(
|
|
() => {
|
|
if (inFlight != null) {
|
|
return;
|
|
}
|
|
inFlight = (async () => {
|
|
const deadline = fencedUntil;
|
|
const renewedUntil = Date.now() + fenceWindowMs;
|
|
const held = await this.renewOwnerAdmission?.(userId, token, new Date(renewedUntil));
|
|
if (held === false) {
|
|
/** The durable entry was absent, so admission may already have opened even
|
|
* when the local deadline has not passed. Reacquire for containment, but
|
|
* retain the lapse so the enclosing deletion re-drains before success. */
|
|
fenceLapsed = true;
|
|
if (releasing) {
|
|
return;
|
|
}
|
|
/** The entry is gone — expired, or pruned by another deletion — so this
|
|
* deletion takes its fence again rather than running on unfenced. */
|
|
await this.fenceOwnerAdmission?.(userId, token, new Date(renewedUntil));
|
|
}
|
|
if (Date.now() >= deadline) {
|
|
/** The write only landed after the deadline it was meant to extend, so
|
|
* admission stood open in between and a child could have taken a lease the
|
|
* drain had already read past. A fence cannot be restored backwards over
|
|
* that gap, so the lapse is recorded rather than papered over. */
|
|
fenceLapsed = true;
|
|
return;
|
|
}
|
|
fencedUntil = renewedUntil;
|
|
})()
|
|
.catch((error) => {
|
|
logger.warn('[subagentThreads] Failed to hold the owner admission fence', error);
|
|
})
|
|
.finally(() => {
|
|
inFlight = undefined;
|
|
});
|
|
},
|
|
Math.max(1, Math.floor(fenceWindowMs / 3)),
|
|
);
|
|
renewal.unref?.();
|
|
const stopRenewal = async (): Promise<void> => {
|
|
clearInterval(renewal);
|
|
await inFlight;
|
|
};
|
|
const fenceHeld = (): boolean =>
|
|
this.fenceOwnerAdmission == null || (!fenceLapsed && Date.now() < fencedUntil);
|
|
try {
|
|
await this.cancelAndDrainForOwner(userId, tenantId);
|
|
/** The drain can outlast the fence window when the database is unreachable, and
|
|
* renewals that keep failing leave the account open to admitting a child against
|
|
* conversations about to disappear. Nothing has been removed yet, so this fails
|
|
* closed and the caller retries once the fence can be held again. */
|
|
if (!fenceHeld()) {
|
|
throw new Error('The subagent admission fence expired before this deletion began.');
|
|
}
|
|
const deleted = await deletion();
|
|
/** Settle a renewal already in flight before deciding whether deletion crossed a
|
|
* gap. Otherwise a late write can report the lapse only after this check and the
|
|
* finally block would release the fence without re-draining. */
|
|
await stopRenewal();
|
|
if (!fenceHeld()) {
|
|
/** The rows are gone, but the gap can leave a child another replica admitted
|
|
* while the fence was down. Re-take the fence and drain that work before this
|
|
* operation may report success. */
|
|
logger.error(
|
|
'[subagentThreads] Owner deletion outlived its admission fence; draining children admitted in the gap',
|
|
);
|
|
const recoveryUntil = Date.now() + fenceWindowMs;
|
|
const reheld = await this.renewOwnerAdmission?.(userId, token, new Date(recoveryUntil));
|
|
if (reheld !== true) {
|
|
await this.fenceOwnerAdmission?.(userId, token, new Date(recoveryUntil));
|
|
}
|
|
if (Date.now() >= recoveryUntil) {
|
|
throw new Error('The subagent admission fence expired while it was being restored.');
|
|
}
|
|
fencedUntil = recoveryUntil;
|
|
fenceLapsed = false;
|
|
await this.cancelAndDrainForOwner(userId, tenantId);
|
|
/** The fence is shared by host-owned execution classes that do not use the
|
|
* subagent lease store. Let the caller re-drain those classes after the same
|
|
* lapse, while the restored fence still prevents fresh admission. */
|
|
await recoverAdditionalOwnerWork?.();
|
|
if (!fenceHeld()) {
|
|
throw new Error('The subagent admission fence expired while recovering this deletion.');
|
|
}
|
|
}
|
|
return deleted;
|
|
} finally {
|
|
releasing = true;
|
|
await stopRenewal();
|
|
/** `clearInterval` stops only future passes. A renewal still waiting on the
|
|
* database would otherwise find its fence released, read that as expiry, and
|
|
* write a fresh one that nothing is left to lift. */
|
|
/** Only this deletion's own fence is lifted: an overlapping deletion that took a
|
|
* later one keeps admission closed until it finishes. */
|
|
await this.releaseOwnerAdmission?.(userId, token).catch((error) => {
|
|
logger.warn('[subagentThreads] Failed to release the owner admission fence', error);
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Cancels local work and waits for every replica's durable lease to drain. Each task
|
|
* is cancelled under one invocation held for the whole drain and only while its
|
|
* owner has not answered: a fresh invocation per poll would retain a replay entry on
|
|
* the owner for every pass, and a task already reported cancelled needs no second
|
|
* command, only its lease to disappear.
|
|
*/
|
|
async cancelAndDrainForOwner(userId: string, tenantId?: string): Promise<void> {
|
|
this.cancelForOwner(userId, tenantId);
|
|
const deadline = Date.now() + this.ownerDrainTimeoutMs;
|
|
const invocations = new Map<string, string>();
|
|
const answered = new Set<string>();
|
|
while (true) {
|
|
const activeLeases = await this.methods.listActiveSubagentThreadLeases({
|
|
user: userId,
|
|
now: new Date(),
|
|
...(tenantId == null ? {} : { tenantId }),
|
|
});
|
|
if (activeLeases.length === 0) {
|
|
return;
|
|
}
|
|
if (Date.now() >= deadline) {
|
|
throw new Error('Timed out draining detached subagent tasks for account deletion.');
|
|
}
|
|
const unanswered = activeLeases.filter(
|
|
({ parentConversationId, taskId }) => !answered.has(drainKey(parentConversationId, taskId)),
|
|
);
|
|
for (let index = 0; index < unanswered.length; index += DELETION_CANCEL_CONCURRENCY) {
|
|
await Promise.all(
|
|
unanswered
|
|
.slice(index, index + DELETION_CANCEL_CONCURRENCY)
|
|
.map(({ parentConversationId, taskId }) =>
|
|
this.cancelDrainedTask(
|
|
{ userId, parentConversationId, taskId, tenantId },
|
|
invocations,
|
|
answered,
|
|
),
|
|
),
|
|
);
|
|
}
|
|
await new Promise<void>((resolve) => setTimeout(resolve, this.ownerDrainPollMs));
|
|
}
|
|
}
|
|
|
|
/** Sends one drained task's cancellation, retrying only unconfirmed deliveries. */
|
|
private async cancelDrainedTask(
|
|
target: { userId: string; parentConversationId: string; taskId: string; tenantId?: string },
|
|
invocations: Map<string, string>,
|
|
answered: Set<string>,
|
|
): Promise<void> {
|
|
const { userId, parentConversationId, taskId, tenantId } = target;
|
|
const key = drainKey(parentConversationId, taskId);
|
|
const invocationId = invocations.get(key) ?? randomUUID();
|
|
invocations.set(key, invocationId);
|
|
const scopeId = serializeScope({
|
|
userId,
|
|
parentConversationId,
|
|
...(tenantId == null ? {} : { tenantId }),
|
|
});
|
|
try {
|
|
const result = await this.controlTask(scopeId, taskId, { action: 'cancel' }, invocationId);
|
|
/** Only the owner confirming the task is stopped ends the commands for it. A
|
|
* `not_found` means its registration is missing while its lease is still live —
|
|
* an unconfirmed delivery, retried once the owner republishes itself. */
|
|
if (result.status === 'cancelled' || result.status === 'not_running') {
|
|
answered.add(key);
|
|
} else if (result.status === 'not_found' && this.cancelUnroutedTask != null) {
|
|
const stopped = await this.cancelUnroutedTask(target);
|
|
if (stopped) {
|
|
answered.add(key);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Retrying an unconfirmed child cancellation', error);
|
|
}
|
|
}
|
|
|
|
private startSharedLeaseHeartbeat(
|
|
scopeId: string,
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
lease: TaskThreadLease,
|
|
): void {
|
|
const shared = lease.shared;
|
|
if (shared == null) {
|
|
return;
|
|
}
|
|
const heartbeat = () => {
|
|
if (shared.lost || shared.heartbeatInFlight != null) {
|
|
return;
|
|
}
|
|
const renewal = (async () => {
|
|
let ownerActive = false;
|
|
try {
|
|
ownerActive = await this.isOwnerActive(scope.userId);
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Failed to verify the child-thread owner', error);
|
|
}
|
|
if (!ownerActive) {
|
|
const task = this.get(scopeId, lease.taskId);
|
|
if (task?.status === 'running') {
|
|
super.control(scopeId, lease.taskId, { action: 'cancel' });
|
|
}
|
|
}
|
|
if (!(await this.renewSharedLeaseFence(scope, threadId, lease))) {
|
|
const task = this.get(scopeId, lease.taskId);
|
|
if (task?.status === 'running') {
|
|
super.control(scopeId, lease.taskId, { action: 'cancel' });
|
|
}
|
|
}
|
|
})().finally(() => {
|
|
if (shared.heartbeatInFlight === renewal) {
|
|
shared.heartbeatInFlight = undefined;
|
|
}
|
|
});
|
|
shared.heartbeatInFlight = renewal;
|
|
};
|
|
shared.heartbeat = setInterval(heartbeat, this.leaseHeartbeatMs);
|
|
}
|
|
|
|
private async renewSharedLease(
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
lease: TaskThreadLease,
|
|
): Promise<boolean> {
|
|
const shared = lease.shared;
|
|
if (shared == null || shared.lost) {
|
|
return false;
|
|
}
|
|
try {
|
|
if (!(await this.isOwnerActive(scope.userId))) {
|
|
return false;
|
|
}
|
|
return this.renewSharedLeaseFence(scope, threadId, lease);
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Failed to verify the child-thread owner', error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private async renewSharedLeaseFence(
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
lease: TaskThreadLease,
|
|
): Promise<boolean> {
|
|
const shared = lease.shared;
|
|
if (shared == null || shared.lost) {
|
|
return false;
|
|
}
|
|
try {
|
|
const deadline = shared.expiresAt;
|
|
const now = new Date();
|
|
const renewedUntil = now.getTime() + this.leaseTtlMs;
|
|
const renewed = await this.methods.renewSubagentThreadLease({
|
|
user: scope.userId,
|
|
conversationId: threadId,
|
|
token: shared.token,
|
|
now,
|
|
expiresAt: new Date(renewedUntil),
|
|
...(scope.tenantId == null ? {} : { tenantId: scope.tenantId }),
|
|
});
|
|
if (!renewed) {
|
|
shared.lost = true;
|
|
return false;
|
|
}
|
|
if (Date.now() >= deadline) {
|
|
/** The renewal filter compares against the `now` captured before the call, so a
|
|
* write that only lands after this lease had expired still succeeds and moves
|
|
* the row forward. An owner drain reading active leases in that gap saw this
|
|
* thread as free, so the executor stops rather than run past a deletion that
|
|
* may already have stepped over it. */
|
|
shared.lost = true;
|
|
return false;
|
|
}
|
|
shared.expiresAt = renewedUntil;
|
|
return true;
|
|
} catch (error) {
|
|
shared.lost = true;
|
|
logger.warn('[subagentThreads] Lost the shared child-thread lease', error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private async stopAndReleaseSharedLease(
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
lease: TaskThreadLease,
|
|
): Promise<void> {
|
|
const shared = lease.shared;
|
|
if (shared == null) {
|
|
return;
|
|
}
|
|
if (shared.heartbeat != null) {
|
|
clearInterval(shared.heartbeat);
|
|
}
|
|
await shared.heartbeatInFlight;
|
|
try {
|
|
await this.methods.releaseSubagentThreadLease({
|
|
user: scope.userId,
|
|
conversationId: threadId,
|
|
token: shared.token,
|
|
...(scope.tenantId == null ? {} : { tenantId: scope.tenantId }),
|
|
});
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Failed to release the shared child-thread lease', error);
|
|
}
|
|
}
|
|
|
|
/** Whether a durable child may be created below the supplied conversation depth. */
|
|
canCreateChildThread(parentDepth: number): boolean {
|
|
return (
|
|
Number.isSafeInteger(parentDepth) && parentDepth >= 0 && parentDepth < this.maxThreadDepth
|
|
);
|
|
}
|
|
|
|
private async prepareThread(
|
|
scopeId: string,
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
isContinuation: boolean,
|
|
request: SubagentTaskStartRequest,
|
|
taskId: string,
|
|
lease: TaskThreadLease,
|
|
): Promise<PreparedThread> {
|
|
if (!(await this.isOwnerActive(scope.userId))) {
|
|
throw new SubagentThreadPublicError('The thread owner is unavailable.');
|
|
}
|
|
const [parent, existing] = await Promise.all([
|
|
this.methods.getConvo(scope.userId, scope.parentConversationId),
|
|
this.methods.getConvo(scope.userId, threadId),
|
|
]);
|
|
if (parent == null || !matchesTenant(parent.tenantId, scope.tenantId)) {
|
|
throw new SubagentThreadPublicError('Parent thread is unavailable.');
|
|
}
|
|
|
|
let createdThread = false;
|
|
let sharedLeaseAcquired = false;
|
|
try {
|
|
let conversation = existing;
|
|
if (conversation == null && isContinuation) {
|
|
throw new SubagentThreadPublicError(
|
|
'Child thread is unavailable for this subagent and parent scope.',
|
|
);
|
|
}
|
|
if (conversation == null) {
|
|
const parentDepth = parent.subagentThread?.depth ?? 0;
|
|
if (!this.canCreateChildThread(parentDepth)) {
|
|
throw new SubagentThreadPublicError(
|
|
`Subagent thread depth exceeds the configured limit of ${this.maxThreadDepth}.`,
|
|
);
|
|
}
|
|
const depth = parentDepth + 1;
|
|
const agentId = childAgentId(request);
|
|
const reserved = await this.methods.reserveSubagentThread({
|
|
user: scope.userId,
|
|
conversationId: threadId,
|
|
...(scope.tenantId == null ? {} : { tenantId: scope.tenantId }),
|
|
conversation: {
|
|
conversationId: threadId,
|
|
endpoint: EModelEndpoint.agents,
|
|
title: `Subagent: ${request.subagentType}`.slice(0, 120),
|
|
...(agentId == null ? {} : { agent_id: agentId }),
|
|
...retentionFields(parent),
|
|
subagentThread: {
|
|
rootConversationId:
|
|
parent.subagentThread?.rootConversationId ?? scope.parentConversationId,
|
|
parentConversationId: scope.parentConversationId,
|
|
parentMessageId: request.parentRunId || request.parentToolCallId,
|
|
parentToolCallId: request.parentToolCallId,
|
|
...(request.parentAgentId == null ? {} : { parentAgentId: request.parentAgentId }),
|
|
subagentType: request.subagentType,
|
|
subagentKind: request.subagentKind as 'agent' | 'graph',
|
|
depth,
|
|
},
|
|
},
|
|
});
|
|
conversation = reserved.conversation;
|
|
createdThread = reserved.created;
|
|
}
|
|
|
|
this.assertContinuation(scope, request, conversation);
|
|
const sharedToken = randomUUID();
|
|
const now = new Date();
|
|
sharedLeaseAcquired = await this.methods.acquireSubagentThreadLease({
|
|
user: scope.userId,
|
|
conversationId: threadId,
|
|
token: sharedToken,
|
|
taskId,
|
|
now,
|
|
expiresAt: new Date(now.getTime() + this.leaseTtlMs),
|
|
...(scope.tenantId == null ? {} : { tenantId: scope.tenantId }),
|
|
});
|
|
if (!sharedLeaseAcquired) {
|
|
throw new SubagentThreadPublicError(
|
|
'This child thread is already being continued by another run.',
|
|
);
|
|
}
|
|
lease.shared = {
|
|
token: sharedToken,
|
|
lost: false,
|
|
expiresAt: now.getTime() + this.leaseTtlMs,
|
|
};
|
|
/** A detached child has no request or stream handle after its parent returns.
|
|
* Keep the lease heartbeat referenced until settlement so Node cannot retire
|
|
* the execution context while its provider promise is still pending. */
|
|
this.startSharedLeaseHeartbeat(scopeId, scope, threadId, lease);
|
|
/** Account deletion can fence the owner after the optimistic probe but before
|
|
* this lease exists. Once the lease is visible, revalidate so deletion either
|
|
* observes and drains us or wins before any provider work can begin. */
|
|
logger.debug('[subagentThreads] Child-thread preparation entered stage', {
|
|
stage: 'owner_recheck',
|
|
taskId,
|
|
threadId,
|
|
});
|
|
if (
|
|
!(await observeSlowPreparation(this.isOwnerActive(scope.userId), {
|
|
stage: 'owner_recheck',
|
|
taskId,
|
|
threadId,
|
|
}))
|
|
) {
|
|
throw new SubagentThreadDeletedError('The thread owner is unavailable.');
|
|
}
|
|
logger.debug('[subagentThreads] Child-thread preparation entered stage', {
|
|
stage: 'transcript_read',
|
|
taskId,
|
|
threadId,
|
|
});
|
|
const allMessages = (await observeSlowPreparation(
|
|
this.methods.getMessages(
|
|
{ conversationId: threadId, user: scope.userId },
|
|
TRANSCRIPT_SELECT,
|
|
{ sort: { createdAt: 1, _id: 1 } },
|
|
),
|
|
{ stage: 'transcript_read', taskId, threadId },
|
|
)) as ThreadMessage[];
|
|
const attemptKey = createSubagentAttemptKey(scopeId, request.idempotencyKey);
|
|
const requestFingerprint = normalizedRequestFingerprint(request);
|
|
const priorAttempt = allMessages.filter(
|
|
(message) => message.subagentTask?.attemptKey === attemptKey,
|
|
);
|
|
if (priorAttempt.length > 0) {
|
|
if (
|
|
priorAttempt.some(
|
|
(message) => message.subagentTask?.requestFingerprint !== requestFingerprint,
|
|
)
|
|
) {
|
|
throw new SubagentThreadPublicError(
|
|
'The same parent tool call was already used with different subagent arguments.',
|
|
);
|
|
}
|
|
const terminal = [...priorAttempt]
|
|
.reverse()
|
|
.find((message) => message.subagentTask?.status !== 'running');
|
|
if (terminal?.subagentTask != null) {
|
|
const canonicalTaskId = terminal.messageId.endsWith(':assistant')
|
|
? terminal.messageId.slice(0, -':assistant'.length)
|
|
: '';
|
|
if (canonicalTaskId === '') {
|
|
throw new Error('The prior subagent result has an invalid task identity.');
|
|
}
|
|
const canonicalStart = priorAttempt.find(
|
|
(message) => message.messageId === `${canonicalTaskId}:user`,
|
|
);
|
|
const taskCreatedAt = durableMessageTime(
|
|
canonicalStart ?? terminal,
|
|
'The prior subagent result has no durable occurrence time.',
|
|
);
|
|
return {
|
|
conversation,
|
|
initialMessages: [],
|
|
initialStoredMessages: [],
|
|
attemptKey,
|
|
taskCreatedAt,
|
|
replay: {
|
|
status: terminal.subagentTask.status as 'completed' | 'error' | 'cancelled',
|
|
taskId: canonicalTaskId,
|
|
...(terminal.parentMessageId?.endsWith(':user') === true &&
|
|
terminal.parentMessageId !== `${canonicalTaskId}:user` && {
|
|
previousTaskId: terminal.parentMessageId.slice(0, -':user'.length),
|
|
}),
|
|
parentRunId: terminal.subagentTask.parentRunId ?? request.parentRunId,
|
|
content:
|
|
terminal.text ??
|
|
(terminal.subagentTask.status === 'completed'
|
|
? 'Subagent task completed.'
|
|
: 'The prior subagent task did not complete successfully.'),
|
|
},
|
|
};
|
|
}
|
|
/** Reaching this point while holding the thread lease proves the original
|
|
* worker no longer owns settlement. Close the abandoned attempt once rather
|
|
* than either re-billing it or leaving every retry permanently "running". */
|
|
const abandoned = priorAttempt[priorAttempt.length - 1];
|
|
const abandonedMessage =
|
|
'Subagent task failed: The prior execution ended before its result could be persisted.';
|
|
const savedAbandoned = await this.methods.saveMessage(
|
|
{ userId: scope.userId, ...retentionContext(conversation) },
|
|
{
|
|
messageId: `${taskId}:assistant`,
|
|
conversationId: threadId,
|
|
parentMessageId: abandoned.messageId,
|
|
sender: request.subagentType,
|
|
text: abandonedMessage,
|
|
endpoint: EModelEndpoint.agents,
|
|
isCreatedByUser: false,
|
|
unfinished: false,
|
|
error: true,
|
|
subagentTask: {
|
|
attemptKey,
|
|
parentRunId: request.parentRunId,
|
|
...(requestFingerprint == null ? {} : { requestFingerprint }),
|
|
status: 'error',
|
|
},
|
|
...(conversation.tenantId == null ? {} : { tenantId: conversation.tenantId }),
|
|
},
|
|
{ context: 'SubagentThreadTaskStore.prepareThread.abandonedAttempt' },
|
|
);
|
|
if (savedAbandoned == null) {
|
|
throw new Error('Unable to close the abandoned subagent attempt.');
|
|
}
|
|
await this.touchAfterMessage(scope, threadId, taskId, 'failed');
|
|
return {
|
|
conversation,
|
|
initialMessages: [],
|
|
initialStoredMessages: [],
|
|
attemptKey,
|
|
taskCreatedAt: durableMessageTime(
|
|
savedAbandoned,
|
|
'The abandoned subagent result has no durable occurrence time.',
|
|
),
|
|
replay: {
|
|
status: 'error',
|
|
content: abandonedMessage,
|
|
taskId,
|
|
...(abandoned.messageId.endsWith(':user') && {
|
|
previousTaskId: abandoned.messageId.slice(0, -':user'.length),
|
|
}),
|
|
parentRunId: request.parentRunId,
|
|
},
|
|
};
|
|
}
|
|
const branch = selectLatestBranch(allMessages);
|
|
const initialMessages = restoreThreadMessages(branch);
|
|
const userMessageId = `${taskId}:user`;
|
|
/** A crashed lease can leave its input row behind. Continue from the latest
|
|
* terminal task row instead of making that incomplete input canonical. */
|
|
let parentMessageId: string = Constants.NO_PARENT;
|
|
for (let index = branch.length - 1; index >= 0; index -= 1) {
|
|
if (branch[index].messageId.endsWith(':assistant')) {
|
|
parentMessageId = branch[index].messageId;
|
|
break;
|
|
}
|
|
}
|
|
logger.debug('[subagentThreads] Child-thread preparation entered stage', {
|
|
stage: 'seed_write',
|
|
taskId,
|
|
threadId,
|
|
});
|
|
const savedUserMessage = await observeSlowPreparation(
|
|
this.methods.saveMessage(
|
|
{ userId: scope.userId, ...retentionContext(conversation) },
|
|
{
|
|
messageId: userMessageId,
|
|
conversationId: threadId,
|
|
parentMessageId,
|
|
sender: 'User',
|
|
text: request.input,
|
|
endpoint: EModelEndpoint.agents,
|
|
isCreatedByUser: true,
|
|
subagentTask: {
|
|
attemptKey,
|
|
parentRunId: request.parentRunId,
|
|
...(requestFingerprint == null ? {} : { requestFingerprint }),
|
|
status: 'running',
|
|
},
|
|
...(conversation.tenantId == null ? {} : { tenantId: conversation.tenantId }),
|
|
},
|
|
{ context: 'SubagentThreadTaskStore.prepareThread' },
|
|
),
|
|
{ stage: 'seed_write', taskId, threadId },
|
|
);
|
|
if (savedUserMessage == null) {
|
|
throw new Error('Unable to persist the child-thread input.');
|
|
}
|
|
await this.flushControlReceiptsForSettlement(scopeId, taskId);
|
|
const currentParent = await this.methods.getConvo(scope.userId, scope.parentConversationId);
|
|
if (currentParent == null || !matchesTenant(currentParent.tenantId, scope.tenantId)) {
|
|
throw new SubagentThreadPublicError('Parent thread is unavailable.');
|
|
}
|
|
return {
|
|
conversation,
|
|
initialMessages,
|
|
initialStoredMessages: mapChatMessagesToStoredMessages(initialMessages),
|
|
attemptKey,
|
|
taskCreatedAt: durableMessageTime(
|
|
savedUserMessage,
|
|
'The child-thread input has no durable occurrence time.',
|
|
),
|
|
userMessageId,
|
|
};
|
|
} catch (error) {
|
|
if (sharedLeaseAcquired) {
|
|
await this.rollbackPreparation(scope, threadId, taskId, createdThread);
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
private assertContinuation(
|
|
scope: SubagentThreadScope,
|
|
request: SubagentTaskStartRequest,
|
|
conversation: IConversation,
|
|
): void {
|
|
if (!this.isContinuationAllowed(scope, request, conversation)) {
|
|
throw new SubagentThreadPublicError(
|
|
'Child thread is unavailable for this subagent and parent scope.',
|
|
);
|
|
}
|
|
}
|
|
|
|
private isContinuationAllowed(
|
|
scope: SubagentThreadScope,
|
|
request: SubagentTaskStartRequest,
|
|
conversation: IConversation,
|
|
): boolean {
|
|
const lineage = conversation.subagentThread;
|
|
const expectedAgentId = childAgentId(request);
|
|
const agentIdentityMatches =
|
|
request.subagentKind === 'graph'
|
|
? conversation.agent_id == null
|
|
: conversation.agent_id === expectedAgentId;
|
|
return !(
|
|
lineage == null ||
|
|
conversation.endpoint !== EModelEndpoint.agents ||
|
|
!agentIdentityMatches ||
|
|
lineage.parentConversationId !== scope.parentConversationId ||
|
|
lineage.parentAgentId !== request.parentAgentId ||
|
|
lineage.subagentType !== request.subagentType ||
|
|
lineage.subagentKind !== request.subagentKind ||
|
|
lineage.depth > this.maxThreadDepth ||
|
|
!matchesTenant(conversation.tenantId, scope.tenantId)
|
|
);
|
|
}
|
|
|
|
private async persistResult(
|
|
scope: SubagentThreadScope,
|
|
request: SubagentTaskStartRequest,
|
|
taskId: string,
|
|
prepared: PreparedThread,
|
|
result: { content: string; messages?: BaseMessage[] },
|
|
detachedUsage: UsageMetadata[],
|
|
): Promise<void> {
|
|
if (prepared.userMessageId == null) {
|
|
throw new Error('The child-thread input was not prepared.');
|
|
}
|
|
await this.flushControlReceiptsForSettlement(request.scopeId, taskId);
|
|
const subagentTranscript = serializeTranscript(
|
|
taskId,
|
|
prepared.initialStoredMessages,
|
|
result.messages,
|
|
);
|
|
const activityProjection =
|
|
subagentTranscript == null
|
|
? undefined
|
|
: projectSubagentActivity(
|
|
subagentTranscript.messagesJson,
|
|
subagentTranscript.mode,
|
|
request.input,
|
|
);
|
|
const subagentActivityProjection =
|
|
activityProjection == null
|
|
? undefined
|
|
: {
|
|
taskId,
|
|
version: 1 as const,
|
|
activityJson: JSON.stringify(activityProjection.activity),
|
|
truncated: activityProjection.truncated,
|
|
};
|
|
const conversation = await this.requireCurrentConversation(
|
|
scope,
|
|
request,
|
|
prepared.conversation.conversationId,
|
|
);
|
|
const usage = this.aggregateDetachedUsage(detachedUsage);
|
|
const savedAssistantMessage = await this.methods.saveMessage(
|
|
{ userId: scope.userId, ...retentionContext(conversation) },
|
|
{
|
|
messageId: `${taskId}:assistant`,
|
|
conversationId: conversation.conversationId,
|
|
parentMessageId: prepared.userMessageId,
|
|
sender: request.subagentType,
|
|
text: result.content,
|
|
endpoint: EModelEndpoint.agents,
|
|
isCreatedByUser: false,
|
|
unfinished: false,
|
|
...(subagentTranscript == null ? {} : { subagentTranscript }),
|
|
...(subagentActivityProjection == null ? {} : { subagentActivityProjection }),
|
|
subagentTask: {
|
|
attemptKey: prepared.attemptKey,
|
|
parentRunId: request.parentRunId,
|
|
...(normalizedRequestFingerprint(request) == null
|
|
? {}
|
|
: { requestFingerprint: normalizedRequestFingerprint(request) }),
|
|
status: 'completed',
|
|
},
|
|
...(usage == null ? {} : { metadata: { usage } }),
|
|
...(conversation.tenantId == null ? {} : { tenantId: conversation.tenantId }),
|
|
},
|
|
{ context: 'SubagentThreadTaskStore.persistResult' },
|
|
);
|
|
if (savedAssistantMessage == null) {
|
|
throw new Error('Unable to persist the child-thread result.');
|
|
}
|
|
await this.touchAfterMessage(scope, conversation.conversationId, taskId, 'completed');
|
|
}
|
|
|
|
private async persistFailure(
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
request: SubagentTaskStartRequest,
|
|
taskId: string,
|
|
error: unknown,
|
|
detachedUsage: UsageMetadata[],
|
|
): Promise<void> {
|
|
await this.flushControlReceiptsForSettlement(request.scopeId, taskId);
|
|
const conversation = await this.currentConversation(scope, request, threadId);
|
|
if (conversation == null || !(await this.taskInputExists(scope, threadId, taskId))) {
|
|
return;
|
|
}
|
|
const usage = this.aggregateDetachedUsage(detachedUsage);
|
|
const savedFailure = await this.methods.saveMessage(
|
|
{ userId: scope.userId, ...retentionContext(conversation) },
|
|
{
|
|
messageId: `${taskId}:assistant`,
|
|
conversationId: threadId,
|
|
parentMessageId: `${taskId}:user`,
|
|
sender: request.subagentType,
|
|
text: safeErrorMessage(error),
|
|
endpoint: EModelEndpoint.agents,
|
|
isCreatedByUser: false,
|
|
unfinished: false,
|
|
error: true,
|
|
subagentTask: {
|
|
attemptKey: createSubagentAttemptKey(request.scopeId, request.idempotencyKey),
|
|
parentRunId: request.parentRunId,
|
|
...(normalizedRequestFingerprint(request) == null
|
|
? {}
|
|
: { requestFingerprint: normalizedRequestFingerprint(request) }),
|
|
status: 'error',
|
|
},
|
|
...(usage == null ? {} : { metadata: { usage } }),
|
|
...(conversation.tenantId == null ? {} : { tenantId: conversation.tenantId }),
|
|
},
|
|
{ context: 'SubagentThreadTaskStore.persistFailure' },
|
|
);
|
|
if (savedFailure == null) {
|
|
throw new Error('Unable to persist the child-thread failure.');
|
|
}
|
|
await this.touchAfterMessage(scope, threadId, taskId, 'failed');
|
|
}
|
|
|
|
private async registerTaskWakeup(
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
request: SubagentTaskStartRequest,
|
|
task: { taskId: string; parentRunId: string; createdAt: number },
|
|
): Promise<void> {
|
|
if (
|
|
this.onTaskPrepared == null ||
|
|
(request as HostSubagentTaskStartRequest).completionDelivery !== SUBAGENT_COMPLETION_DELIVERY
|
|
) {
|
|
return;
|
|
}
|
|
const admitted = await this.onTaskPrepared({
|
|
userId: scope.userId,
|
|
parentConversationId: scope.parentConversationId,
|
|
parentMessageId: task.parentRunId,
|
|
...(request.parentAgentId == null ? {} : { parentAgentId: request.parentAgentId }),
|
|
...(scope.tenantId == null ? {} : { tenantId: scope.tenantId }),
|
|
taskId: task.taskId,
|
|
threadId,
|
|
subagentType: request.subagentType,
|
|
createdAt: task.createdAt,
|
|
});
|
|
if (admitted !== false) {
|
|
this.wakeupTaskIds.add(task.taskId);
|
|
}
|
|
}
|
|
|
|
private async persistCancellation(
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
request: SubagentTaskStartRequest,
|
|
taskId: string,
|
|
detachedUsage: UsageMetadata[],
|
|
): Promise<void> {
|
|
await this.flushControlReceiptsForSettlement(request.scopeId, taskId);
|
|
const conversation = await this.currentConversation(scope, request, threadId);
|
|
if (conversation == null || !(await this.taskInputExists(scope, threadId, taskId))) {
|
|
return;
|
|
}
|
|
const usage = this.aggregateDetachedUsage(detachedUsage);
|
|
const savedCancellation = await this.methods.saveMessage(
|
|
{ userId: scope.userId, ...retentionContext(conversation) },
|
|
{
|
|
messageId: `${taskId}:assistant`,
|
|
conversationId: threadId,
|
|
parentMessageId: `${taskId}:user`,
|
|
sender: request.subagentType,
|
|
text: 'Subagent task was cancelled.',
|
|
endpoint: EModelEndpoint.agents,
|
|
isCreatedByUser: false,
|
|
unfinished: false,
|
|
subagentTask: {
|
|
attemptKey: createSubagentAttemptKey(request.scopeId, request.idempotencyKey),
|
|
parentRunId: request.parentRunId,
|
|
...(normalizedRequestFingerprint(request) == null
|
|
? {}
|
|
: { requestFingerprint: normalizedRequestFingerprint(request) }),
|
|
status: 'cancelled',
|
|
},
|
|
...(usage == null ? {} : { metadata: { usage } }),
|
|
...(conversation.tenantId == null ? {} : { tenantId: conversation.tenantId }),
|
|
},
|
|
{ context: 'SubagentThreadTaskStore.persistCancellation' },
|
|
);
|
|
if (savedCancellation == null) {
|
|
throw new Error('Unable to persist the child-thread cancellation.');
|
|
}
|
|
await this.touchAfterMessage(scope, threadId, taskId, 'cancelled');
|
|
}
|
|
|
|
private async currentConversation(
|
|
scope: SubagentThreadScope,
|
|
request: SubagentTaskStartRequest,
|
|
threadId: string,
|
|
): Promise<IConversation | null> {
|
|
const [ownerActive, parent, conversation] = await Promise.all([
|
|
this.isOwnerActive(scope.userId),
|
|
this.methods.getConvo(scope.userId, scope.parentConversationId),
|
|
this.methods.getConvo(scope.userId, threadId),
|
|
]);
|
|
if (conversation == null || !this.isContinuationAllowed(scope, request, conversation)) {
|
|
return null;
|
|
}
|
|
if (!ownerActive) {
|
|
return null;
|
|
}
|
|
if (parent != null && matchesTenant(parent.tenantId, scope.tenantId)) {
|
|
return conversation;
|
|
}
|
|
await this.methods.deleteConvos(scope.userId, { conversationId: threadId }).catch((error) => {
|
|
logger.warn('[subagentThreads] Failed to remove an orphaned child thread', error);
|
|
});
|
|
return null;
|
|
}
|
|
|
|
private cancelMatchingThreads(
|
|
matches: (scope: SubagentThreadScope, threadId: string) => boolean,
|
|
): number {
|
|
let cancelled = 0;
|
|
for (const [lockKey, lease] of this.activeThreads) {
|
|
const separator = lockKey.lastIndexOf('\u0000');
|
|
if (separator < 0 || lease.taskId === '') {
|
|
continue;
|
|
}
|
|
const scopeId = lockKey.slice(0, separator);
|
|
const threadId = lockKey.slice(separator + 1);
|
|
const scope = parseScope(scopeId);
|
|
if (!matches(scope, threadId)) {
|
|
continue;
|
|
}
|
|
const result = this.control(scopeId, lease.taskId, { action: 'cancel' });
|
|
if (result.status === 'cancelled') {
|
|
cancelled += 1;
|
|
}
|
|
}
|
|
return cancelled;
|
|
}
|
|
|
|
private async requireCurrentConversation(
|
|
scope: SubagentThreadScope,
|
|
request: SubagentTaskStartRequest,
|
|
threadId: string,
|
|
): Promise<IConversation> {
|
|
const conversation = await this.currentConversation(scope, request, threadId);
|
|
if (conversation == null) {
|
|
throw new SubagentThreadDeletedError('Child thread was deleted before settlement.');
|
|
}
|
|
return conversation;
|
|
}
|
|
|
|
private async taskInputExists(
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
taskId: string,
|
|
): Promise<boolean> {
|
|
const messages = await this.methods.getMessages(
|
|
{ conversationId: threadId, user: scope.userId, messageId: `${taskId}:user` },
|
|
'messageId',
|
|
{ limit: 1 },
|
|
);
|
|
return messages.length > 0;
|
|
}
|
|
|
|
private async rollbackPreparation(
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
taskId: string,
|
|
createdThread: boolean,
|
|
): Promise<void> {
|
|
try {
|
|
if (createdThread) {
|
|
await this.methods.deleteConvos(scope.userId, { conversationId: threadId });
|
|
return;
|
|
}
|
|
await this.deleteTaskMessages(scope, threadId, taskId);
|
|
} catch (cleanupError) {
|
|
logger.error('[subagentThreads] Failed to roll back child-thread setup', cleanupError);
|
|
}
|
|
}
|
|
|
|
private async deleteTaskMessages(
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
taskId: string,
|
|
): Promise<void> {
|
|
await this.methods.deleteMessages({
|
|
user: scope.userId,
|
|
conversationId: threadId,
|
|
messageId: { $in: [`${taskId}:user`, `${taskId}:assistant`] },
|
|
});
|
|
}
|
|
|
|
private async touchAfterMessage(
|
|
scope: SubagentThreadScope,
|
|
threadId: string,
|
|
taskId: string,
|
|
outcome: 'cancelled' | 'completed' | 'failed',
|
|
): Promise<void> {
|
|
try {
|
|
const saved = await this.methods.saveConvo(
|
|
{ userId: scope.userId },
|
|
{ conversationId: threadId },
|
|
{ context: 'SubagentThreadTaskStore.touchAfterMessage', noUpsert: true },
|
|
);
|
|
if (saved == null) {
|
|
throw new SubagentThreadDeletedError('Child thread was deleted before settlement.');
|
|
}
|
|
if ('message' in saved) {
|
|
throw new Error('Unable to refresh the child thread.');
|
|
}
|
|
} catch (error) {
|
|
if (error instanceof SubagentThreadDeletedError) {
|
|
await this.deleteTaskMessages(scope, threadId, taskId);
|
|
throw error;
|
|
}
|
|
logger.error(`[subagentThreads] Failed to refresh ${outcome} child thread`, error);
|
|
}
|
|
this.notifyTaskSettled(scope, taskId);
|
|
}
|
|
|
|
private notifyTaskSettled(
|
|
scope: SubagentThreadScope,
|
|
taskId: string,
|
|
previousTaskId?: string,
|
|
): void {
|
|
if (!this.wakeupTaskIds.delete(taskId)) {
|
|
return;
|
|
}
|
|
try {
|
|
this.onTaskSettled?.(
|
|
scope.userId,
|
|
scope.parentConversationId,
|
|
previousTaskId == null ? [taskId] : [taskId, previousTaskId],
|
|
);
|
|
} catch (error) {
|
|
logger.warn('[subagentThreads] Settled-task listener failed', error);
|
|
}
|
|
}
|
|
|
|
private aggregateDetachedUsage(detachedUsage: UsageMetadata[]) {
|
|
return aggregateEmittedUsage(
|
|
detachedUsage.map((entry) => ({ ...entry, usage_type: 'subagent' as const })),
|
|
);
|
|
}
|
|
}
|
|
|
|
const REQUIRED_THREAD_METHODS = [
|
|
'acquireSubagentThreadLease',
|
|
'claimSubagentTaskResult',
|
|
'countActiveSubagentThreadLeases',
|
|
'deleteConvos',
|
|
'deleteMessages',
|
|
'getConvo',
|
|
'getSubagentTaskControlReplay',
|
|
'getMessages',
|
|
'listActiveSubagentThreadLeases',
|
|
'recordSubagentTaskControlReceipt',
|
|
'releaseSubagentThreadLease',
|
|
'renewSubagentThreadLease',
|
|
'reserveSubagentThread',
|
|
'saveConvo',
|
|
'saveMessage',
|
|
] as const;
|
|
|
|
export function createSubagentThreadTaskStore(
|
|
methods: Pick<
|
|
ConversationMethods,
|
|
| 'acquireSubagentThreadLease'
|
|
| 'countActiveSubagentThreadLeases'
|
|
| 'deleteConvos'
|
|
| 'getConvo'
|
|
| 'listActiveSubagentThreadLeases'
|
|
| 'releaseSubagentThreadLease'
|
|
| 'reserveSubagentThread'
|
|
| 'renewSubagentThreadLease'
|
|
| 'saveConvo'
|
|
> &
|
|
Pick<
|
|
MessageMethods,
|
|
| 'claimSubagentTaskResult'
|
|
| 'deleteMessages'
|
|
| 'getSubagentTaskControlReplay'
|
|
| 'getMessages'
|
|
| 'recordSubagentTaskControlReceipt'
|
|
| 'saveMessage'
|
|
>,
|
|
options?: SubagentThreadTaskStoreOptions,
|
|
): SubagentThreadTaskStore {
|
|
/** The host wires this from JavaScript, where the parameter type checks nothing. A
|
|
* method missing there would otherwise surface as a routed failure at claim time,
|
|
* long after startup, so the omission is caught here instead. */
|
|
const missing = REQUIRED_THREAD_METHODS.filter(
|
|
(name) => typeof (methods as Record<string, unknown>)[name] !== 'function',
|
|
);
|
|
if (missing.length > 0) {
|
|
throw new Error(`Subagent thread task store is missing methods: ${missing.join(', ')}`);
|
|
}
|
|
return new SubagentThreadTaskStore(methods, options);
|
|
}
|
|
|
|
type CompletionWakeupStore = SubagentTaskStore &
|
|
Pick<SubagentThreadTaskStore, 'claimTask' | 'controlTask' | 'hasTasks' | 'listTasks'>;
|
|
|
|
const completionWakeupStores = new WeakMap<SubagentThreadTaskStore, CompletionWakeupStore>();
|
|
|
|
function completionWakeupStore(store: SubagentThreadTaskStore): CompletionWakeupStore {
|
|
const existing = completionWakeupStores.get(store);
|
|
if (existing != null) {
|
|
return existing;
|
|
}
|
|
const adapter: CompletionWakeupStore = {
|
|
supportsThreadContinuation: store.supportsThreadContinuation,
|
|
start: (request) => {
|
|
const hostRequest: HostSubagentTaskStartRequest = {
|
|
...request,
|
|
completionDelivery: SUBAGENT_COMPLETION_DELIVERY,
|
|
};
|
|
return store.start(hostRequest);
|
|
},
|
|
get: (scopeId, taskId) => store.get(scopeId, taskId),
|
|
list: (scopeId) => store.list(scopeId),
|
|
claim: (scopeId, taskId) => store.claim(scopeId, taskId),
|
|
control: (scopeId, taskId, command) => store.control(scopeId, taskId, command),
|
|
claimTask: (scopeId, taskId, invocationId) => store.claimTask(scopeId, taskId, invocationId),
|
|
controlTask: (scopeId, taskId, command, invocationId) =>
|
|
store.controlTask(scopeId, taskId, command, invocationId),
|
|
hasTasks: (scopeId) => store.hasTasks(scopeId),
|
|
listTasks: (scopeId) => store.listTasks(scopeId),
|
|
};
|
|
completionWakeupStores.set(store, adapter);
|
|
return adapter;
|
|
}
|
|
|
|
export function buildSubagentThreadTaskConfig(
|
|
store: SubagentThreadTaskStore,
|
|
scope: Omit<SubagentThreadScope, 'version'>,
|
|
options: { completionWakeups?: boolean } = {},
|
|
): HostSubagentTaskConfig {
|
|
const taskStore = options.completionWakeups === true ? completionWakeupStore(store) : store;
|
|
return {
|
|
store: taskStore,
|
|
scopeId: serializeScope(scope),
|
|
...(options.completionWakeups === true
|
|
? { completionDelivery: SUBAGENT_COMPLETION_DELIVERY }
|
|
: {}),
|
|
};
|
|
}
|