mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-09-21 07:36:25 +00:00
📬 feat: Serialize Bound Actor Event Mailboxes (#15260)
* feat: serialize bound actor event mailboxes * test: harden actor mailbox batch ordering * fix: treat batch roots as mailbox authority * test: pin mailbox replay semantics * fix: preserve mailbox authority through settlement
This commit is contained in:
parent
68fc46a055
commit
52fa88fc54
16 changed files with 493 additions and 21 deletions
|
|
@ -173,6 +173,16 @@ immediately before dispatch, so queued events do not persist stale chat topology
|
|||
is its default ordering lane. A short-lived internal trigger token plus a second binding lookup is
|
||||
required to pass the child-thread write guard; possessing a binding id alone grants no access.
|
||||
|
||||
Set `endpoints.agents.eventDriven.actorMailbox: true` only after every API replica runs a release
|
||||
that understands terminal-handling mailbox blockers. The flag defaults to false for rolling
|
||||
deployment safety, and `ENABLE_AGENT_EVENT_ACTOR_MAILBOX` remains a compatibility fallback. Once
|
||||
enabled, a bound actor's next delivery stays queued after the current delivery reaches transport
|
||||
success and does not dispatch until that child generation records `applied`,
|
||||
`completed_no_action`, `failed`, or `cancelled`. Different bindings remain independent and can run
|
||||
in parallel. Existing coalesced batches occupy one mailbox position and retain each member's
|
||||
individual receipt. An active mailbox record does not receive its normal success TTL; the 90-day
|
||||
retention window begins only after terminal handling is recorded.
|
||||
|
||||
### Coalescing observational child events
|
||||
|
||||
Sources that can prove several bound `continue` events are interchangeable observations may opt
|
||||
|
|
|
|||
|
|
@ -35,6 +35,9 @@ export interface PreparedAgentTriggerDelivery {
|
|||
coalesceKey?: string;
|
||||
coalesceFrom?: Date;
|
||||
coalesceUntil?: Date;
|
||||
/** Persisted rollout marker: keep this bound actor lane queued until the
|
||||
* admitted child turn records an authoritative terminal outcome. */
|
||||
awaitTerminalHandling?: boolean;
|
||||
}
|
||||
|
||||
export class AgentTriggerDeliveryError extends TypeError {
|
||||
|
|
|
|||
|
|
@ -106,7 +106,9 @@ describe('createAgentTriggerDeliveryEngine', () => {
|
|||
},
|
||||
input: 'Take the turn.',
|
||||
});
|
||||
const store = storeWith({ claimNext: jest.fn(async () => delivery({ envelope })) });
|
||||
const store = storeWith({
|
||||
claimNext: jest.fn(async () => delivery({ envelope, awaitTerminalHandling: true })),
|
||||
});
|
||||
const result: AgentTriggerExecutionResult = {
|
||||
mode: 'continue',
|
||||
status: 'started',
|
||||
|
|
@ -123,6 +125,7 @@ describe('createAgentTriggerDeliveryEngine', () => {
|
|||
|
||||
expect(store.complete).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
awaitTerminalHandling: true,
|
||||
handling: {
|
||||
status: 'started',
|
||||
conversationId: 'conversation-1',
|
||||
|
|
@ -545,6 +548,28 @@ describe('createAgentTriggerDeliveryEngine', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('rechecks an active actor turn without a tight delivery-lease polling loop', async () => {
|
||||
const store = storeWith({
|
||||
findEarlierUnsettled: jest.fn(async () => ({
|
||||
availableAt: START,
|
||||
reason: 'active_handling' as const,
|
||||
})),
|
||||
});
|
||||
const dispatch = jest.fn(async () => successResult());
|
||||
const engine = createAgentTriggerDeliveryEngine(
|
||||
{ store, dispatch, now: () => START },
|
||||
{ concurrency: 1 },
|
||||
);
|
||||
|
||||
await engine.runTick();
|
||||
|
||||
expect(store.release).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ availableAt: new Date(START.getTime() + 5_000) }),
|
||||
);
|
||||
expect(store.beginAttempt).not.toHaveBeenCalled();
|
||||
expect(dispatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('starts independent deliveries up to the configured concurrency', async () => {
|
||||
let releaseDispatch: (() => void) | undefined;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const DEFAULT_RETRY_CAP_MS = 5 * 60_000;
|
|||
const DEFAULT_TICK_MS = 1_000;
|
||||
const DEFAULT_MAX_IDLE_TICK_MS = 15_000;
|
||||
const ORDERING_RECHECK_MS = 250;
|
||||
const ACTIVE_HANDLING_RECHECK_MS = 5_000;
|
||||
const DEFAULT_DEFER_MS = 5_000;
|
||||
const MAX_RETRY_AFTER_MS = 24 * 60 * 60_000;
|
||||
|
||||
|
|
@ -97,6 +98,7 @@ export interface AgentTriggerDeliveryRecord {
|
|||
batchMemberIds?: Array<{ toString(): string } | string>;
|
||||
batchRootId?: { toString(): string } | string;
|
||||
batchMembersSettledAt?: Date;
|
||||
awaitTerminalHandling?: boolean;
|
||||
leaseBy?: string;
|
||||
leaseUntil?: Date;
|
||||
lastError?: AgentTriggerDeliveryFailure;
|
||||
|
|
@ -115,6 +117,7 @@ export interface AgentTriggerDeliveryRecord {
|
|||
export interface AgentTriggerOrderingBlock {
|
||||
availableAt: Date;
|
||||
leaseUntil?: Date;
|
||||
reason?: 'active_handling';
|
||||
}
|
||||
|
||||
export interface AgentTriggerDeliveryStore {
|
||||
|
|
@ -157,6 +160,7 @@ export interface AgentTriggerDeliveryStore {
|
|||
result: AgentTriggerExecutionResult;
|
||||
settledAt: Date;
|
||||
handling?: AgentTriggerDeliveryRecord['handling'];
|
||||
awaitTerminalHandling?: true;
|
||||
}) => Promise<boolean>;
|
||||
retry: (input: {
|
||||
id: string;
|
||||
|
|
@ -338,7 +342,9 @@ export function createAgentTriggerDeliveryEngine(
|
|||
|
||||
const block = await deps.store.findEarlierUnsettled(delivery);
|
||||
if (block != null) {
|
||||
const recheckAt = now().getTime() + ORDERING_RECHECK_MS;
|
||||
const recheckAt =
|
||||
now().getTime() +
|
||||
(block.reason === 'active_handling' ? ACTIVE_HANDLING_RECHECK_MS : ORDERING_RECHECK_MS);
|
||||
const nextCheck =
|
||||
block.leaseUntil == null ? Math.max(recheckAt, block.availableAt.getTime()) : recheckAt;
|
||||
noteEligibleAt(new Date(nextCheck));
|
||||
|
|
@ -506,6 +512,7 @@ export function createAgentTriggerDeliveryEngine(
|
|||
attempt,
|
||||
result,
|
||||
settledAt,
|
||||
...(delivery.awaitTerminalHandling === true && { awaitTerminalHandling: true }),
|
||||
...(handling != null && { handling }),
|
||||
});
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,29 @@ const envelope = () =>
|
|||
input: 'Handle the ready resource.',
|
||||
});
|
||||
|
||||
const boundEnvelope = () =>
|
||||
createAgentTriggerEnvelope({
|
||||
mode: 'continue',
|
||||
requestId: 'request-bound-1',
|
||||
deliveryId: 'delivery-bound-1',
|
||||
receivedAt: 20,
|
||||
principal: { id: '507f1f77bcf86cd799439011', tenantId: 'tenant-1' },
|
||||
target: {
|
||||
agentId: 'agent-1',
|
||||
conversationId: 'child-conversation-1',
|
||||
parentMessageId: 'parent-message-1',
|
||||
bindingId: 'binding-1',
|
||||
sourceKeyId: 'source-key-1',
|
||||
},
|
||||
event: {
|
||||
id: 'event-bound-1',
|
||||
type: 'game.turn',
|
||||
occurredAt: 10,
|
||||
source: { id: 'source-key-1', type: 'remote_api_key' },
|
||||
},
|
||||
input: 'Make the next move.',
|
||||
});
|
||||
|
||||
function deliveryRecord(overrides: Partial<AgentTriggerStoredRecord> = {}) {
|
||||
return {
|
||||
id: 'delivery-row-1',
|
||||
|
|
@ -182,6 +205,46 @@ describe('durable agent trigger service', () => {
|
|||
await service.stop();
|
||||
});
|
||||
|
||||
it('persists terminal handling serialization only for opted-in bound continuations', async () => {
|
||||
const methods = deliveryMethods();
|
||||
const service = createAgentTriggerService({
|
||||
methods,
|
||||
actorMailboxEnabled: () => true,
|
||||
deliveryOptions: { concurrency: 1, tickMs: 60_000 },
|
||||
});
|
||||
await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
|
||||
|
||||
await service.enqueue(boundEnvelope());
|
||||
await service.enqueue(envelope());
|
||||
|
||||
expect(methods.enqueueAgentTriggerDelivery).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ awaitTerminalHandling: true }),
|
||||
);
|
||||
expect(methods.enqueueAgentTriggerDelivery).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.not.objectContaining({ awaitTerminalHandling: expect.anything() }),
|
||||
);
|
||||
await service.stop();
|
||||
});
|
||||
|
||||
it('does not persist mailbox semantics before the rollout is enabled', async () => {
|
||||
const methods = deliveryMethods();
|
||||
const service = createAgentTriggerService({
|
||||
methods,
|
||||
actorMailboxEnabled: () => false,
|
||||
deliveryOptions: { concurrency: 1, tickMs: 60_000 },
|
||||
});
|
||||
await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
|
||||
|
||||
await service.enqueue(boundEnvelope());
|
||||
|
||||
expect(methods.enqueueAgentTriggerDelivery).toHaveBeenCalledWith(
|
||||
expect.not.objectContaining({ awaitTerminalHandling: expect.anything() }),
|
||||
);
|
||||
await service.stop();
|
||||
});
|
||||
|
||||
it('rejects enqueue before persistence when the principal was deleted', async () => {
|
||||
const methods = deliveryMethods();
|
||||
const service = createAgentTriggerService({
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ export interface AgentTriggerServiceDeps {
|
|||
purgeRecoveryIntervalMs?: number;
|
||||
purgeRecoveryLimit?: number;
|
||||
coalescingEnabled?: () => boolean;
|
||||
actorMailboxEnabled?: () => boolean;
|
||||
}
|
||||
|
||||
export interface AgentTriggerDeliveryReceipt {
|
||||
|
|
@ -195,6 +196,8 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
const purgeRecoveryLimit = deps.purgeRecoveryLimit ?? DEFAULT_PURGE_RECOVERY_LIMIT;
|
||||
const coalescingEnabled =
|
||||
deps.coalescingEnabled ?? (() => isEnabled(process.env.ENABLE_AGENT_EVENT_COALESCING));
|
||||
const actorMailboxEnabled =
|
||||
deps.actorMailboxEnabled ?? (() => isEnabled(process.env.ENABLE_AGENT_EVENT_ACTOR_MAILBOX));
|
||||
if (!Number.isSafeInteger(userDrainTimeoutMs) || userDrainTimeoutMs <= 0) {
|
||||
throw new TypeError('userDrainTimeoutMs must be a positive integer');
|
||||
}
|
||||
|
|
@ -406,8 +409,19 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
|
|||
throw new AgentTriggerDeliveryError('Agent event coalescing is not enabled on this server');
|
||||
}
|
||||
const prepared = prepareAgentTriggerDelivery(envelope, options);
|
||||
const awaitTerminalHandling =
|
||||
actorMailboxEnabled() &&
|
||||
prepared.envelope.mode === 'continue' &&
|
||||
prepared.envelope.target.bindingId != null &&
|
||||
prepared.envelope.target.sourceKeyId != null;
|
||||
const durableDelivery: PreparedAgentTriggerDelivery = {
|
||||
...prepared,
|
||||
...(awaitTerminalHandling && { awaitTerminalHandling: true }),
|
||||
};
|
||||
await requireActivePrincipal(String(prepared.user));
|
||||
const queued = await runAsSystem(async () => methods.enqueueAgentTriggerDelivery(prepared));
|
||||
const queued = await runAsSystem(async () =>
|
||||
methods.enqueueAgentTriggerDelivery(durableDelivery),
|
||||
);
|
||||
try {
|
||||
await requireActivePrincipal(String(prepared.user));
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ describe('configureAgentEventRuntime', () => {
|
|||
delete process.env.ENABLE_AGENT_EVENT_CHILD_TURNS;
|
||||
delete process.env.ENABLE_SUBAGENT_COMPLETION_WAKEUPS;
|
||||
delete process.env.ENABLE_AGENT_EVENT_COALESCING;
|
||||
delete process.env.ENABLE_AGENT_EVENT_ACTOR_MAILBOX;
|
||||
delete process.env.AGENT_TRIGGERS_SELF_URL;
|
||||
});
|
||||
|
||||
|
|
@ -20,12 +21,14 @@ describe('configureAgentEventRuntime', () => {
|
|||
childTurns: true,
|
||||
completionWakeups: false,
|
||||
coalescing: true,
|
||||
actorMailbox: true,
|
||||
selfUrl: 'https://triggers.internal',
|
||||
});
|
||||
|
||||
expect(process.env.ENABLE_AGENT_EVENT_CHILD_TURNS).toBe('true');
|
||||
expect(process.env.ENABLE_SUBAGENT_COMPLETION_WAKEUPS).toBe('false');
|
||||
expect(process.env.ENABLE_AGENT_EVENT_COALESCING).toBe('true');
|
||||
expect(process.env.ENABLE_AGENT_EVENT_ACTOR_MAILBOX).toBe('true');
|
||||
expect(process.env.AGENT_TRIGGERS_SELF_URL).toBe('https://triggers.internal');
|
||||
});
|
||||
|
||||
|
|
@ -33,6 +36,7 @@ describe('configureAgentEventRuntime', () => {
|
|||
process.env.ENABLE_AGENT_EVENT_CHILD_TURNS = 'true';
|
||||
process.env.ENABLE_SUBAGENT_COMPLETION_WAKEUPS = 'true';
|
||||
process.env.ENABLE_AGENT_EVENT_COALESCING = 'true';
|
||||
process.env.ENABLE_AGENT_EVENT_ACTOR_MAILBOX = 'true';
|
||||
process.env.AGENT_TRIGGERS_SELF_URL = 'https://legacy.internal';
|
||||
|
||||
configureAgentEventRuntime(undefined);
|
||||
|
|
@ -40,6 +44,7 @@ describe('configureAgentEventRuntime', () => {
|
|||
expect(process.env.ENABLE_AGENT_EVENT_CHILD_TURNS).toBe('true');
|
||||
expect(process.env.ENABLE_SUBAGENT_COMPLETION_WAKEUPS).toBe('true');
|
||||
expect(process.env.ENABLE_AGENT_EVENT_COALESCING).toBe('true');
|
||||
expect(process.env.ENABLE_AGENT_EVENT_ACTOR_MAILBOX).toBe('true');
|
||||
expect(process.env.AGENT_TRIGGERS_SELF_URL).toBe('https://legacy.internal');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ export const configureAgentEventRuntime = (config?: AgentEventRuntimeConfig): vo
|
|||
setBooleanEnvironmentFallback('ENABLE_AGENT_EVENT_CHILD_TURNS', config?.childTurns);
|
||||
setBooleanEnvironmentFallback('ENABLE_SUBAGENT_COMPLETION_WAKEUPS', config?.completionWakeups);
|
||||
setBooleanEnvironmentFallback('ENABLE_AGENT_EVENT_COALESCING', config?.coalescing);
|
||||
setBooleanEnvironmentFallback('ENABLE_AGENT_EVENT_ACTOR_MAILBOX', config?.actorMailbox);
|
||||
if (config?.selfUrl != null) {
|
||||
process.env.AGENT_TRIGGERS_SELF_URL = config.selfUrl;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ describe('agent event runtime config', () => {
|
|||
childTurns: true,
|
||||
completionWakeups: false,
|
||||
coalescing: true,
|
||||
actorMailbox: true,
|
||||
checkpointForks: true,
|
||||
selfUrl: 'https://triggers.internal',
|
||||
},
|
||||
|
|
@ -94,6 +95,7 @@ describe('agent event runtime config', () => {
|
|||
childTurns: true,
|
||||
completionWakeups: false,
|
||||
coalescing: true,
|
||||
actorMailbox: true,
|
||||
checkpointForks: true,
|
||||
selfUrl: 'https://triggers.internal',
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1051,6 +1051,9 @@ export const agentsEndpointSchema = baseEndpointSchema
|
|||
completionWakeups: z.boolean().optional(),
|
||||
/** Enable only after every API worker can consume coalesced deliveries. */
|
||||
coalescing: z.boolean().optional(),
|
||||
/** Keep each bound actor's durable delivery lane queued through the
|
||||
* admitted child turn's authoritative terminal outcome. */
|
||||
actorMailbox: z.boolean().optional(),
|
||||
/** Reuse a bound event actor's committed checkpoint through isolated
|
||||
* per-invocation forks. Keep off until every API worker runs an SDK
|
||||
* and host adapter that understand the fork lifecycle. */
|
||||
|
|
|
|||
|
|
@ -103,6 +103,32 @@ describe('agent trigger delivery methods', () => {
|
|||
expect(await Delivery.countDocuments()).toBe(1);
|
||||
});
|
||||
|
||||
it('keeps the original mailbox rollout semantics on idempotent replay', async () => {
|
||||
const enabled = enqueueInput({ awaitTerminalHandling: true });
|
||||
const first = await methods.enqueueAgentTriggerDelivery(enabled);
|
||||
const disabledReplay = await methods.enqueueAgentTriggerDelivery({
|
||||
...enabled,
|
||||
awaitTerminalHandling: undefined,
|
||||
});
|
||||
|
||||
expect(first.delivery.awaitTerminalHandling).toBe(true);
|
||||
expect(disabledReplay).toMatchObject({
|
||||
replayed: true,
|
||||
delivery: { id: first.delivery.id, awaitTerminalHandling: true },
|
||||
});
|
||||
|
||||
const disabled = enqueueInput();
|
||||
const second = await methods.enqueueAgentTriggerDelivery(disabled);
|
||||
const enabledReplay = await methods.enqueueAgentTriggerDelivery({
|
||||
...disabled,
|
||||
awaitTerminalHandling: true,
|
||||
});
|
||||
|
||||
expect(second.delivery.awaitTerminalHandling).toBeUndefined();
|
||||
expect(enabledReplay).toMatchObject({ replayed: true, delivery: { id: second.delivery.id } });
|
||||
expect(enabledReplay.delivery.awaitTerminalHandling).toBeUndefined();
|
||||
});
|
||||
|
||||
it('projects public status while enforcing API key, owner, and tenant in the query', async () => {
|
||||
const user = new mongoose.Types.ObjectId();
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(
|
||||
|
|
@ -226,6 +252,7 @@ describe('agent trigger delivery methods', () => {
|
|||
coalesceFrom,
|
||||
coalesceUntil,
|
||||
availableAt: coalesceUntil,
|
||||
awaitTerminalHandling: true,
|
||||
envelopeBytes: 128,
|
||||
envelope: {
|
||||
mode: 'continue',
|
||||
|
|
@ -289,6 +316,7 @@ describe('agent trigger delivery methods', () => {
|
|||
attempt: attempt!,
|
||||
result,
|
||||
settledAt: coalesceUntil,
|
||||
awaitTerminalHandling: true,
|
||||
handling: {
|
||||
status: 'started',
|
||||
conversationId: 'child-thread',
|
||||
|
|
@ -312,6 +340,30 @@ describe('agent trigger delivery methods', () => {
|
|||
expect(receipts).toHaveLength(4);
|
||||
expect(receipts.every((receipt) => receipt?.status === 'succeeded')).toBe(true);
|
||||
expect(receipts.every((receipt) => receipt?.handling?.status === 'started')).toBe(true);
|
||||
expect(
|
||||
(
|
||||
await Delivery.find({ _id: { $in: queued.map(({ delivery }) => delivery.id) } }).lean()
|
||||
).every((row) => row.expiresAt == null),
|
||||
).toBe(true);
|
||||
|
||||
const later = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({
|
||||
user,
|
||||
orderingKey: 'commentary-lane',
|
||||
awaitTerminalHandling: true,
|
||||
availableAt: coalesceUntil,
|
||||
}),
|
||||
);
|
||||
const laterClaim = await methods.claimNextAgentTriggerDelivery({
|
||||
workerId: 'worker-later',
|
||||
claimToken: 'claim-later',
|
||||
now: coalesceUntil,
|
||||
leaseUntil: new Date(coalesceUntil.getTime() + 60_000),
|
||||
});
|
||||
expect(laterClaim?.id).toBe(later.delivery.id);
|
||||
await expect(methods.findEarlierAgentTriggerDelivery(laterClaim!)).resolves.toMatchObject({
|
||||
reason: 'active_handling',
|
||||
});
|
||||
|
||||
await expect(
|
||||
methods.settleAgentTriggerHandlingOutcome({
|
||||
|
|
@ -322,11 +374,24 @@ describe('agent trigger delivery methods', () => {
|
|||
settledAt: new Date(coalesceUntil.getTime() + 1_000),
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
await expect(methods.findEarlierAgentTriggerDelivery(laterClaim!)).resolves.toBeNull();
|
||||
|
||||
let terminal = await Delivery.find({ orderingKey: 'commentary-lane' }).lean();
|
||||
expect(terminal.every((row) => row.handling?.status === 'completed_no_action')).toBe(true);
|
||||
const batchIds = new Set(queued.map(({ delivery }) => delivery.id));
|
||||
expect(
|
||||
terminal
|
||||
.filter((row) => batchIds.has(String(row._id)))
|
||||
.every((row) => row.handling?.status === 'completed_no_action'),
|
||||
).toBe(true);
|
||||
|
||||
const recoveringMember = terminal.find((row) => String(row._id) !== String(root!._id));
|
||||
const recoveringMember = terminal.find(
|
||||
(row) => batchIds.has(String(row._id)) && String(row._id) !== String(root!._id),
|
||||
);
|
||||
await Delivery.updateOne(
|
||||
{ _id: recoveringMember!._id },
|
||||
{ $set: { 'handling.status': 'started' } },
|
||||
);
|
||||
await expect(methods.findEarlierAgentTriggerDelivery(laterClaim!)).resolves.toBeNull();
|
||||
await Delivery.updateOne({ _id: recoveringMember!._id }, { $unset: { handling: 1 } });
|
||||
await expect(
|
||||
methods.settleAgentTriggerHandlingOutcome({
|
||||
|
|
@ -338,7 +403,44 @@ describe('agent trigger delivery methods', () => {
|
|||
}),
|
||||
).resolves.toBe(true);
|
||||
terminal = await Delivery.find({ orderingKey: 'commentary-lane' }).lean();
|
||||
expect(terminal.every((row) => row.handling?.status === 'completed_no_action')).toBe(true);
|
||||
expect(
|
||||
terminal
|
||||
.filter((row) => batchIds.has(String(row._id)))
|
||||
.every((row) => row.handling?.status === 'completed_no_action'),
|
||||
).toBe(true);
|
||||
expect(
|
||||
terminal
|
||||
.filter((row) => batchIds.has(String(row._id)))
|
||||
.every(
|
||||
(row) =>
|
||||
row.expiresAt?.getTime() === coalesceUntil.getTime() + 1_000 + 90 * 24 * 60 * 60_000,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('promotes mailbox semantics when an enabled delivery joins an unmarked batch root', async () => {
|
||||
const user = new mongoose.Types.ObjectId();
|
||||
const coalesceUntil = new Date(Date.now() + 60_000);
|
||||
const shared = {
|
||||
user,
|
||||
orderingKey: 'mixed-rollout-batch',
|
||||
coalesceKey: 'trigger_batch_mixed_rollout',
|
||||
coalesceFrom: new Date(coalesceUntil.getTime() - 750),
|
||||
coalesceUntil,
|
||||
availableAt: coalesceUntil,
|
||||
envelopeBytes: 128,
|
||||
};
|
||||
const first = await methods.enqueueAgentTriggerDelivery(enqueueInput(shared));
|
||||
await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({ ...shared, awaitTerminalHandling: true }),
|
||||
);
|
||||
|
||||
const root = await Delivery.findById(first.delivery.id).lean();
|
||||
expect(root).toMatchObject({
|
||||
status: 'pending',
|
||||
awaitTerminalHandling: true,
|
||||
batchSize: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it('recovers batch receipts and lane cleanup after root settlement was interrupted', async () => {
|
||||
|
|
@ -1163,6 +1265,135 @@ describe('agent trigger delivery methods', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('keeps a binding lane queued until the earlier child turn settles', async () => {
|
||||
const user = new mongoose.Types.ObjectId();
|
||||
const first = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({ user, orderingKey: 'binding-lane', awaitTerminalHandling: true }),
|
||||
);
|
||||
const firstClaim = await methods.claimNextAgentTriggerDelivery({
|
||||
workerId: 'worker-1',
|
||||
claimToken: 'claim-1',
|
||||
now: START,
|
||||
leaseUntil: new Date(START.getTime() + 60_000),
|
||||
});
|
||||
const firstAttempt = await methods.beginAgentTriggerDeliveryAttempt({
|
||||
id: firstClaim!.id,
|
||||
workerId: 'worker-1',
|
||||
claimToken: 'claim-1',
|
||||
now: START,
|
||||
});
|
||||
const generationCreatedAt = START.getTime() + 1_000;
|
||||
await methods.completeAgentTriggerDelivery({
|
||||
id: firstClaim!.id,
|
||||
workerId: 'worker-1',
|
||||
claimToken: 'claim-1',
|
||||
attempt: firstAttempt!,
|
||||
result: {
|
||||
mode: 'continue',
|
||||
status: 'started',
|
||||
conversationId: 'actor-thread',
|
||||
streamId: 'actor-thread',
|
||||
generationCreatedAt,
|
||||
},
|
||||
settledAt: new Date(generationCreatedAt),
|
||||
awaitTerminalHandling: true,
|
||||
handling: {
|
||||
status: 'started',
|
||||
conversationId: 'actor-thread',
|
||||
streamId: 'actor-thread',
|
||||
generationCreatedAt,
|
||||
startedAt: new Date(generationCreatedAt),
|
||||
},
|
||||
});
|
||||
const second = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({ user, orderingKey: 'binding-lane', awaitTerminalHandling: true }),
|
||||
);
|
||||
const secondClaim = await methods.claimNextAgentTriggerDelivery({
|
||||
workerId: 'worker-2',
|
||||
claimToken: 'claim-2',
|
||||
now: new Date(generationCreatedAt),
|
||||
leaseUntil: new Date(generationCreatedAt + 60_000),
|
||||
});
|
||||
|
||||
expect(firstClaim?.id).toBe(first.delivery.id);
|
||||
expect(second.delivery.laneSequence).toBe(first.delivery.laneSequence + 1);
|
||||
await expect(methods.findEarlierAgentTriggerDelivery(secondClaim!)).resolves.toMatchObject({
|
||||
availableAt: START,
|
||||
reason: 'active_handling',
|
||||
});
|
||||
await expect(Delivery.findById(first.delivery.id).lean()).resolves.not.toHaveProperty(
|
||||
'expiresAt',
|
||||
);
|
||||
|
||||
await expect(
|
||||
methods.settleAgentTriggerHandlingOutcome({
|
||||
deliveryKey: first.delivery.deliveryKey,
|
||||
conversationId: 'actor-thread',
|
||||
generationCreatedAt,
|
||||
status: 'applied',
|
||||
settledAt: new Date(generationCreatedAt + 1_000),
|
||||
action: { toolName: 'submit_action' },
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
await expect(Delivery.findById(first.delivery.id).lean()).resolves.toMatchObject({
|
||||
expiresAt: new Date(generationCreatedAt + 1_000 + 90 * 24 * 60 * 60_000),
|
||||
});
|
||||
await expect(methods.findEarlierAgentTriggerDelivery(secondClaim!)).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('reclaims a binding lane only after its admitted child turn settles', async () => {
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(
|
||||
enqueueInput({ orderingKey: 'settling-binding-lane', awaitTerminalHandling: true }),
|
||||
);
|
||||
const claim = await methods.claimNextAgentTriggerDelivery({
|
||||
workerId: 'worker-1',
|
||||
claimToken: 'claim-1',
|
||||
now: START,
|
||||
leaseUntil: new Date(START.getTime() + 60_000),
|
||||
});
|
||||
const attempt = await methods.beginAgentTriggerDeliveryAttempt({
|
||||
id: claim!.id,
|
||||
workerId: 'worker-1',
|
||||
claimToken: 'claim-1',
|
||||
now: START,
|
||||
});
|
||||
const generationCreatedAt = START.getTime() + 1_000;
|
||||
await methods.completeAgentTriggerDelivery({
|
||||
id: claim!.id,
|
||||
workerId: 'worker-1',
|
||||
claimToken: 'claim-1',
|
||||
attempt: attempt!,
|
||||
result: {
|
||||
mode: 'continue',
|
||||
status: 'started',
|
||||
conversationId: 'actor-thread',
|
||||
streamId: 'actor-thread',
|
||||
generationCreatedAt,
|
||||
},
|
||||
settledAt: new Date(generationCreatedAt),
|
||||
awaitTerminalHandling: true,
|
||||
handling: {
|
||||
status: 'started',
|
||||
conversationId: 'actor-thread',
|
||||
streamId: 'actor-thread',
|
||||
generationCreatedAt,
|
||||
startedAt: new Date(generationCreatedAt),
|
||||
},
|
||||
});
|
||||
|
||||
await expect(LaneSequence.findById('settling-binding-lane')).resolves.not.toBeNull();
|
||||
await expect(
|
||||
methods.settleAgentTriggerHandlingOutcome({
|
||||
deliveryKey: queued.delivery.deliveryKey,
|
||||
conversationId: 'actor-thread',
|
||||
generationCreatedAt,
|
||||
status: 'completed_no_action',
|
||||
settledAt: new Date(generationCreatedAt + 1_000),
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
await expect(LaneSequence.findById('settling-binding-lane')).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('records retry and success outcomes and expires only successful rows', async () => {
|
||||
const queued = await methods.enqueueAgentTriggerDelivery(enqueueInput());
|
||||
const first = await methods.claimNextAgentTriggerDelivery({
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ export interface EnqueueAgentTriggerDeliveryInput {
|
|||
coalesceKey?: string;
|
||||
coalesceFrom?: Date;
|
||||
coalesceUntil?: Date;
|
||||
awaitTerminalHandling?: boolean;
|
||||
}
|
||||
|
||||
export interface AgentTriggerDeliveryFence {
|
||||
|
|
@ -99,6 +100,7 @@ export interface AgentTriggerDeliveryMethods {
|
|||
result: unknown;
|
||||
settledAt: Date;
|
||||
handling?: AgentTriggerHandlingState;
|
||||
awaitTerminalHandling?: true;
|
||||
},
|
||||
) => Promise<boolean>;
|
||||
settleAgentTriggerHandlingOutcome: (
|
||||
|
|
@ -196,6 +198,9 @@ function toRecord(delivery: IAgentTriggerDelivery): AgentTriggerDeliveryRecord {
|
|||
...(delivery.batchMembersSettledAt != null && {
|
||||
batchMembersSettledAt: delivery.batchMembersSettledAt,
|
||||
}),
|
||||
...(delivery.awaitTerminalHandling != null && {
|
||||
awaitTerminalHandling: delivery.awaitTerminalHandling,
|
||||
}),
|
||||
...(delivery.handling != null && { handling: delivery.handling }),
|
||||
};
|
||||
}
|
||||
|
|
@ -315,6 +320,9 @@ export function createAgentTriggerDeliveryMethods(
|
|||
availableAt: staged.coalesceUntil,
|
||||
},
|
||||
$push: { batchMemberIds: staged._id },
|
||||
...(staged.awaitTerminalHandling === true && {
|
||||
$set: { awaitTerminalHandling: true },
|
||||
}),
|
||||
},
|
||||
{ new: true, sort: { laneSequence: -1, _id: -1 } },
|
||||
)
|
||||
|
|
@ -325,6 +333,20 @@ export function createAgentTriggerDeliveryMethods(
|
|||
.lean<IAgentTriggerDelivery>();
|
||||
}
|
||||
}
|
||||
if (
|
||||
batchRoot?._id != null &&
|
||||
staged.awaitTerminalHandling === true &&
|
||||
batchRoot.awaitTerminalHandling !== true
|
||||
) {
|
||||
const promoted = await Delivery().updateOne(
|
||||
{ _id: batchRoot._id, batchMemberIds: staged._id },
|
||||
{ $set: { awaitTerminalHandling: true } },
|
||||
);
|
||||
if (promoted.matchedCount !== 1) {
|
||||
throw new Error('Failed to promote terminal handling onto the trigger batch root');
|
||||
}
|
||||
batchRoot.awaitTerminalHandling = true;
|
||||
}
|
||||
}
|
||||
const published = await Delivery().updateOne(
|
||||
{ _id: publisherDeliveryId, orderingKey: lane._id, status: 'staging' },
|
||||
|
|
@ -657,7 +679,15 @@ export function createAgentTriggerDeliveryMethods(
|
|||
}
|
||||
const stillRetained = await Delivery().exists({
|
||||
orderingKey,
|
||||
status: { $in: ['staging', 'batched', 'pending', 'leased', 'dead'] },
|
||||
$or: [
|
||||
{ status: { $in: ['staging', 'batched', 'pending', 'leased', 'dead'] } },
|
||||
{
|
||||
status: 'succeeded',
|
||||
batchRootId: { $exists: false },
|
||||
awaitTerminalHandling: true,
|
||||
'handling.status': 'started',
|
||||
},
|
||||
],
|
||||
});
|
||||
if (stillRetained != null) {
|
||||
await LaneSequence().updateOne(
|
||||
|
|
@ -774,18 +804,28 @@ export function createAgentTriggerDeliveryMethods(
|
|||
const earlier = await Delivery()
|
||||
.findOne({
|
||||
orderingKey: delivery.orderingKey,
|
||||
status: { $in: ['pending', 'leased'] },
|
||||
laneSequence: { $lt: delivery.laneSequence },
|
||||
$or: [
|
||||
{ status: { $in: ['pending', 'leased'] } },
|
||||
{
|
||||
status: 'succeeded',
|
||||
batchRootId: { $exists: false },
|
||||
awaitTerminalHandling: true,
|
||||
'handling.status': 'started',
|
||||
},
|
||||
],
|
||||
})
|
||||
.sort({ laneSequence: 1 })
|
||||
.select('availableAt leaseUntil')
|
||||
.lean<Pick<IAgentTriggerDelivery, 'availableAt' | 'leaseUntil'>>();
|
||||
.select('availableAt leaseUntil status handling.status')
|
||||
.lean<Pick<IAgentTriggerDelivery, 'availableAt' | 'leaseUntil' | 'status' | 'handling'>>();
|
||||
if (earlier == null) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
availableAt: earlier.availableAt,
|
||||
...(earlier.leaseUntil != null && { leaseUntil: earlier.leaseUntil }),
|
||||
...(earlier.status === 'succeeded' &&
|
||||
earlier.handling?.status === 'started' && { reason: 'active_handling' as const }),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -831,7 +871,15 @@ export function createAgentTriggerDeliveryMethods(
|
|||
{ 'handling.status': { $exists: false } },
|
||||
],
|
||||
},
|
||||
{ $set: { handling: root.handling } },
|
||||
{
|
||||
$set: {
|
||||
handling: root.handling,
|
||||
...(status !== 'started' &&
|
||||
root.handling.settledAt != null && {
|
||||
expiresAt: new Date(root.handling.settledAt.getTime() + SUCCESS_RETENTION_MS),
|
||||
}),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -843,6 +891,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
| 'batchMemberIds'
|
||||
| 'batchMembersSettledAt'
|
||||
| 'requeueCount'
|
||||
| 'awaitTerminalHandling'
|
||||
| 'handling'
|
||||
>,
|
||||
input: {
|
||||
|
|
@ -861,6 +910,8 @@ export function createAgentTriggerDeliveryMethods(
|
|||
if (memberIds.length > 0) {
|
||||
const error = input.error == null ? undefined : normalizeFailure(input.error);
|
||||
const batchRootRequeueCount = root.requeueCount ?? 0;
|
||||
const awaitsTerminalHandling =
|
||||
root.awaitTerminalHandling === true && root.handling?.status === 'started';
|
||||
const settlement =
|
||||
input.status === 'succeeded'
|
||||
? {
|
||||
|
|
@ -868,7 +919,9 @@ export function createAgentTriggerDeliveryMethods(
|
|||
attempts: input.attempt,
|
||||
result: input.result,
|
||||
settledAt: input.settledAt,
|
||||
expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
|
||||
...(!awaitsTerminalHandling && {
|
||||
expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
|
||||
}),
|
||||
batchRootId: root._id,
|
||||
...(root.handling != null && { handling: root.handling }),
|
||||
}
|
||||
|
|
@ -896,7 +949,9 @@ export function createAgentTriggerDeliveryMethods(
|
|||
leaseBy: 1,
|
||||
leaseUntil: 1,
|
||||
claimToken: 1,
|
||||
...(input.status === 'succeeded' ? { lastError: 1 } : { result: 1, expiresAt: 1 }),
|
||||
...(input.status === 'succeeded'
|
||||
? { lastError: 1, ...(awaitsTerminalHandling && { expiresAt: 1 }) }
|
||||
: { result: 1, expiresAt: 1 }),
|
||||
},
|
||||
$push: {
|
||||
history: {
|
||||
|
|
@ -1033,21 +1088,37 @@ export function createAgentTriggerDeliveryMethods(
|
|||
result: unknown;
|
||||
settledAt: Date;
|
||||
handling?: AgentTriggerHandlingState;
|
||||
awaitTerminalHandling?: true;
|
||||
},
|
||||
): Promise<boolean> {
|
||||
const awaitsTerminalHandling =
|
||||
input.awaitTerminalHandling === true && input.handling?.status === 'started';
|
||||
const completed = await Delivery()
|
||||
.findOneAndUpdate(
|
||||
fence(input),
|
||||
{
|
||||
...fence(input),
|
||||
...(input.awaitTerminalHandling === true
|
||||
? { awaitTerminalHandling: true }
|
||||
: { awaitTerminalHandling: { $ne: true } }),
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
status: 'succeeded',
|
||||
result: input.result,
|
||||
settledAt: input.settledAt,
|
||||
expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
|
||||
...(!awaitsTerminalHandling && {
|
||||
expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
|
||||
}),
|
||||
laneCleanupPendingAt: input.settledAt,
|
||||
...(input.handling != null && { handling: input.handling }),
|
||||
},
|
||||
$unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, lastError: 1 },
|
||||
$unset: {
|
||||
leaseBy: 1,
|
||||
leaseUntil: 1,
|
||||
claimToken: 1,
|
||||
lastError: 1,
|
||||
...(awaitsTerminalHandling && { expiresAt: 1 }),
|
||||
},
|
||||
$push: {
|
||||
history: {
|
||||
$each: [
|
||||
|
|
@ -1065,7 +1136,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
{ new: true },
|
||||
)
|
||||
.select(
|
||||
'_id orderingKey laneCleanupPendingAt batchMemberIds batchMembersSettledAt requeueCount handling',
|
||||
'_id orderingKey laneCleanupPendingAt batchMemberIds batchMembersSettledAt requeueCount awaitTerminalHandling handling',
|
||||
)
|
||||
.lean<
|
||||
Pick<
|
||||
|
|
@ -1076,6 +1147,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
| 'batchMemberIds'
|
||||
| 'batchMembersSettledAt'
|
||||
| 'requeueCount'
|
||||
| 'awaitTerminalHandling'
|
||||
| 'handling'
|
||||
>
|
||||
>();
|
||||
|
|
@ -1110,6 +1182,7 @@ export function createAgentTriggerDeliveryMethods(
|
|||
const terminalHandling = {
|
||||
'handling.status': input.status,
|
||||
'handling.settledAt': input.settledAt,
|
||||
expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
|
||||
...(error != null && { 'handling.error': error }),
|
||||
...(input.action != null && { 'handling.action': input.action }),
|
||||
};
|
||||
|
|
@ -1130,8 +1203,13 @@ export function createAgentTriggerDeliveryMethods(
|
|||
},
|
||||
{ new: true },
|
||||
)
|
||||
.select('_id orderingKey batchMemberIds handling')
|
||||
.lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'batchMemberIds' | 'handling'>>();
|
||||
.select('_id orderingKey batchMemberIds awaitTerminalHandling handling')
|
||||
.lean<
|
||||
Pick<
|
||||
IAgentTriggerDelivery,
|
||||
'_id' | 'orderingKey' | 'batchMemberIds' | 'awaitTerminalHandling' | 'handling'
|
||||
>
|
||||
>();
|
||||
|
||||
let authoritative = terminal;
|
||||
if (authoritative == null) {
|
||||
|
|
@ -1141,8 +1219,13 @@ export function createAgentTriggerDeliveryMethods(
|
|||
'handling.conversationId': input.conversationId,
|
||||
'handling.generationCreatedAt': input.generationCreatedAt,
|
||||
})
|
||||
.select('_id orderingKey batchMemberIds handling')
|
||||
.lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'batchMemberIds' | 'handling'>>();
|
||||
.select('_id orderingKey batchMemberIds awaitTerminalHandling handling')
|
||||
.lean<
|
||||
Pick<
|
||||
IAgentTriggerDelivery,
|
||||
'_id' | 'orderingKey' | 'batchMemberIds' | 'awaitTerminalHandling' | 'handling'
|
||||
>
|
||||
>();
|
||||
const replayed =
|
||||
existing?.handling?.status === input.status &&
|
||||
existing.handling.error === error &&
|
||||
|
|
@ -1155,6 +1238,13 @@ export function createAgentTriggerDeliveryMethods(
|
|||
}
|
||||
|
||||
await propagateBatchHandling(authoritative);
|
||||
if (authoritative.awaitTerminalHandling === true) {
|
||||
await LaneSequence().updateOne(
|
||||
{ _id: authoritative.orderingKey },
|
||||
{ $set: { cleanupRequestedAt: input.settledAt } },
|
||||
);
|
||||
await reclaimLaneIfInactive(authoritative.orderingKey);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
|
|||
batchRootId: { type: Schema.Types.ObjectId, ref: 'AgentTriggerDelivery' },
|
||||
batchRootRequeueCount: { type: Number, min: 0 },
|
||||
batchMembersSettledAt: { type: Date },
|
||||
awaitTerminalHandling: { type: Boolean },
|
||||
handling: { type: handlingSchema },
|
||||
leaseBy: { type: String },
|
||||
leaseUntil: { type: Date },
|
||||
|
|
@ -102,6 +103,14 @@ triggerDeliverySchema.index({ deliveryKey: 1 }, { unique: true });
|
|||
triggerDeliverySchema.index({ status: 1, availableAt: 1, createdAt: 1 });
|
||||
triggerDeliverySchema.index({ status: 1, leaseUntil: 1, createdAt: 1 });
|
||||
triggerDeliverySchema.index({ orderingKey: 1, status: 1, laneSequence: 1 });
|
||||
triggerDeliverySchema.index({
|
||||
orderingKey: 1,
|
||||
awaitTerminalHandling: 1,
|
||||
status: 1,
|
||||
'handling.status': 1,
|
||||
batchRootId: 1,
|
||||
laneSequence: 1,
|
||||
});
|
||||
triggerDeliverySchema.index({ batchRootId: 1 }, { sparse: true });
|
||||
triggerDeliverySchema.index(
|
||||
{ orderingKey: 1, coalesceKey: 1, status: 1, coalesceUntil: 1 },
|
||||
|
|
|
|||
|
|
@ -63,6 +63,9 @@ export interface IAgentTriggerDelivery {
|
|||
batchRootId?: Types.ObjectId;
|
||||
batchRootRequeueCount?: number;
|
||||
batchMembersSettledAt?: Date;
|
||||
/** Keeps this binding lane serialized until its admitted child turn reaches
|
||||
* an authoritative terminal handling outcome. */
|
||||
awaitTerminalHandling?: boolean;
|
||||
handling?: AgentTriggerHandlingState;
|
||||
leaseBy?: string;
|
||||
leaseUntil?: Date;
|
||||
|
|
@ -147,4 +150,5 @@ export interface AgentTriggerDeliveryClaim extends AgentTriggerDeliveryRecord {
|
|||
export interface AgentTriggerOrderingBlock {
|
||||
availableAt: Date;
|
||||
leaseUntil?: Date;
|
||||
reason?: 'active_handling';
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue